From 80be4b573a4abe0489a29dc0d80798067c1d20f1 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Tue, 27 Oct 2026 17:01:38 +0200 Subject: [PATCH 001/112] Python support: transpiler, runtime, stdlib and native benchmark gate Python 3 sources transpile to Java 17 (maven/python-transpiler) and run on maven/python-runtime on every Codename One target. Generators, async, match, dataclasses, the builtins and a stdlib wave are covered by a conformance suite diffed against CPython on the JVM and natively through ParparVM. Constructs a translated program cannot run (eval, exec, compile, __import__, frame hooks) are build errors naming file and line. ParparVM: a frameless method returning a value after several try blocks that end on one label ran a single END_TRY and left the other blocks on the thread's try stack, so a later throw longjmp'd into a returned frame. The folded return paths now unwind the stack (FramelessTryReturnIntegrationTest). Co-Authored-By: Claude Opus 5.5 --- .github/workflows/pr.yml | 26 +- .github/workflows/python-bench.yml | 191 + CLAUDE.md | 9 +- NOTICE | 16 + maven/pom.xml | 3 + maven/python-runtime/pom.xml | 181 + maven/python-runtime/spotbugs-exclude.xml | 93 + .../com/codename1/python/runtime/Args.java | 111 + .../codename1/python/runtime/AttrSite.java | 487 ++ .../python/runtime/BuiltinSubclass.java | 262 + .../codename1/python/runtime/Builtins.java | 1198 ++++ .../python/runtime/Cn1FileSystem.java | 145 + .../com/codename1/python/runtime/Codecs.java | 126 + .../com/codename1/python/runtime/Erf.java | 192 + .../python/runtime/ExceptionGroups.java | 292 + .../codename1/python/runtime/Formatting.java | 612 +++ .../com/codename1/python/runtime/Hashing.java | 127 + .../python/runtime/MathFunctions.java | 627 +++ .../java/com/codename1/python/runtime/Py.java | 2071 +++++++ .../com/codename1/python/runtime/PyBool.java | 47 + .../codename1/python/runtime/PyBuiltin.java | 147 + .../codename1/python/runtime/PyByteArray.java | 397 ++ .../com/codename1/python/runtime/PyBytes.java | 643 +++ .../python/runtime/PyClassMethod.java | 55 + .../codename1/python/runtime/PyComplex.java | 362 ++ .../com/codename1/python/runtime/PyDict.java | 453 ++ .../codename1/python/runtime/PyDictView.java | 120 + .../codename1/python/runtime/PyException.java | 66 + .../python/runtime/PyFileSystem.java | 75 + .../com/codename1/python/runtime/PyFloat.java | 390 ++ .../codename1/python/runtime/PyFunction.java | 478 ++ .../codename1/python/runtime/PyGenerator.java | 462 ++ .../codename1/python/runtime/PyInstance.java | 621 +++ .../com/codename1/python/runtime/PyInt.java | 610 +++ .../com/codename1/python/runtime/PyIter.java | 59 + .../codename1/python/runtime/PyJavaIter.java | 63 + .../python/runtime/PyJavaObject.java | 63 + .../com/codename1/python/runtime/PyList.java | 920 ++++ .../com/codename1/python/runtime/PyMatch.java | 231 + .../python/runtime/PyMemoryView.java | 245 + .../codename1/python/runtime/PyMethod.java | 95 + .../codename1/python/runtime/PyModule.java | 245 + .../com/codename1/python/runtime/PyNone.java | 60 + .../codename1/python/runtime/PyObject.java | 344 ++ .../codename1/python/runtime/PyProperty.java | 108 + .../com/codename1/python/runtime/PyRange.java | 112 + .../codename1/python/runtime/PySeqIter.java | 57 + .../com/codename1/python/runtime/PySet.java | 587 ++ .../com/codename1/python/runtime/PySlice.java | 133 + .../python/runtime/PyStaticMethod.java | 60 + .../com/codename1/python/runtime/PyStr.java | 348 ++ .../com/codename1/python/runtime/PySuper.java | 79 + .../com/codename1/python/runtime/PyTuple.java | 171 + .../com/codename1/python/runtime/PyType.java | 977 ++++ .../codename1/python/runtime/PyWeakRef.java | 119 + .../com/codename1/python/runtime/Shape.java | 83 + .../codename1/python/runtime/StdModules.java | 485 ++ .../codename1/python/runtime/StrFormat.java | 158 + .../codename1/python/runtime/TypeMethods.java | 1160 ++++ .../com/codename1/python/runtime/Typed.java | 350 ++ .../com/codename1/python/runtime/Utf8.java | 133 + maven/python-transpiler/pom.xml | 172 + maven/python-transpiler/spotbugs-exclude.xml | 26 + .../python/transpiler/analyze/Desugar.java | 389 ++ .../transpiler/analyze/ScopeBuilder.java | 469 ++ .../codename1/python/transpiler/api/Main.java | 82 + .../transpiler/api/PythonSyntaxError.java | 46 + .../transpiler/api/PythonTranspiler.java | 328 ++ .../codename1/python/transpiler/ast/Node.java | 685 +++ .../python/transpiler/ast/Scope.java | 160 + .../python/transpiler/codegen/Closed.java | 160 + .../python/transpiler/codegen/DefAssign.java | 387 ++ .../python/transpiler/codegen/Emitter.java | 4849 +++++++++++++++++ .../transpiler/codegen/GenLowering.java | 706 +++ .../python/transpiler/codegen/Infer.java | 1644 ++++++ .../python/transpiler/codegen/Layouts.java | 307 ++ .../python/transpiler/codegen/Linearize.java | 772 +++ .../transpiler/codegen/ListRecycling.java | 307 ++ .../transpiler/codegen/ShadowTypes.java | 119 + .../python/transpiler/codegen/Ty.java | 212 + .../python/transpiler/codegen/TypedEmit.java | 2666 +++++++++ .../python/transpiler/parser/Lexer.java | 533 ++ .../python/transpiler/parser/Parser.java | 1604 ++++++ .../python/transpiler/parser/Token.java | 60 + .../transpiler/parser/UnicodeNames.java | 105 + .../com/codename1/python/stdlib/_strptime.py | 217 + .../com/codename1/python/stdlib/abc.py | 75 + .../com/codename1/python/stdlib/asyncio.py | 920 ++++ .../com/codename1/python/stdlib/base64.py | 308 ++ .../com/codename1/python/stdlib/binascii.py | 164 + .../com/codename1/python/stdlib/bisect.py | 46 + .../com/codename1/python/stdlib/cmath.py | 219 + .../python/stdlib/collections/__init__.py | 633 +++ .../python/stdlib/collections/abc.py | 481 ++ .../com/codename1/python/stdlib/contextlib.py | 330 ++ .../com/codename1/python/stdlib/copy.py | 144 + .../com/codename1/python/stdlib/csv.py | 692 +++ .../codename1/python/stdlib/dataclasses.py | 511 ++ .../com/codename1/python/stdlib/datetime.py | 1258 +++++ .../com/codename1/python/stdlib/decimal.py | 2163 ++++++++ .../com/codename1/python/stdlib/enum.py | 521 ++ .../com/codename1/python/stdlib/fnmatch.py | 111 + .../com/codename1/python/stdlib/fractions.py | 460 ++ .../com/codename1/python/stdlib/functools.py | 250 + .../com/codename1/python/stdlib/glob.py | 90 + .../com/codename1/python/stdlib/hashlib.py | 358 ++ .../com/codename1/python/stdlib/heapq.py | 107 + .../codename1/python/stdlib/html/__init__.py | 73 + .../codename1/python/stdlib/html/entities.py | 2503 +++++++++ .../com/codename1/python/stdlib/io.py | 730 +++ .../com/codename1/python/stdlib/itertools.py | 288 + .../com/codename1/python/stdlib/json.py | 397 ++ .../python/stdlib/logging/__init__.py | 1043 ++++ .../com/codename1/python/stdlib/numbers.py | 257 + .../com/codename1/python/stdlib/operator.py | 122 + .../codename1/python/stdlib/os/__init__.py | 344 ++ .../com/codename1/python/stdlib/os/path.py | 245 + .../com/codename1/python/stdlib/pathlib.py | 447 ++ .../com/codename1/python/stdlib/pprint.py | 552 ++ .../com/codename1/python/stdlib/queue.py | 154 + .../com/codename1/python/stdlib/random.py | 486 ++ .../codename1/python/stdlib/re/__init__.py | 455 ++ .../com/codename1/python/stdlib/re/_engine.py | 314 ++ .../com/codename1/python/stdlib/re/_parser.py | 724 +++ .../com/codename1/python/stdlib/secrets.py | 58 + .../com/codename1/python/stdlib/shlex.py | 229 + .../com/codename1/python/stdlib/statistics.py | 651 +++ .../com/codename1/python/stdlib/string.py | 280 + .../com/codename1/python/stdlib/struct.py | 290 + .../com/codename1/python/stdlib/textwrap.py | 217 + .../com/codename1/python/stdlib/time.py | 327 ++ .../com/codename1/python/stdlib/types.py | 140 + .../com/codename1/python/stdlib/typing.py | 663 +++ .../python/stdlib/urllib/__init__.py | 4 + .../codename1/python/stdlib/urllib/parse.py | 564 ++ .../com/codename1/python/stdlib/uuid.py | 375 ++ .../com/codename1/python/stdlib/warnings.py | 284 + .../com/codename1/python/stdlib/weakref.py | 495 ++ .../python/transpiler/ConformanceTest.java | 172 + .../python/transpiler/JvmFileSystem.java | 118 + .../python/transpiler/UnsupportedTest.java | 90 + .../src/test/resources/conformance/async_.out | 12 + .../src/test/resources/conformance/async_.py | 76 + .../test/resources/conformance/async_gen.out | 17 + .../test/resources/conformance/async_gen.py | 55 + .../src/test/resources/conformance/attrs_.out | 15 + .../src/test/resources/conformance/attrs_.py | 80 + .../test/resources/conformance/base64_.out | 23 + .../src/test/resources/conformance/base64_.py | 44 + .../src/test/resources/conformance/basics.out | 20 + .../src/test/resources/conformance/basics.py | 28 + .../conformance/builtin_subclass.out | 12 + .../resources/conformance/builtin_subclass.py | 103 + .../test/resources/conformance/bytearray_.out | 19 + .../test/resources/conformance/bytearray_.py | 40 + .../test/resources/conformance/classes.out | 13 + .../src/test/resources/conformance/classes.py | 110 + .../resources/conformance/collections_.out | 29 + .../resources/conformance/collections_.py | 155 + .../test/resources/conformance/complex_.out | 15 + .../test/resources/conformance/complex_.py | 17 + .../test/resources/conformance/control.out | 20 + .../src/test/resources/conformance/control.py | 104 + .../src/test/resources/conformance/csv_.out | 26 + .../src/test/resources/conformance/csv_.py | 67 + .../resources/conformance/dataclasses_.out | 17 + .../resources/conformance/dataclasses_.py | 138 + .../test/resources/conformance/datetime_.out | 43 + .../test/resources/conformance/datetime_.py | 93 + .../test/resources/conformance/decimal_.out | 56 + .../test/resources/conformance/decimal_.py | 71 + .../src/test/resources/conformance/enum_.out | 17 + .../src/test/resources/conformance/enum_.py | 79 + .../test/resources/conformance/errors_.out | 72 + .../src/test/resources/conformance/errors_.py | 177 + .../resources/conformance/except_star.out | 54 + .../test/resources/conformance/except_star.py | 103 + .../test/resources/conformance/fractions_.out | 18 + .../test/resources/conformance/fractions_.py | 27 + .../test/resources/conformance/generators.out | 48 + .../test/resources/conformance/generators.py | 222 + .../test/resources/conformance/imports_.out | 17 + .../resources/conformance/imports_/main.py | 34 + .../conformance/imports_/pkg/__init__.py | 5 + .../resources/conformance/imports_/pkg/a.py | 6 + .../conformance/imports_/pkg/allstar.py | 3 + .../conformance/imports_/pkg/star.py | 3 + .../conformance/imports_/pkg/sub/__init__.py | 1 + .../conformance/imports_/pkg/sub/b.py | 7 + .../test/resources/conformance/itertools_.out | 33 + .../test/resources/conformance/itertools_.py | 125 + .../src/test/resources/conformance/json_.out | 56 + .../src/test/resources/conformance/json_.py | 71 + .../test/resources/conformance/logging_.out | 23 + .../test/resources/conformance/logging_.py | 104 + .../src/test/resources/conformance/match_.out | 25 + .../src/test/resources/conformance/match_.py | 91 + .../src/test/resources/conformance/math2_.out | 35 + .../src/test/resources/conformance/math2_.py | 44 + .../resources/conformance/metaclasses.out | 12 + .../test/resources/conformance/metaclasses.py | 157 + .../test/resources/conformance/numbers.out | 87 + .../src/test/resources/conformance/numbers.py | 354 ++ .../src/test/resources/conformance/open_.out | 32 + .../src/test/resources/conformance/open_.py | 86 + .../src/test/resources/conformance/os_.out | 37 + .../src/test/resources/conformance/os_.py | 74 + .../test/resources/conformance/pathlib_.out | 33 + .../test/resources/conformance/pathlib_.py | 67 + .../test/resources/conformance/pprint_.out | 354 ++ .../src/test/resources/conformance/pprint_.py | 46 + .../resources/conformance/random_hashlib.out | 55 + .../resources/conformance/random_hashlib.py | 51 + .../src/test/resources/conformance/re_.out | 153 + .../src/test/resources/conformance/re_.py | 72 + .../src/test/resources/conformance/scopes.out | 14 + .../src/test/resources/conformance/scopes.py | 119 + .../resources/conformance/statistics_.out | 34 + .../test/resources/conformance/statistics_.py | 32 + .../conformance/stdlib_collections.out | 31 + .../conformance/stdlib_collections.py | 143 + .../test/resources/conformance/strings.out | 26 + .../src/test/resources/conformance/strings.py | 55 + .../test/resources/conformance/struct_.out | 34 + .../src/test/resources/conformance/struct_.py | 37 + .../test/resources/conformance/textproc_.out | 44 + .../test/resources/conformance/textproc_.py | 82 + .../test/resources/conformance/textwrap_.out | 24 + .../test/resources/conformance/textwrap_.py | 26 + .../test/resources/conformance/typing_abc.out | 36 + .../test/resources/conformance/typing_abc.py | 189 + .../src/test/resources/conformance/uuid_.out | 24 + .../src/test/resources/conformance/uuid_.py | 36 + .../test/resources/conformance/weakref_.out | 23 + .../test/resources/conformance/weakref_.py | 60 + scripts/check-cast-semantics.sh | 2 + scripts/copyright-header-exclusions.txt | 1 + scripts/python-bench/README.md | 60 + scripts/python-bench/build_parparvm.sh | 75 + scripts/python-bench/conformance_parparvm.sh | 27 + .../com/codename1/io/FileSystemStorage.java | 112 + scripts/python-bench/kernels/_harness.py | 35 + scripts/python-bench/kernels/dicts_strings.py | 27 + scripts/python-bench/kernels/fannkuch.py | 39 + scripts/python-bench/kernels/nbody.py | 87 + scripts/python-bench/kernels/richards.py | 217 + scripts/python-bench/kernels/spectral_norm.py | 47 + scripts/python-bench/parparvm_table.py | 122 + scripts/python-bench/run_bench.py | 242 + vm/ByteCodeTranslator/src/cn1_globals.h | 6 +- .../tools/translator/BytecodeMethod.java | 14 +- .../translator/JavascriptNativeRegistry.java | 1 + .../src/javascript/parparvm_runtime.js | 9 + vm/ByteCodeTranslator/src/nativeMethods.m | 41 + vm/JavaAPI/src/java/lang/String.java | 33 + .../src/java/util/stream/DoublePipeline.java | 8 +- .../src/java/util/stream/IntPipeline.java | 8 +- .../src/java/util/stream/LongPipeline.java | 8 +- .../FramelessTryReturnIntegrationTest.java | 132 + 259 files changed, 67169 insertions(+), 18 deletions(-) create mode 100644 .github/workflows/python-bench.yml create mode 100644 maven/python-runtime/pom.xml create mode 100644 maven/python-runtime/spotbugs-exclude.xml create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/Args.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/AttrSite.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/BuiltinSubclass.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/Builtins.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/Cn1FileSystem.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/Codecs.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/Erf.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/ExceptionGroups.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/Formatting.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/Hashing.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/MathFunctions.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/Py.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/PyBool.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/PyBuiltin.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/PyByteArray.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/PyBytes.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/PyClassMethod.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/PyComplex.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/PyDict.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/PyDictView.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/PyException.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/PyFileSystem.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/PyFloat.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/PyFunction.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/PyGenerator.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/PyInstance.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/PyInt.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/PyIter.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/PyJavaIter.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/PyJavaObject.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/PyList.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/PyMatch.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/PyMemoryView.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/PyMethod.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/PyModule.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/PyNone.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/PyObject.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/PyProperty.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/PyRange.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/PySeqIter.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/PySet.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/PySlice.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/PyStaticMethod.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/PyStr.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/PySuper.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/PyTuple.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/PyType.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/PyWeakRef.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/Shape.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/StrFormat.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/TypeMethods.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/Typed.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/Utf8.java create mode 100644 maven/python-transpiler/pom.xml create mode 100644 maven/python-transpiler/spotbugs-exclude.xml create mode 100644 maven/python-transpiler/src/main/java/com/codename1/python/transpiler/analyze/Desugar.java create mode 100644 maven/python-transpiler/src/main/java/com/codename1/python/transpiler/analyze/ScopeBuilder.java create mode 100644 maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/Main.java create mode 100644 maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonSyntaxError.java create mode 100644 maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java create mode 100644 maven/python-transpiler/src/main/java/com/codename1/python/transpiler/ast/Node.java create mode 100644 maven/python-transpiler/src/main/java/com/codename1/python/transpiler/ast/Scope.java create mode 100644 maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Closed.java create mode 100644 maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/DefAssign.java create mode 100644 maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Emitter.java create mode 100644 maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/GenLowering.java create mode 100644 maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Infer.java create mode 100644 maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Layouts.java create mode 100644 maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Linearize.java create mode 100644 maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/ListRecycling.java create mode 100644 maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/ShadowTypes.java create mode 100644 maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Ty.java create mode 100644 maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/TypedEmit.java create mode 100644 maven/python-transpiler/src/main/java/com/codename1/python/transpiler/parser/Lexer.java create mode 100644 maven/python-transpiler/src/main/java/com/codename1/python/transpiler/parser/Parser.java create mode 100644 maven/python-transpiler/src/main/java/com/codename1/python/transpiler/parser/Token.java create mode 100644 maven/python-transpiler/src/main/java/com/codename1/python/transpiler/parser/UnicodeNames.java create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/_strptime.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/abc.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/asyncio.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/base64.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/binascii.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/bisect.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/cmath.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/collections/__init__.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/collections/abc.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/contextlib.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/copy.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/csv.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/dataclasses.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/datetime.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/decimal.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/enum.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/fnmatch.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/fractions.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/functools.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/glob.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/hashlib.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/heapq.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/html/__init__.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/html/entities.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/io.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/itertools.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/json.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/logging/__init__.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/numbers.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/operator.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/os/__init__.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/os/path.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/pathlib.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/pprint.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/queue.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/random.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/re/__init__.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/re/_engine.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/re/_parser.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/secrets.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/shlex.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/statistics.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/string.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/struct.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/textwrap.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/time.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/types.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/typing.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/urllib/__init__.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/urllib/parse.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/uuid.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/warnings.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/weakref.py create mode 100644 maven/python-transpiler/src/test/java/com/codename1/python/transpiler/ConformanceTest.java create mode 100644 maven/python-transpiler/src/test/java/com/codename1/python/transpiler/JvmFileSystem.java create mode 100644 maven/python-transpiler/src/test/java/com/codename1/python/transpiler/UnsupportedTest.java create mode 100644 maven/python-transpiler/src/test/resources/conformance/async_.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/async_.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/async_gen.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/async_gen.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/attrs_.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/attrs_.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/base64_.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/base64_.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/basics.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/basics.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/builtin_subclass.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/builtin_subclass.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/bytearray_.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/bytearray_.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/classes.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/classes.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/collections_.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/collections_.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/complex_.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/complex_.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/control.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/control.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/csv_.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/csv_.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/dataclasses_.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/dataclasses_.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/datetime_.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/datetime_.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/decimal_.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/decimal_.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/enum_.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/enum_.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/errors_.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/errors_.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/except_star.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/except_star.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/fractions_.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/fractions_.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/generators.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/generators.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/imports_.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/imports_/main.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/imports_/pkg/__init__.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/imports_/pkg/a.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/imports_/pkg/allstar.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/imports_/pkg/star.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/imports_/pkg/sub/__init__.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/imports_/pkg/sub/b.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/itertools_.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/itertools_.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/json_.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/json_.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/logging_.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/logging_.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/match_.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/match_.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/math2_.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/math2_.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/metaclasses.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/metaclasses.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/numbers.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/numbers.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/open_.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/open_.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/os_.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/os_.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/pathlib_.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/pathlib_.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/pprint_.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/pprint_.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/random_hashlib.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/random_hashlib.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/re_.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/re_.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/scopes.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/scopes.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/statistics_.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/statistics_.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/stdlib_collections.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/stdlib_collections.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/strings.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/strings.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/struct_.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/struct_.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/textproc_.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/textproc_.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/textwrap_.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/textwrap_.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/typing_abc.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/typing_abc.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/uuid_.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/uuid_.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/weakref_.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/weakref_.py create mode 100644 scripts/python-bench/README.md create mode 100755 scripts/python-bench/build_parparvm.sh create mode 100755 scripts/python-bench/conformance_parparvm.sh create mode 100644 scripts/python-bench/hostfs/com/codename1/io/FileSystemStorage.java create mode 100644 scripts/python-bench/kernels/_harness.py create mode 100644 scripts/python-bench/kernels/dicts_strings.py create mode 100644 scripts/python-bench/kernels/fannkuch.py create mode 100644 scripts/python-bench/kernels/nbody.py create mode 100644 scripts/python-bench/kernels/richards.py create mode 100644 scripts/python-bench/kernels/spectral_norm.py create mode 100755 scripts/python-bench/parparvm_table.py create mode 100644 scripts/python-bench/run_bench.py create mode 100644 vm/tests/src/test/java/com/codename1/tools/translator/FramelessTryReturnIntegrationTest.java diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 194431d2822..2e761511d27 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -402,7 +402,15 @@ jobs: -Dcn1.binaries="${CN1_BINARIES}" \ -Pcompile-android \ -DbackendAnalysis \ - -pl android,ios,project-model,build-engine,codenameone-maven-plugin,codenameone-gradle-plugin,backend,backend-test,backend-parparvm,javac,android-res-compiler,android-compat,swing-compat,javafx-compat,fxml-compiler,desktop-compat-runtime -am verify + -pl android,ios,project-model,build-engine,codenameone-maven-plugin,codenameone-gradle-plugin,backend,backend-test,backend-parparvm,javac,android-res-compiler,android-compat,swing-compat,javafx-compat,fxml-compiler,desktop-compat-runtime,python-transpiler -am verify + # The Python runtime compiles at release 17, so it cannot join the JDK 8 + # reactor above; it runs against the core that reactor installed. It ships + # inside applications, so it is held to the same zero-findings gate. + - name: Run SpotBugs for the Python runtime + if: ${{ matrix.java-version == 8 }} + working-directory: maven + run: | + JAVA_HOME="${JAVA_HOME_17}" mvn -B -DskipTests=true -Pflutter-modules -pl python-runtime verify - name: Type-check the SQLCipher-backed database package if: ${{ matrix.java-version == 8 }} env: @@ -620,7 +628,7 @@ jobs: - name: Generate static analysis HTML summaries if: ${{ !cancelled() && matrix.java-version == 8 }} env: - QUALITY_REPORT_TARGET_DIRS: maven/core-unittests/target:maven/android/target:maven/ios/target:vm/ByteCodeTranslator/target:maven/codenameone-maven-plugin/target:maven/build-engine/target:maven/project-model/target:maven/codenameone-gradle-plugin/target:maven/build-hint-catalog/target:maven/build-hint-tools/target:maven/backend/target:maven/backend-test/target:maven/backend-parparvm/target:maven/javac/target:maven/android-res-compiler/target:maven/android-compat/target:maven/swing-compat/target:maven/javafx-compat/target:maven/fxml-compiler/target:maven/desktop-compat-runtime/target + QUALITY_REPORT_TARGET_DIRS: maven/core-unittests/target:maven/android/target:maven/ios/target:vm/ByteCodeTranslator/target:maven/codenameone-maven-plugin/target:maven/build-engine/target:maven/project-model/target:maven/codenameone-gradle-plugin/target:maven/build-hint-catalog/target:maven/build-hint-tools/target:maven/backend/target:maven/backend-test/target:maven/backend-parparvm/target:maven/javac/target:maven/android-res-compiler/target:maven/android-compat/target:maven/swing-compat/target:maven/javafx-compat/target:maven/fxml-compiler/target:maven/desktop-compat-runtime/target:maven/python-transpiler/target:maven/python-runtime/target QUALITY_REPORT_SERVER_URL: ${{ github.server_url }} QUALITY_REPORT_REPOSITORY: ${{ github.repository }} QUALITY_REPORT_REF: ${{ github.event.pull_request.head.sha || github.sha }} @@ -776,16 +784,16 @@ jobs: - name: Generate quality report summary if: ${{ !cancelled() && matrix.java-version == 8 }} env: - QUALITY_REPORT_TARGET_DIRS: maven/core-unittests/target:maven/android/target:maven/ios/target:vm/ByteCodeTranslator/target:maven/codenameone-maven-plugin/target:maven/build-engine/target:maven/project-model/target:maven/codenameone-gradle-plugin/target:maven/build-hint-catalog/target:maven/build-hint-tools/target:maven/backend/target:maven/backend-test/target:maven/backend-parparvm/target:maven/javac/target:maven/android-res-compiler/target:maven/android-compat/target:maven/swing-compat/target:maven/javafx-compat/target:maven/fxml-compiler/target:maven/desktop-compat-runtime/target + QUALITY_REPORT_TARGET_DIRS: maven/core-unittests/target:maven/android/target:maven/ios/target:vm/ByteCodeTranslator/target:maven/codenameone-maven-plugin/target:maven/build-engine/target:maven/project-model/target:maven/codenameone-gradle-plugin/target:maven/build-hint-catalog/target:maven/build-hint-tools/target:maven/backend/target:maven/backend-test/target:maven/backend-parparvm/target:maven/javac/target:maven/android-res-compiler/target:maven/android-compat/target:maven/swing-compat/target:maven/javafx-compat/target:maven/fxml-compiler/target:maven/desktop-compat-runtime/target:maven/python-transpiler/target:maven/python-runtime/target # Every project listed here must produce a SpotBugs report and that # report must be empty. A missing report fails the gate too, so a # SpotBugs run that silently stops running is not mistaken for clean. - QUALITY_REPORT_REQUIRED_SPOTBUGS: core-unittests:android:ios:ByteCodeTranslator:codenameone-maven-plugin:build-engine:project-model:codenameone-gradle-plugin:build-hint-catalog:build-hint-tools:backend:backend-test:javac:android-res-compiler:android-compat:swing-compat:javafx-compat:fxml-compiler:desktop-compat-runtime - # PMD and Checkstyle run on the core and on the backend runtime, and each - # must produce its report: without one, the forbidden-rule gate below it - # has nothing to read and passes. - QUALITY_REPORT_REQUIRED_PMD: core-unittests:backend:backend-test:backend-parparvm - QUALITY_REPORT_REQUIRED_CHECKSTYLE: core-unittests:backend:backend-test + QUALITY_REPORT_REQUIRED_SPOTBUGS: core-unittests:android:ios:ByteCodeTranslator:codenameone-maven-plugin:build-engine:project-model:codenameone-gradle-plugin:build-hint-catalog:build-hint-tools:backend:backend-test:javac:android-res-compiler:android-compat:swing-compat:javafx-compat:fxml-compiler:desktop-compat-runtime:python-transpiler:python-runtime + # PMD and Checkstyle run on the core, the backend runtime and the Python + # transpiler and runtime, and each must produce its report: without one, + # the forbidden-rule gate below it has nothing to read and passes. + QUALITY_REPORT_REQUIRED_PMD: core-unittests:backend:backend-test:backend-parparvm:python-transpiler:python-runtime + QUALITY_REPORT_REQUIRED_CHECKSTYLE: core-unittests:backend:backend-test:python-transpiler:python-runtime QUALITY_REPORT_SERVER_URL: ${{ github.server_url }} QUALITY_REPORT_REPOSITORY: ${{ github.repository }} QUALITY_REPORT_REF: ${{ github.event.pull_request.head.sha || github.sha }} diff --git a/.github/workflows/python-bench.yml b/.github/workflows/python-bench.yml new file mode 100644 index 00000000000..27dfe48e43e --- /dev/null +++ b/.github/workflows/python-bench.yml @@ -0,0 +1,191 @@ +name: Python benchmark + +# Runs the kernels in scripts/python-bench/kernels twice -- by CPython and +# transpiled to Java by Codename One -- checks that both compute the same +# result, and gates the ours/CPython ratio of compute time, process wall time +# and peak memory against scripts/python-bench/baseline/ratios.json. +# +# Two arms: the JVM (the simulator and desktop targets) and ParparVM (the +# native translation iOS and the native desktop targets ship), each gated on +# its own baseline. The ParparVM arm also runs the conformance programs as +# native executables, where a bad cast is a crash rather than an exception. +# Android and JavaScript join as the Python support grows. +# +# It also runs the transpiler's conformance tests, which need the Java 17 +# runtime and so cannot run in pr.yml's JDK 8 reactor. + +on: + pull_request: + paths: + - 'maven/python-transpiler/**' + - 'maven/python-runtime/**' + - 'scripts/python-bench/**' + - '.github/workflows/python-bench.yml' + # The ParparVM arm translates and compiles with these. + - 'vm/ByteCodeTranslator/**' + - 'vm/JavaAPI/**' + - 'vm/selfhost/compile-dist.sh' + workflow_dispatch: + +permissions: + contents: read + +env: + # Pinned, like flutter-bench's FLUTTER_REF: the ratios are against this + # interpreter, and moving it is a deliberate re-baseline. + CPYTHON_VERSION: '3.13.9' + +jobs: + # JDK 8 for the core (the container bakes both JDKs and cn1-binaries, as + # flutter-bench's framework job explains), JDK 17 for the runtime. + framework: + name: framework + runs-on: ubuntu-latest + container: ghcr.io/codenameone/codenameone/pr-ci-container:latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + + - name: Link cn1-binaries + run: ln -s /opt/cn1-binaries ../cn1-binaries + + - name: Build the core and the transpiler on JDK 8 + run: | + cd maven + JAVA_HOME="$JAVA_HOME_8" bash ../scripts/ci/retry.sh mvn -B -q install -DskipTests \ + -Dmaven.javadoc.skip=true -Dmaven.source.skip=true \ + -Dmaven.repo.local="$GITHUB_WORKSPACE/m2" \ + -pl core,python-transpiler -am + + # Without -am: core was installed above, and re-resolving it here would + # rebuild it on the wrong JDK. + - name: Build the runtime and run the conformance tests on JDK 17 + run: | + cd maven + JAVA_HOME="$JAVA_HOME_17" bash ../scripts/ci/retry.sh mvn -B install \ + -Dmaven.javadoc.skip=true -Dmaven.source.skip=true \ + -Dmaven.repo.local="$GITHUB_WORKSPACE/m2" \ + -pl python-runtime,python-transpiler + + - name: Build the ParparVM translator and its class library on JDK 8 + run: | + JAVA_HOME="$JAVA_HOME_8" bash scripts/ci/retry.sh mvn -B -q install -DskipTests \ + -Dmaven.javadoc.skip=true -Dmaven.source.skip=true \ + -Dmaven.repo.local="$GITHUB_WORKSPACE/m2" \ + -f vm/pom.xml -pl JavaAPI,ByteCodeTranslator -am + + - name: Pack the repository + run: tar -C "$GITHUB_WORKSPACE" -czf framework-m2.tgz m2 + + - name: Upload the repository + uses: actions/upload-artifact@v4 + with: + name: python-bench-framework + path: framework-m2.tgz + retention-days: 1 + + measure: + name: jvm + needs: framework + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + + - name: Set up JDK 17 + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 17 + + - name: Set up CPython + uses: actions/setup-python@v5 + with: + python-version: ${{ env.CPYTHON_VERSION }} + + - name: Download the framework + uses: actions/download-artifact@v4 + with: + name: python-bench-framework + + - name: Unpack the framework + run: tar -xzf framework-m2.tgz + + - name: Measure and gate + run: | + repo="$GITHUB_WORKSPACE/m2/com/codenameone" + python3 scripts/python-bench/run_bench.py \ + --java-home "$JAVA_HOME" \ + --runtime-jar "$repo/codenameone-python-runtime/8.0-SNAPSHOT/codenameone-python-runtime-8.0-SNAPSHOT.jar" \ + --core-jar "$repo/codenameone-core/8.0-SNAPSHOT/codenameone-core-8.0-SNAPSHOT.jar" \ + --transpiler-cp "$repo/codenameone-python-transpiler/8.0-SNAPSHOT/codenameone-python-transpiler-8.0-SNAPSHOT.jar" \ + --json python-bench.json --check | tee -a "$GITHUB_STEP_SUMMARY" + + - name: Upload the report + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v4 + with: + name: python-bench-report + path: python-bench.json + + parparvm: + name: parparvm + needs: framework + runs-on: ubuntu-24.04 + timeout-minutes: 60 + env: + PYBENCH_M2: ${{ github.workspace }}/m2 + steps: + - uses: actions/checkout@v4 + + - name: Install clang + run: | + bash scripts/ci/apt-get-update.sh + sudo apt-get install -y clang time + + # The translator runs on JDK 8, the generated Java compiles on JDK 17. + - name: Set up JDK 8 + uses: actions/setup-java@v4 + with: + distribution: zulu + java-version: '8' + + - name: Remember JDK 8 + run: echo "JDK_8_HOME=$JAVA_HOME" >> "$GITHUB_ENV" + + - name: Set up JDK 17 + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 17 + + - name: Remember JDK 17 + run: echo "JAVA17_HOME=$JAVA_HOME" >> "$GITHUB_ENV" + + - name: Set up CPython + uses: actions/setup-python@v5 + with: + python-version: ${{ env.CPYTHON_VERSION }} + + - name: Download the framework + uses: actions/download-artifact@v4 + with: + name: python-bench-framework + + - name: Unpack the framework + run: tar -xzf framework-m2.tgz + + - name: Conformance, natively + run: scripts/python-bench/conformance_parparvm.sh + + - name: Measure and gate + run: | + python3 scripts/python-bench/parparvm_table.py --json python-bench-parparvm.json --check \ + | tee -a "$GITHUB_STEP_SUMMARY" + + - name: Upload the report + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v4 + with: + name: python-bench-parparvm-report + path: python-bench-parparvm.json diff --git a/CLAUDE.md b/CLAUDE.md index d5d36659cb7..af9bedb3494 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -113,8 +113,9 @@ PR CI (`.github/workflows/pr.yml`, Java 8 leg) runs SpotBugs over `codenameone-maven-plugin`, `build-engine` (the Maven-free build logic both build plugins share), `project-model` (the project-layout API), `android-res-compiler` and `android-compat` (the Android compatibility -runtime), `codenameone-gradle-plugin` and `javac` (the in-tree Java compiler, -`vm/JavaCompiler`), then enforces the result in +runtime), `codenameone-gradle-plugin`, `javac` (the in-tree Java compiler, +`vm/JavaCompiler`) and `python-transpiler`, plus `python-runtime` in a step of +its own on JDK 17 (it compiles at release 17), then enforces the result in `.github/scripts/generate-quality-report.py`. `cil-translator` and `unity-compat` (the Unity compatibility translator and runtime) are held to the same script with SpotBugs, PMD and Checkstyle all three, but in @@ -133,6 +134,7 @@ any of them cannot skip it. `maven/build-engine/`, `maven/project-model/`, `maven/android-res-compiler/`, `maven/android-compat/`, `maven/codenameone-gradle-plugin/`, `maven/javac/`, + `maven/python-transpiler/`, `maven/python-runtime/`, `maven/cil-translator/`, `maven/unity-compat/`), scoped to the class or method it applies to and with a comment explaining why. Keep the generated report at zero rather than tolerating known noise. @@ -374,7 +376,8 @@ The scope is what a translation actually sees -- `maven/core`, `maven/ios`, `vm/JavaAPI` and `maven/android-compat`, whose runtime an application with Android sources ships relocated inside its own classes, and `maven/swing-compat` and `maven/javafx-compat`, which are relocated into an -application with desktop sources the same way. **The Android port is not covered**, and adding it back would be a +application with desktop sources the same way. `maven/python-runtime` is in +scope too: an application with Python sources ships it. **The Android port is not covered**, and adding it back would be a mistake: ART implements `CHECKCAST` to spec, so a `catch (Throwable)` around a `(NotificationManager) getSystemService(...)` there is live, correct code, and `Ports/Android` is never translated. That is the same reason `Ports/CLDC11` is diff --git a/NOTICE b/NOTICE index afbcc0dc85d..fd4e0d6641b 100644 --- a/NOTICE +++ b/NOTICE @@ -228,3 +228,19 @@ file is distributed under the GNU General Public License version 2 with the keeps OpenJDK's notice: Copyright (c) 1996, 2013, Oracle and/or its affiliates. All rights reserved. + +================================================================================ +FDLIBM erf and erfc +================================================================================ + +The class com.codename1.python.runtime.Erf, in maven/python-runtime, is a Java +rewrite of s_erf.c from Sun Microsystems' FDLIBM, the C library the platform +math libraries derive their erf and erfc from, so that Python's math.erf and +math.erfc agree with CPython's. It retains FDLIBM's notice: + + Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + + Developed at SunPro, a Sun Microsystems, Inc. business. + Permission to use, copy, modify, and distribute this + software is freely granted, provided that this notice + is preserved. diff --git a/maven/pom.xml b/maven/pom.xml index 0e9749d93bf..72271629408 100644 --- a/maven/pom.xml +++ b/maven/pom.xml @@ -112,6 +112,8 @@ exactly the JDK the plugin is built with. --> dart-transpiler + + python-transpiler sqlite-jdbc backend @@ -712,6 +714,7 @@ dart-runtime flutter-runtime + python-runtime diff --git a/maven/python-runtime/pom.xml b/maven/python-runtime/pom.xml new file mode 100644 index 00000000000..021ae973cc0 --- /dev/null +++ b/maven/python-runtime/pom.xml @@ -0,0 +1,181 @@ + + + + + com.codenameone + codenameone + 8.0-SNAPSHOT + + 4.0.0 + + codenameone-python-runtime + 8.0-SNAPSHOT + jar + codenameone-python-runtime + + Java implementation of the Python object model and builtins used by Python + code transpiled to Java by the Codename One Python transpiler. App-grade + code: depends only on codenameone-core and compiles to Java 17 + bytecode, which all Codename One targets accept. + + + + UTF-8 + 17 + 17 + + + + + + maven-compiler-plugin + + + 17 + 17 + 17 + + + + com.github.spotbugs + spotbugs-maven-plugin + 4.7.3.4 + + Max + Low + false + true + ${project.build.directory} + ${project.basedir}/spotbugs-exclude.xml + + + + spotbugs + verify + + spotbugs + + + + + + + org.apache.maven.plugins + maven-pmd-plugin + 3.21.0 + + false + false + 17 + + ${project.basedir}/../core-unittests/pmd.xml + + + + + pmd + verify + + pmd + + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + 3.3.0 + + ${project.basedir}/../core-unittests/checkstyle.xml + ${project.build.sourceEncoding} + false + false + false + false + ${project.build.directory}/checkstyle-result.xml + xml + + + + validate + verify + + checkstyle + + + + checkstyle-zero-errors + verify + + check + + + true + error + 0 + true + + + + + + + + + + + java17-fork + + + env.JAVA17_HOME + + + + + + maven-compiler-plugin + + + 17 + 17 + 17 + true + ${env.JAVA17_HOME}/bin/javac + + + + maven-surefire-plugin + + ${env.JAVA17_HOME}/bin/java + + + + + + + + + + com.codenameone + codenameone-core + provided + + + org.junit.jupiter + junit-jupiter + test + + + diff --git a/maven/python-runtime/spotbugs-exclude.xml b/maven/python-runtime/spotbugs-exclude.xml new file mode 100644 index 00000000000..d78931a00c0 --- /dev/null +++ b/maven/python-runtime/spotbugs-exclude.xml @@ -0,0 +1,93 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Args.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Args.java new file mode 100644 index 00000000000..629b0a4d6f1 --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Args.java @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +/// Argument access for builtins implemented in Java: positional arguments by +/// index and keyword arguments by name, with CPython's error messages. +final class Args { + final String fn; + final Object[] args; + final String[] kwNames; + final int npos; + + Args(String fn, Object[] args, String[] kwNames) { + this.fn = fn; + this.args = args; + this.kwNames = kwNames; + this.npos = args.length - kwNames.length; + } + + /// Binds `args` to `names` positionally or by keyword; an unbound name + /// takes its entry of `defaults`, and a null default makes it required. + static Object[] bind(String fn, Object[] args, String[] kwNames, String[] names, Object[] defaults) { + Args a = new Args(fn, args, kwNames).check(names.length, names); + Object[] r = new Object[names.length]; + for (int i = 0; i < names.length; i++) { + r[i] = defaults[i] == null ? a.required(i, names[i]) : a.get(i, names[i], defaults[i]); + } + return r; + } + + /// The argument at position `i`, else keyword `name`, else `def`. + Object get(int i, String name, Object def) { + if (i >= 0 && i < npos) { + return args[i]; + } + Object k = kw(name); + return k != null ? k : def; + } + + Object kw(String name) { + if (name == null) { + return null; + } + for (int i = 0; i < kwNames.length; i++) { + if (kwNames[i].equals(name)) { + return args[npos + i]; + } + } + return null; + } + + Object required(int i, String name) { + Object v = get(i, name, null); + if (v == null) { + throw Py.typeError(fn + "() missing required argument '" + name + "' (pos " + (i + 1) + ")"); + } + return v; + } + + /// Rejects keywords not in `allowed` and more than `maxPos` positionals. + Args check(int maxPos, String... allowed) { + if (npos > maxPos) { + throw Py.typeError(fn + "() takes at most " + maxPos + " argument" + (maxPos == 1 ? "" : "s") + + " (" + npos + " given)"); + } + for (String kwName : kwNames) { + boolean ok = false; + for (String e : allowed) { + if (e.equals(kwName)) { + ok = true; + break; + } + } + if (!ok) { + throw Py.typeError(fn + "() got an unexpected keyword argument '" + kwName + "'"); + } + } + return this; + } + + /// The positional arguments only. + Object[] positional() { + if (kwNames.length == 0) { + return args; + } + Object[] r = new Object[npos]; + System.arraycopy(args, 0, r, 0, npos); + return r; + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/AttrSite.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/AttrSite.java new file mode 100644 index 00000000000..783c6ff9e84 --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/AttrSite.java @@ -0,0 +1,487 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +/// An inline cache for one attribute access in generated code. +/// +/// The transpiler gives every `obj.name` read, `obj.name = v` store and +/// `obj.name(...)` call its own site. The site remembers what it found the +/// last time -- the receiver's class, the class version, the instance's shape +/// and where the answer lives -- and as long as the next receiver matches, the +/// lookup is one comparison and an array access instead of a walk of the +/// method resolution order and a hash lookup per class. A site remembers up +/// to [#WAYS] receiver layouts, because one call written against a base +/// class routinely sees several subclasses (`task.run()` over every kind of +/// task); past that it replaces entries in turn. +/// +/// Correctness rests on two identities. A class's version changes whenever +/// any class namespace changes (`PyType.version`), so a cached method, +/// property check or `__setattr__` check can never outlive the class it was +/// taken from. And a shape fixes which attribute names an instance holds, so +/// the same shape means the same slot -- and, for a class attribute, that the +/// instance still does not shadow it. Anything the cache does not model goes +/// the generic way, which is always correct. +public final class AttrSite { + /// How many receiver layouts a site remembers. + static final int WAYS = 4; + + private static final int NONE = 0; + private static final int SLOT = 1; + private static final int CLASS_ATTR = 2; + private static final int BUILTIN = 3; + + private final String name; + /// A builtin method this site calls directly when the receiver is that + /// exact builtin type: Python forbids replacing a builtin type's methods, + /// so the lookup, the argument array and the bound call can all be skipped. + private final int intrinsic; + private static final int I_NONE = 0; + private static final int I_GET = 1; + private static final int I_APPEND = 2; + private static final int I_UPPER = 3; + private static final int I_LOWER = 4; + private static final int I_SPLIT = 5; + private static final int I_JOIN = 6; + + // Reads and calls. An entry with a null shape is a non-instance receiver + // type (BUILTIN or NONE). All entries are valid only while getVersion is + // the current class version. + private int getVersion = -1; + private int getUsed; + private int getNext; + private final PyType[] getTypes = new PyType[WAYS]; + private final Shape[] getShapes = new Shape[WAYS]; + private final int[] getKinds = new int[WAYS]; + private final int[] getSlots = new int[WAYS]; + private final Object[] getValues = new Object[WAYS]; + + // Stores: a slot hit when the instance already has the attribute, or a + // shape transition when the store adds it. + private int setVersion = -1; + private int setUsed; + private int setNext; + private final PyType[] setTypes = new PyType[WAYS]; + private final Shape[] setShapes = new Shape[WAYS]; + private final int[] setSlots = new int[WAYS]; + private final Shape[] setTos = new Shape[WAYS]; + + // The same entries again as plain fields, which the hit paths compare: + // a field is a load, an array element a bounds-checked call in + // translated code, and these sites run for every attribute access. + // gfN is the function a call takes the receiver for (CLASS_ATTR holding + // a PyFunction), else null; gkN is SLOT for a slot read, else NONE. + private PyType gt0; + private PyType gt1; + private PyType gt2; + private PyType gt3; + private Shape gs0; + private Shape gs1; + private Shape gs2; + private Shape gs3; + private int gk0; + private int gk1; + private int gk2; + private int gk3; + private int gi0; + private int gi1; + private int gi2; + private int gi3; + private Object gf0; + private Object gf1; + private Object gf2; + private Object gf3; + private PyType st0; + private PyType st1; + private PyType st2; + private PyType st3; + private Shape ss0; + private Shape ss1; + private Shape ss2; + private Shape ss3; + private int si0; + private int si1; + private int si2; + private int si3; + private Shape sto0; + private Shape sto1; + private Shape sto2; + private Shape sto3; + + public AttrSite(String name) { + this.name = name.intern(); + this.intrinsic = intrinsicOf(name); + } + + private static int intrinsicOf(String name) { + switch (name) { + case "get": + return I_GET; + case "append": + return I_APPEND; + case "upper": + return I_UPPER; + case "lower": + return I_LOWER; + case "split": + return I_SPLIT; + case "join": + return I_JOIN; + default: + return I_NONE; + } + } + + /// The read entry for this receiver layout, or -1. + private int findGet(PyType t, Shape shape) { + if (getVersion != PyType.version) { + getVersion = PyType.version; + getUsed = 0; + getNext = 0; + gt0 = gt1 = gt2 = gt3 = null; + return -1; + } + for (int k = 0; k < getUsed; k++) { + if (getTypes[k] == t && getShapes[k] == shape) { //NOPMD CompareObjectsWithEquals - identity is meant + return k; + } + } + return -1; + } + + private int addGet(PyType t, Shape shape, int kind, int slot, Object value) { + int k; + if (getUsed < WAYS) { + k = getUsed++; + } else { + k = getNext; + getNext = (getNext + 1) % WAYS; + } + getTypes[k] = t; + getShapes[k] = shape; + getKinds[k] = kind; + getSlots[k] = slot; + getValues[k] = value; + int fast = kind == SLOT ? SLOT : NONE; + Object f = kind == CLASS_ATTR && value instanceof PyFunction ? value : null; + if (k == 0) { + gt0 = t; gs0 = shape; gk0 = fast; gi0 = slot; gf0 = f; + } else if (k == 1) { + gt1 = t; gs1 = shape; gk1 = fast; gi1 = slot; gf1 = f; + } else if (k == 2) { + gt2 = t; gs2 = shape; gk2 = fast; gi2 = slot; gf2 = f; + } else { + gt3 = t; gs3 = shape; gk3 = fast; gi3 = slot; gf3 = f; + } + return k; + } + + // ---- reads --------------------------------------------------------------- + + public Object get(Object o) { + if (o instanceof PyInstance) { + PyInstance i = (PyInstance) o; + if (getVersion == PyType.version) { + PyType t = i.cls; + Shape s = i.shape; + if (gs0 == s && gt0 == t && gk0 == SLOT) { //NOPMD CompareObjectsWithEquals - identity is meant + return i.slots[gi0]; + } + if (gs1 == s && gt1 == t && gk1 == SLOT) { //NOPMD CompareObjectsWithEquals - identity is meant + return i.slots[gi1]; + } + if (gs2 == s && gt2 == t && gk2 == SLOT) { //NOPMD CompareObjectsWithEquals - identity is meant + return i.slots[gi2]; + } + if (gs3 == s && gt3 == t && gk3 == SLOT) { //NOPMD CompareObjectsWithEquals - identity is meant + return i.slots[gi3]; + } + } + return getSlow(i); + } + return Py.obj(o).getAttr(name); + } + + private Object getSlow(PyInstance i) { + int k = findGet(i.cls, i.shape); + if (k >= 0) { + int kind = getKinds[k]; + if (kind == SLOT) { + return i.slots[getSlots[k]]; + } + if (kind == CLASS_ATTR) { + return Py.bindAttr(getValues[k], i); + } + return i.getAttr(name); + } + return fillGet(i); + } + + private Object fillGet(PyInstance i) { + Shape shape = i.shape; + Object result = i.getAttr(name); + if (getVersion != PyType.version || i.shape != shape) { //NOPMD CompareObjectsWithEquals - identity is meant + // The read ran code (a __getattr__, a property) that changed what + // the cache would key on; remember nothing. + return result; + } + PyType cls = i.cls; + int kind = NONE; + int slot = 0; + Object value = null; + Object clsAttr = cls.lookup(name); + // A compiled field is read by the generated code itself; this site + // only sees it unset, and must not remember that. + if (!(clsAttr instanceof PyProperty) && !cls.hasCustomGetattribute() && i.fieldIndex(name) < 0) { + slot = shape.indexOf(name); + if (slot >= 0) { + kind = SLOT; + } else if (clsAttr != null) { + kind = CLASS_ATTR; + value = clsAttr; + } + } + addGet(cls, shape, kind, slot, value); + return result; + } + + // ---- stores -------------------------------------------------------------- + + public void set(Object o, Object v) { + if (o instanceof PyInstance) { + PyInstance i = (PyInstance) o; + if (setVersion == PyType.version) { + PyType t = i.cls; + Shape s = i.shape; + if (ss0 == s && st0 == t && sto0 == null) { //NOPMD CompareObjectsWithEquals - identity is meant + i.slots[si0] = v; + return; + } + if (ss1 == s && st1 == t && sto1 == null) { //NOPMD CompareObjectsWithEquals - identity is meant + i.slots[si1] = v; + return; + } + if (ss2 == s && st2 == t && sto2 == null) { //NOPMD CompareObjectsWithEquals - identity is meant + i.slots[si2] = v; + return; + } + if (ss3 == s && st3 == t && sto3 == null) { //NOPMD CompareObjectsWithEquals - identity is meant + i.slots[si3] = v; + return; + } + } + setSlow(i, v); + return; + } + Py.obj(o).setAttr(name, v); + } + + private void setSlow(PyInstance i, Object v) { + if (setVersion == PyType.version) { + PyType t = i.cls; + Shape s = i.shape; + for (int k = 0; k < setUsed; k++) { + if (setTypes[k] == t && setShapes[k] == s) { //NOPMD CompareObjectsWithEquals - identity is meant + Shape to = setTos[k]; + if (to == null) { + i.slots[setSlots[k]] = v; + } else { + i.append(to, v); + } + return; + } + } + } else { + setVersion = PyType.version; + setUsed = 0; + setNext = 0; + st0 = st1 = st2 = st3 = null; + } + fillSet(i, v); + } + + private void addSet(PyType t, Shape shape, int slot, Shape to) { + int k; + if (setUsed < WAYS) { + k = setUsed++; + } else { + k = setNext; + setNext = (setNext + 1) % WAYS; + } + setTypes[k] = t; + setShapes[k] = shape; + setSlots[k] = slot; + setTos[k] = to; + if (k == 0) { + st0 = t; ss0 = shape; si0 = slot; sto0 = to; + } else if (k == 1) { + st1 = t; ss1 = shape; si1 = slot; sto1 = to; + } else if (k == 2) { + st2 = t; ss2 = shape; si2 = slot; sto2 = to; + } else { + st3 = t; ss3 = shape; si3 = slot; sto3 = to; + } + } + + private void fillSet(PyInstance i, Object v) { + PyType cls = i.cls; + Object clsAttr = cls.lookup(name); + Object sa = cls.lookup("__setattr__"); + boolean plain = !(clsAttr instanceof PyProperty) && (sa == null || sa instanceof PyBuiltin) + && i.fieldIndex(name) < 0; + Shape before = i.shape; + i.setAttr(name, v); + if (!plain || setVersion != PyType.version) { + return; + } + Shape after = i.shape; + if (before == after) { //NOPMD CompareObjectsWithEquals - identity is meant + addSet(cls, before, after.indexOf(name), null); + } else { + // Adding the attribute moved the instance to a new shape. Remember + // both the transition (the next new instance) and the slot on the + // new shape (the next store to an instance already initialised). + addSet(cls, before, 0, after); + addSet(cls, after, after.indexOf(name), null); + } + } + + // ---- calls --------------------------------------------------------------- + + public Object call0(Object o) { + if (intrinsic != I_NONE && o instanceof String) { + String v = ((String) o); + if (intrinsic == I_UPPER) { + return Py.str(TypeMethods.mapCase(v, 1)); + } + if (intrinsic == I_LOWER) { + return Py.str(TypeMethods.mapCase(v, 0)); + } + if (intrinsic == I_SPLIT) { + return TypeMethods.split(v, null, -1); + } + } + Object f = method(o); + return f != null ? Py.obj(f).call1(o) : Py.obj(Py.obj(o).getAttr(name)).call0(); + } + + public Object call1(Object o, Object a) { + // instanceof rather than getClass(): both classes are final, and + // getClass() is a call on ParparVM. + if (intrinsic != I_NONE && o != null) { + if (intrinsic == I_APPEND && o instanceof PyList) { + ((PyList) o).append(a); + return Py.NONE; + } + if (o instanceof String) { + if (intrinsic == I_SPLIT) { + return TypeMethods.split(((String) o), a, -1); + } + if (intrinsic == I_JOIN) { + return TypeMethods.join(((String) o), a); + } + } + if (intrinsic == I_GET && o instanceof PyDict && o.getClass() == PyDict.class) { + Object v = ((PyDict) o).getOrNull(a); + return v != null ? v : Py.NONE; + } + } + Object f = method(o); + return f != null ? Py.obj(f).call2(o, a) : Py.obj(Py.obj(o).getAttr(name)).call1(a); + } + + public Object call2(Object o, Object a, Object b) { + if (intrinsic == I_GET && o != null && o.getClass() == PyDict.class) { + Object v = ((PyDict) o).getOrNull(a); + return v != null ? v : b; + } + Object f = method(o); + return f != null ? Py.obj(f).call3(o, a, b) : Py.obj(Py.obj(o).getAttr(name)).call2(a, b); + } + + public Object call3(Object o, Object a, Object b, Object c) { + Object f = method(o); + return f != null ? Py.obj(f).call4(o, a, b, c) : Py.obj(Py.obj(o).getAttr(name)).call3(a, b, c); + } + + public Object callN(Object o, Object[] args) { + Object f = method(o); + return f != null ? Py.obj(f).call(Py.prepend(o, args), Py.NO_KW) : Py.obj(Py.obj(o).getAttr(name)).call(args, Py.NO_KW); + } + + /// The function to call with the receiver prepended, when the call needs + /// no bound method; otherwise null and the caller goes the generic way. + private Object method(Object o) { + if (o instanceof PyInstance) { + PyInstance i = (PyInstance) o; + if (getVersion == PyType.version) { + PyType t = i.cls; + Shape s = i.shape; + if (gs0 == s && gt0 == t) { //NOPMD CompareObjectsWithEquals - identity is meant + return gf0; + } + if (gs1 == s && gt1 == t) { //NOPMD CompareObjectsWithEquals - identity is meant + return gf1; + } + if (gs2 == s && gt2 == t) { //NOPMD CompareObjectsWithEquals - identity is meant + return gf2; + } + if (gs3 == s && gt3 == t) { //NOPMD CompareObjectsWithEquals - identity is meant + return gf3; + } + } + int k = findGet(i.cls, i.shape); + if (k >= 0) { + return getKinds[k] == CLASS_ATTR && getValues[k] instanceof PyFunction ? getValues[k] : null; + } + Object clsAttr = i.cls.lookup(name); + if (i.cls.hasCustomGetattribute() || clsAttr instanceof PyProperty || i.fieldIndex(name) >= 0) { + addGet(i.cls, i.shape, NONE, 0, null); + return null; + } + int slot = i.shape.indexOf(name); + if (slot >= 0) { + addGet(i.cls, i.shape, SLOT, slot, null); + return null; + } + if (clsAttr != null) { + addGet(i.cls, i.shape, CLASS_ATTR, 0, clsAttr); + return clsAttr instanceof PyFunction ? clsAttr : null; + } + addGet(i.cls, i.shape, NONE, 0, null); + return null; + } + if (o instanceof PyType || o instanceof PyModule || o instanceof PySuper) { + return null; + } + PyType t = Py.typeOf(o); + int k = findGet(t, null); + if (k >= 0) { + return getKinds[k] == BUILTIN ? getValues[k] : null; + } + Object m = t.lookup(name); + int kind = m instanceof PyBuiltin ? BUILTIN : NONE; + addGet(t, null, kind, 0, m); + return kind == BUILTIN ? m : null; + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/BuiltinSubclass.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/BuiltinSubclass.java new file mode 100644 index 00000000000..5b36b904028 --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/BuiltinSubclass.java @@ -0,0 +1,262 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +/// Subclasses of the builtin value types: `class Stack(list)`, +/// `class Celsius(float)`, `class Counter(dict)`. +/// +/// An instance of such a class is a [PyInstance] whose [PyInstance#base] +/// holds the builtin value it extends: a `PyList` for a list subclass, a +/// `Long` for an int subclass. A method of the builtin type called on it +/// runs on that value (see `PyBuiltin.method`), and the dunder methods +/// installed here give the builtin types the `__getitem__`, `__add__`, ... +/// that `super()` and the instance's operators resolve to. A dunder the +/// subclass defines itself comes first in its MRO, so it wins as in CPython. +final class BuiltinSubclass { + private BuiltinSubclass() { + } + + interface Binary { + Object apply(PyObject self, Object other); + } + + interface Unary { + Object apply(PyObject self); + } + + /// Whether instances of `t` can be extended by a class. + static boolean subclassable(PyType t) { + return t == Py.INT_TYPE || t == Py.FLOAT_TYPE || t == Py.STR_TYPE || t == Py.BYTES_TYPE //NOPMD CompareObjectsWithEquals - identity is meant + || t == Py.LIST_TYPE || t == Py.TUPLE_TYPE || t == Py.DICT_TYPE || t == Py.SET_TYPE //NOPMD CompareObjectsWithEquals - identity is meant + || t == Py.FROZENSET_TYPE || t == Py.PROPERTY_TYPE; //NOPMD CompareObjectsWithEquals - identity is meant + } + + /// Whether `t` is mutable: its value is created empty with the instance + /// and filled by `__init__`; an immutable one is built by `__new__`. + static boolean mutable(PyType t) { + return t == Py.LIST_TYPE || t == Py.DICT_TYPE || t == Py.SET_TYPE || t == Py.PROPERTY_TYPE; //NOPMD CompareObjectsWithEquals - identity is meant + } + + private static void binary(PyType t, String name, Binary op) { + t.define(name, PyBuiltin.method(t, name, a -> { + if (a.length != 2) { + throw Py.typeError("expected 1 argument, got " + (a.length - 1)); + } + // The other operand may be a subclass instance too (two IntEnum + // members compared): the built-in operation sees its value. + Object other = a[1] instanceof PyInstance && ((PyInstance) a[1]).base != null + ? ((PyInstance) a[1]).base : a[1]; + Object r = op.apply(Py.obj(a[0]), other); + return r == null ? Py.NOT_IMPLEMENTED : r; + })); + } + + private static void unary(PyType t, String name, Unary op) { + t.define(name, PyBuiltin.method(t, name, a -> op.apply(Py.obj(a[0])))); + } + + private static void arithmetic(PyType t) { + binary(t, "__add__", PyObject::add); + binary(t, "__radd__", PyObject::radd); + binary(t, "__sub__", PyObject::sub); + binary(t, "__rsub__", PyObject::rsub); + binary(t, "__mul__", PyObject::mul); + binary(t, "__rmul__", PyObject::rmul); + binary(t, "__truediv__", PyObject::truediv); + binary(t, "__rtruediv__", PyObject::rtruediv); + binary(t, "__floordiv__", PyObject::floordiv); + binary(t, "__rfloordiv__", PyObject::rfloordiv); + binary(t, "__mod__", PyObject::mod); + binary(t, "__rmod__", PyObject::rmod); + binary(t, "__pow__", PyObject::pow); + binary(t, "__rpow__", PyObject::rpow); + unary(t, "__neg__", PyObject::neg); + unary(t, "__pos__", PyObject::pos); + unary(t, "__abs__", s -> Builtins.lookup("abs") instanceof PyObject + ? ((PyObject) Builtins.lookup("abs")).call(new Object[] {s}, Py.NO_KW) : null); + unary(t, "__float__", s -> Py.ofFloat(s.asDouble())); + unary(t, "__bool__", s -> Py.bool(s.isTrue())); + } + + private static void bits(PyType t) { + binary(t, "__and__", PyObject::and); + binary(t, "__rand__", PyObject::rand); + binary(t, "__or__", PyObject::or); + binary(t, "__ror__", PyObject::ror); + binary(t, "__xor__", PyObject::xor); + binary(t, "__rxor__", PyObject::rxor); + } + + private static void compare(PyType t) { + binary(t, "__eq__", PyObject::eq); + binary(t, "__ne__", (s, o) -> { + Object r = s.eq(o); + return r == null || r == Py.NOT_IMPLEMENTED ? null : Py.bool(!Py.isTrue(r)); //NOPMD CompareObjectsWithEquals - identity is meant + }); + binary(t, "__lt__", PyObject::lt); + binary(t, "__le__", PyObject::le); + binary(t, "__gt__", PyObject::gt); + binary(t, "__ge__", PyObject::ge); + unary(t, "__repr__", s -> Py.str(s.repr())); + // The receiver reaches here already unwrapped to the built-in value, + // so this formats the value, never back through a subclass. + t.define("__format__", PyBuiltin.method(t, "__format__", + a -> Py.str(Formatting.format(a[0], a.length > 1 ? Py.toStr(a[1]) : "")))); + if (mutable(t)) { + t.define("__hash__", Py.NONE); + } else { + unary(t, "__hash__", s -> PyInt.of(s.hash())); + } + } + + private static void container(PyType t) { + unary(t, "__len__", s -> PyInt.of(s.len())); + unary(t, "__iter__", PyObject::iter); + binary(t, "__contains__", (s, x) -> Py.bool(s.contains(x))); + binary(t, "__getitem__", PyObject::getItem); + if (t == Py.LIST_TYPE || t == Py.DICT_TYPE || t == Py.BYTEARRAY_TYPE || t == Py.MEMORYVIEW_TYPE) { //NOPMD CompareObjectsWithEquals - identity is meant + t.define("__setitem__", PyBuiltin.method(t, "__setitem__", a -> { + Py.obj(a[0]).setItem(a[1], a[2]); + return Py.NONE; + })); + t.define("__delitem__", PyBuiltin.method(t, "__delitem__", a -> { + Py.obj(a[0]).delItem(a[1]); + return Py.NONE; + })); + } + } + + /// `T.__new__(cls, *args)` of an immutable builtin `T`. + private static void newMethod(final PyType t) { + t.define("__new__", new PyStaticMethod(new PyBuiltin("__new__", (args, kw) -> { + if (args.length == 0 || !(args[0] instanceof PyType)) { + throw Py.notATypeObject(t.name, args); + } + PyType cls = (PyType) args[0]; + if (!cls.isSubtype(t)) { + throw Py.typeError(t.name + ".__new__(" + cls.name + "): " + cls.name + " is not a subtype of " + + t.name); + } + Object[] rest = new Object[args.length - 1]; + System.arraycopy(args, 1, rest, 0, rest.length); + Object v = t.factory.create(t, rest, kw); + if (cls == t) { //NOPMD CompareObjectsWithEquals - identity is meant + return v; + } + PyInstance o = cls.newInstance(); + o.base = v; + return o; + }))); + } + + /// `T.__init__(self, *args)` of a mutable builtin `T`: refills the value. + private static void initMethod(final PyType t) { + t.define("__init__", PyBuiltin.methodKw(t, "__init__", (args, kw) -> { + Object[] rest = new Object[args.length - 1]; + System.arraycopy(args, 1, rest, 0, rest.length); + Object fresh = t.factory.create(t, rest, kw); + Object self = args[0]; + if (self instanceof PyProperty && fresh instanceof PyProperty) { + ((PyProperty) self).fill((PyProperty) fresh); + } else if (self instanceof PyList) { + PyList l = (PyList) self; + l.clear(); + l.extend(fresh); + } else if (self instanceof PyDict) { + PyDict d = (PyDict) self; + d.clear(); + d.update(fresh); + } else if (self instanceof PySet) { + PySet s = (PySet) self; + s.clear(); + Object it = Py.iter(fresh); + for (Object v = Py.next(it); v != null; v = Py.next(it)) { + s.addItem(v); + } + } + return Py.NONE; + })); + } + + static void install() { + // Not subclassable here, but their protocol is reachable by name. + PyType[] buffers = {Py.BYTEARRAY_TYPE, Py.MEMORYVIEW_TYPE}; + for (PyType buffer : buffers) { + container(buffer); + binary(buffer, "__eq__", PyObject::eq); + unary(buffer, "__repr__", x -> Py.str(x.repr())); + } + Py.BYTEARRAY_TYPE.define("__hash__", Py.NONE); + binary(Py.BYTEARRAY_TYPE, "__add__", PyObject::add); + initMethod(Py.PROPERTY_TYPE); + PyType[] all = {Py.INT_TYPE, Py.FLOAT_TYPE, Py.STR_TYPE, Py.BYTES_TYPE, Py.LIST_TYPE, Py.TUPLE_TYPE, + Py.DICT_TYPE, Py.SET_TYPE, Py.FROZENSET_TYPE}; + for (PyType e : all) { + PyType t = e; + compare(t); + if (mutable(t)) { + initMethod(t); + } else { + newMethod(t); + } + if (t == Py.INT_TYPE || t == Py.FLOAT_TYPE) { //NOPMD CompareObjectsWithEquals - identity is meant + arithmetic(t); + if (t == Py.INT_TYPE) { //NOPMD CompareObjectsWithEquals - identity is meant + bits(t); + binary(t, "__lshift__", PyObject::lshift); + binary(t, "__rshift__", PyObject::rshift); + unary(t, "__int__", s -> PyInt.of(s.asLong())); + unary(t, "__index__", s -> PyInt.of(s.asLong())); + unary(t, "__invert__", PyObject::invert); + } + } else { + container(t); + if (t == Py.SET_TYPE || t == Py.FROZENSET_TYPE) { //NOPMD CompareObjectsWithEquals - identity is meant + bits(t); + binary(t, "__sub__", PyObject::sub); + binary(t, "__rsub__", PyObject::rsub); + } else if (t == Py.DICT_TYPE) { //NOPMD CompareObjectsWithEquals - identity is meant + binary(t, "__or__", PyObject::or); + binary(t, "__ror__", PyObject::ror); + } else { + binary(t, "__add__", PyObject::add); + binary(t, "__mul__", PyObject::mul); + binary(t, "__rmul__", PyObject::rmul); + if (t == Py.STR_TYPE || t == Py.BYTES_TYPE) { //NOPMD CompareObjectsWithEquals - identity is meant + binary(t, "__mod__", PyObject::mod); + } + } + if (t == Py.LIST_TYPE) { //NOPMD CompareObjectsWithEquals - identity is meant + binary(t, "__iadd__", (s, o) -> { + ((PyList) s).extend(o); + return s; + }); + } + } + if (t == Py.STR_TYPE) { //NOPMD CompareObjectsWithEquals - identity is meant + unary(t, "__str__", s -> Py.str(s.str())); + } + } + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Builtins.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Builtins.java new file mode 100644 index 00000000000..2b69a3f1225 --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Builtins.java @@ -0,0 +1,1198 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +import java.util.HashMap; +import java.util.Map; + +/// The `builtins` module: the builtin functions, the constructors of the +/// builtin types, and the methods of `object` and `BaseException`. +/// +/// The transpiler resolves a name that no enclosing scope assigns to the +/// matching public field here, so `len(x)` compiles to a direct reference to +/// [#LEN] rather than a dictionary lookup. +public final class Builtins { + /// Where `print` writes. Replaceable, for tests and for apps that route + /// output into the UI. + public interface Output { + void write(String s); + } + + private static Output out = new Output() { + @Override + public void write(String s) { + System.out.print(s); + } + }; + + /// Writes to the platform output, past any replaced `sys.stdout`. + static void writeOut(String s) { + out.write(s); + } + + static final Map NAMES = new HashMap(); + + public static final PyBuiltin OBJECT_NEW = PyBuiltin.positional("__new__", a -> { + if (a.length == 0 || !(a[0] instanceof PyType)) { + throw Py.notATypeObject("object", a); + } + return ((PyType) a[0]).newInstance(); + }); + /// Marks an optional argument that was not passed. + private static final Object ABSENT = new PyStr(""); + public static final PyBuiltin OBJECT_INIT = PyBuiltin.positional("__init__", a -> Py.NONE); + + public static final Object PRINT = new PyBuiltin("print", (args, kw) -> { + Args a = new Args("print", args, kw).check(Integer.MAX_VALUE, "sep", "end", "file", "flush"); + Object sepO = a.kw("sep"); + Object endO = a.kw("end"); + String sep = sepO == null || sepO == Py.NONE ? " " : Py.toStr(sepO); //NOPMD CompareObjectsWithEquals - identity is meant + String end = endO == null || endO == Py.NONE ? "\n" : Py.toStr(endO); //NOPMD CompareObjectsWithEquals - identity is meant + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < a.npos; i++) { + if (i > 0) { + sb.append(sep); + } + sb.append(Py.toStr(args[i])); + } + sb.append(end); + Object file = a.kw("file"); + if (file == null || file == Py.NONE) { //NOPMD CompareObjectsWithEquals - identity is meant + // A program that replaced sys.stdout (contextlib.redirect_stdout) + // prints there. + file = StdModules.redirectedStdout(); + } + if (file != null) { + Py.callMethod(file, "write", Py.str(sb.toString())); + } else { + out.write(sb.toString()); + } + return Py.NONE; + }); + + public static final Object LEN = PyBuiltin.positional("len", a -> PyInt.of(Py.len(one("len", a)))); + public static final Object REPR = PyBuiltin.positional("repr", a -> Py.str(Py.toRepr(one("repr", a)))); + public static final Object ASCII = PyBuiltin.positional("ascii", a -> Py.str(asciiOnly(Py.toRepr(one("ascii", a))))); + public static final Object HASH = PyBuiltin.positional("hash", a -> PyInt.of(Py.hashOf(one("hash", a)))); + public static final Object ID = PyBuiltin.positional("id", a -> PyInt.of(System.identityHashCode(one("id", a)))); + public static final Object ABS = PyBuiltin.positional("abs", a -> abs(one("abs", a))); + public static final Object CALLABLE = PyBuiltin.positional("callable", a -> Py.bool(isCallable(one("callable", a)))); + public static final Object ITER = PyBuiltin.positional("iter", a -> iter(a)); + public static final Object NEXT = PyBuiltin.positional("next", a -> next(a)); + public static final Object AITER = PyBuiltin.positional("aiter", a -> Py.callMethod(one("aiter", a), "__aiter__")); + // anext with a default is an awaitable that answers the default on StopAsyncIteration; the + // asyncio module supplies that wrapper, so the builtin covers the one-argument form. + public static final Object ANEXT = PyBuiltin.positional("anext", a -> Py.callMethod(one("anext", a), "__anext__")); + public static final Object ORD = PyBuiltin.positional("ord", a -> ord(one("ord", a))); + public static final Object CHR = PyBuiltin.positional("chr", a -> { + long c = Py.asLong(one("chr", a)); + if (c < 0 || c > 0x10ffff) { + throw Py.valueError("chr() arg not in range(0x110000)"); + } + return Py.str(new String(Character.toChars((int) c))); + }); + public static final Object HEX = PyBuiltin.positional("hex", a -> Py.str(Formatting.format(one("hex", a), "#x"))); + public static final Object OCT = PyBuiltin.positional("oct", a -> Py.str(Formatting.format(one("oct", a), "#o"))); + public static final Object BIN = PyBuiltin.positional("bin", a -> Py.str(Formatting.format(one("bin", a), "#b"))); + public static final Object FORMAT = PyBuiltin.positional("format", a -> Py.format(a[0], a.length > 1 ? Py.toStr(a[1]) : "")); + public static final Object DIVMOD = PyBuiltin.positional("divmod", a -> PyTuple.of(Py.floordiv(a[0], a[1]), Py.mod(a[0], a[1]))); + public static final Object POW = PyBuiltin.positional("pow", a -> pow(a)); + public static final Object ROUND = new PyBuiltin("round", (args, kw) -> round(new Args("round", args, kw).check(2, "number", "ndigits"))); + public static final Object ISINSTANCE = PyBuiltin.positional("isinstance", a -> Py.bool(Py.isinstance(a[0], a[1]))); + public static final Object ISSUBCLASS = PyBuiltin.positional("issubclass", a -> Py.bool(issubclass(a[0], a[1]))); + public static final Object GETATTR = PyBuiltin.positional("getattr", a -> getattr(a)); + public static final Object SETATTR = PyBuiltin.positional("setattr", a -> { + Py.obj(a[0]).setAttr(Py.toStr(a[1]), a[2]); + return Py.NONE; + }); + public static final Object HASATTR = PyBuiltin.positional("hasattr", a -> { + try { + return Py.bool(Py.obj(a[0]).getAttrOrNull(Py.toStr(a[1])) != null); + } catch (PyException e) { + if (e.matches(Py.ATTRIBUTE_ERROR)) { + return Py.FALSE; + } + throw e; + } + }); + public static final Object DELATTR = PyBuiltin.positional("delattr", a -> { + Py.obj(a[0]).delAttr(Py.toStr(a[1])); + return Py.NONE; + }); + public static final Object VARS = PyBuiltin.positional("vars", a -> Py.obj(one("vars", a)).getAttr("__dict__")); + public static final Object DIR = PyBuiltin.positional("dir", a -> dir(one("dir", a))); + public static final Object SUM = new PyBuiltin("sum", (args, kw) -> sum(new Args("sum", args, kw).check(2, "start"))); + public static final Object MIN = new PyBuiltin("min", (args, kw) -> minMax("min", args, kw, true)); + public static final Object MAX = new PyBuiltin("max", (args, kw) -> minMax("max", args, kw, false)); + public static final Object ANY = PyBuiltin.positional("any", a -> { + Object it = Py.iter(one("any", a)); + for (Object v = Py.next(it); v != null; v = Py.next(it)) { + if (Py.isTrue(v)) { + return Py.TRUE; + } + } + return Py.FALSE; + }); + public static final Object ALL = PyBuiltin.positional("all", a -> { + Object it = Py.iter(one("all", a)); + for (Object v = Py.next(it); v != null; v = Py.next(it)) { + if (!Py.isTrue(v)) { + return Py.FALSE; + } + } + return Py.TRUE; + }); + public static final Object SORTED = new PyBuiltin("sorted", (args, kw) -> { + Args a = new Args("sorted", args, kw).check(1, "key", "reverse"); + PyList l = new PyList(); + l.extend(a.required(0, "iterable")); + Object rev = a.kw("reverse"); + l.sort(a.kw("key"), rev != null && Py.isTrue(rev)); + return l; + }); + public static final Object REVERSED = PyBuiltin.positional("reversed", a -> reversed(one("reversed", a))); + public static final Object ENUMERATE = new PyBuiltin("enumerate", (args, kw) -> { + Args a = new Args("enumerate", args, kw).check(2, "iterable", "start"); + final Object it = Py.iter(a.required(0, "iterable")); + Object start = a.get(1, "start", PyInt.of(0)); + final Object[] n = {start}; + return new PyJavaIter(() -> { + Object v = Py.next(it); + if (v == null) { + return null; + } + Object r = PyTuple.of(n[0], v); + n[0] = Py.add(n[0], PyInt.of(1)); + return r; + }); + }); + public static final Object ZIP = new PyBuiltin("zip", (args, kw) -> { + Args a = new Args("zip", args, kw).check(Integer.MAX_VALUE, "strict"); + final Object[] its = new Object[a.npos]; + for (int i = 0; i < its.length; i++) { + its[i] = Py.iter(args[i]); + } + Object strictO = a.kw("strict"); + final boolean strict = strictO != null && Py.isTrue(strictO); + return new PyJavaIter(() -> { + if (its.length == 0) { + return null; + } + Object[] row = new Object[its.length]; + for (int i = 0; i < its.length; i++) { + row[i] = Py.next(its[i]); + if (row[i] == null) { + if (strict) { + if (i > 0) { + throw Py.valueError("zip() argument " + (i + 1) + " is shorter than argument 1"); + } + for (int j = 1; j < its.length; j++) { + if (Py.next(its[j]) != null) { + throw Py.valueError("zip() argument " + (j + 1) + " is longer than argument 1"); + } + } + } + return null; + } + } + return new PyTuple(row); + }); + }); + public static final Object MAP = PyBuiltin.positional("map", a -> { + final Object f = a[0]; + final Object[] its = new Object[a.length - 1]; + for (int i = 0; i < its.length; i++) { + its[i] = Py.iter(a[i + 1]); + } + return new PyJavaIter(() -> { + Object[] row = new Object[its.length]; + for (int i = 0; i < its.length; i++) { + row[i] = Py.next(its[i]); + if (row[i] == null) { + return null; + } + } + return Py.obj(f).call(row, Py.NO_KW); + }); + }); + public static final Object FILTER = PyBuiltin.positional("filter", a -> { + final Object f = a[0]; + final Object it = Py.iter(a[1]); + return new PyJavaIter(() -> { + for (Object v = Py.next(it); v != null; v = Py.next(it)) { + if (f == Py.NONE ? Py.isTrue(v) : Py.isTrue(Py.obj(f).call(new Object[] {v}, Py.NO_KW))) { //NOPMD CompareObjectsWithEquals - identity is meant + return v; + } + } + return null; + }); + }); + public static final Object SUPER = PyBuiltin.positional("super", a -> { + if (a.length != 2 || !(a[0] instanceof PyType)) { + throw Py.runtimeError("super(): no arguments"); + } + return new PySuper((PyType) a[0], a[1]); + }); + public static final Object INPUT = PyBuiltin.positional("input", a -> { + throw Py.error(Py.OS_ERROR, "input() has no console on this platform"); + }); + + private Builtins() { + } + + /// Redirects `print`. + public static void setOutput(Output o) { + out = o; + } + + /// Looks up a builtin by name, for `builtins.x` and `getattr(builtins, x)`. + public static Object lookup(String name) { + return NAMES.get(name); + } + + // ---- installation ---------------------------------------------------- + + static void install() { + PyType o = Py.OBJECT_TYPE; + o.define("__new__", OBJECT_NEW); + o.define("__init__", OBJECT_INIT); + o.define("__setattr__", PyBuiltin.positional("__setattr__", a -> { + if (a[0] instanceof PyInstance) { + ((PyInstance) a[0]).setAttrDirect(Py.toStr(a[1]), a[2]); + } else if (a[0] instanceof PyType) { + // What a metaclass's super().__setattr__ reaches: store, + // never back through the metaclass. + ((PyType) a[0]).setAttrDirect(Py.toStr(a[1]), a[2]); + } else { + Py.obj(a[0]).setAttr(Py.toStr(a[1]), a[2]); + } + return Py.NONE; + })); + o.define("__getattribute__", PyBuiltin.positional("__getattribute__", a -> { + // The lookup without __getattr__: what a class's own + // __getattribute__ or __getattr__ reaches for. + String name = Py.toStr(a[1]); + Object v = a[0] instanceof PyObject ? ((PyObject) a[0]).getAttrOrNull(name) + : Py.bindAttr(Py.typeOf(a[0]).lookup(name), a[0]); + if (v == null) { + throw Py.attributeError("'" + Py.typeOf(a[0]).name + "' object has no attribute '" + name + "'"); + } + return v; + })); + o.define("__delattr__", PyBuiltin.positional("__delattr__", a -> { + if (a[0] instanceof PyInstance) { + ((PyInstance) a[0]).delAttrDirect(Py.toStr(a[1])); + } else if (a[0] instanceof PyType) { + ((PyType) a[0]).delAttrDirect(Py.toStr(a[1])); + } else { + Py.obj(a[0]).delAttr(Py.toStr(a[1])); + } + return Py.NONE; + })); + o.define("__repr__", PyBuiltin.positional("__repr__", a -> Py.str("<" + Py.typeOf(a[0]).name + + " object at 0x" + Integer.toHexString(System.identityHashCode(a[0])) + ">"))); + o.define("__str__", PyBuiltin.positional("__str__", a -> Py.str(Py.toRepr(a[0])))); + installExceptions(); + installFactories(); + TypeMethods.install(); + PyGenerator.install(); + ExceptionGroups.install(); + BuiltinSubclass.install(); + PyBytes.install(); + PyComplex.install(); + PyByteArray.install(); + PyMemoryView.install(); + names(); + } + + private static void installExceptions() { + PyType b = Py.BASE_EXCEPTION; + b.define("__init__", PyBuiltin.method(b, "__init__", a -> { + Object[] rest = new Object[a.length - 1]; + System.arraycopy(a, 1, rest, 0, rest.length); + if (a[0] instanceof PyInstance) { + ((PyInstance) a[0]).setAttrDirect("args", new PyTuple(rest)); + } + return Py.NONE; + })); + b.define("__new__", new PyStaticMethod(PyBuiltin.positional("__new__", a -> { + if (a.length == 0 || !(a[0] instanceof PyType)) { + throw Py.notATypeObject("BaseException", a); + } + PyType t = (PyType) a[0]; + if (t == Py.OS_ERROR && a.length >= 3 && Py.isInt(a[1])) { //NOPMD CompareObjectsWithEquals - identity is meant + // OSError(errno, strerror) is the subclass for that errno. + t = osErrorFor(Py.asLong(a[1])); + } + PyInstance e = t.newInstance(); + Object[] rest = new Object[a.length - 1]; + System.arraycopy(a, 1, rest, 0, rest.length); + e.setAttrDirect("args", new PyTuple(rest)); + return e; + }))); + b.define("__str__", PyBuiltin.positional("__str__", a -> { + Object[] args = exceptionArgs(a[0]); + if (args.length == 0) { + return Py.EMPTY_STR; + } + if (args.length == 1) { + if (Py.typeOf(a[0]).isSubtype(Py.KEY_ERROR)) { + return Py.str(Py.toRepr(args[0])); + } + return Py.str(Py.toStr(args[0])); + } + return Py.str(new PyTuple(args).repr()); + })); + b.define("__repr__", PyBuiltin.positional("__repr__", a -> { + Object[] args = exceptionArgs(a[0]); + String inner = args.length == 1 ? Py.toRepr(args[0]) : Py.joinRepr("", args, args.length, ""); + return Py.str(Py.typeOf(a[0]).name + "(" + inner + ")"); + })); + installOsError(); + b.define("with_traceback", PyBuiltin.positional("with_traceback", a -> a[0])); + b.define("__cause__", Py.NONE); + b.define("__context__", Py.NONE); + b.define("__suppress_context__", Py.FALSE); + b.define("__traceback__", Py.NONE); + Py.STOP_ITERATION.define("value", new PyProperty(PyBuiltin.positional("value", a -> { + Object[] args = exceptionArgs(a[0]); + return args.length == 0 ? Py.NONE : args[0]; + }), null, null)); + } + + /// OSError(errno, strerror[, filename[, winerror[, filename2]]]) keeps + /// its parts as attributes, and `args` is just (errno, strerror). + /// The errno numbers POSIX systems share; the ones that differ between + /// Linux and Darwin (the socket errors, EAGAIN) stay plain OSError. + private static PyType osErrorFor(long errno) { + switch ((int) errno) { + case 1: + case 13: + return Py.PERMISSION_ERROR; + case 2: + return Py.FILE_NOT_FOUND_ERROR; + case 3: + return Py.PROCESS_LOOKUP_ERROR; + case 4: + return Py.INTERRUPTED_ERROR; + case 10: + return Py.CHILD_PROCESS_ERROR; + case 17: + return Py.FILE_EXISTS_ERROR; + case 20: + return Py.NOT_A_DIRECTORY_ERROR; + case 21: + return Py.IS_A_DIRECTORY_ERROR; + default: + return Py.OS_ERROR; + } + } + + private static void installOsError() { + PyType t = Py.OS_ERROR; + t.define("__init__", PyBuiltin.method(t, "__init__", a -> { + if (!(a[0] instanceof PyInstance)) { + return Py.NONE; + } + PyInstance e = (PyInstance) a[0]; + int n = a.length - 1; + Object[] args = new Object[n]; + System.arraycopy(a, 1, args, 0, n); + Object none = Py.NONE; + e.setAttrDirect("errno", n >= 2 && n <= 5 ? a[1] : none); + e.setAttrDirect("strerror", n >= 2 && n <= 5 ? a[2] : none); + e.setAttrDirect("filename", n >= 3 && n <= 5 ? a[3] : none); + e.setAttrDirect("filename2", n == 5 ? a[5] : none); + if (n >= 3 && n <= 5) { + args = new Object[] {a[1], a[2]}; + } + e.setAttrDirect("args", new PyTuple(args)); + return Py.NONE; + })); + t.define("__str__", PyBuiltin.positional("__str__", a -> { + if (a[0] instanceof PyInstance) { + PyInstance e = (PyInstance) a[0]; + Object errno = orNone(e.own("errno")); + Object strerror = orNone(e.own("strerror")); + Object filename = orNone(e.own("filename")); + Object filename2 = orNone(e.own("filename2")); + String head = "[Errno " + Py.toStr(errno) + "] " + Py.toStr(strerror); + if (filename != Py.NONE) { //NOPMD CompareObjectsWithEquals - identity is meant + head += ": " + Py.toRepr(filename); + if (filename2 != Py.NONE) { //NOPMD CompareObjectsWithEquals - identity is meant + head += " -> " + Py.toRepr(filename2); + } + return Py.str(head); + } + if (errno != Py.NONE && strerror != Py.NONE) { //NOPMD CompareObjectsWithEquals - identity is meant + return Py.str(head); + } + } + return Py.call(Py.BASE_EXCEPTION.lookup("__str__"), a[0]); + })); + } + + static Object[] exceptionArgs(Object e) { + Object args = ((PyInstance) e).own("args"); + return args instanceof PyTuple ? ((PyTuple) args).items : Py.NO_ARGS; + } + + private static void installFactories() { + Py.METHOD_TYPE.factory = (t, args, kw) -> { + if (args.length != 2 || kw.length != 0) { + throw Py.typeError("method expected 2 arguments, got " + args.length); + } + if (!isCallable(args[0])) { + throw Py.typeError("first argument must be callable"); + } + if (args[1] == Py.NONE) { + throw Py.typeError("instance must not be None"); + } + return new PyMethod(args[1], args[0]); + }; + Py.INT_TYPE.factory = (t, args, kw) -> { + Args a = new Args("int", args, kw).check(2, "base"); + Object x = a.get(0, "x", null); + Object base = a.get(1, "base", null); + if (x == null) { + return PyInt.of(0); + } + if (base != null) { + return PyInt.parse(intText(x, true), (int) Py.asLong(base)); + } + return toInt(x); + }; + Py.BOOL_TYPE.factory = (t, args, kw) -> Py.bool(args.length > 0 && Py.isTrue(args[0])); + Py.FLOAT_TYPE.factory = (t, args, kw) -> args.length == 0 ? Py.ofFloat(0) : toFloat(args[0]); + Py.STR_TYPE.factory = (t, args, kw) -> { + Object[] p = Args.bind("str", args, kw, new String[] {"object", "encoding", "errors"}, + new Object[] {Py.EMPTY_STR, ABSENT, ABSENT}); + if (p[1] == ABSENT && p[2] == ABSENT) { + return Py.str(Py.toStr(p[0])); + } + if (!(p[0] instanceof PyBytes)) { + throw Py.typeError("decoding to str: need a bytes-like object, " + Py.typeOf(p[0]).name + " found"); + } + return Py.str(Codecs.decode(((PyBytes) p[0]).value, p[1] == ABSENT ? "utf-8" : Py.toStr(p[1]), + p[2] == ABSENT ? "strict" : Py.toStr(p[2]))); + }; + Py.LIST_TYPE.factory = (t, args, kw) -> { + PyList l = new PyList(); + if (args.length > 0) { + l.extend(args[0]); + } + return l; + }; + Py.TUPLE_TYPE.factory = (t, args, kw) -> args.length == 0 ? PyTuple.EMPTY + : args[0] instanceof PyTuple ? args[0] : new PyTuple(Py.toArray(args[0])); + Py.DICT_TYPE.factory = (t, args, kw) -> { + PyDict d = new PyDict(); + int npos = args.length - kw.length; + if (npos > 0) { + d.update(args[0]); + } + for (int i = 0; i < kw.length; i++) { + d.setItem(Py.str(kw[i]), args[npos + i]); + } + return d; + }; + Py.SET_TYPE.factory = (t, args, kw) -> args.length == 0 ? new PySet() : PySet.of(args[0], false); + Py.FROZENSET_TYPE.factory = (t, args, kw) -> args.length == 0 ? new PySet(true) : PySet.of(args[0], true); + Py.RANGE_TYPE.factory = (t, args, kw) -> { + if (args.length == 1) { + return new PyRange(0, Py.index(args[0]), 1); + } + if (args.length == 2) { + return new PyRange(Py.index(args[0]), Py.index(args[1]), 1); + } + if (args.length == 3) { + return new PyRange(Py.index(args[0]), Py.index(args[1]), Py.index(args[2])); + } + throw Py.typeError("range expected at most 3 arguments, got " + args.length); + }; + Py.SLICE_TYPE.factory = (t, args, kw) -> { + if (args.length == 1) { + return new PySlice(null, args[0], null); + } + return new PySlice(args[0], args[1], args.length > 2 ? args[2] : null); + }; + Py.TYPE_TYPE.factory = (t, args, kw) -> { + if (args.length == 1 && kw.length == 0) { + return Py.typeOf(args[0]); + } + if (args.length - kw.length != 3) { + throw Py.typeError("type() takes 1 or 3 arguments"); + } + Object[] bs = Py.toArray(args[1]); + PyType[] bases = new PyType[bs.length]; + for (int i = 0; i < bs.length; i++) { + if (!(bs[i] instanceof PyType)) { + throw Py.typeError("bases must be types"); + } + bases[i] = (PyType) bs[i]; + } + PyType w = PyType.winner(Py.TYPE_TYPE, bases); + if (w != Py.TYPE_TYPE) { //NOPMD CompareObjectsWithEquals - identity is meant + return w.call(args, kw); + } + return PyType.typeNew(Py.prepend(Py.TYPE_TYPE, args), kw); + }; + Py.TYPE_TYPE.define("__new__", new PyStaticMethod(new PyBuiltin("__new__", PyType::typeNew))); + Py.TYPE_TYPE.define("__call__", PyBuiltin.methodKw(Py.TYPE_TYPE, "__call__", (args, kw) -> { + Object[] rest = new Object[args.length - 1]; + System.arraycopy(args, 1, rest, 0, rest.length); + return ((PyType) args[0]).construct(rest, kw); + })); + Py.OBJECT_TYPE.define("__init_subclass__", new PyClassMethod(new PyBuiltin("__init_subclass__", (args, kw) -> { + if (kw.length > 0) { + String owner = args.length > 0 && args[0] instanceof PyType ? ((PyType) args[0]).name + "." : ""; + throw Py.typeError(owner + "__init_subclass__() takes no keyword arguments"); + } + return Py.NONE; + }))); + Py.PROPERTY_TYPE.factory = (t, args, kw) -> { + Args a = new Args("property", args, kw).check(4, "fget", "fset", "fdel", "doc"); + return new PyProperty(a.get(0, "fget", null), a.get(1, "fset", null), a.get(2, "fdel", null)); + }; + // Callables expose __call__, so `type(f).__dict__` answers the way + // collections.abc.Callable's check reads it. + PyType[] callables = {Py.FUNCTION_TYPE, Py.BUILTIN_TYPE, Py.METHOD_TYPE}; + for (PyType callable : callables) { + callable.define("__call__", new PyBuiltin("__call__", (args, kw) -> { + Object[] rest = new Object[args.length - 1]; + System.arraycopy(args, 1, rest, 0, rest.length); + return Py.obj(args[0]).call(rest, kw); + })); + } + // The descriptor protocol, spelled out for code that asks for it by + // name (enum and dataclasses tell members from methods this way). + Py.PROPERTY_TYPE.define("__get__", PyBuiltin.positional("__get__", a -> { + PyProperty p = property(a[0]); + return a.length < 2 || a[1] == Py.NONE ? a[0] : p.get(a[1]); + })); + Py.PROPERTY_TYPE.define("__set__", PyBuiltin.positional("__set__", a -> { + property(a[0]).set(a[1], a[2]); + return Py.NONE; + })); + Py.PROPERTY_TYPE.define("__delete__", PyBuiltin.positional("__delete__", a -> { + property(a[0]).delete(a[1]); + return Py.NONE; + })); + // On the type rather than only on PyProperty, so an instance of a + // property subclass has them too; a copy keeps the receiver's class. + Py.PROPERTY_TYPE.define("getter", PyBuiltin.positional("getter", a -> { + PyProperty p = property(a[0]); + return propertyCopy(a[0], a[1], p.fset, p.fdel); + })); + Py.PROPERTY_TYPE.define("setter", PyBuiltin.positional("setter", a -> { + PyProperty p = property(a[0]); + return propertyCopy(a[0], p.fget, a[1], p.fdel); + })); + Py.PROPERTY_TYPE.define("deleter", PyBuiltin.positional("deleter", a -> { + PyProperty p = property(a[0]); + return propertyCopy(a[0], p.fget, p.fset, a[1]); + })); + String[] parts = {"fget", "fset", "fdel"}; + for (int i = 0; i < parts.length; i++) { + final int which = i; + Py.PROPERTY_TYPE.define(parts[i], new PyProperty(PyBuiltin.positional(parts[i], a -> { + PyProperty p = property(a[0]); + Object v = which == 0 ? p.fget : which == 1 ? p.fset : p.fdel; + return v == null ? Py.NONE : v; + }), null, null)); + } + PyType[] binders = {Py.FUNCTION_TYPE, Py.CLASSMETHOD_TYPE, Py.STATICMETHOD_TYPE}; + for (PyType binder : binders) { + final PyType bt = binder; + bt.define("__get__", PyBuiltin.positional("__get__", a -> { + Object inst = a.length > 1 ? a[1] : Py.NONE; + if (inst == Py.NONE) { //NOPMD CompareObjectsWithEquals - identity is meant + if (bt == Py.FUNCTION_TYPE) { //NOPMD CompareObjectsWithEquals - identity is meant + return a[0]; + } + inst = a.length > 2 ? a[2] : Py.NONE; + if (inst == Py.NONE) { //NOPMD CompareObjectsWithEquals - identity is meant + return a[0]; + } + } + return Py.bindAttr(a[0], inst); + })); + } + Py.CLASSMETHOD_TYPE.factory = (t, args, kw) -> new PyClassMethod(args[0]); + Py.STATICMETHOD_TYPE.factory = (t, args, kw) -> new PyStaticMethod(args[0]); + } + + /// `type(self)(fget, fset, fdel)`: what getter, setter and deleter give. + private static Object propertyCopy(Object self, Object fget, Object fset, Object fdel) { + if (self instanceof PyProperty) { + return new PyProperty(fget, fset, fdel); + } + return Py.call(Py.typeOf(self), fget == null ? Py.NONE : fget, fset == null ? Py.NONE : fset, + fdel == null ? Py.NONE : fdel); + } + + private static PyProperty property(Object o) { + if (o instanceof PyInstance && ((PyInstance) o).base instanceof PyProperty) { + // An instance of a property subclass. + return (PyProperty) ((PyInstance) o).base; + } + if (!(o instanceof PyProperty)) { + throw Py.typeError("descriptor requires a 'property' object but received a '" + Py.typeOf(o).name + "'"); + } + return (PyProperty) o; + } + + private static void names() { + PyType[] types = {Py.OBJECT_TYPE, Py.TYPE_TYPE, Py.INT_TYPE, Py.BOOL_TYPE, Py.FLOAT_TYPE, Py.COMPLEX_TYPE, Py.STR_TYPE, Py.BYTES_TYPE, + Py.BYTEARRAY_TYPE, Py.MEMORYVIEW_TYPE, + Py.LIST_TYPE, Py.TUPLE_TYPE, Py.DICT_TYPE, Py.SET_TYPE, Py.FROZENSET_TYPE, Py.RANGE_TYPE, + Py.SLICE_TYPE, Py.PROPERTY_TYPE, Py.CLASSMETHOD_TYPE, Py.STATICMETHOD_TYPE, + Py.BASE_EXCEPTION, Py.SYSTEM_EXIT, Py.KEYBOARD_INTERRUPT, Py.GENERATOR_EXIT, Py.EXCEPTION, + Py.STOP_ITERATION, Py.ARITHMETIC_ERROR, Py.ZERO_DIVISION_ERROR, Py.OVERFLOW_ERROR, + Py.ASSERTION_ERROR, Py.ATTRIBUTE_ERROR, Py.LOOKUP_ERROR, Py.INDEX_ERROR, Py.KEY_ERROR, + Py.NAME_ERROR, Py.UNBOUND_LOCAL_ERROR, Py.RUNTIME_ERROR, Py.NOT_IMPLEMENTED_ERROR, + Py.RECURSION_ERROR, Py.TYPE_ERROR, Py.VALUE_ERROR, Py.UNICODE_ERROR, Py.IMPORT_ERROR, + Py.MODULE_NOT_FOUND_ERROR, Py.OS_ERROR, Py.FILE_NOT_FOUND_ERROR, Py.MEMORY_ERROR}; + for (PyType e : types) { + NAMES.put(e.name, e); + } + for (PyType t : Py.EXCEPTION_TYPES) { + NAMES.put(t.name, t); + } + // What the transpiler rewrites a `match` case into (see PyMatch). + NAMES.put("__cn1_match__", PyBuiltin.positional("__cn1_match__", a -> PyMatch.match(a[0], a[1]))); + ExceptionGroups.names(NAMES); + // open() is io.open, written in Python; the transpiler bundles `io` + // into any program that mentions open. + NAMES.put("open", new PyBuiltin("open", (args, kw) -> + Py.obj(PyModule.load("io").getAttr("open")).call(args, kw))); + NAMES.put("EnvironmentError", Py.OS_ERROR); + NAMES.put("IOError", Py.OS_ERROR); + NAMES.put("None", Py.NONE); + NAMES.put("True", Py.TRUE); + NAMES.put("False", Py.FALSE); + NAMES.put("NotImplemented", Py.NOT_IMPLEMENTED); + NAMES.put("Ellipsis", Py.ELLIPSIS); + Object[] fns = {PRINT, LEN, REPR, ASCII, HASH, ID, ABS, CALLABLE, ITER, NEXT, ORD, CHR, HEX, OCT, BIN, + FORMAT, DIVMOD, POW, ROUND, ISINSTANCE, ISSUBCLASS, GETATTR, SETATTR, HASATTR, DELATTR, VARS, DIR, + SUM, MIN, MAX, ANY, ALL, SORTED, REVERSED, ENUMERATE, ZIP, MAP, FILTER, SUPER, INPUT, AITER, ANEXT}; + for (Object fn : fns) { + NAMES.put(((PyBuiltin) fn).name, fn); + } + NAMES.put("EnvironmentError", Py.OS_ERROR); + NAMES.put("IOError", Py.OS_ERROR); + } + + // ---- implementations ------------------------------------------------- + + private static Object one(String fn, Object[] a) { + if (a.length != 1) { + throw Py.typeError(fn + "() takes exactly one argument (" + a.length + " given)"); + } + return a[0]; + } + + static Object toInt(Object x) { + if (x instanceof PyBool) { + return PyInt.of(((PyBool) x).value); + } + if (Py.isInt(x)) { + return x; + } + if (Py.isFloat(x)) { + double d = Py.asDouble(x); + if (Double.isNaN(d)) { + throw Py.valueError("cannot convert float NaN to integer"); + } + if (Double.isInfinite(d)) { + throw Py.overflowError("cannot convert float infinity to integer"); + } + if (Math.abs(d) < 9.2e18) { + return PyInt.of((long) d); + } + return Py.mul(PyInt.parse(Formatting.fixed(Math.abs(d), 0), 10), PyInt.of(d < 0 ? -1 : 1)); + } + if (x instanceof String || x instanceof PyBytes || x instanceof PyByteArray) { + return PyInt.parse(intText(x, false), 10); + } + Object m = Py.typeOf(x).lookup("__int__"); + if (m == null) { + m = Py.typeOf(x).lookup("__index__"); + } + if (m != null) { + return Py.obj(m).call(new Object[] {x}, Py.NO_KW); + } + throw Py.typeError("int() argument must be a string, a bytes-like object or a real number, not '" + + Py.typeOf(x).name + "'"); + } + + /// The text `int()` parses: a str, or the ASCII of a bytes-like value. + private static String intText(Object x, boolean explicitBase) { + if (x instanceof String) { + return (String) x; + } + if (x instanceof PyStr) { + return ((PyStr) x).value; + } + byte[] b = x instanceof PyBytes || x instanceof PyByteArray ? PyBytes.like(x) : null; + if (b == null) { + if (explicitBase) { + throw Py.typeError("int() can't convert non-string with explicit base"); + } + throw Py.typeError("int() argument must be a string, a bytes-like object or a real number, not '" + + Py.typeOf(x).name + "'"); + } + char[] c = new char[b.length]; + for (int i = 0; i < b.length; i++) { + c[i] = (char) (b[i] & 0xff); + } + return new String(c); + } + + static Object toFloat(Object x) { + if (Py.isFloat(x)) { + return x; + } + if (Py.isInt(x)) { + return Py.ofFloat(Py.asDouble(x)); + } + if (x instanceof String) { + String raw = ((String) x).trim(); + String s = raw.replace("_", ""); + String l = TypeMethods.asciiLower(s); + if ("nan".equals(l) || "+nan".equals(l) || "-nan".equals(l)) { + return Py.ofFloat(Double.NaN); + } + if ("inf".equals(l) || "+inf".equals(l) || "infinity".equals(l) || "+infinity".equals(l)) { + return Py.ofFloat(Double.POSITIVE_INFINITY); + } + if ("-inf".equals(l) || "-infinity".equals(l)) { + return Py.ofFloat(Double.NEGATIVE_INFINITY); + } + // Python's grammar, checked here: a device's parseDouble may + // accept more than the JDK's ("1+" parsed as 1.0 on ParparVM). + if (decimalLiteral(raw)) { + try { + return Py.ofFloat(Double.parseDouble(s)); + } catch (NumberFormatException e) { + // reported below with Python's message + } + } + throw Py.valueError("could not convert string to float: " + Py.toRepr(x)); + } + return Py.ofFloat(Py.asDouble(x)); + } + + /// `[sign] (digits [. [digits]] | . digits) [e [sign] digits]`, where + /// digits may hold single underscores between digits. + static boolean decimalLiteral(String s) { + int n = s.length(); + int i = 0; + if (i < n && (s.charAt(i) == '+' || s.charAt(i) == '-')) { + i++; + } + int d = digits(s, i); + boolean any = d > i; + i = d; + if (i < n && s.charAt(i) == '.') { + int f = digits(s, i + 1); + any |= f > i + 1; + i = f; + } + if (!any) { + return false; + } + if (i < n && (s.charAt(i) == 'e' || s.charAt(i) == 'E')) { + i++; + if (i < n && (s.charAt(i) == '+' || s.charAt(i) == '-')) { + i++; + } + int e = digits(s, i); + if (e <= i) { + return false; + } + i = e; + } + return i == n; + } + + /// The end of a run of digits starting at `i` (`i` itself when there is + /// none). An underscore not between two digits ends the run, and so is + /// left over for the caller to reject. + private static int digits(String s, int i) { + int j = i; + while (j < s.length()) { + char c = s.charAt(j); + if (c >= '0' && c <= '9') { + j++; + } else if (c == '_' && j > i && j + 1 < s.length() && s.charAt(j + 1) >= '0' && s.charAt(j + 1) <= '9') { + j++; + } else { + break; + } + } + return j; + } + + private static Object abs(Object x) { + if (x instanceof PyComplex) { + return Py.ofFloat(PyComplex.abs((PyComplex) x)); + } + if (Py.isFloat(x)) { + return Py.ofFloat(Math.abs(Py.asDouble(x))); + } + if (Py.isInt(x)) { + PyInt i = Py.intView(x); + return i.isTrue() && Py.ltBool(i, PyInt.of(0)) ? i.neg() : PyInt.view(0).add(i); + } + Object m = Py.typeOf(x).lookup("__abs__"); + if (m == null) { + throw Py.typeError("bad operand type for abs(): '" + Py.typeOf(x).name + "'"); + } + return Py.obj(m).call(new Object[] {x}, Py.NO_KW); + } + + static boolean isCallable(Object x) { + return x instanceof PyFunction || x instanceof PyBuiltin || x instanceof PyMethod || x instanceof PyType + || x instanceof PyStaticMethod || x instanceof PyInstance && Py.typeOf(x).lookup("__call__") != null; + } + + private static Object iter(Object[] a) { + if (a.length == 2) { + final Object f = a[0]; + final Object sentinel = a[1]; + return new PyJavaIter(() -> { + Object v = Py.obj(f).call(Py.NO_ARGS, Py.NO_KW); + return Py.eqBool(v, sentinel) ? null : v; + }); + } + return Py.iter(one("iter", a)); + } + + private static Object next(Object[] a) { + Object v = Py.next(a[0]); + if (v == null) { + if (a.length > 1) { + return a[1]; + } + if (a[0] instanceof PyGenerator) { + // A finished generator's StopIteration carries its return value. + throw PyGenerator.stopIteration(((PyGenerator) a[0]).returned()); + } + throw new PyException(Py.STOP_ITERATION.call(Py.NO_ARGS, Py.NO_KW)); + } + return v; + } + + private static Object ord(Object x) { + if (!(x instanceof String) || PyStr.codePoints((String) x) != 1) { + throw Py.typeError("ord() expected a character, but string of length " + + (x instanceof String ? PyStr.codePoints((String) x) : 0) + " found"); + } + return PyInt.of(((String) x).codePointAt(0)); + } + + private static String asciiOnly(String s) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < s.length(); i++) { + int c = s.codePointAt(i); + if (c > 0xffff) { + i++; + } + if (c < 128) { + sb.append((char) c); + } else if (c < 256) { + sb.append("\\x"); + PyStr.hex(sb, c, 2); + } else if (c < 0x10000) { + sb.append("\\u"); + PyStr.hex(sb, c, 4); + } else { + sb.append("\\U"); + PyStr.hex(sb, c, 8); + } + } + return sb.toString(); + } + + private static Object pow(Object[] a) { + if (a.length == 3 && a[2] != Py.NONE) { //NOPMD CompareObjectsWithEquals - identity is meant + PyInt base = Py.intView(toInt(a[0])); + PyInt exp = Py.intView(toInt(a[1])); + PyInt mod = Py.intView(toInt(a[2])); + com.codename1.util.BigInteger m = mod.toBig(); + if (exp.toBig().signum() < 0) { + return PyInt.of(base.toBig().modInverse(m).modPow(exp.toBig().negate(), m)); + } + com.codename1.util.BigInteger r = base.toBig().modPow(exp.toBig(), m.abs()); + if (m.signum() < 0 && r.signum() != 0) { + r = r.add(m); + } + return PyInt.of(r); + } + return Py.pow(a[0], a[1]); + } + + private static Object round(Args a) { + Object x = a.required(0, "number"); + Object nd = a.get(1, "ndigits", Py.NONE); + if (x instanceof PyInstance) { + Object m = Py.typeOf(x).lookup("__round__"); + if (m != null) { + return nd == Py.NONE ? Py.obj(m).call(new Object[] {x}, Py.NO_KW) : Py.obj(m).call(new Object[] {x, nd}, Py.NO_KW); //NOPMD CompareObjectsWithEquals - identity is meant + } + } + if (Py.isInt(x)) { + if (nd == Py.NONE || Py.asLong(nd) >= 0) { //NOPMD CompareObjectsWithEquals - identity is meant + return PyInt.view(0).add(x); + } + Object p = PyInt.view(10).pow(PyInt.of(-Py.asLong(nd))); + Object half = Py.floordiv(p, PyInt.of(2)); + Object q = Py.floordiv(x, p); + Object r = Py.mod(x, p); + if (Py.ltBool(half, r) || Py.eqBool(r, half) && Py.isTrue(Py.mod(q, PyInt.of(2)))) { + q = Py.add(q, PyInt.of(1)); + } + return Py.mul(q, p); + } + double d = Py.asDouble(x); + if (nd == Py.NONE) { //NOPMD CompareObjectsWithEquals - identity is meant + double r = Math.rint(d); + return toInt(Py.ofFloat(r)); + } + long n = Py.asLong(nd); + if (Double.isNaN(d) || Double.isInfinite(d) || d == 0) { + return Py.ofFloat(d); + } + if (n >= 0) { + if (n > 300) { + return Py.ofFloat(d); + } + String s = Formatting.fixed(Math.abs(d), (int) n); + double r = Double.parseDouble(s); + return Py.ofFloat(d < 0 ? -r : r); + } + double p = Math.pow(10, -n); + double r = Math.rint(d / p) * p; + return Py.ofFloat(r); + } + + private static boolean issubclass(Object c, Object t) { + if (!(c instanceof PyType)) { + throw Py.typeError("issubclass() arg 1 must be a class"); + } + if (t instanceof PyTuple) { + Object[] items = ((PyTuple) t).items; + for (Object item : items) { + if (issubclass(c, item)) { + return true; + } + } + return false; + } + if (!(t instanceof PyType)) { + throw Py.typeError("issubclass() arg 2 must be a class, a tuple of classes, or a union"); + } + return ((PyType) t).subclassCheck((PyType) c); + } + + private static Object getattr(Object[] a) { + if (a.length == 3) { + try { + Object v = Py.obj(a[0]).getAttrOrNull(Py.toStr(a[1])); + return v != null ? v : a[2]; + } catch (PyException e) { + if (e.matches(Py.ATTRIBUTE_ERROR)) { + return a[2]; + } + throw e; + } + } + return Py.obj(a[0]).getAttr(Py.toStr(a[1])); + } + + private static Object dir(Object o) { + PyList l = new PyList(); + if (o instanceof PyModule) { + // A module lists its namespace, never its type's attributes. + Object it = ((PyModule) o).globals().iter(); + for (Object k = Py.next(it); k != null; k = Py.next(it)) { + l.append(k); + } + l.sort(null, false); + return l; + } + java.util.Set seen = new java.util.HashSet(); + if (o instanceof PyInstance) { + seen.addAll(java.util.Arrays.asList(((PyInstance) o).attributeNames())); + } + PyType t = o instanceof PyType ? (PyType) o : Py.typeOf(o); + for (int i = 0; i < t.mro.length; i++) { + seen.addAll(t.mro[i].dict.keySet()); + } + for (String s : seen) { + l.append(Py.str(s)); + } + l.sort(null, false); + return l; + } + + private static Object sum(Args a) { + Object it = Py.iter(a.required(0, "iterable")); + Object acc = a.get(1, "start", PyInt.of(0)); + if (acc instanceof String) { + throw Py.typeError("sum() can't sum strings [use ''.join(seq) instead]"); + } + // Integer runs stay in a long; floats use Neumaier summation as CPython 3.12 does. + long ia = 0; + boolean intMode = Py.isInt(acc) && !Py.intView(acc).isBig() && !(acc instanceof PyBool); + if (intMode) { + ia = Py.intView(acc).value; + } + double fs = 0; + double fc = 0; + boolean floatMode = false; + for (Object v = Py.next(it); v != null; v = Py.next(it)) { + if (intMode) { + if (Py.isInt(v) && !Py.intView(v).isBig()) { + long b = Py.intView(v).value; + long r = ia + b; + if (((ia ^ r) & (b ^ r)) >= 0) { + ia = r; + continue; + } + } + acc = PyInt.of(ia); + intMode = false; + if (Py.isFloat(v)) { + floatMode = true; + fs = Py.asDouble(acc); + fc = 0; + } + } + if (floatMode) { + if (Py.isFloat(v) || Py.isInt(v)) { + double x = Py.asDouble(v); + double t = fs + x; + if (Math.abs(fs) >= Math.abs(x)) { + fc += (fs - t) + x; + } else { + fc += (x - t) + fs; + } + fs = t; + continue; + } + acc = Py.ofFloat(fs + fc); + floatMode = false; + } + if (Py.isFloat(acc) && Py.isFloat(v) && !floatMode) { + floatMode = true; + fs = Py.asDouble(acc); + fc = 0; + double x = Py.asDouble(v); + double t = fs + x; + fc += Math.abs(fs) >= Math.abs(x) ? (fs - t) + x : (x - t) + fs; + fs = t; + continue; + } + acc = Py.add(acc, v); + } + if (intMode) { + return PyInt.of(ia); + } + if (floatMode) { + return Py.ofFloat(fs + fc); + } + return acc; + } + + private static Object minMax(String fn, Object[] args, String[] kw, boolean min) { + Args a = new Args(fn, args, kw).check(Integer.MAX_VALUE, "key", "default"); + Object key = a.kw("key"); + if (key == Py.NONE) { //NOPMD CompareObjectsWithEquals - identity is meant + key = null; + } + Object def = a.kw("default"); + Object it; + if (a.npos == 1) { + it = Py.iter(args[0]); + } else { + if (def != null) { + throw Py.typeError("Cannot specify a default for " + fn + "() with multiple positional arguments"); + } + it = new PyTuple(a.positional()).iter(); + } + Object best = null; + Object bestKey = null; + for (Object v = Py.next(it); v != null; v = Py.next(it)) { + Object k = key == null ? v : Py.obj(key).call(new Object[] {v}, Py.NO_KW); + if (best == null || (min ? Py.ltBool(k, bestKey) : Py.ltBool(bestKey, k))) { + best = v; + bestKey = k; + } + } + if (best == null) { + if (def != null) { + return def; + } + throw Py.valueError(fn + "() iterable argument is empty"); + } + return best; + } + + private static Object reversed(Object seq) { + Object m = Py.typeOf(seq).lookup("__reversed__"); + if (m != null) { + return Py.obj(m).call(new Object[] {seq}, Py.NO_KW); + } + if (seq instanceof PyInstance && ((PyInstance) seq).base != null) { + // A subclass of a built-in reverses as its built-in value does. + Object base = ((PyInstance) seq).base; + if (base instanceof PyDict || base instanceof PyList || base instanceof PyTuple) { + return reversed(base); + } + } + if (seq instanceof PyDict) { + Object[] keys = ((PyDict) seq).keyArray(); + return reversedArray(keys); + } + if (seq instanceof PyList || seq instanceof PyTuple) { + return reversedArray(Py.toArray(seq)); + } + final Object s = seq; + final int[] i = {Py.len(seq)}; + return new PyJavaIter(() -> i[0] <= 0 ? null : Py.obj(s).getItem(PyInt.of(--i[0]))); + } + + private static Object reversedArray(final Object[] items) { + final int[] i = {items.length}; + return new PyJavaIter(() -> i[0] <= 0 ? null : items[--i[0]]); + } + + /// An attribute an instance does not have, read as `None`. + private static Object orNone(Object o) { + return o == null ? Py.NONE : o; + } + +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Cn1FileSystem.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Cn1FileSystem.java new file mode 100644 index 00000000000..381f6d3a845 --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Cn1FileSystem.java @@ -0,0 +1,145 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +import com.codename1.io.FileSystemStorage; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +/// The file system of a Codename One application: `FileSystemStorage`, +/// whose paths are `file://` URLs. +final class Cn1FileSystem extends PyFileSystem { + private static final String SCHEME = "file://"; + + private static FileSystemStorage fs() { + return FileSystemStorage.getInstance(); + } + + private static String url(String path) { + return path.startsWith(SCHEME) ? path : SCHEME + path; + } + + @Override + public String home() { + String h = fs().getAppHomePath(); + if (h.startsWith(SCHEME)) { + h = h.substring(SCHEME.length()); + } + while (h.length() > 1 && h.endsWith("/")) { + h = h.substring(0, h.length() - 1); + } + return h; + } + + @Override + public InputStream openInput(String path) throws IOException { + return fs().openInputStream(url(path)); + } + + @Override + public OutputStream openOutput(String path, boolean append) throws IOException { + String u = url(path); + if (append && fs().exists(u)) { + return fs().openOutputStream(u, (int) fs().getLength(u)); + } + return fs().openOutputStream(u); + } + + @Override + public boolean exists(String path) { + return fs().exists(url(path)); + } + + @Override + public boolean isDirectory(String path) { + return fs().isDirectory(url(path)); + } + + @Override + public String[] list(String path) throws IOException { + String[] names = fs().listFiles(url(path)); + String[] r = new String[names.length]; + for (int i = 0; i < names.length; i++) { + String n = names[i]; + // Some ports mark a directory with a trailing separator. + r[i] = n.endsWith("/") ? n.substring(0, n.length() - 1) : n; + } + return r; + } + + @Override + public void mkdir(String path) throws IOException { + fs().mkdir(url(path)); + if (!fs().exists(url(path))) { + throw new IOException("mkdir failed"); + } + } + + @Override + public void delete(String path) throws IOException { + fs().delete(url(path)); + } + + @Override + public void rename(String from, String to) throws IOException { + int a = from.lastIndexOf('/'); + int b = to.lastIndexOf('/'); + if (a != b || !from.substring(0, a + 1).equals(to.substring(0, b + 1))) { + // FileSystemStorage renames within a directory only; a move + // is a copy and a delete. + InputStream in = openInput(from); + try { + OutputStream out = openOutput(to, false); + try { + byte[] buf = new byte[8192]; + for (int n = in.read(buf); n > 0; n = in.read(buf)) { + out.write(buf, 0, n); + } + } finally { + out.close(); + } + } finally { + in.close(); + } + delete(from); + return; + } + if (fs().exists(url(to))) { + fs().delete(url(to)); + } + fs().rename(url(from), to.substring(b + 1)); + } + + @Override + public long size(String path) { + return fs().getLength(url(path)); + } + + @Override + public long lastModified(String path) { + return fs().getLastModified(url(path)); + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Codecs.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Codecs.java new file mode 100644 index 00000000000..212817ca129 --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Codecs.java @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +import java.io.ByteArrayOutputStream; + +/// `str.encode` / `bytes.decode` for the encodings Codename One supports on +/// every platform: UTF-8, ASCII and Latin-1. Any other name is a +/// LookupError rather than a silent fallback, so a program that needs +/// another charset fails where it asks for it. +final class Codecs { + private Codecs() { + } + + /// A normalised codec name: `utf-8`, `ascii` or `latin-1`. + static String normalize(String encoding) { + String e = TypeMethods.asciiLower(encoding).replace('_', '-'); + if ("utf-8".equals(e) || "utf8".equals(e) || "u8".equals(e) || "utf".equals(e)) { + return "utf-8"; + } + if ("ascii".equals(e) || "us-ascii".equals(e) || "646".equals(e)) { + return "ascii"; + } + if ("latin-1".equals(e) || "latin1".equals(e) || "iso-8859-1".equals(e) || "iso8859-1".equals(e) + || "l1".equals(e) || "8859".equals(e) || "cp819".equals(e)) { + return "latin-1"; + } + throw Py.error(Py.LOOKUP_ERROR, "unknown encoding: " + encoding); + } + + static byte[] encode(String s, String encoding, String errors) { + String enc = normalize(encoding); + if ("utf-8".equals(enc)) { + return Utf8.encode(s); + } + int limit = "ascii".equals(enc) ? 0x80 : 0x100; + ByteArrayOutputStream out = new ByteArrayOutputStream(s.length()); + int i = 0; + while (i < s.length()) { + int cp = Character.codePointAt(s, i); + int pos = i; + i += Character.charCount(cp); + if (cp < limit) { + out.write(cp); + } else if ("ignore".equals(errors)) { + continue; + } else if ("replace".equals(errors)) { + out.write('?'); + } else { + throw Py.error(Py.UNICODE_ENCODE_ERROR, "'" + (limit == 0x80 ? "ascii" : "latin-1") + + "' codec can't encode character '" + escape(cp) + "' in position " + + PyStr.cpIndex(s, pos) + ": ordinal not in range(" + limit + ")"); + } + } + return out.toByteArray(); + } + + static String decode(byte[] b, String encoding, String errors) { + String enc = normalize(encoding); + if ("utf-8".equals(enc)) { + return Utf8.decode(b, errors); + } + boolean ascii = "ascii".equals(enc); + StringBuilder sb = new StringBuilder(b.length); + for (int i = 0; i < b.length; i++) { + int c = b[i] & 0xff; + if (ascii && c >= 0x80) { + if ("ignore".equals(errors)) { + continue; + } + if ("replace".equals(errors)) { + sb.append('\uFFFD'); + continue; + } + throw Py.error(Py.UNICODE_DECODE_ERROR, "'ascii' codec can't decode byte 0x" + Integer.toHexString(c) + + " in position " + i + ": ordinal not in range(128)"); + } + sb.append((char) c); + } + return sb.toString(); + } + + /// `repr`'s escape of a code point: backslash-x for Latin-1, backslash-u + /// below 0x10000, backslash-U above. + private static String escape(int cp) { + if (cp < 0x100) { + String h = Integer.toHexString(cp); + return "\\x" + (h.length() < 2 ? "0" + h : h); + } + if (cp < 0x10000) { + return "\\u" + hex4(cp); + } + String h = Integer.toHexString(cp); + return "\\U" + "00000000".substring(h.length()) + h; + } + + private static String hex4(int cp) { + String h = Integer.toHexString(cp); + StringBuilder sb = new StringBuilder(); + for (int k = h.length(); k < 4; k++) { + sb.append('0'); + } + return sb.append(h).toString(); + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Erf.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Erf.java new file mode 100644 index 00000000000..567996605ee --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Erf.java @@ -0,0 +1,192 @@ +/* + * Ported from the Sun Microsystems FDLIBM C-library (s_erf.c). + * (Freely Distributable Library for Math) + * ==================================================== + * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + * + * Developed at SunPro, a Sun Microsystems, Inc. business. + * Permission to use, copy, modify, and distribute this + * software is freely granted, provided that this notice + * is preserved. + * ==================================================== + */ +package com.codename1.python.runtime; + +import com.codename1.util.MathUtil; + +/// `math.erf` and `math.erfc`, a Java rewrite of FDLIBM's `s_erf.c`. +/// +/// CPython calls the C library's functions, and the C libraries it runs on +/// derive theirs from this code, so a series of our own disagreed with +/// CPython in the last bit and that showed in `statistics.NormalDist`. +final class Erf { + private static final double TINY = 1e-300; + private static final double ERX = 8.45062911510467529297e-01; + private static final double EFX = 1.28379167095512586316e-01; + private static final double EFX8 = 1.02703333676410069053e+00; + private static final double PP0 = 1.28379167095512558561e-01; + private static final double PP1 = -3.25042107247001499370e-01; + private static final double PP2 = -2.84817495755985104766e-02; + private static final double PP3 = -5.77027029648944159157e-03; + private static final double PP4 = -2.37630166566501626084e-05; + private static final double QQ1 = 3.97917223959155352819e-01; + private static final double QQ2 = 6.50222499887672944485e-02; + private static final double QQ3 = 5.08130628187576562776e-03; + private static final double QQ4 = 1.32494738004321644526e-04; + private static final double QQ5 = -3.96022827877536812320e-06; + private static final double PA0 = -2.36211856075265944077e-03; + private static final double PA1 = 4.14856118683748331666e-01; + private static final double PA2 = -3.72207876035701323847e-01; + private static final double PA3 = 3.18346619901161753674e-01; + private static final double PA4 = -1.10894694282396677476e-01; + private static final double PA5 = 3.54783043256182359371e-02; + private static final double PA6 = -2.16637559486879084300e-03; + private static final double QA1 = 1.06420880400844228286e-01; + private static final double QA2 = 5.40397917702171048937e-01; + private static final double QA3 = 7.18286544141962662868e-02; + private static final double QA4 = 1.26171219808761642112e-01; + private static final double QA5 = 1.36370839120290507362e-02; + private static final double QA6 = 1.19844998467991074170e-02; + private static final double RA0 = -9.86494403484714822705e-03; + private static final double RA1 = -6.93858572707181764372e-01; + private static final double RA2 = -1.05586262253232909814e+01; + private static final double RA3 = -6.23753324503260060396e+01; + private static final double RA4 = -1.62396669462573470355e+02; + private static final double RA5 = -1.84605092906711035994e+02; + private static final double RA6 = -8.12874355063065934246e+01; + private static final double RA7 = -9.81432934416914548592e+00; + private static final double SA1 = 1.96512716674392571292e+01; + private static final double SA2 = 1.37657754143519042600e+02; + private static final double SA3 = 4.34565877475229228821e+02; + private static final double SA4 = 6.45387271733267880336e+02; + private static final double SA5 = 4.29008140027567833386e+02; + private static final double SA6 = 1.08635005541779435134e+02; + private static final double SA7 = 6.57024977031928170135e+00; + private static final double SA8 = -6.04244152148580987438e-02; + private static final double RB0 = -9.86494292470009928597e-03; + private static final double RB1 = -7.99283237680523006574e-01; + private static final double RB2 = -1.77579549177547519889e+01; + private static final double RB3 = -1.60636384855821916062e+02; + private static final double RB4 = -6.37566443368389627722e+02; + private static final double RB5 = -1.02509513161107724954e+03; + private static final double RB6 = -4.83519191608651397019e+02; + private static final double SB1 = 3.03380607434824582924e+01; + private static final double SB2 = 3.25792512996573918826e+02; + private static final double SB3 = 1.53672958608443695994e+03; + private static final double SB4 = 3.19985821950859553908e+03; + private static final double SB5 = 2.55305040643316442583e+03; + private static final double SB6 = 4.74528541206955367215e+02; + private static final double SB7 = -2.24409524465858183362e+01; + + private Erf() { + } + + private static int high(double x) { + return (int) (Double.doubleToRawLongBits(x) >> 32); + } + + private static double clearLow(double x) { + return Double.longBitsToDouble(Double.doubleToRawLongBits(x) & 0xffffffff00000000L); + } + + private static double smallP(double z) { + return PP0 + z * (PP1 + z * (PP2 + z * (PP3 + z * PP4))); + } + + private static double smallQ(double z) { + return 1.0 + z * (QQ1 + z * (QQ2 + z * (QQ3 + z * (QQ4 + z * QQ5)))); + } + + private static double midP(double s) { + return PA0 + s * (PA1 + s * (PA2 + s * (PA3 + s * (PA4 + s * (PA5 + s * PA6))))); + } + + private static double midQ(double s) { + return 1.0 + s * (QA1 + s * (QA2 + s * (QA3 + s * (QA4 + s * (QA5 + s * QA6))))); + } + + /// `exp(-x*x - 0.5625) * exp(...)/x` for the tails, `x` positive. + private static double tail(double x, int ix, int bound) { + double s = 1.0 / (x * x); + double r; + double ss; + if (ix < bound) { + r = RA0 + s * (RA1 + s * (RA2 + s * (RA3 + s * (RA4 + s * (RA5 + s * (RA6 + s * RA7)))))); + ss = 1.0 + s * (SA1 + s * (SA2 + s * (SA3 + s * (SA4 + s * (SA5 + s * (SA6 + s * (SA7 + + s * SA8))))))); + } else { + r = RB0 + s * (RB1 + s * (RB2 + s * (RB3 + s * (RB4 + s * (RB5 + s * RB6))))); + ss = 1.0 + s * (SB1 + s * (SB2 + s * (SB3 + s * (SB4 + s * (SB5 + s * (SB6 + s * SB7)))))); + } + double z = clearLow(x); + return MathUtil.exp(-z * z - 0.5625) * MathUtil.exp((z - x) * (z + x) + r / ss) / x; + } + + static double erf(double x) { + int hx = high(x); + int ix = hx & 0x7fffffff; + if (ix >= 0x7ff00000) { + if (Double.isNaN(x)) { + return x; + } + return hx < 0 ? -1.0 : 1.0; + } + if (ix < 0x3feb0000) { + if (ix < 0x3e300000) { + if (ix < 0x00800000) { + return 0.125 * (8.0 * x + EFX8 * x); + } + return x + EFX * x; + } + double z = x * x; + return x + x * (smallP(z) / smallQ(z)); + } + if (ix < 0x3ff40000) { + double s = Math.abs(x) - 1.0; + double pq = midP(s) / midQ(s); + return hx >= 0 ? ERX + pq : -ERX - pq; + } + if (ix >= 0x40180000) { + return hx >= 0 ? 1.0 - TINY : TINY - 1.0; + } + double r = tail(Math.abs(x), ix, 0x4006DB6E); + return hx >= 0 ? 1.0 - r : r - 1.0; + } + + static double erfc(double x) { + int hx = high(x); + int ix = hx & 0x7fffffff; + if (ix >= 0x7ff00000) { + if (Double.isNaN(x)) { + return x; + } + return hx < 0 ? 2.0 : 0.0; + } + if (ix < 0x3feb0000) { + if (ix < 0x3c700000) { + return 1.0 - x; + } + double z = x * x; + double y = smallP(z) / smallQ(z); + if (hx < 0x3fd00000) { + return 1.0 - (x + x * y); + } + double r = x * y; + r += x - 0.5; + return 0.5 - r; + } + if (ix < 0x3ff40000) { + double s = Math.abs(x) - 1.0; + double pq = midP(s) / midQ(s); + return hx >= 0 ? (1.0 - ERX) - pq : 1.0 + (ERX + pq); + } + if (ix < 0x403c0000) { + if (hx < 0 && ix >= 0x40180000) { + return 2.0 - TINY; + } + double r = tail(Math.abs(x), ix, 0x4006DB6D); + return hx > 0 ? r : 2.0 - r; + } + return hx > 0 ? TINY * TINY : 2.0 - TINY; + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/ExceptionGroups.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/ExceptionGroups.java new file mode 100644 index 00000000000..3a40e7deddb --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/ExceptionGroups.java @@ -0,0 +1,292 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/// `BaseExceptionGroup`, `ExceptionGroup` and `except*`. +/// +/// The transpiler rewrites a `try` with `except*` clauses into an ordinary +/// `try` whose single handler drives an [ExceptStar]: each clause takes the +/// part of the group its type matches, an exception its body raises is +/// recorded, and `finish` raises what is left, as PEP 654 specifies. +final class ExceptionGroups { + private ExceptionGroups() { + } + + static void install() { + PyType b = Py.BASE_EXCEPTION_GROUP; + b.define("__new__", new PyStaticMethod(PyBuiltin.positional("__new__", a -> { + if (a.length != 3 || !(a[0] instanceof PyType)) { + throw Py.typeError("BaseExceptionGroup.__new__() takes exactly 2 arguments (" + (a.length - 1) + + " given)"); + } + return create((PyType) a[0], a[1], a[2]); + }))); + b.define("__init__", PyBuiltin.positional("__init__", a -> Py.NONE)); + b.define("__str__", PyBuiltin.positional("__str__", a -> { + int n = exceptions(a[0]).length; + return Py.str(Py.toStr(message(a[0])) + " (" + n + " sub-exception" + (n == 1 ? "" : "s") + ")"); + })); + b.define("split", PyBuiltin.positional("split", a -> { + Object[] r = split(a[0], a[1]); + return new PyTuple(new Object[] {r[0] == null ? Py.NONE : r[0], r[1] == null ? Py.NONE : r[1]}); + })); + b.define("subgroup", PyBuiltin.positional("subgroup", a -> { + Object m = split(a[0], a[1])[0]; + return m == null ? Py.NONE : m; + })); + b.define("derive", PyBuiltin.positional("derive", a -> create(Py.BASE_EXCEPTION_GROUP, message(a[0]), a[1]))); + } + + static void names(Map names) { + names.put("__cn1_except_star__", PyBuiltin.positional("__cn1_except_star__", a -> new ExceptStar(a[0]))); + names.put("__cn1_except_star_take__", PyBuiltin.positional("__cn1_except_star_take__", + a -> star(a[0]).take(a[1]))); + names.put("__cn1_except_star_raised__", PyBuiltin.positional("__cn1_except_star_raised__", a -> { + star(a[0]).raised.add(a[1]); + return Py.NONE; + })); + names.put("__cn1_except_star_finish__", PyBuiltin.positional("__cn1_except_star_finish__", a -> { + star(a[0]).finish(); + return Py.NONE; + })); + } + + private static ExceptStar star(Object o) { + if (o instanceof ExceptStar) { + return (ExceptStar) o; + } + throw Py.typeError("internal error: not an except* state"); + } + + /// `BaseExceptionGroup(message, exceptions)`: an `ExceptionGroup` when + /// `cls` is the base and every exception is an `Exception`. + static PyInstance create(PyType cls, Object message, Object excs) { + if (!(message instanceof String)) { + throw Py.typeError("argument 1 must be str, not " + Py.typeOf(message).name); + } + Object[] items = Py.toArray(excs); + if (items.length == 0) { + throw Py.valueError("second argument (exceptions) must be a non-empty sequence"); + } + boolean allExceptions = true; + for (int i = 0; i < items.length; i++) { + if (!Py.isinstance(items[i], Py.BASE_EXCEPTION)) { + throw Py.valueError("Item " + i + " of second argument (exceptions) is not an exception"); + } + allExceptions &= Py.isinstance(items[i], Py.EXCEPTION); + } + if (cls == Py.BASE_EXCEPTION_GROUP && allExceptions) { //NOPMD CompareObjectsWithEquals - identity is meant + cls = Py.EXCEPTION_GROUP; + } else if (cls.isSubtype(Py.EXCEPTION_GROUP) && !allExceptions) { + throw Py.typeError("Cannot nest BaseExceptions in an ExceptionGroup"); + } + PyInstance e = cls.newInstance(); + e.setAttrDirect("args", new PyTuple(new Object[] {message, excs})); + e.setAttrDirect("message", message); + e.setAttrDirect("exceptions", new PyTuple(items)); + return e; + } + + static boolean isGroup(Object e) { + return Py.isinstance(e, Py.BASE_EXCEPTION_GROUP); + } + + private static Object message(Object eg) { + return Py.obj(eg).getAttr("message"); + } + + private static Object[] exceptions(Object eg) { + return Py.toArray(Py.obj(eg).getAttr("exceptions")); + } + + private static boolean matches(Object e, Object cond) { + if (cond instanceof PyType || cond instanceof PyTuple) { + return Py.isinstance(e, cond); + } + return Py.isTrue(Py.call(cond, e)); + } + + /// `eg.split(cond)`: the matching part and the rest, either null. + static Object[] split(Object eg, Object cond) { + if (matches(eg, cond)) { + return new Object[] {eg, null}; + } + List ms = new ArrayList(); + List rs = new ArrayList(); + Object[] items = exceptions(eg); + for (Object item : items) { + Object e = item; + if (isGroup(e)) { + Object[] r = split(e, cond); + if (r[0] != null) { + ms.add(r[0]); + } + if (r[1] != null) { + rs.add(r[1]); + } + } else if (matches(e, cond)) { + ms.add(e); + } else { + rs.add(e); + } + } + return new Object[] {derive(eg, ms), derive(eg, rs)}; + } + + private static Object derive(Object eg, List excs) { + if (excs.isEmpty()) { + return null; + } + PyList l = new PyList(); + for (int i = 0; i < excs.size(); i++) { //NOPMD ForLoopCanBeForeach - indexed: a for-each allocates an iterator + l.append(excs.get(i)); + } + Object d = Py.callMethod(eg, "derive", l); + Object notes = Py.obj(eg).getAttrOrNull("__notes__"); + if (notes != null) { + Py.obj(d).setAttr("__notes__", notes); + } + return d; + } + + /// The leaf exceptions of `e`, a group or a single exception. + static void leaves(Object e, List out) { + if (!isGroup(e)) { + out.add(e); + return; + } + Object[] items = exceptions(e); + for (Object item : items) { + leaves(item, out); + } + } + + /// The part of group `eg` whose leaves are among `keep`, in its + /// structure, or null. + static Object select(Object eg, List keep) { + List kept = new ArrayList(); + Object[] items = exceptions(eg); + boolean all = true; + for (Object item : items) { + Object e = item; + Object k = isGroup(e) ? select(e, keep) : keep.contains(e) ? e : null; + if (k != null) { + kept.add(k); + } + all &= k == e; //NOPMD CompareObjectsWithEquals - identity is meant + } + return all ? eg : derive(eg, kept); + } + + /// The state of one `except*` dispatch. + static final class ExceptStar extends PyObject { + private final Object original; + private Object rest; + private final Object whole; + private boolean taken; + final List raised = new ArrayList(); + private final List matched = new ArrayList(); + + ExceptStar(Object exc) { + original = exc; + rest = isGroup(exc) ? exc : create(Py.BASE_EXCEPTION_GROUP, Py.EMPTY_STR, new PyList(new Object[] {exc})); + whole = rest; + } + + @Override + public PyType type() { + return Py.OBJECT_TYPE; + } + + Object take(Object type) { + Object[] ts = type instanceof PyTuple ? ((PyTuple) type).items : new Object[] {type}; + for (Object x : ts) { + if (x instanceof PyType && ((PyType) x).isSubtype(Py.BASE_EXCEPTION_GROUP)) { + throw Py.typeError("catching ExceptionGroup with except* is not allowed. Use except instead."); + } + } + if (rest == null) { + return Py.NONE; + } + Object[] r = split(rest, type); + if (r[0] == null) { + return Py.NONE; + } + taken = true; + rest = r[1]; + matched.add(r[0]); + return r[0]; + } + + void finish() { + List fresh = new ArrayList(); + List keepLeaves = new ArrayList(); + boolean reraised = false; + for (int i = 0; i < raised.size(); i++) { //NOPMD ForLoopCanBeForeach - indexed: a for-each allocates an iterator + Object r = raised.get(i); + if (matched.contains(r)) { + // A bare `raise`: that part stays unhandled, in the + // original group's structure. + leaves(r, keepLeaves); + reraised = true; + } else { + fresh.add(r); + } + } + Object keep; + if (!taken) { + keep = original; + } else if (reraised) { + if (rest != null) { + leaves(rest, keepLeaves); + } + keep = select(whole, keepLeaves); + } else { + keep = rest; + } + if (fresh.isEmpty()) { + if (keep != null) { + throw new PyException(keep); + } + return; + } + if (keep == null && fresh.size() == 1) { + // One new exception and nothing left over propagates as it is. + throw new PyException(fresh.get(0)); + } + PyList l = new PyList(); + for (int i = 0; i < fresh.size(); i++) { //NOPMD ForLoopCanBeForeach - indexed: a for-each allocates an iterator + l.append(fresh.get(i)); + } + if (keep != null) { + l.append(keep); + } + throw new PyException(create(Py.BASE_EXCEPTION_GROUP, Py.EMPTY_STR, l)); + } + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Formatting.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Formatting.java new file mode 100644 index 00000000000..7f548b3646c --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Formatting.java @@ -0,0 +1,612 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +import com.codename1.util.BigInteger; + +/// The format-spec mini-language (`format()`, f-strings, `str.format`) and +/// printf-style `%` formatting. +/// +/// Float conversions are computed from the exact binary value of the double, +/// so `f"{2.675:.2f}"` gives `2.67` exactly as CPython does; nothing here +/// depends on `String.format`, which the device class library lacks. +final class Formatting { + + private Formatting() { + } + + /// A parsed `[[fill]align][sign][z][#][0][width][grouping][.precision][type]`. + static final class Spec { + char fill = ' '; + char align; + char sign = '-'; + boolean alt; + boolean zero; + int width = -1; + char grouping; + int precision = -1; + char type; + + static Spec parse(String s) { + Spec p = new Spec(); + int i = 0; + int n = s.length(); + if (n >= 2 && "<>=^".indexOf(s.charAt(1)) >= 0) { + p.fill = s.charAt(0); + p.align = s.charAt(1); + i = 2; + } else if (n >= 1 && "<>=^".indexOf(s.charAt(0)) >= 0) { + p.align = s.charAt(0); + i = 1; + } + if (i < n && "+- ".indexOf(s.charAt(i)) >= 0) { + p.sign = s.charAt(i++); + } + if (i < n && s.charAt(i) == 'z') { + i++; + } + if (i < n && s.charAt(i) == '#') { + p.alt = true; + i++; + } + if (i < n && s.charAt(i) == '0') { + p.zero = true; + i++; + } + int w = 0; + boolean hasW = false; + while (i < n && Character.isDigit(s.charAt(i))) { + w = w * 10 + (s.charAt(i++) - '0'); + hasW = true; + } + if (hasW) { + p.width = w; + } + if (i < n && (s.charAt(i) == ',' || s.charAt(i) == '_')) { + p.grouping = s.charAt(i++); + } + if (i < n && s.charAt(i) == '.') { + i++; + int pr = 0; + boolean hasP = false; + while (i < n && Character.isDigit(s.charAt(i))) { + pr = pr * 10 + (s.charAt(i++) - '0'); + hasP = true; + } + if (!hasP) { + throw Py.valueError("Format specifier missing precision"); + } + p.precision = pr; + } + if (i < n) { + p.type = s.charAt(i++); + } + if (i != n) { + throw Py.valueError("Invalid format specifier '" + s + "'"); + } + if (p.zero && p.align == 0) { + p.fill = '0'; + p.align = '='; + } + return p; + } + } + + private static PyException unknownCode(Spec p, Object v) { + return Py.valueError("Unknown format code '" + p.type + "' for object of type '" + Py.typeOf(v).name + "'"); + } + + static String format(Object v, String spec) { + if (v instanceof PyInstance) { + Object m = Py.typeOf(v).lookup("__format__"); + if (m != null) { + return Py.strValue(Py.obj(m).call(new Object[] {v, Py.str(spec)}, Py.NO_KW), "__format__"); + } + if (spec.length() == 0) { + return Py.toStr(v); + } + if (((PyInstance) v).base != null) { + // A subclass of a builtin formats as its value. + v = ((PyInstance) v).base; + } + } + if (spec.length() == 0) { + return Py.toStr(v); + } + Spec p = Spec.parse(spec); + if (Py.isInt(v) && !(v instanceof PyBool && p.type == 0)) { + if (p.type == 0 || "dboxXcn".indexOf(p.type) >= 0) { + return formatInt(Py.intView(v), p); + } + if ("eEfFgG%".indexOf(p.type) < 0) { + throw unknownCode(p, v); + } + return formatFloat(Py.asDouble(v), p); + } + if (Py.isFloat(v)) { + if (p.type != 0 && "eEfFgGn%".indexOf(p.type) < 0) { + throw unknownCode(p, v); + } + return formatFloat(Py.asDouble(v), p); + } + String s = Py.toStr(v); + if (p.type != 0 && p.type != 's') { + throw unknownCode(p, v); + } + if (p.precision >= 0 && s.length() > p.precision) { + s = s.substring(0, p.precision); + } + return pad("", s, p, '<'); + } + + private static String formatInt(PyInt v, Spec p) { + BigInteger b = v.toBig(); + boolean neg = b.signum() < 0; + BigInteger a = neg ? b.negate() : b; + String digits; + String prefix = ""; + switch (p.type) { + case 'b': + digits = a.toString(2); + prefix = p.alt ? "0b" : ""; + break; + case 'o': + digits = a.toString(8); + prefix = p.alt ? "0o" : ""; + break; + case 'x': + digits = a.toString(16); + prefix = p.alt ? "0x" : ""; + break; + case 'X': + digits = upper(a.toString(16)); + prefix = p.alt ? "0X" : ""; + break; + case 'c': + return pad("", new String(Character.toChars((int) v.asLong())), p, '<'); + default: + digits = a.toString(); + break; + } + if (p.grouping != 0) { + digits = group(digits, p.grouping, p.type == 'b' || p.type == 'o' || p.type == 'x' || p.type == 'X' ? 4 : 3); + } + return pad(signOf(neg, p) + prefix, digits, p, '>'); + } + + static String upper(String s) { + StringBuilder sb = new StringBuilder(s.length()); + for (int i = 0; i < s.length(); i++) { + char c = s.charAt(i); + sb.append(c >= 'a' && c <= 'z' ? (char) (c - 32) : c); + } + return sb.toString(); + } + + private static String signOf(boolean neg, Spec p) { + if (neg) { + return "-"; + } + return p.sign == '+' ? "+" : (p.sign == ' ' ? " " : ""); + } + + private static String group(String digits, char sep, int every) { + int dot = digits.indexOf('.'); + String intPart = dot >= 0 ? digits.substring(0, dot) : digits; + String rest = dot >= 0 ? digits.substring(dot) : ""; + StringBuilder sb = new StringBuilder(); + int n = intPart.length(); + for (int i = 0; i < n; i++) { + if (i > 0 && (n - i) % every == 0) { + sb.append(sep); + } + sb.append(intPart.charAt(i)); + } + return sb.append(rest).toString(); + } + + private static String pad(String sign, String body, Spec p, char defaultAlign) { + int len = sign.length() + body.length(); + if (p.width <= len) { + return sign + body; + } + int fillN = p.width - len; + StringBuilder sb = new StringBuilder(p.width); + char align = p.align == 0 ? defaultAlign : p.align; + switch (align) { + case '<': + sb.append(sign).append(body); + repeat(sb, p.fill, fillN); + break; + case '^': + repeat(sb, p.fill, fillN / 2); + sb.append(sign).append(body); + repeat(sb, p.fill, fillN - fillN / 2); + break; + case '=': + sb.append(sign); + repeat(sb, p.fill, fillN); + sb.append(body); + break; + default: + repeat(sb, p.fill, fillN); + sb.append(sign).append(body); + break; + } + return sb.toString(); + } + + private static void repeat(StringBuilder sb, char c, int n) { + for (int i = 0; i < n; i++) { + sb.append(c); + } + } + + // ---- floats ---------------------------------------------------------- + + private static String formatFloat(double d, Spec p) { + boolean neg = d < 0 || d == 0 && 1 / d < 0; + double a = Math.abs(d); + String body; + char type = p.type; + if (Double.isNaN(d) || Double.isInfinite(d)) { + body = Double.isNaN(d) ? "nan" : "inf"; + if (type == 'F' || type == 'E' || type == 'G') { + body = upper(body); + } + return pad(signOf(neg && !Double.isNaN(d), p), body, p, '>'); + } + switch (type) { + case 'f': + case 'F': + body = fixed(a, p.precision < 0 ? 6 : p.precision); + if (p.alt && body.indexOf('.') < 0) { + body += "."; + } + break; + case 'e': + case 'E': + body = exponent(a, p.precision < 0 ? 6 : p.precision, p.alt); + if (type == 'E') { + body = upper(body); + } + break; + case '%': + body = fixed(a * 100, p.precision < 0 ? 6 : p.precision) + "%"; + break; + case 'g': + case 'G': + case 'n': + body = general(a, p.precision < 0 ? 6 : Math.max(1, p.precision), p.alt); + if (type == 'G') { + body = upper(body); + } + break; + default: + if (p.precision >= 0) { + body = general(a, Math.max(1, p.precision), true); + body = trimGeneral(body, true); + } else { + body = PyFloat.repr(a); + } + break; + } + if (p.grouping != 0) { + int e = body.indexOf('e'); + body = e >= 0 ? group(body.substring(0, e), p.grouping, 3) + body.substring(e) + : group(body, p.grouping, 3); + } + return pad(signOf(neg, p), body, p, '>'); + } + + /// The exact decimal value of a non-negative finite double, as digits and + /// the power of ten of the LAST digit. + private static Object[] exact(double a) { + if (a == 0) { + return new Object[] {"0", Integer.valueOf(0)}; + } + long bits = Double.doubleToLongBits(a); + int exp = (int) ((bits >> 52) & 0x7ff); + long mant = bits & 0xfffffffffffffL; + if (exp == 0) { + exp = 1; + } else { + mant |= 1L << 52; + } + exp -= 1075; + BigInteger m = BigInteger.valueOf(mant); + if (exp >= 0) { + return new Object[] {m.shiftLeft(exp).toString(), Integer.valueOf(0)}; + } + // m * 2^exp = m * 5^-exp / 10^-exp + BigInteger scaled = m.multiply(BigInteger.valueOf(5).pow(-exp)); + return new Object[] {scaled.toString(), Integer.valueOf(exp)}; + } + + /// Rounds the digit string so its last digit has power `targetExp` + /// (half to even on the exact value). Returns `{digits, lastExp}`. + private static Object[] roundTo(String digits, int lastExp, int targetExp) { + if (targetExp <= lastExp) { + StringBuilder sb = new StringBuilder(digits); + for (int i = targetExp; i < lastExp; i++) { + sb.append('0'); + } + return new Object[] {sb.toString(), Integer.valueOf(targetExp)}; + } + int drop = targetExp - lastExp; + if (drop > digits.length()) { + return new Object[] {"0", Integer.valueOf(targetExp)}; + } + String keep = digits.substring(0, digits.length() - drop); + String gone = digits.substring(digits.length() - drop); + boolean up; + char first = gone.charAt(0); + if (first > '5') { + up = true; + } else if (first < '5') { + up = false; + } else { + boolean rest = false; + for (int i = 1; i < gone.length(); i++) { + if (gone.charAt(i) != '0') { + rest = true; + break; + } + } + if (rest) { + up = true; + } else { + int lastKeep = keep.length() == 0 ? 0 : keep.charAt(keep.length() - 1) - '0'; + up = (lastKeep & 1) == 1; + } + } + if (keep.length() == 0) { + keep = "0"; + } + if (up) { + keep = new BigInteger(keep).add(BigInteger.ONE).toString(); + } + return new Object[] {keep, Integer.valueOf(targetExp)}; + } + + static String fixed(double a, int prec) { + Object[] ex = exact(a); + Object[] r = roundTo((String) ex[0], ((Integer) ex[1]).intValue(), -prec); + String digits = (String) r[0]; + if (prec == 0) { + return digits; + } + while (digits.length() <= prec) { + digits = "0" + digits; + } + return digits.substring(0, digits.length() - prec) + "." + digits.substring(digits.length() - prec); + } + + /// `{mantissaDigits (prec+1 of them), decimalExponent}` for `e` formatting. + private static Object[] sci(double a, int prec) { + if (a == 0) { + StringBuilder z = new StringBuilder("0"); + for (int i = 0; i < prec; i++) { + z.append('0'); + } + return new Object[] {z.toString(), Integer.valueOf(0)}; + } + Object[] ex = exact(a); + String digits = (String) ex[0]; + int lastExp = ((Integer) ex[1]).intValue(); + int sigLen = digits.length(); + int lead = 0; + while (lead < sigLen - 1 && digits.charAt(lead) == '0') { + lead++; + } + digits = digits.substring(lead); + int topExp = lastExp + digits.length() - 1; + Object[] r = roundTo(digits, lastExp, topExp - prec); + String d = (String) r[0]; + if (d.length() > prec + 1) { + topExp++; + d = d.substring(0, prec + 1); + } + return new Object[] {d, Integer.valueOf(topExp)}; + } + + static String exponent(double a, int prec, boolean alt) { + Object[] s = sci(a, prec); + String d = (String) s[0]; + int e = ((Integer) s[1]).intValue(); + StringBuilder sb = new StringBuilder(); + sb.append(d.charAt(0)); + if (prec > 0 || alt) { + sb.append('.'); + } + sb.append(d, 1, d.length()); + sb.append('e').append(e < 0 ? '-' : '+'); + int ae = Math.abs(e); + if (ae < 10) { + sb.append('0'); + } + return sb.append(ae).toString(); + } + + static String general(double a, int prec, boolean alt) { + int e; + if (a == 0) { + e = 0; + } else { + e = ((Integer) sci(a, prec - 1)[1]).intValue(); + } + String body; + if (e >= -4 && e < prec) { + body = fixed(a, prec - 1 - e); + } else { + body = exponent(a, prec - 1, alt); + } + return trimGeneral(body, alt); + } + + private static String trimGeneral(String body, boolean alt) { + if (alt) { + return body; + } + int e = body.indexOf('e'); + String mant = e >= 0 ? body.substring(0, e) : body; + String tail = e >= 0 ? body.substring(e) : ""; + if (mant.indexOf('.') >= 0) { + int end = mant.length(); + while (mant.charAt(end - 1) == '0') { + end--; + } + if (mant.charAt(end - 1) == '.') { + end--; + } + mant = mant.substring(0, end); + } + return mant + tail; + } + + // ---- printf style ---------------------------------------------------- + + static String percent(String fmt, Object args) { + Object[] items; + PyDict mapping = null; + if (args instanceof PyTuple) { + items = ((PyTuple) args).items; + } else { + items = new Object[] {args}; + if (args instanceof PyDict) { + mapping = (PyDict) args; + } + } + StringBuilder sb = new StringBuilder(); + int ai = 0; + int n = fmt.length(); + for (int i = 0; i < n; i++) { + char c = fmt.charAt(i); + if (c != '%') { + sb.append(c); + continue; + } + i++; + if (i >= n) { + throw Py.valueError("incomplete format"); + } + Object value = null; + if (fmt.charAt(i) == '(') { + int close = fmt.indexOf(')', i); + String key = fmt.substring(i + 1, close); + if (mapping == null) { + throw Py.typeError("format requires a mapping"); + } + value = mapping.getItem(Py.str(key)); + i = close + 1; + } + StringBuilder spec = new StringBuilder(); + boolean left = false; + while (i < n && "-+ #0".indexOf(fmt.charAt(i)) >= 0) { + char f = fmt.charAt(i++); + if (f == '-') { + left = true; + } else { + spec.append(f == '0' && left ? "" : String.valueOf(f)); + } + } + StringBuilder width = new StringBuilder(); + while (i < n && (Character.isDigit(fmt.charAt(i)) || fmt.charAt(i) == '*')) { + if (fmt.charAt(i) == '*') { + width.append(Py.asLong(items[ai++])); + i++; + } else { + width.append(fmt.charAt(i++)); + } + } + String prec = ""; + if (i < n && fmt.charAt(i) == '.') { + i++; + StringBuilder pr = new StringBuilder("."); + while (i < n && Character.isDigit(fmt.charAt(i))) { + pr.append(fmt.charAt(i++)); + } + if (pr.length() == 1) { + pr.append('0'); + } + prec = pr.toString(); + } + char conv = fmt.charAt(i); + if (conv == '%') { + sb.append('%'); + continue; + } + if (value == null) { + if (ai >= items.length) { + throw Py.typeError("not enough arguments for format string"); + } + value = items[ai++]; + } + if (conv != 's' && conv != 'r' && conv != 'a' && value instanceof PyInstance + && ((PyInstance) value).base != null) { + // A number conversion formats a subclass's value (an IntEnum). + value = ((PyInstance) value).base; + } + String flags = spec.toString(); + String align = left ? "<" : ""; + // printf-style padding right-justifies text too, unlike format(). + String textAlign = left ? "<" : ">"; + switch (conv) { + case 's': + sb.append(format(Py.str(Py.toStr(value)), textAlign + width + prec)); + break; + case 'r': + case 'a': + sb.append(format(Py.str(Py.toRepr(value)), textAlign + width + prec)); + break; + case 'd': + case 'i': + case 'u': { + Object iv = Py.isFloat(value) ? PyInt.of((long) Py.asDouble(value)) : value; + if (!(Py.isInt(iv))) { + throw Py.typeError("%" + conv + " format: a real number is required, not " + Py.typeOf(value).name); + } + sb.append(format(iv, align + flags.replace("#", "") + width + "d")); + break; + } + case 'c': + sb.append(value instanceof String ? Py.toStr(value) : new String(Character.toChars((int) Py.asLong(value)))); + break; + default: + if ("xXo".indexOf(conv) >= 0) { + sb.append(format(value, align + flags + width + conv)); + } else if ("eEfFgG".indexOf(conv) >= 0) { + sb.append(format(Py.ofFloat(Py.asDouble(value)), align + flags + width + prec + conv)); + } else { + throw Py.valueError("unsupported format character '" + conv + "'"); + } + break; + } + } + if (ai < items.length && mapping == null) { + throw Py.typeError("not all arguments converted during string formatting"); + } + return sb.toString(); + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Hashing.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Hashing.java new file mode 100644 index 00000000000..e688807f5c6 --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Hashing.java @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +import com.codename1.util.BigInteger; + +/// CPython's hash functions for the types whose hash is deterministic there: +/// numbers and tuples. Matching them is what makes a set of numbers or +/// tuples iterate (and print) in the same order as on CPython, and keeps +/// `hash(1) == hash(1.0) == hash(True)` as the language requires. +final class Hashing { + static final int BITS = 61; + /// 2**61 - 1, the prime CPython reduces numeric hashes by. + static final long MODULUS = (1L << BITS) - 1; + private static final BigInteger BIG_MODULUS = BigInteger.valueOf(MODULUS); + private static final long INF = 314159; + + private static final long XXPRIME_1 = -7046029288634856825L; + private static final long XXPRIME_2 = -4417276706812531889L; + private static final long XXPRIME_5 = 2870177450012600261L; + + private Hashing() { + } + + private static long fix(long h) { + return h == -1 ? -2 : h; + } + + static long ofLong(long v) { + if (v >= 0) { + return fix(v % MODULUS); + } + if (v == Long.MIN_VALUE) { + // |MIN_VALUE| = 2**63 = 4 * 2**61, which is 4 modulo 2**61 - 1. + return -4; + } + return fix(-((-v) % MODULUS)); + } + + static long ofBig(BigInteger v) { + long r = v.abs().mod(BIG_MODULUS).longValue(); + return fix(v.signum() < 0 ? -r : r); + } + + /// `_Py_HashDouble`. NaN hashes by identity in CPython; callers pass the + /// object for that case. + static long ofDouble(double v, Object identity) { + if (Double.isNaN(v)) { + return System.identityHashCode(identity); + } + if (Double.isInfinite(v)) { + return v > 0 ? INF : -INF; + } + if (v == 0) { + return 0; + } + int sign = 1; + double m = v; + if (m < 0) { + sign = -1; + m = -m; + } + // frexp: m in [0.5, 1) and v = m * 2**e. + long bits = Double.doubleToLongBits(m); + int exp = (int) ((bits >>> 52) & 0x7ff); + int e; + if (exp == 0) { + // Subnormal: scale into the normal range first. + m *= 18014398509481984.0; // 2**54 + bits = Double.doubleToLongBits(m); + exp = (int) ((bits >>> 52) & 0x7ff); + e = exp - 1022 - 54; + } else { + e = exp - 1022; + } + m = Double.longBitsToDouble((bits & 0x800fffffffffffffL) | (1022L << 52)); + long x = 0; + while (m != 0) { + x = ((x << 28) & MODULUS) | x >>> (BITS - 28); + m *= 268435456.0; // 2**28 + e -= 28; + long y = (long) m; + m -= y; + x += y; + if (x >= MODULUS) { + x -= MODULUS; + } + } + e = e >= 0 ? e % BITS : BITS - 1 - ((-1 - e) % BITS); + x = ((x << e) & MODULUS) | x >>> (BITS - e); + return fix(x * sign); + } + + /// The xxHash-based tuple hash of CPython 3.8+. + static long ofTuple(Object[] items) { + long acc = XXPRIME_5; + for (Object item : items) { + long lane = Py.hashOf(item); + acc += lane * XXPRIME_2; + acc = (acc << 31) | (acc >>> 33); + acc *= XXPRIME_1; + } + acc += items.length ^ (XXPRIME_5 ^ 3527539L); + return acc == -1 ? 1546275796L : acc; + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/MathFunctions.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/MathFunctions.java new file mode 100644 index 00000000000..3c5fe340e3e --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/MathFunctions.java @@ -0,0 +1,627 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +import com.codename1.util.MathUtil; + +/// The `math` functions beyond the elementary ones: hyperbolic, error and +/// gamma functions, float decomposition and the integer combinatorics. +/// Device class libraries have no `Math.sinh`/`cbrt`/`expm1`, so each is +/// computed here from `exp` and `log`, with the series a small argument +/// needs to keep its precision. +final class MathFunctions { + private static final double LN2 = 0.6931471805599453; + + private MathFunctions() { + } + + private interface D1 { + double f(double x); + } + + private static void fn(PyModule m, String name, final D1 f) { + fn(m, name, f, null); + } + + /// `domain` is the ValueError message for an argument outside the + /// function's domain, with `%s` for the argument. + private static void fn(PyModule m, final String name, final D1 f, final String domain) { + m.setAttr(name, PyBuiltin.positional(name, a -> { + if (a.length != 1) { + throw Py.typeError("math." + name + "() takes exactly one argument (" + a.length + " given)"); + } + double x = Py.asDouble(a[0]); + double r = f.f(x); + if (Double.isNaN(r) && !Double.isNaN(x)) { + if (domain != null) { + int at = domain.indexOf("%s"); + throw Py.valueError(domain.substring(0, at) + Py.toRepr(Py.ofFloat(x)) + domain.substring(at + 2)); + } + throw Py.valueError("math domain error"); + } + if (Double.isInfinite(r) && !Double.isInfinite(x)) { + throw Py.error(Py.OVERFLOW_ERROR, "math range error"); + } + return Py.ofFloat(r); + })); + } + + static double expm1(double x) { + if (x == 0) { + // Keeps the sign of a zero, as the C library does. + return x; + } + if (Math.abs(x) < 1e-5) { + return x + x * x / 2 + x * x * x / 6; + } + if (Math.abs(x) < 0.5) { + // exp(x)-1 = 2 tanh(x/2) / (1 - tanh(x/2)), no cancellation. + double u = MathUtil.exp(x); + if (u == 1.0) { + return x; + } + double um1 = u - 1.0; + return um1 * x / MathUtil.log(u); + } + return MathUtil.exp(x) - 1; + } + + static double log1p(double x) { + if (x == 0) { + // Keeps the sign of a zero, as the C library does. + return x; + } + if (x == -1) { + return Double.NEGATIVE_INFINITY; + } + if (x < -1) { + return Double.NaN; + } + double u = 1.0 + x; + if (u == 1.0) { + return x; + } + return MathUtil.log(u) * x / (u - 1.0); + } + + private static double sinh(double x) { + if (x == 0) { + // Keeps the sign of a zero, as the C library does. + return x; + } + if (Math.abs(x) < 0.5) { + double e = expm1(Math.abs(x)); + double r = (e + e / (e + 1)) / 2; + return x < 0 ? -r : r; + } + double e = MathUtil.exp(Math.abs(x)); + double r = (e - 1 / e) / 2; + return x < 0 ? -r : r; + } + + private static double cosh(double x) { + double e = MathUtil.exp(Math.abs(x)); + return (e + 1 / e) / 2; + } + + private static double tanh(double x) { + if (x == 0) { + // Keeps the sign of a zero, as the C library does. + return x; + } + double ax = Math.abs(x); + if (ax > 22) { + return x < 0 ? -1 : 1; + } + double e = expm1(2 * ax); + double r = e / (e + 2); + return x < 0 || (x == 0 && 1 / x < 0) ? -r : r; + } + + private static double asinh(double x) { + if (x == 0) { + // Keeps the sign of a zero, as the C library does. + return x; + } + double ax = Math.abs(x); + double r; + if (ax > 1e8) { + r = MathUtil.log(ax) + LN2; + } else if (ax > 2) { + r = MathUtil.log(2 * ax + 1 / (Math.sqrt(x * x + 1) + ax)); + } else { + double t = x * x; + r = log1p(ax + t / (1 + Math.sqrt(1 + t))); + } + return x < 0 ? -r : r; + } + + private static double acosh(double x) { + if (x < 1) { + return Double.NaN; + } + if (x > 1e8) { + return MathUtil.log(x) + LN2; + } + if (x > 2) { + return MathUtil.log(2 * x - 1 / (x + Math.sqrt(x * x - 1))); + } + double t = x - 1; + return log1p(t + Math.sqrt(2 * t + t * t)); + } + + private static double atanh(double x) { + if (x == 0) { + // Keeps the sign of a zero, as the C library does. + return x; + } + double ax = Math.abs(x); + if (ax >= 1) { + return Double.NaN; + } + double r; + if (ax < 0.5) { + double t = ax + ax; + r = 0.5 * log1p(t + t * ax / (1 - ax)); + } else { + r = 0.5 * log1p((ax + ax) / (1 - ax)); + } + return x < 0 ? -r : r; + } + + private static double cbrt(double x) { + if (x == 0 || Double.isNaN(x) || Double.isInfinite(x)) { + return x; + } + double ax = Math.abs(x); + double y = MathUtil.pow(ax, 1.0 / 3); + y = y - (y * y * y - ax) / (3 * y * y); + return x < 0 ? -y : y; + } + + private static final double LANCZOS_G = 6.024680040776729583740234375; + private static final double LANCZOS_G_MINUS_HALF = 5.524680040776729583740234375; + private static final double[] LANCZOS_NUM = { + 23531376880.410759688572007674451636754734846804940, + 42919803642.649098768957899047001988850926355848959, + 35711959237.355668049440185451547166705960488635843, + 17921034426.037209699919755754458931112671403265390, + 6039542586.3520280050642916443072979210699388420708, + 1439720407.3117216736632230727949123939715485786772, + 248874557.86205415651146038641322942321632125127801, + 31426415.585400194380614231628318205362874684987640, + 2876370.6289353724412254090516208496135991145378768, + 186056.26539522349504029498971604569928220784236328, + 8071.6720023658162106380029022722506138218516325024, + 210.82427775157934587250973392071336271166969580291, + 2.5066282746310002701649081771338373386264310793408, + }; + private static final double[] LANCZOS_DEN = { + 0.0, 39916800.0, 120543840.0, 150917976.0, 105258076.0, 45995730.0, + 13339535.0, 2637558.0, 357423.0, 32670.0, 1925.0, 66.0, 1.0, + }; + private static final double[] GAMMA_INTEGRAL = { + 1.0, 1.0, 2.0, 6.0, 24.0, 120.0, 720.0, 5040.0, 40320.0, 362880.0, 3628800.0, 39916800.0, + 479001600.0, 6227020800.0, 87178291200.0, 1307674368000.0, 20922789888000.0, 355687428096000.0, + 6402373705728000.0, 121645100408832000.0, 2432902008176640000.0, 51090942171709440000.0, + 1124000727777607680000.0, + }; + + private static double lanczosSum(double x) { + double num = 0; + double den = 0; + if (x < 5) { + for (int i = 12; i >= 0; i--) { + num = num * x + LANCZOS_NUM[i]; + den = den * x + LANCZOS_DEN[i]; + } + } else { + for (int i = 0; i < 13; i++) { + num = num / x + LANCZOS_NUM[i]; + den = den / x + LANCZOS_DEN[i]; + } + } + return num / den; + } + + private static double sinpi(double x) { + double y = Math.abs(x) % 2.0; + long n = Math.round(2.0 * y); + double r; + switch ((int) n) { + case 0: + r = Math.sin(Math.PI * y); + break; + case 1: + r = Math.cos(Math.PI * (y - 0.5)); + break; + case 2: + r = Math.sin(Math.PI * (1.0 - y)); + break; + case 3: + r = -Math.cos(Math.PI * (y - 1.5)); + break; + default: + r = Math.sin(Math.PI * (y - 2.0)); + break; + } + return (x < 0 ? -1.0 : 1.0) * r; + } + + static double gamma(double x) { + if (Double.isNaN(x) || x == Double.POSITIVE_INFINITY) { + return x; + } + if (x == Double.NEGATIVE_INFINITY || (x == Math.floor(x) && x <= 0)) { + return Double.NaN; + } + if (x == Math.floor(x) && x <= GAMMA_INTEGRAL.length) { + return GAMMA_INTEGRAL[(int) x - 1]; + } + double absx = Math.abs(x); + if (absx < 1e-20) { + return 1.0 / x; + } + if (absx > 200.0) { + if (x < 0) { + return 0.0 / sinpi(x); + } + return Double.POSITIVE_INFINITY; + } + double y = absx + LANCZOS_G_MINUS_HALF; + double q; + if (absx > LANCZOS_G_MINUS_HALF) { + q = y - absx; + q = q - LANCZOS_G_MINUS_HALF; + } else { + q = y - LANCZOS_G_MINUS_HALF; + q = q - absx; + } + double z = q * LANCZOS_G / y; + double r; + if (x < 0.0) { + r = -Math.PI / sinpi(absx) / absx * MathUtil.exp(y) / lanczosSum(absx); + r -= z * r; + if (absx < 140.0) { + r /= MathUtil.pow(y, absx - 0.5); + } else { + double sqrtpow = MathUtil.pow(y, absx / 2.0 - 0.25); + r /= sqrtpow; + r /= sqrtpow; + } + } else { + r = lanczosSum(absx) / MathUtil.exp(y); + r += z * r; + if (absx < 140.0) { + r *= MathUtil.pow(y, absx - 0.5); + } else { + double sqrtpow = MathUtil.pow(y, absx / 2.0 - 0.25); + r *= sqrtpow; + r *= sqrtpow; + } + } + return r; + } + + static double lgamma(double x) { + if (Double.isNaN(x)) { + return x; + } + if (Double.isInfinite(x)) { + return Double.POSITIVE_INFINITY; + } + if (x == Math.floor(x) && x <= 2.0) { + if (x <= 0.0) { + return Double.NaN; + } + return 0.0; + } + double absx = Math.abs(x); + if (absx < 1e-20) { + return -MathUtil.log(absx); + } + double r = MathUtil.log(lanczosSum(absx)) - LANCZOS_G; + r += (absx - 0.5) * (MathUtil.log(absx + LANCZOS_G - 0.5) - 1); + if (x < 0.0) { + r = 1.1447298858494002 - MathUtil.log(Math.abs(sinpi(absx))) - MathUtil.log(absx) - r; + } + return r; + } + + private static double ldexp(double m, long e) { + if (m == 0 || Double.isInfinite(m) || Double.isNaN(m)) { + return m; + } + long bits = Double.doubleToLongBits(m); + long me = ((bits >>> 52) & 0x7ff) - 1023; + if (me == -1023) { + // A subnormal m: make it normal first, which is exact. + m *= 0x1p54; + e -= 54; + bits = Double.doubleToLongBits(m); + me = ((bits >>> 52) & 0x7ff) - 1023; + } + if (e > 2200) { + e = 2200; + } else if (e < -2200) { + e = -2200; + } + long ne = me + e; + if (ne > 1023) { + return m > 0 ? Double.POSITIVE_INFINITY : Double.NEGATIVE_INFINITY; + } + if (ne < -1075) { + // Below half the smallest subnormal: rounds to a signed zero. + return m > 0 ? 0.0 : -0.0; + } + long mantissa = bits & 0x800fffffffffffffL; + if (ne >= -1022) { + return Double.longBitsToDouble(mantissa | ((ne + 1023) << 52)); + } + // A subnormal result: build the exact value 2**64 times larger, + // which is normal, and scale it down in one multiplication so it + // is rounded exactly once. + return Double.longBitsToDouble(mantissa | ((ne + 64 + 1023) << 52)) * 0x1p-64; + } + + private static double ulp(double x) { + if (Double.isNaN(x)) { + return x; + } + x = Math.abs(x); + if (Double.isInfinite(x)) { + return x; + } + double next = nextUp(x); + if (Double.isInfinite(next)) { + return x - nextDown(x); + } + return next - x; + } + + private static double nextUp(double x) { + if (Double.isNaN(x) || x == Double.POSITIVE_INFINITY) { + return x; + } + if (x == 0) { + return Double.MIN_VALUE; + } + long b = Double.doubleToLongBits(x); + return Double.longBitsToDouble(x > 0 ? b + 1 : b - 1); + } + + private static double nextDown(double x) { + return -nextUp(-x); + } + + private static double remainder(double x, double y) { + if (Double.isNaN(x) || Double.isNaN(y)) { + return Double.NaN; + } + if (Double.isInfinite(x) || y == 0) { + return Double.NaN; + } + if (Double.isInfinite(y)) { + return x; + } + double absx = Math.abs(x); + double absy = Math.abs(y); + double m = absx % absy; + double c = absy - m; + double r; + if (m < c) { + r = m; + } else if (m > c) { + r = -c; + } else { + // A tie: the quotient rounds to even. + r = Math.floor(absx / absy) % 2 == 0 ? m : -c; + } + return x < 0 ? -r : r; + } + + private static Object index(Object o) { + if (Py.isInt(o)) { + return o; + } + throw Py.typeError("'" + Py.typeOf(o).name + "' object cannot be interpreted as an integer"); + } + + static void install(final PyModule m) { + fn(m, "sinh", MathFunctions::sinh); + fn(m, "cosh", MathFunctions::cosh); + fn(m, "tanh", MathFunctions::tanh); + fn(m, "asinh", MathFunctions::asinh); + fn(m, "acosh", MathFunctions::acosh, "expected argument value not less than 1, got %s"); + fn(m, "atanh", MathFunctions::atanh, "expected a number between -1 and 1, got %s"); + fn(m, "cbrt", MathFunctions::cbrt); + fn(m, "exp2", x -> MathUtil.pow(2, x)); + fn(m, "expm1", MathFunctions::expm1); + fn(m, "log1p", MathFunctions::log1p, "expected argument value > -1, got %s"); + fn(m, "erf", Erf::erf); + fn(m, "erfc", Erf::erfc); + fn(m, "gamma", MathFunctions::gamma, "expected a noninteger or positive integer, got %s"); + fn(m, "lgamma", MathFunctions::lgamma, "expected a noninteger or positive integer, got %s"); + m.setAttr("frexp", PyBuiltin.positional("frexp", a -> { + double x = Py.asDouble(a[0]); + if (x == 0 || Double.isNaN(x) || Double.isInfinite(x)) { + return new PyTuple(new Object[] {Py.ofFloat(x), PyInt.of(0)}); + } + int e = 0; + double ax = Math.abs(x); + if (ax < 0x1p-1000) { + ax *= 0x1p100; + e -= 100; + } + long bits = Double.doubleToLongBits(ax); + int be = (int) ((bits >> 52) & 0x7ff); + if (be == 0) { + ax *= 0x1p60; + e -= 60; + bits = Double.doubleToLongBits(ax); + be = (int) ((bits >> 52) & 0x7ff); + } + e += be - 1022; + double mant = Double.longBitsToDouble((bits & 0x800fffffffffffffL) | (1022L << 52)); + return new PyTuple(new Object[] {Py.ofFloat(x < 0 ? -mant : mant), PyInt.of(e)}); + })); + m.setAttr("ldexp", PyBuiltin.positional("ldexp", a -> { + if (!Py.isInt(a[1])) { + throw Py.typeError("Expected an int as second argument to ldexp."); + } + double x = Py.asDouble(a[0]); + Object e = a[1]; + long n; + com.codename1.util.BigInteger big = Py.intView(e).toBig(); + if (big.bitLength() > 62) { + n = big.signum() < 0 ? -10000 : 10000; + } else { + n = big.longValue(); + } + double r = ldexp(x, n); + if (Double.isInfinite(r) && !Double.isInfinite(x)) { + throw Py.error(Py.OVERFLOW_ERROR, "math range error"); + } + return Py.ofFloat(r); + })); + m.setAttr("modf", PyBuiltin.positional("modf", a -> { + double x = Py.asDouble(a[0]); + if (Double.isInfinite(x)) { + return new PyTuple(new Object[] {Py.ofFloat(x > 0 ? 0.0 : -0.0), Py.ofFloat(x)}); + } + double ip = x < 0 ? Math.ceil(x) : Math.floor(x); + double fp = x - ip; + if (fp == 0 && x < 0 || (x == 0 && 1 / x < 0)) { + fp = -0.0; + } + return new PyTuple(new Object[] {Py.ofFloat(fp), Py.ofFloat(ip)}); + })); + m.setAttr("fma", PyBuiltin.positional("fma", a -> { + // Not fused: device class libraries have no exact fma. + double r = Py.asDouble(a[0]) * Py.asDouble(a[1]) + Py.asDouble(a[2]); + return Py.ofFloat(r); + })); + m.setAttr("remainder", PyBuiltin.positional("remainder", a -> { + double x = Py.asDouble(a[0]); + double y = Py.asDouble(a[1]); + double r = remainder(x, y); + if (Double.isNaN(r) && !Double.isNaN(x) && !Double.isNaN(y)) { + throw Py.valueError("math domain error"); + } + return Py.ofFloat(r); + })); + m.setAttr("nextafter", new PyBuiltin("nextafter", (args, kw) -> { + Args a = new Args("nextafter", args, kw).check(3, "x", "y", "steps"); + double x = Py.asDouble(a.get(0, "x", null)); + double y = Py.asDouble(a.get(1, "y", null)); + Object stepsArg = a.get(2, "steps", Py.NONE); + if (stepsArg != Py.NONE) { //NOPMD CompareObjectsWithEquals - identity is meant + long steps = Py.asLong(stepsArg); + if (steps < 0) { + throw Py.valueError("steps must be a non-negative integer"); + } + for (long i = 0; i < steps && x != y; i++) { + x = Double.isNaN(x) || Double.isNaN(y) ? Double.NaN : (y > x ? nextUp(x) : nextDown(x)); + } + return Py.ofFloat(x); + } + if (Double.isNaN(x) || Double.isNaN(y)) { + return Py.ofFloat(Double.NaN); + } + if (x == y) { + return Py.ofFloat(y); + } + return Py.ofFloat(y > x ? nextUp(x) : nextDown(x)); + })); + fn(m, "ulp", MathFunctions::ulp); + m.setAttr("dist", PyBuiltin.positional("dist", a -> { + Object[] p = Py.toArray(a[0]); + Object[] q = Py.toArray(a[1]); + if (p.length != q.length) { + throw Py.valueError("both points must have the same number of dimensions"); + } + double max = 0; + double[] d = new double[p.length]; + for (int i = 0; i < p.length; i++) { + d[i] = Math.abs(Py.asDouble(p[i]) - Py.asDouble(q[i])); + max = Math.max(max, d[i]); + } + if (max == 0 || Double.isInfinite(max)) { + return Py.ofFloat(max); + } + double s = 0; + for (double o : d) { + double v = o / max; + s += v * v; + } + return Py.ofFloat(max * Math.sqrt(s)); + })); + m.setAttr("lcm", PyBuiltin.positional("lcm", a -> { + com.codename1.util.BigInteger r = com.codename1.util.BigInteger.ONE; + for (int i = 0; i < a.length; i++) { + com.codename1.util.BigInteger v = Py.intView(index(a[i])).toBig().abs(); + if (v.signum() == 0) { + return PyInt.of(0); + } + r = r.divide(r.gcd(v)).multiply(v); + } + return PyInt.of(r); + })); + m.setAttr("perm", PyBuiltin.positional("perm", a -> { + long n = Py.asLong(index(a[0])); + if (n < 0) { + // CPython's wording: perm(n) is factorial(n). + throw Py.valueError(a.length > 1 && a[1] != Py.NONE ? "n must be a non-negative integer" + : "factorial() not defined for negative values"); + } + long k = n; + if (a.length > 1 && a[1] != Py.NONE) { + k = Py.asLong(index(a[1])); + if (k < 0) { + throw Py.valueError("k must be a non-negative integer"); + } + } + if (k > n) { + return PyInt.of(0); + } + com.codename1.util.BigInteger r = com.codename1.util.BigInteger.ONE; + for (long i = n - k + 1; i <= n; i++) { + r = r.multiply(com.codename1.util.BigInteger.valueOf(i)); + } + return PyInt.of(r); + })); + m.setAttr("sumprod", PyBuiltin.positional("sumprod", a -> { + Object[] p = Py.toArray(a[0]); + Object[] q = Py.toArray(a[1]); + if (p.length != q.length) { + throw Py.valueError("Inputs are not the same length"); + } + Object total = PyInt.of(0); + for (int i = 0; i < p.length; i++) { + total = Py.add(total, Py.mul(p[i], q[i])); + } + return total; + })); + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Py.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Py.java new file mode 100644 index 00000000000..8ce47d9b0a4 --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Py.java @@ -0,0 +1,2071 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +import java.util.ArrayList; +import java.util.List; + +/// The operations transpiled code calls: operators with CPython's dispatch +/// rules, attribute access, calls, iteration helpers, and the builtin types +/// and exceptions. +/// +/// Everything here is static and the transpiler emits direct calls, so the +/// common cases (`int + int`, `list[i]`) are a type test and an inlined +/// virtual call away from the arithmetic. +public final class Py { + public static final Object[] NO_ARGS = new Object[0]; + public static final String[] NO_KW = new String[0]; + + // ---- types ----------------------------------------------------------- + // Order matters: `object` and `type` first, every other type derives. + + public static final PyType OBJECT_TYPE = new PyType("object", new PyType[0], null); + public static final PyType TYPE_TYPE = builtinType("type", null); + public static final PyType NONE_TYPE = builtinType("NoneType", null); + public static final PyType INT_TYPE = builtinType("int", null); + public static final PyType BOOL_TYPE = new PyType("bool", new PyType[] {INT_TYPE}, null); + public static final PyType FLOAT_TYPE = builtinType("float", null); + public static final PyType COMPLEX_TYPE = builtinType("complex", null); + public static final PyType STR_TYPE = builtinType("str", null); + public static final PyType BYTES_TYPE = builtinType("bytes", null); + public static final PyType BYTEARRAY_TYPE = builtinType("bytearray", null); + public static final PyType MEMORYVIEW_TYPE = builtinType("memoryview", null); + public static final PyType WEAKREF_TYPE = builtinType("ReferenceType", null); + public static final PyType LIST_TYPE = builtinType("list", null); + public static final PyType TUPLE_TYPE = builtinType("tuple", null); + public static final PyType DICT_TYPE = builtinType("dict", null); + public static final PyType SET_TYPE = builtinType("set", null); + public static final PyType FROZENSET_TYPE = builtinType("frozenset", null); + public static final PyType RANGE_TYPE = builtinType("range", null); + public static final PyType SLICE_TYPE = builtinType("slice", null); + public static final PyType FUNCTION_TYPE = builtinType("function", null); + public static final PyType BUILTIN_TYPE = builtinType("builtin_function_or_method", null); + public static final PyType METHOD_TYPE = builtinType("method", null); + public static final PyType PROPERTY_TYPE = builtinType("property", null); + public static final PyType CLASSMETHOD_TYPE = builtinType("classmethod", null); + public static final PyType STATICMETHOD_TYPE = builtinType("staticmethod", null); + public static final PyType ITERATOR_TYPE = builtinType("iterator", null); + public static final PyType GENERATOR_TYPE = builtinType("generator", null); + public static final PyType COROUTINE_TYPE = builtinType("coroutine", null); + public static final PyType ASYNC_GENERATOR_TYPE = builtinType("async_generator", null); + public static final PyType MODULE_TYPE = builtinType("module", null); + public static final PyType NOT_IMPLEMENTED_TYPE = builtinType("NotImplementedType", null); + public static final PyType ELLIPSIS_TYPE = builtinType("ellipsis", null); + + // ---- singletons ------------------------------------------------------ + + public static final Object NONE = new PyNone("None", null); + public static final Object NOT_IMPLEMENTED = new PyNone("NotImplemented", NOT_IMPLEMENTED_TYPE); + public static final Object ELLIPSIS = new PyNone("Ellipsis", ELLIPSIS_TYPE); + public static final PyBool TRUE = new PyBool(true); + public static final PyBool FALSE = new PyBool(false); + public static final String EMPTY_STR = ""; + static final String[] ASCII = new String[128]; + + // ---- exceptions ------------------------------------------------------ + + /// Every builtin exception type, in declaration order, for the builtins table. + static final List EXCEPTION_TYPES = new ArrayList(); + public static final PyType BASE_EXCEPTION = exceptionType("BaseException", OBJECT_TYPE); + public static final PyType SYSTEM_EXIT = exceptionType("SystemExit", BASE_EXCEPTION); + public static final PyType KEYBOARD_INTERRUPT = exceptionType("KeyboardInterrupt", BASE_EXCEPTION); + public static final PyType GENERATOR_EXIT = exceptionType("GeneratorExit", BASE_EXCEPTION); + public static final PyType EXCEPTION = exceptionType("Exception", BASE_EXCEPTION); + public static final PyType BASE_EXCEPTION_GROUP = exceptionType("BaseExceptionGroup", BASE_EXCEPTION); + public static final PyType EXCEPTION_GROUP = exceptionGroupType(); + public static final PyType STOP_ITERATION = exceptionType("StopIteration", EXCEPTION); + public static final PyType ARITHMETIC_ERROR = exceptionType("ArithmeticError", EXCEPTION); + public static final PyType ZERO_DIVISION_ERROR = exceptionType("ZeroDivisionError", ARITHMETIC_ERROR); + public static final PyType OVERFLOW_ERROR = exceptionType("OverflowError", ARITHMETIC_ERROR); + public static final PyType ASSERTION_ERROR = exceptionType("AssertionError", EXCEPTION); + public static final PyType ATTRIBUTE_ERROR = exceptionType("AttributeError", EXCEPTION); + public static final PyType LOOKUP_ERROR = exceptionType("LookupError", EXCEPTION); + public static final PyType INDEX_ERROR = exceptionType("IndexError", LOOKUP_ERROR); + public static final PyType KEY_ERROR = exceptionType("KeyError", LOOKUP_ERROR); + public static final PyType NAME_ERROR = exceptionType("NameError", EXCEPTION); + public static final PyType UNBOUND_LOCAL_ERROR = exceptionType("UnboundLocalError", NAME_ERROR); + public static final PyType RUNTIME_ERROR = exceptionType("RuntimeError", EXCEPTION); + public static final PyType NOT_IMPLEMENTED_ERROR = exceptionType("NotImplementedError", RUNTIME_ERROR); + public static final PyType RECURSION_ERROR = exceptionType("RecursionError", RUNTIME_ERROR); + public static final PyType TYPE_ERROR = exceptionType("TypeError", EXCEPTION); + public static final PyType VALUE_ERROR = exceptionType("ValueError", EXCEPTION); + public static final PyType UNICODE_ERROR = exceptionType("UnicodeError", VALUE_ERROR); + public static final PyType IMPORT_ERROR = exceptionType("ImportError", EXCEPTION); + public static final PyType MODULE_NOT_FOUND_ERROR = exceptionType("ModuleNotFoundError", IMPORT_ERROR); + public static final PyType OS_ERROR = exceptionType("OSError", EXCEPTION); + public static final PyType FILE_NOT_FOUND_ERROR = exceptionType("FileNotFoundError", OS_ERROR); + public static final PyType MEMORY_ERROR = exceptionType("MemoryError", EXCEPTION); + public static final PyType UNICODE_ENCODE_ERROR = exceptionType("UnicodeEncodeError", UNICODE_ERROR); + public static final PyType UNICODE_DECODE_ERROR = exceptionType("UnicodeDecodeError", UNICODE_ERROR); + public static final PyType UNICODE_TRANSLATE_ERROR = exceptionType("UnicodeTranslateError", UNICODE_ERROR); + public static final PyType FLOATING_POINT_ERROR = exceptionType("FloatingPointError", ARITHMETIC_ERROR); + public static final PyType STOP_ASYNC_ITERATION = exceptionType("StopAsyncIteration", EXCEPTION); + public static final PyType EOF_ERROR = exceptionType("EOFError", EXCEPTION); + public static final PyType BUFFER_ERROR = exceptionType("BufferError", EXCEPTION); + public static final PyType REFERENCE_ERROR = exceptionType("ReferenceError", EXCEPTION); + public static final PyType SYSTEM_ERROR = exceptionType("SystemError", EXCEPTION); + public static final PyType SYNTAX_ERROR = exceptionType("SyntaxError", EXCEPTION); + public static final PyType INDENTATION_ERROR = exceptionType("IndentationError", SYNTAX_ERROR); + public static final PyType TAB_ERROR = exceptionType("TabError", INDENTATION_ERROR); + public static final PyType PYTHON_FINALIZATION_ERROR = exceptionType("PythonFinalizationError", RUNTIME_ERROR); + public static final PyType FILE_EXISTS_ERROR = exceptionType("FileExistsError", OS_ERROR); + public static final PyType PERMISSION_ERROR = exceptionType("PermissionError", OS_ERROR); + public static final PyType IS_A_DIRECTORY_ERROR = exceptionType("IsADirectoryError", OS_ERROR); + public static final PyType NOT_A_DIRECTORY_ERROR = exceptionType("NotADirectoryError", OS_ERROR); + public static final PyType TIMEOUT_ERROR = exceptionType("TimeoutError", OS_ERROR); + public static final PyType INTERRUPTED_ERROR = exceptionType("InterruptedError", OS_ERROR); + public static final PyType BLOCKING_IO_ERROR = exceptionType("BlockingIOError", OS_ERROR); + public static final PyType CHILD_PROCESS_ERROR = exceptionType("ChildProcessError", OS_ERROR); + public static final PyType PROCESS_LOOKUP_ERROR = exceptionType("ProcessLookupError", OS_ERROR); + public static final PyType CONNECTION_ERROR = exceptionType("ConnectionError", OS_ERROR); + public static final PyType BROKEN_PIPE_ERROR = exceptionType("BrokenPipeError", CONNECTION_ERROR); + public static final PyType CONNECTION_ABORTED_ERROR = exceptionType("ConnectionAbortedError", CONNECTION_ERROR); + public static final PyType CONNECTION_REFUSED_ERROR = exceptionType("ConnectionRefusedError", CONNECTION_ERROR); + public static final PyType CONNECTION_RESET_ERROR = exceptionType("ConnectionResetError", CONNECTION_ERROR); + public static final PyType WARNING = exceptionType("Warning", EXCEPTION); + public static final PyType USER_WARNING = exceptionType("UserWarning", WARNING); + public static final PyType DEPRECATION_WARNING = exceptionType("DeprecationWarning", WARNING); + public static final PyType PENDING_DEPRECATION_WARNING = exceptionType("PendingDeprecationWarning", WARNING); + public static final PyType SYNTAX_WARNING = exceptionType("SyntaxWarning", WARNING); + public static final PyType RUNTIME_WARNING = exceptionType("RuntimeWarning", WARNING); + public static final PyType FUTURE_WARNING = exceptionType("FutureWarning", WARNING); + public static final PyType IMPORT_WARNING = exceptionType("ImportWarning", WARNING); + public static final PyType UNICODE_WARNING = exceptionType("UnicodeWarning", WARNING); + public static final PyType BYTES_WARNING = exceptionType("BytesWarning", WARNING); + public static final PyType RESOURCE_WARNING = exceptionType("ResourceWarning", WARNING); + public static final PyType ENCODING_WARNING = exceptionType("EncodingWarning", WARNING); + + private static PyType builtinType(String name, PyType.Factory f) { + return new PyType(name, new PyType[] {OBJECT_TYPE}, f); + } + + private static PyType exceptionGroupType() { + PyType t = new PyType("ExceptionGroup", new PyType[] {BASE_EXCEPTION_GROUP, EXCEPTION}, null); + EXCEPTION_TYPES.add(t); + return t; + } + + private static PyType exceptionType(String name, PyType base) { + PyType t = new PyType(name, new PyType[] {base}, null); + EXCEPTION_TYPES.add(t); + return t; + } + + static { + for (int i = 0; i < ASCII.length; i++) { + ASCII[i] = String.valueOf((char) i); + } + Builtins.install(); + } + + private Py() { + } + + // ---- value representation ------------------------------------------- + // + // A Python value is a Java Object. An int that fits in a long is a + // java.lang.Long and a float is a java.lang.Double; every other value + // (bool, big ints, str, containers, instances) is a PyObject. ParparVM + // keeps a Long within [-2^60, 2^60) and many Doubles as tagged immediates + // that never reach the heap, so int and float arithmetic does not + // allocate there. PyInt and PyFloat remain the implementations of int and + // float behaviour: obj() hands a Long or Double to them as a short-lived + // view on the paths that are not worth a fast path of their own. + + /// The PyObject behaviour of any value. + public static PyObject obj(Object o) { + if (o instanceof PyObject) { + return (PyObject) o; + } + if (o instanceof Long) { + return PyInt.view(((Long) o).longValue()); + } + if (o instanceof Double) { + return new PyFloat(((Double) o).doubleValue()); + } + if (o instanceof String) { + return new PyStr((String) o); + } + if (o == null) { + throw new NullPointerException("null Python value"); + } + return new PyJavaObject(o); + } + + /// `attr`, found on a class, as read through `self`: a function binds, + /// and a value the runtime holds as a plain Java object (an int, a float, + /// a str) is returned as it is, never as the transient view [#obj] makes. + public static Object bindAttr(Object attr, Object self) { + if (attr instanceof PyProperty) { + return ((PyProperty) attr).get(self); + } + return attr instanceof PyObject ? ((PyObject) attr).bind(self) : attr; + } + + /// True for a Python int, bool included. + public static boolean isInt(Object o) { + return o instanceof Long || o instanceof PyInt; + } + + public static PyType typeOf(Object o) { + if (o instanceof Long) { + return INT_TYPE; + } + if (o instanceof Double) { + return FLOAT_TYPE; + } + if (o instanceof String) { + return STR_TYPE; + } + return obj(o).type(); + } + + public static long asLong(Object o) { + if (o instanceof Long) { + return ((Long) o).longValue(); + } + return obj(o).asLong(); + } + + public static double asDouble(Object o) { + if (o instanceof Double) { + return ((Double) o).doubleValue(); + } + if (o instanceof Long) { + return ((Long) o).longValue(); + } + return obj(o).asDouble(); + } + + public static String toStr(Object o) { + if (o instanceof Long) { + return Long.toString(((Long) o).longValue()); + } + if (o instanceof String) { + return (String) o; + } + return obj(o).str(); + } + + public static String toRepr(Object o) { + if (o instanceof Long) { + return Long.toString(((Long) o).longValue()); + } + if (o instanceof String) { + return PyStr.repr((String) o); + } + return obj(o).repr(); + } + + public static long hashOf(Object o) { + if (o instanceof Long) { + return Hashing.ofLong(((Long) o).longValue()); + } + if (o instanceof String) { + return o.hashCode(); + } + return obj(o).hash(); + } + + /// CPython's error for `T.__new__(x)` when `x` is not a type. + static PyException notATypeObject(String owner, Object[] args) { + if (args.length == 0) { + return typeError(owner + ".__new__(): not enough arguments"); + } + return typeError(owner + ".__new__(X): X is not a type object (" + typeOf(args[0]).name + ")"); + } + + /// The hash of a dict key (`role` "a dict key") or a set element: an + /// unhashable one is named as such, as CPython does. + static long keyHash(Object o, String role) { + if (o instanceof Long) { + return Hashing.ofLong(((Long) o).longValue()); + } + if (o instanceof String) { + return o.hashCode(); + } + try { + return obj(o).hash(); + } catch (PyException e) { + String m = e.matches(TYPE_ERROR) ? toStr(e.value) : ""; + if (m.startsWith("unhashable type: ")) { + throw typeError("cannot use '" + typeOf(o).name + "' as " + role + " (" + m + ")"); //NOPMD PreserveStackTrace - CPython's error has no cause + } + throw e; + } + } + + /// `f(o)` where `f` is what the name `len` holds: the builtin's work + /// without a call when it is the builtin. + public static Object len(Object f, Object o) { + if (f == Builtins.LEN) { //NOPMD CompareObjectsWithEquals - identity is meant + return PyInt.of(len(o)); + } + return call(f, o); + } + + public static int len(Object o) { + if (o instanceof PyObject) { + return ((PyObject) o).len(); + } + if (o instanceof String) { + return PyStr.codePoints((String) o); + } + return obj(o).len(); + } + + // ---- boxing ---------------------------------------------------------- + + public static PyBool bool(boolean b) { + return b ? TRUE : FALSE; + } + + public static Object of(long v) { + return Long.valueOf(v); + } + + public static Object of(double v) { + return Double.valueOf(v); + } + + public static Object ofFloat(double v) { + return Double.valueOf(v); + } + + /// True for a Python float. + public static boolean isFloat(Object o) { + return o instanceof Double || o instanceof PyFloat; + } + + /// The PyInt behaviour of an int (callers have checked [#isInt(Object)]). + public static PyInt intView(Object o) { + if (o instanceof PyInt) { + return (PyInt) o; + } + return PyInt.view(((Long) o).longValue()); + } + + /// A Python str: the Java string itself. + public static String str(String s) { + return s; + } + + /// The one-character string `c`, shared for ASCII as CPython shares it. + public static String chr(char c) { + return c < 128 ? ASCII[c] : String.valueOf(c); + } + + /// Converts a Java value to its Python counterpart: the reverse of + /// [PyObject#toJava()]. Unknown Java objects are wrapped as [PyJavaObject]. + public static Object wrap(Object o) { + if (o == null) { + return NONE; + } + if (o instanceof PyObject) { + return (PyObject) o; + } + if (o instanceof String) { + return str((String) o); + } + if (o instanceof Boolean) { + return bool(((Boolean) o).booleanValue()); + } + if (o instanceof Long || o instanceof Double) { + return o; + } + if (o instanceof Integer || o instanceof Short || o instanceof Byte) { + return PyInt.of(((Number) o).longValue()); + } + if (o instanceof Float) { + return Py.ofFloat(((Float) o).doubleValue()); + } + if (o instanceof com.codename1.util.BigInteger) { + return PyInt.of((com.codename1.util.BigInteger) o); + } + if (o instanceof Character) { + return str(String.valueOf(((Character) o).charValue())); + } + return new PyJavaObject(o); + } + + // ---- errors ---------------------------------------------------------- + + /// Instantiates exception type `t` with `msg` and wraps it for `throw`. + public static PyException error(PyType t, String msg) { + return new PyException(t.call(new Object[] {str(msg)}, NO_KW)); + } + + public static PyException typeError(String m) { + return error(TYPE_ERROR, m); + } + + public static PyException valueError(String m) { + return error(VALUE_ERROR, m); + } + + public static PyException indexError(String m) { + return error(INDEX_ERROR, m); + } + + public static PyException attributeError(String m) { + return error(ATTRIBUTE_ERROR, m); + } + + public static PyException zeroDivision(String m) { + return error(ZERO_DIVISION_ERROR, m); + } + + public static PyException overflowError(String m) { + return error(OVERFLOW_ERROR, m); + } + + public static PyException runtimeError(String m) { + return error(RUNTIME_ERROR, m); + } + + public static PyException nameError(String name) { + return error(NAME_ERROR, "name '" + name + "' is not defined"); + } + + public static PyException unboundLocal(String name) { + return error(UNBOUND_LOCAL_ERROR, "cannot access local variable '" + name + + "' where it is not associated with a value"); + } + + public static PyException keyError(Object key) { + return new PyException(KEY_ERROR.call(new Object[] {key}, NO_KW)); + } + + /// `raise x`: `x` is an exception instance or class. + public static PyException raise(Object x) { + if (x instanceof PyType) { + PyType t = (PyType) x; + if (!t.isSubtype(BASE_EXCEPTION)) { + return typeError("exceptions must derive from BaseException"); + } + x = t.call(NO_ARGS, NO_KW); + } else if (!Py.typeOf(x).isSubtype(BASE_EXCEPTION)) { + return typeError("exceptions must derive from BaseException"); + } + return new PyException(x); + } + + /// `raise x from cause`. + public static PyException raise(Object x, Object cause) { + PyException e = raise(x); + if (e.value instanceof PyInstance) { + ((PyInstance) e.value).setAttrDirect("__cause__", cause instanceof PyType + ? Py.obj(cause).call(NO_ARGS, NO_KW) : cause); + ((PyInstance) e.value).setAttrDirect("__suppress_context__", TRUE); + } + return e; + } + + /// The exception the innermost running `except` clause handles, for + /// `sys.exception()` and `sys.exc_info()`; `None` outside one. + private static Object handling = NONE; + + /// Marks `e` as the exception being handled and answers the one it + /// replaces, which [#exitHandler(Object)] restores. + public static Object enterHandler(PyException e) { + Object prev = handling; + handling = e.value; + return prev; + } + + public static void exitHandler(Object prev) { + handling = prev; + } + + static Object handling() { + return handling; + } + + /// Converts anything a `try` caught into a Python exception, so an + /// `except` clause sees Java failures under their Python names. + public static PyException convert(Throwable t) { + if (t instanceof PyException) { + return (PyException) t; + } + if (t instanceof StackOverflowError) { + return error(RECURSION_ERROR, "maximum recursion depth exceeded"); + } + if (t instanceof OutOfMemoryError) { + return error(MEMORY_ERROR, ""); + } + if (t instanceof ArithmeticException) { + return error(ZERO_DIVISION_ERROR, String.valueOf(t.getMessage())); + } + if (t instanceof IndexOutOfBoundsException) { + return error(INDEX_ERROR, String.valueOf(t.getMessage())); + } + if (t instanceof IllegalArgumentException) { + return error(VALUE_ERROR, String.valueOf(t.getMessage())); + } + if (t instanceof java.io.IOException) { + return error(OS_ERROR, String.valueOf(t.getMessage())); + } + return error(RUNTIME_ERROR, t.toString()); + } + + // ---- truth and comparison -------------------------------------------- + + public static boolean isTrue(Object o) { + if (o == TRUE) { + return true; + } + if (o == FALSE || o == NONE) { + return false; + } + if (o instanceof Long) { + return ((Long) o).longValue() != 0; + } + if (o instanceof Double) { + return ((Double) o).doubleValue() != 0; + } + if (o instanceof String) { + return ((String) o).length() != 0; + } + return obj(o).isTrue(); + } + + public static Object not(Object o) { + return bool(!isTrue(o)); + } + + public static boolean eqBool(Object a, Object b) { + if (a instanceof Long && b instanceof Long) { + return ((Long) a).longValue() == ((Long) b).longValue(); + } + if (a == b) { //NOPMD CompareObjectsWithEquals - identity is meant + return !(Py.isFloat(a)) || !Double.isNaN(Py.asDouble(a)); + } + if (a instanceof String) { + return b instanceof String && a.equals(b); + } + return isTrue(eq(a, b)); + } + + public static Object eq(Object a, Object b) { + if (a instanceof String && b instanceof String) { + return bool(a.equals(b)); + } + if (a instanceof Long) { + long x = ((Long) a).longValue(); + if (b instanceof Long) { + long y = ((Long) b).longValue(); + return bool(x == y); + } else if (b instanceof Double) { + double y = ((Double) b).doubleValue(); + if (x == (x << 11 >> 11)) { + return bool(x == y); + } + } + } else if (a instanceof Double) { + double x = ((Double) a).doubleValue(); + if (b instanceof Double) { + double y = ((Double) b).doubleValue(); + return bool(x == y); + } else if (b instanceof Long) { + long y = ((Long) b).longValue(); + if (y == (y << 11 >> 11)) { + return bool(x == y); + } + } + } + Object r = Py.obj(a).eq(b); + if (r == null) { + r = Py.obj(b).eq(a); + } + return r != null ? r : bool(a == b); //NOPMD CompareObjectsWithEquals - identity is meant + } + + public static Object ne(Object a, Object b) { + if (a instanceof PyInstance) { + Object m = Py.typeOf(a).lookup("__ne__"); + if (m != null) { + Object r = Py.obj(m).call(new Object[] {a, b}, NO_KW); + if (r != NOT_IMPLEMENTED) { + return r; + } + } + } + return not(eq(a, b)); + } + + public static Object lt(Object a, Object b) { + if (a instanceof Long) { + long x = ((Long) a).longValue(); + if (b instanceof Long) { + long y = ((Long) b).longValue(); + return bool(x < y); + } else if (b instanceof Double) { + double y = ((Double) b).doubleValue(); + if (x == (x << 11 >> 11)) { + return bool(x < y); + } + } + } else if (a instanceof Double) { + double x = ((Double) a).doubleValue(); + if (b instanceof Double) { + double y = ((Double) b).doubleValue(); + return bool(x < y); + } else if (b instanceof Long) { + long y = ((Long) b).longValue(); + if (y == (y << 11 >> 11)) { + return bool(x < y); + } + } + } + Object r = Py.obj(a).lt(b); + if (r == null) { + r = Py.obj(b).gt(a); + } + if (r == null) { + throw cmpError("<", a, b); + } + return r; + } + + public static Object le(Object a, Object b) { + if (a instanceof Long) { + long x = ((Long) a).longValue(); + if (b instanceof Long) { + long y = ((Long) b).longValue(); + return bool(x <= y); + } else if (b instanceof Double) { + double y = ((Double) b).doubleValue(); + if (x == (x << 11 >> 11)) { + return bool(x <= y); + } + } + } else if (a instanceof Double) { + double x = ((Double) a).doubleValue(); + if (b instanceof Double) { + double y = ((Double) b).doubleValue(); + return bool(x <= y); + } else if (b instanceof Long) { + long y = ((Long) b).longValue(); + if (y == (y << 11 >> 11)) { + return bool(x <= y); + } + } + } + Object r = Py.obj(a).le(b); + if (r == null) { + r = Py.obj(b).ge(a); + } + if (r == null) { + throw cmpError("<=", a, b); + } + return r; + } + + public static Object gt(Object a, Object b) { + if (a instanceof Long) { + long x = ((Long) a).longValue(); + if (b instanceof Long) { + long y = ((Long) b).longValue(); + return bool(x > y); + } else if (b instanceof Double) { + double y = ((Double) b).doubleValue(); + if (x == (x << 11 >> 11)) { + return bool(x > y); + } + } + } else if (a instanceof Double) { + double x = ((Double) a).doubleValue(); + if (b instanceof Double) { + double y = ((Double) b).doubleValue(); + return bool(x > y); + } else if (b instanceof Long) { + long y = ((Long) b).longValue(); + if (y == (y << 11 >> 11)) { + return bool(x > y); + } + } + } + Object r = Py.obj(a).gt(b); + if (r == null) { + r = Py.obj(b).lt(a); + } + if (r == null) { + throw cmpError(">", a, b); + } + return r; + } + + public static Object ge(Object a, Object b) { + if (a instanceof Long) { + long x = ((Long) a).longValue(); + if (b instanceof Long) { + long y = ((Long) b).longValue(); + return bool(x >= y); + } else if (b instanceof Double) { + double y = ((Double) b).doubleValue(); + if (x == (x << 11 >> 11)) { + return bool(x >= y); + } + } + } else if (a instanceof Double) { + double x = ((Double) a).doubleValue(); + if (b instanceof Double) { + double y = ((Double) b).doubleValue(); + return bool(x >= y); + } else if (b instanceof Long) { + long y = ((Long) b).longValue(); + if (y == (y << 11 >> 11)) { + return bool(x >= y); + } + } + } + Object r = Py.obj(a).ge(b); + if (r == null) { + r = Py.obj(b).le(a); + } + if (r == null) { + throw cmpError(">=", a, b); + } + return r; + } + + public static boolean ltBool(Object a, Object b) { + if (a instanceof Long && b instanceof Long) { + return ((Long) a).longValue() < ((Long) b).longValue(); + } + if (a instanceof Double && b instanceof Double) { + return ((Double) a).doubleValue() < ((Double) b).doubleValue(); + } + if (a instanceof String && b instanceof String) { + return PyStr.compare((String) a, (String) b) < 0; + } + return isTrue(lt(a, b)); + } + + private static PyException cmpError(String op, Object a, Object b) { + return typeError("'" + op + "' not supported between instances of '" + Py.typeOf(a).name + + "' and '" + Py.typeOf(b).name + "'"); + } + + public static Object is(Object a, Object b) { + return bool(identical(a, b)); + } + + public static Object isNot(Object a, Object b) { + return bool(!identical(a, b)); + } + + /// `is`. Small ints and one-character strings are interned as in CPython; + /// beyond that two equal numbers are `is`-identical only if they are the + /// same object, also as in CPython. + public static boolean identical(Object a, Object b) { + return a == b; //NOPMD CompareObjectsWithEquals - identity is meant + } + + public static Object in(Object x, Object container) { + return bool(Py.obj(container).contains(x)); + } + + public static Object notIn(Object x, Object container) { + return bool(!Py.obj(container).contains(x)); + } + + // ---- arithmetic ------------------------------------------------------ + + private static PyException opError(String op, Object a, Object b) { + return typeError("unsupported operand type(s) for " + op + ": '" + Py.typeOf(a).name + + "' and '" + Py.typeOf(b).name + "'"); + } + + /// Whether `b`'s reflected method must be tried first: CPython does so + /// when `b`'s type is a proper subclass of `a`'s. + private static boolean rightFirst(Object a, Object b) { + PyType ta = Py.typeOf(a); + PyType tb = Py.typeOf(b); + return ta != tb && tb.isSubtype(ta) && b instanceof PyInstance; //NOPMD CompareObjectsWithEquals - identity is meant + } + + /// `d[k] = d.get(k, dflt) + inc`, probing an exact dict once. + public static void dictCount(Object d, Object k, Object dflt, Object inc) { + if (d instanceof PyDict && ((PyDict) d).type() == DICT_TYPE) { + ((PyDict) d).count(k, dflt, inc); + return; + } + Object v = add(call(getattr(d, "get"), k, dflt), inc); + setitem(d, k, v); + } + + public static Object add(Object a, Object b) { + if (a instanceof String && b instanceof String) { + return ((String) a).concat((String) b); + } + if (a instanceof Long) { + long x = ((Long) a).longValue(); + if (b instanceof Long) { + long y = ((Long) b).longValue(); + long r = x + y; + if (((x ^ r) & (y ^ r)) >= 0) { + return Long.valueOf(r); + } + } else if (b instanceof Double) { + double y = ((Double) b).doubleValue(); + return Double.valueOf(x + y); + } + } else if (a instanceof Double) { + double x = ((Double) a).doubleValue(); + if (b instanceof Double) { + double y = ((Double) b).doubleValue(); + return Double.valueOf(x + y); + } else if (b instanceof Long) { + long y = ((Long) b).longValue(); + return Double.valueOf(x + y); + } + } + if (Py.isInt(a) && Py.isInt(b)) { + PyInt x = Py.intView(a); + PyInt y = Py.intView(b); + if (x.big == null && y.big == null) { + long r = x.value + y.value; + if (((x.value ^ r) & (y.value ^ r)) >= 0) { + return PyInt.of(r); + } + } + } + if (rightFirst(a, b)) { + Object r = Py.obj(b).radd(a); + if (r != null) { + return r; + } + } + Object r = Py.obj(a).add(b); + if (r == null) { + r = Py.obj(b).radd(a); + if (r == null) { + if (a instanceof String && !(b instanceof String)) { + throw typeError("can only concatenate str (not \"" + Py.typeOf(b).name + "\") to str"); + } + if (a instanceof PyList && !(b instanceof PyList)) { + throw typeError("can only concatenate list (not \"" + Py.typeOf(b).name + "\") to list"); + } + throw opError("+", a, b); + } + } + return r; + } + + public static Object sub(Object a, Object b) { + if (a instanceof Long) { + long x = ((Long) a).longValue(); + if (b instanceof Long) { + long y = ((Long) b).longValue(); + long r = x - y; + if (((x ^ y) & (x ^ r)) >= 0) { + return Long.valueOf(r); + } + } else if (b instanceof Double) { + double y = ((Double) b).doubleValue(); + return Double.valueOf(x - y); + } + } else if (a instanceof Double) { + double x = ((Double) a).doubleValue(); + if (b instanceof Double) { + double y = ((Double) b).doubleValue(); + return Double.valueOf(x - y); + } else if (b instanceof Long) { + long y = ((Long) b).longValue(); + return Double.valueOf(x - y); + } + } + if (Py.isInt(a) && Py.isInt(b)) { + PyInt x = Py.intView(a); + PyInt y = Py.intView(b); + if (x.big == null && y.big == null) { + long r = x.value - y.value; + if (((x.value ^ y.value) & (x.value ^ r)) >= 0) { + return PyInt.of(r); + } + } + } + Object r = rightFirst(a, b) ? Py.obj(b).rsub(a) : null; + if (r == null) { + r = Py.obj(a).sub(b); + } + if (r == null) { + r = Py.obj(b).rsub(a); + } + if (r == null) { + throw opError("-", a, b); + } + return r; + } + + public static Object mul(Object a, Object b) { + if (a instanceof Long) { + long x = ((Long) a).longValue(); + if (b instanceof Long) { + long y = ((Long) b).longValue(); + if (x == (int) x && y == (int) y) { + return Long.valueOf(x * y); + } + // Wider operands: the product is exact when dividing it back + // gives the operand (MIN_VALUE * -1 is the one case that lies). + long r = x * y; + if (y != 0 && r / y == x && !(x == Long.MIN_VALUE && y == -1) && !(y == Long.MIN_VALUE && x == -1)) { + return Long.valueOf(r); + } + } else if (b instanceof Double) { + double y = ((Double) b).doubleValue(); + return Double.valueOf(x * y); + } + } else if (a instanceof Double) { + double x = ((Double) a).doubleValue(); + if (b instanceof Double) { + double y = ((Double) b).doubleValue(); + return Double.valueOf(x * y); + } else if (b instanceof Long) { + long y = ((Long) b).longValue(); + return Double.valueOf(x * y); + } + } + Object r = rightFirst(a, b) ? Py.obj(b).rmul(a) : null; + if (r == null) { + r = Py.obj(a).mul(b); + } + if (r == null) { + r = Py.obj(b).rmul(a); + } + if (r == null) { + // A sequence with a non-int: CPython reports the repeat's error. + if (isSequence(a) || isSequence(b)) { + throw typeError("can't multiply sequence by non-int of type '" + + Py.typeOf(isSequence(a) ? b : a).name + "'"); + } + throw opError("*", a, b); + } + return r; + } + + public static Object truediv(Object a, Object b) { + if (a instanceof Long) { + long x = ((Long) a).longValue(); + if (b instanceof Long) { + long y = ((Long) b).longValue(); + if (y != 0 && x == (x << 11 >> 11) && y == (y << 11 >> 11)) { + return Double.valueOf((double) x / (double) y); + } + } else if (b instanceof Double) { + double y = ((Double) b).doubleValue(); + if (y != 0) { + return Double.valueOf(x / y); + } + } + } else if (a instanceof Double) { + double x = ((Double) a).doubleValue(); + if (b instanceof Double) { + double y = ((Double) b).doubleValue(); + if (y != 0) { + return Double.valueOf(x / y); + } + } else if (b instanceof Long) { + long y = ((Long) b).longValue(); + if (y != 0) { + return Double.valueOf(x / y); + } + } + } + Object r = Py.obj(a).truediv(b); + if (r == null) { + r = Py.obj(b).rtruediv(a); + } + if (r == null) { + throw opError("/", a, b); + } + return r; + } + + public static Object floordiv(Object a, Object b) { + if (a instanceof Long) { + long x = ((Long) a).longValue(); + if (b instanceof Long) { + long y = ((Long) b).longValue(); + if (y != 0 && !(x == Long.MIN_VALUE && y == -1)) { + long q = x / y; + if ((x % y != 0) && ((x ^ y) < 0)) { + q--; + } + return Long.valueOf(q); + } + } + } + Object r = Py.obj(a).floordiv(b); + if (r == null) { + r = Py.obj(b).rfloordiv(a); + } + if (r == null) { + throw opError("//", a, b); + } + return r; + } + + public static Object mod(Object a, Object b) { + if (a instanceof String) { + return Formatting.percent((String) a, b); + } + if (a instanceof Long) { + long x = ((Long) a).longValue(); + if (b instanceof Long) { + long y = ((Long) b).longValue(); + if (y != 0 && y != -1) { + long m = x % y; + if (m != 0 && ((m ^ y) < 0)) { + m += y; + } + return Long.valueOf(m); + } + } + } + Object r = Py.obj(a).mod(b); + if (r == null) { + r = Py.obj(b).rmod(a); + } + if (r == null) { + throw opError("%", a, b); + } + return r; + } + + public static Object pow(Object a, Object b) { + if (a instanceof Double) { + double x = ((Double) a).doubleValue(); + if (b instanceof Double) { + double y = ((Double) b).doubleValue(); + if (x > 0 || y == Math.floor(y)) { + double r = Math.pow(x, y); + if (!Double.isInfinite(r) && !Double.isNaN(r)) { + return Double.valueOf(r); + } + } + } else if (b instanceof Long) { + long y = ((Long) b).longValue(); + if (x != 0 || y >= 0) { + double r = Math.pow(x, y); + if (!Double.isInfinite(r) && !Double.isNaN(r)) { + return Double.valueOf(r); + } + } + } + } + Object r = Py.obj(a).pow(b); + if (r == null) { + r = Py.obj(b).rpow(a); + } + if (r == null) { + throw opError("** or pow()", a, b); + } + return r; + } + + public static Object lshift(Object a, Object b) { + if (a instanceof Long) { + long x = ((Long) a).longValue(); + if (b instanceof Long) { + long y = ((Long) b).longValue(); + if (y >= 0 && y < 63 && (x << y >> y) == x) { + return Long.valueOf(x << y); + } + } + } + Object r = Py.obj(a).lshift(b); + if (r == null) { + throw opError("<<", a, b); + } + return r; + } + + public static Object rshift(Object a, Object b) { + if (a instanceof Long) { + long x = ((Long) a).longValue(); + if (b instanceof Long) { + long y = ((Long) b).longValue(); + if (y >= 0) { + return Long.valueOf(y > 63 ? (x < 0 ? -1 : 0) : x >> y); + } + } + } + Object r = Py.obj(a).rshift(b); + if (r == null) { + throw opError(">>", a, b); + } + return r; + } + + public static Object and(Object a, Object b) { + if (a instanceof Long) { + long x = ((Long) a).longValue(); + if (b instanceof Long) { + long y = ((Long) b).longValue(); + return Long.valueOf(x & y); + } + } + Object r = Py.obj(a).and(b); + if (r == null) { + r = Py.obj(b).rand(a); + } + if (r == null) { + throw opError("&", a, b); + } + return r; + } + + public static Object or(Object a, Object b) { + if (a instanceof Long) { + long x = ((Long) a).longValue(); + if (b instanceof Long) { + long y = ((Long) b).longValue(); + return Long.valueOf(x | y); + } + } + Object r = Py.obj(a).or(b); + if (r == null) { + r = Py.obj(b).ror(a); + } + if (r == null) { + throw opError("|", a, b); + } + return r; + } + + public static Object xor(Object a, Object b) { + if (a instanceof Long) { + long x = ((Long) a).longValue(); + if (b instanceof Long) { + long y = ((Long) b).longValue(); + return Long.valueOf(x ^ y); + } + } + Object r = Py.obj(a).xor(b); + if (r == null) { + r = Py.obj(b).rxor(a); + } + if (r == null) { + throw opError("^", a, b); + } + return r; + } + + /// `a += b`: the in-place method when the type has one, else `a + b`. + public static Object iadd(Object a, Object b) { + Object r = a instanceof PyObject ? ((PyObject) a).iadd(b) : null; + return r != null ? r : add(a, b); + } + + public static Object isub(Object a, Object b) { + Object r = a instanceof PyObject ? ((PyObject) a).inplace("__isub__", b) : null; + return r != null ? r : sub(a, b); + } + + public static Object imul(Object a, Object b) { + Object r = a instanceof PyObject ? ((PyObject) a).inplace("__imul__", b) : null; + return r != null ? r : mul(a, b); + } + + public static Object itruediv(Object a, Object b) { + Object r = a instanceof PyObject ? ((PyObject) a).inplace("__itruediv__", b) : null; + return r != null ? r : truediv(a, b); + } + + public static Object ifloordiv(Object a, Object b) { + Object r = a instanceof PyObject ? ((PyObject) a).inplace("__ifloordiv__", b) : null; + return r != null ? r : floordiv(a, b); + } + + public static Object imod(Object a, Object b) { + Object r = a instanceof PyObject ? ((PyObject) a).inplace("__imod__", b) : null; + return r != null ? r : mod(a, b); + } + + public static Object ipow(Object a, Object b) { + Object r = a instanceof PyObject ? ((PyObject) a).inplace("__ipow__", b) : null; + return r != null ? r : pow(a, b); + } + + public static Object ilshift(Object a, Object b) { + Object r = a instanceof PyObject ? ((PyObject) a).inplace("__ilshift__", b) : null; + return r != null ? r : lshift(a, b); + } + + public static Object irshift(Object a, Object b) { + Object r = a instanceof PyObject ? ((PyObject) a).inplace("__irshift__", b) : null; + return r != null ? r : rshift(a, b); + } + + public static Object iand(Object a, Object b) { + Object r = a instanceof PyObject ? ((PyObject) a).inplace("__iand__", b) : null; + return r != null ? r : and(a, b); + } + + public static Object ior(Object a, Object b) { + Object r = a instanceof PyObject ? ((PyObject) a).inplace("__ior__", b) : null; + return r != null ? r : or(a, b); + } + + public static Object ixor(Object a, Object b) { + Object r = a instanceof PyObject ? ((PyObject) a).inplace("__ixor__", b) : null; + return r != null ? r : xor(a, b); + } + + public static Object neg(Object a) { + if (a instanceof Long && ((Long) a).longValue() != Long.MIN_VALUE) { + return Long.valueOf(-((Long) a).longValue()); + } + if (a instanceof Double) { + return Double.valueOf(-((Double) a).doubleValue()); + } + return Py.obj(a).neg(); + } + + public static Object pos(Object a) { + return Py.obj(a).pos(); + } + + public static Object invert(Object a) { + return Py.obj(a).invert(); + } + + /// Bitwise results of two bools stay bool, as in CPython. + static Object wrapIntResult(PyInt a, PyInt b, long v) { + if (a instanceof PyBool && b instanceof PyBool) { + return bool(v != 0); + } + return PyInt.of(v); + } + + // ---- attributes, items, calls ---------------------------------------- + + public static Object getattr(Object o, String name) { + return Py.obj(o).getAttr(name); + } + + public static void setattr(Object o, String name, Object v) { + Py.obj(o).setAttr(name, v); + } + + public static Object getitem(Object o, Object key) { + if (o instanceof PyList) { + PyList l = (PyList) o; + if (key instanceof Long) { + long i = ((Long) key).longValue(); + if (i >= 0 && i < l.size) { + long[] li = l.ints; + if (li != null) { + return Long.valueOf(li[(int) i]); + } + if (l.floats == null) { + return l.items[(int) i]; + } + } + } + return l.getItem(key); + } + return Py.obj(o).getItem(key); + } + + // `f(...)` as a raw double when `f` can answer one without running its + // body (PyFunction.PureD1..4); NaN means "make the ordinary call". + + /// callD for int arguments held unboxed. + public static double callDL(Object f, long a) { + return f instanceof PyFunction ? ((PyFunction) f).callL1(a) : Double.NaN; + } + + public static double callDL(Object f, long a, long b) { + return f instanceof PyFunction ? ((PyFunction) f).callL2(a, b) : Double.NaN; + } + + public static double callD(Object f, Object a) { + return f instanceof PyFunction ? ((PyFunction) f).callD1(a) : Double.NaN; + } + + public static double callD(Object f, Object a, Object b) { + return f instanceof PyFunction ? ((PyFunction) f).callD2(a, b) : Double.NaN; + } + + public static double callD(Object f, Object a, Object b, Object c) { + return f instanceof PyFunction ? ((PyFunction) f).callD3(a, b, c) : Double.NaN; + } + + public static double callD(Object f, Object a, Object b, Object c, Object d) { + return f instanceof PyFunction ? ((PyFunction) f).callD4(a, b, c, d) : Double.NaN; + } + + private static boolean isSequence(Object o) { + return o instanceof String || o instanceof PyList || o instanceof PyTuple || o instanceof PyBytes + || o instanceof PyByteArray; + } + + /// `o[i]` for an index known when the code was generated. + public static Object getitem(Object o, long i) { + if (o instanceof PyList) { + PyList l = (PyList) o; + if (i >= 0 && i < l.size) { + return l.get((int) i); + } + } else if (o instanceof PyTuple) { + Object[] items = ((PyTuple) o).items; + if (i >= 0 && i < items.length) { + return items[(int) i]; + } + } + return Py.obj(o).getItem(Long.valueOf(i)); + } + + static String[] argv = {""}; + + /// What `sys.argv` holds: the program name, then the arguments a + /// launcher passes. Must be called before `sys` is first imported. + public static void setArgv(String program, String[] args) { + String[] a = new String[args.length + 1]; + a[0] = program; + System.arraycopy(args, 0, a, 1, args.length); + argv = a; + } + + /// `o[a:b:c]`, with NONE for a missing part. + public static Object getslice(Object o, Object a, Object b, Object c) { + if (o instanceof PyList) { + return ((PyList) o).getSlice(a, b, c); + } + return Py.obj(o).getItem(new PySlice(a, b, c)); + } + + /// `o[a:b:c]` for a local whose previous list `old` nothing else can + /// see any more: a list slice is copied into `old` rather than a new + /// list. Anything else is an ordinary [#getslice(Object, Object, Object, Object)]. + public static Object recycleSlice(Object old, Object o, Object a, Object b, Object c) { + if (o instanceof PyList) { + if (old instanceof PyList && ((PyList) old).owned && old != o) { //NOPMD CompareObjectsWithEquals - identity is meant + return ((PyList) o).sliceInto((PyList) old, a, b, c); + } + PyList r = ((PyList) o).getSlice(a, b, c); + r.owned = true; + return r; + } + return getslice(o, a, b, c); + } + + /// `[]` for a local whose previous list `old` nothing else can see any + /// more: that list is emptied and handed back, keeping its array. + public static PyList recycleList(Object old) { + if (old instanceof PyList && ((PyList) old).owned) { + PyList l = (PyList) old; + l.reset(); + return l; + } + PyList l = new PyList(); + l.owned = true; + return l; + } + + /// The iterator of `for w in s.split():`. A str is split lazily + /// (TypeMethods.Words); anything else has its `split` called as written. + public static PyObject splitWords(Object s) { + if (s instanceof String) { + return new TypeMethods.Words((String) s); + } + return iter(callMethod(s, "split")); + } + + /// The iterator of `for x in s.split(sep):`, lazy for a str split on a + /// non-empty str; anything else has its `split` called as written. + public static PyObject splitOn(Object s, Object sep) { + if (s instanceof String && sep instanceof String && ((String) sep).length() > 0) { + return new TypeMethods.Pieces((String) s, (String) sep); + } + return iter(callMethod(s, "split", sep)); + } + + /// The position `i` names in a list or tuple of `n` items, or -1 when + /// `i` is not an int in range. + private static int floatIndex(Object i, int n) { + if (!(i instanceof Long)) { + return -1; + } + long k = ((Long) i).longValue(); + if (k < 0) { + k += n; + } + return k < 0 || k >= n ? -1 : (int) k; + } + + /// Whether `o[i]` is a float of a list or tuple, readable through + /// [#floatAt(Object, Object)]. Never runs user code, so a generated + /// float region can test through it and still give up without side + /// effects. + public static boolean isFloatAt(Object o, Object i) { + if (o instanceof PyList) { + PyList l = (PyList) o; + int k = floatIndex(i, l.size); + return k >= 0 && l.isFloatAt(k); + } + if (o instanceof PyTuple) { + Object[] items = ((PyTuple) o).items; + int k = floatIndex(i, items.length); + return k >= 0 && items[k] instanceof Double; + } + return false; + } + + /// `o[i]` unboxed, once [#isFloatAt(Object, Object)] said it is a float. + public static double floatAt(Object o, Object i) { + if (o instanceof PyList) { + PyList l = (PyList) o; + return l.floatAt(floatIndex(i, l.size)); + } + Object[] items = ((PyTuple) o).items; + return ((Double) items[floatIndex(i, items.length)]).doubleValue(); + } + + /// `o[i]` unboxed when it is a float of a list or tuple, else NaN: the + /// generated region then leaves for the general path, which also + /// handles an item that really is NaN. One call where isFloatAt and + /// floatAt took two. + public static double floatItem(Object o, Object i) { + if (!(i instanceof Long)) { + return Double.NaN; + } + return floatItem(o, ((Long) i).longValue()); + } + + /// floatItem for an index the region holds unboxed. The common case, an + /// unboxed list read in range, is all this method does, so that the C + /// compiler can inline it into the region; everything else is + /// floatItemSlow. + private static final double[] NO_FLOATS = new double[0]; + + /// The unboxed items of a list that holds only floats, which a loop + /// that cannot grow or retype the list reads directly; an empty array + /// for anything else. + public static double[] floatArray(Object o) { + double[] f = o instanceof PyList ? ((PyList) o).floats : null; + return f != null ? f : NO_FLOATS; + } + + /// How many items [#floatArray] holds: the list's length, or 0 when the + /// list is not all floats, so every index then misses and the reader + /// takes its general path. + public static int floatCount(Object o) { + return o instanceof PyList && ((PyList) o).floats != null ? ((PyList) o).size : 0; + } + + public static double floatItem(Object o, long k) { + if (o instanceof PyList) { + PyList l = (PyList) o; + double[] f = l.floats; + if (f != null && k >= 0 && k < l.size) { + return f[(int) k]; + } + } + return floatItemSlow(o, k); + } + + private static double floatItemSlow(Object o, long k) { + if (o instanceof PyList) { + PyList l = (PyList) o; + int n = l.size; + if (k < 0) { + k += n; + } + if (k < 0 || k >= n) { + return Double.NaN; + } + double[] f = l.floats; + if (f != null) { + return f[(int) k]; + } + if (l.ints != null) { + return Double.NaN; + } + Object v = l.items[(int) k]; + return v instanceof Double ? ((Double) v).doubleValue() : Double.NaN; + } + if (o instanceof PyTuple) { + Object[] items = ((PyTuple) o).items; + if (k < 0) { + k += items.length; + } + if (k < 0 || k >= items.length) { + return Double.NaN; + } + Object v = items[(int) k]; + return v instanceof Double ? ((Double) v).doubleValue() : Double.NaN; + } + return Double.NaN; + } + + /// setFloat for an index the region holds unboxed. + public static void setFloat(Object o, long k, double v) { + // Only the unboxed in-range store here, so it inlines into the region. + if (o instanceof PyList) { + PyList l = (PyList) o; + double[] f = l.floats; + if (f != null && k >= 0 && k < l.size) { + f[(int) k] = v; + return; + } + } + setFloatSlow(o, k, v); + } + + private static void setFloatSlow(Object o, long k, double v) { + if (o instanceof PyList) { + PyList l = (PyList) o; + long i = k < 0 ? k + l.size : k; + if (i >= 0 && i < l.size) { + l.setFloat((int) i, v); + return; + } + } + setitem(o, Long.valueOf(k), Double.valueOf(v)); + } + + /// `o[i] = v` for a float a region computed: an unboxed list stores it + /// unboxed, anything else as `setitem` would. + public static void setFloat(Object o, Object i, double v) { + if (o instanceof PyList) { + PyList l = (PyList) o; + int k = floatIndex(i, l.size); + if (k >= 0) { + l.setFloat(k, v); + return; + } + } + setitem(o, i, Double.valueOf(v)); + } + + /// Whether `f(start, stop, step)` is the builtin `range` over ints a + /// generated loop can count itself. The bounds stay well inside a long, + /// so stepping past the end can never wrap around. + public static boolean isIntRange(Object f, Object start, Object stop, Object step) { + if (f != RANGE_TYPE || !(start instanceof Long) || !(stop instanceof Long) || !(step instanceof Long)) { + return false; + } + long a = ((Long) start).longValue(); + long b = ((Long) stop).longValue(); + long c = ((Long) step).longValue(); + long lim = 1L << 61; + return c != 0 && a > -lim && a < lim && b > -lim && b < lim && c > -lim && c < lim; + } + + /// `o[a:b:c] = v[d:e:f]`, the source first as Python evaluates it. Two + /// lists copy directly, without the intermediate list. + public static void setsliceFrom(Object v, Object d, Object e, Object f, Object o, Object a, Object b, Object c) { + if (v instanceof PyList && o instanceof PyList) { + ((PyList) o).setSliceFrom(a, b, c, (PyList) v, d, e, f); + return; + } + Object src = getslice(v, d, e, f); + setslice(o, a, b, c, src); + } + + /// `o[a:b:c] = v`, with NONE for a missing part. + public static void setslice(Object o, Object a, Object b, Object c, Object v) { + if (o instanceof PyList) { + ((PyList) o).setSlice(a, b, c, v); + return; + } + Py.obj(o).setItem(new PySlice(a, b, c), v); + } + + public static void setitem(Object o, Object key, Object v) { + if (o instanceof PyList) { + PyList l = (PyList) o; + if (key instanceof Long) { + long i = ((Long) key).longValue(); + if (i >= 0 && i < l.size) { + long[] li = l.ints; + if (li != null) { + if (v instanceof Long) { + li[(int) i] = ((Long) v).longValue(); + return; + } + } else if (l.floats == null) { + l.items[(int) i] = v; + return; + } + } + } + l.setItem(key, v); + return; + } + Py.obj(o).setItem(key, v); + } + + /// One-argument call; `len(x)` skips the argument array and the builtin. + public static Object call(Object f, Object a) { + if (f == Builtins.LEN) { //NOPMD CompareObjectsWithEquals - identity is meant + return Long.valueOf(len(a)); + } + return Py.obj(f).call1(a); + } + + public static Object call(Object f) { + return Py.obj(f).call0(); + } + + public static Object call(Object f, Object a, Object b) { + return Py.obj(f).call2(a, b); + } + + public static Object call(Object f, Object a, Object b, Object c) { + return Py.obj(f).call3(a, b, c); + } + + public static Object call(Object f, Object a, Object b, Object c, Object d) { + return Py.obj(f).call4(a, b, c, d); + } + + public static Object call(Object f, Object... args) { + return Py.obj(f).call(args, NO_KW); + } + + public static Object callKw(Object f, Object[] args, String[] kwNames) { + return Py.obj(f).call(args, kwNames); + } + + /// `obj.name(args)`, without materialising the bound method when `name` + /// is a plain function on the class. + public static Object callMethod(Object o, String name, Object... args) { + if (o instanceof PyInstance) { + PyInstance inst = (PyInstance) o; + if (inst.own(name) == null) { + Object m = inst.cls.lookup(name); + if (m instanceof PyFunction) { + return Py.obj(m).call(prepend(o, args), NO_KW); + } + } + } else if (!(o instanceof PyType) && !(o instanceof PyModule) && !(o instanceof PySuper)) { + // A super proxy reports object as its type; its attributes come + // from the MRO after the class it was made for, never from there. + Object m = Py.typeOf(o).lookup(name); + if (m instanceof PyBuiltin) { + return Py.obj(m).call(prepend(o, args), NO_KW); + } + } + return Py.obj(Py.obj(o).getAttr(name)).call(args, NO_KW); + } + + /// A call whose arguments contain `*seq` or `**map` unpacking. `parts` + /// lists each argument; `kinds` says 0 = plain, 1 = star, 2 = double star, + /// 3 = keyword (name in `names` at the same index). + public static Object callSpread(Object f, Object[] parts, int[] kinds, String[] names) { + List pos = new ArrayList(); + List kwn = new ArrayList(); + List kwv = new ArrayList(); + for (int i = 0; i < parts.length; i++) { + switch (kinds[i]) { + case 0: + pos.add(parts[i]); + break; + case 1: { + Object it = Py.iter(parts[i]); + for (Object v = Py.next(it); v != null; v = Py.next(it)) { + pos.add(v); + } + break; + } + case 2: { + Object m = parts[i]; + Object[] keys = m instanceof PyDict ? ((PyDict) m).keyArray() + : toArray(Py.obj(Py.obj(m).getAttr("keys")).call(NO_ARGS, NO_KW)); + for (Object key : keys) { + if (!(key instanceof String)) { + throw typeError("keywords must be strings"); + } + String kn = ((String) key); + if (kwn.contains(kn)) { + throw typeError("got multiple values for keyword argument '" + kn + "'"); + } + kwn.add(kn); + kwv.add(Py.obj(m).getItem(key)); + } + break; + } + default: + kwn.add(names[i]); + kwv.add(parts[i]); + break; + } + } + Object[] args = new Object[pos.size() + kwv.size()]; + for (int i = 0; i < pos.size(); i++) { + args[i] = pos.get(i); + } + for (int i = 0; i < kwv.size(); i++) { + args[pos.size() + i] = kwv.get(i); + } + return Py.obj(f).call(args, kwn.toArray(new String[kwn.size()])); + } + + public static Object[] prepend(Object first, Object[] rest) { + Object[] r = new Object[rest.length + 1]; + r[0] = first; + System.arraycopy(rest, 0, r, 1, rest.length); + return r; + } + + /// An integer index; `__index__` for other types. + public static long index(Object o) { + if (o instanceof Long) { + return ((Long) o).longValue(); + } + if (Py.isInt(o)) { + return Py.asLong(o); + } + if (o instanceof PyInstance) { + return Py.asLong(o); + } + throw typeError("indices must be integers or slices, not " + Py.typeOf(o).name); + } + + // ---- iteration ------------------------------------------------------- + + public static PyObject iter(Object o) { + Object r = obj(o).iter(); + if (r instanceof PyObject) { + return (PyObject) r; + } + throw typeError("iter() returned non-iterator of type '" + typeOf(r).name + "'"); + } + + /// The next item of iterator `it`, or null when it is exhausted. + public static Object next(Object it) { + return ((PyObject) it).next(); + } + + /// Materialises any iterable. + public static Object[] toArray(Object o) { + if (o instanceof PyTuple) { + return ((PyTuple) o).items; + } + if (o instanceof PyList) { + return ((PyList) o).toArray(); + } + PyList l = new PyList(); + l.extend(o); + return l.toArray(); + } + + /// `a, b = value`: exactly `n` items or `ValueError`. + public static Object[] unpack(Object o, int n) { + if (o instanceof PyTuple) { + Object[] items = ((PyTuple) o).items; + if (items.length == n) { + return items; + } + throw unpackError(items.length, n); + } + Object it = Py.iter(o); + Object[] r = new Object[n]; + for (int i = 0; i < n; i++) { + Object v = Py.next(it); + if (v == null) { + throw valueError("not enough values to unpack (expected " + n + ", got " + i + ")"); + } + r[i] = v; + } + if (Py.next(it) != null) { + throw valueError("too many values to unpack (expected " + n + ")"); + } + return r; + } + + /// [#unpack(Object, int)] for a target made of plain names: storing to + /// those cannot change a list being unpacked, so its own array is read + /// in place instead of being copied. The caller only reads the result. + public static Object[] unpackNames(Object o, int n) { + if (o instanceof PyList) { + PyList l = (PyList) o; + if (l.size == n) { + return l.floats != null || l.ints != null ? l.toArray() : l.items; + } + throw unpackError(l.size, n); + } + if (o instanceof PyTuple) { + // A tuple never changes, and the generated code only reads the + // array it gets back, at once: its own items need no copy. + Object[] items = ((PyTuple) o).items; + if (items.length == n) { + return items; + } + } + return unpack(o, n); + } + + /// `a, *b, c = value`: `before` items, a list, then `after` items. + public static Object[] unpackStar(Object o, int before, int after) { + Object[] all = toArray(o); + if (all.length < before + after) { + throw valueError("not enough values to unpack (expected at least " + (before + after) + + ", got " + all.length + ")"); + } + Object[] r = new Object[before + 1 + after]; + System.arraycopy(all, 0, r, 0, before); + Object[] mid = new Object[all.length - before - after]; + System.arraycopy(all, before, mid, 0, mid.length); + r[before] = new PyList(mid); + System.arraycopy(all, all.length - after, r, before + 1, after); + return r; + } + + private static PyException unpackError(int got, int n) { + if (got < n) { + return valueError("not enough values to unpack (expected " + n + ", got " + got + ")"); + } + return valueError("too many values to unpack (expected " + n + ")"); + } + + // ---- literals -------------------------------------------------------- + + /// `[]`, without the empty varargs array. + public static PyList list() { + return new PyList(); + } + + public static PyList list(Object... items) { + return PyList.of(items); + } + + public static PyTuple tuple(Object... items) { + return PyTuple.of(items); + } + + /// `{k0: v0, k1: v1, ...}` from alternating keys and values. + public static PyDict dict(Object... kv) { + PyDict d = new PyDict(); + for (int i = 0; i < kv.length; i += 2) { + d.setItem(kv[i], kv[i + 1]); + } + return d; + } + + public static PySet set(Object... items) { + PySet s = new PySet(); + for (Object item : items) { + s.addItem(item); + } + return s; + } + + public static PySlice slice(Object a, Object b, Object c) { + return new PySlice(a, b, c); + } + + // ---- strings --------------------------------------------------------- + + public static String strOf(Object o) { + return Py.toStr(o); + } + + /// The string a `__str__`/`__repr__` override returned, type checked. + static String strValue(Object r, String method) { + if (!(r instanceof String)) { + throw typeError(method + " returned non-string (type " + Py.typeOf(r).name + ")"); + } + return ((String) r); + } + + /// `format(value, spec)`, the engine of f-strings. + public static Object format(Object v, String spec) { + return str(Formatting.format(v, spec)); + } + + /// Joins the pieces of an f-string. + public static String concat(Object... parts) { + StringBuilder sb = new StringBuilder(); + for (Object e : parts) { + Object part = e; + sb.append(part instanceof String ? ((String) part) : strOf(part)); + } + return sb.toString(); + } + + static String joinRepr(String open, Object[] items, int n, String close) { + StringBuilder sb = new StringBuilder(open); + for (int i = 0; i < n; i++) { + if (i > 0) { + sb.append(", "); + } + sb.append(Py.toRepr(items[i])); + } + return sb.append(close).toString(); + } + + private static final List REPR_STACK = new ArrayList(); + + /// Recursion guard for containers that contain themselves. + static boolean enterRepr(Object o) { + for (int i = 0; i < REPR_STACK.size(); i++) { //NOPMD ForLoopCanBeForeach - indexed: a for-each allocates an iterator + if (REPR_STACK.get(i) == o) { //NOPMD CompareObjectsWithEquals - identity is meant + return true; + } + } + REPR_STACK.add(o); + return false; + } + + static void exitRepr(Object o) { + REPR_STACK.remove(REPR_STACK.size() - 1); + } + + static boolean seqEquals(Object[] a, int na, Object[] b, int nb) { + if (na != nb) { + return false; + } + for (int i = 0; i < na; i++) { + if (a[i] != b[i] && !eqBool(a[i], b[i])) { //NOPMD CompareObjectsWithEquals - identity is meant + return false; + } + } + return true; + } + + static int seqCompare(Object[] a, int na, Object[] b, int nb) { + int n = Math.min(na, nb); + for (int i = 0; i < n; i++) { + if (a[i] != b[i] && !eqBool(a[i], b[i])) { //NOPMD CompareObjectsWithEquals - identity is meant + return ltBool(a[i], b[i]) ? -1 : 1; + } + } + return na - nb; + } + + // ---- type tests ------------------------------------------------------ + + /// A mapping argument as the dict holding it. + public static PyDict toDict(Object o) { + if (o instanceof PyDict) { + return (PyDict) o; + } + if (o instanceof PyInstance && ((PyInstance) o).base instanceof PyDict) { + return (PyDict) ((PyInstance) o).base; + } + PyDict d = new PyDict(); + d.update(o); + return d; + } + + public static boolean isinstance(Object o, Object t) { + if (t instanceof PyTuple) { + Object[] items = ((PyTuple) t).items; + for (Object item : items) { + if (isinstance(o, item)) { + return true; + } + } + return false; + } + if (!(t instanceof PyType)) { + throw typeError("isinstance() arg 2 must be a type, a tuple of types, or a union"); + } + return ((PyType) t).instanceCheck(o); + } + + /// The items of a display with `*x` unpacking: `star[i]` marks the parts + /// to iterate rather than take whole. + public static Object[] spread(Object[] parts, boolean[] star) { + PyList out = new PyList(parts.length); + for (int i = 0; i < parts.length; i++) { + if (star[i]) { + Object it = iter(parts[i]); + for (Object v = Py.next(it); v != null; v = Py.next(it)) { + out.append(v); + } + } else { + out.append(parts[i]); + } + } + return out.toArray(); + } + + /// A dict display with `**m` entries: `kv` holds key/value pairs, and a + /// `null` key marks a mapping to merge in at that position. + public static PyDict dictSpread(Object... kv) { + PyDict d = new PyDict(); + for (int i = 0; i < kv.length; i += 2) { + if (kv[i] == null) { + Object m = kv[i + 1]; + Object keys = iter(callMethod(m, "keys")); + for (Object k = Py.next(keys); k != null; k = Py.next(keys)) { + d.setItem(k, getitem(m, k)); + } + } else { + d.setItem(kv[i], kv[i + 1]); + } + } + return d; + } + + /// `from m import *` into `target`: `__all__` when defined, else every + /// name that does not start with an underscore. + public static void importStar(PyModule target, PyModule m) { + Object all = m.getAttrOrNull("__all__"); + if (all != null) { + Object it = iter(all); + for (Object n = Py.next(it); n != null; n = Py.next(it)) { + String name = strOf(n); + target.setAttr(name, importFrom(m, name)); + } + return; + } + String[] names = m.exportedNames(); + for (String e : names) { + if (!e.startsWith("_")) { + Object v = m.getAttrOrNull(e); + if (v != null) { + target.setAttr(e, v); + } + } + } + } + + // ---- modules --------------------------------------------------------- + + /// `import name`. + public static PyModule importModule(String name) { + return PyModule.load(name); + } + + /// `from module import name`. + public static Object importFrom(PyModule m, String name) { + Object v = m.getAttrOrNull(name); + if (v == null) { + PyModule sub = PyModule.find(m.name + "." + name); + if (sub != null) { + return sub; + } + throw error(IMPORT_ERROR, "cannot import name '" + name + "' from '" + m.name + "'"); + } + return v; + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyBool.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyBool.java new file mode 100644 index 00000000000..c24dfcea33d --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyBool.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +/// `bool`, a subclass of `int` with exactly two instances. +public final class PyBool extends PyInt { + + PyBool(boolean v) { + super(v ? 1 : 0); + } + + @Override + public PyType type() { + return Py.BOOL_TYPE; + } + + @Override + public String repr() { + return value != 0 ? "True" : "False"; + } + + @Override + public Object toJava() { + return Boolean.valueOf(value != 0); + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyBuiltin.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyBuiltin.java new file mode 100644 index 00000000000..abd17fa08d2 --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyBuiltin.java @@ -0,0 +1,147 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +/// A function or method implemented in Java: the builtins, and the methods of +/// the builtin types. As a class attribute it binds like a Python function. +public final class PyBuiltin extends PyObject { + + /// The implementation; receives the arguments as passed. + public interface Impl { + Object call(Object[] args, String[] kwNames); + } + + /// An implementation that takes positional arguments only. + public interface Positional { + Object call(Object[] args); + } + + public final String name; + final Impl impl; + + public PyBuiltin(String name, Impl impl) { + this.name = name; + this.impl = impl; + } + + /// A builtin that rejects keyword arguments, as most C functions do. + public static PyBuiltin positional(final String name, final Positional p) { + return new PyBuiltin(name, new Impl() { + @Override + public Object call(Object[] args, String[] kwNames) { + if (kwNames.length != 0) { + throw Py.typeError(name + "() takes no keyword arguments"); + } + return p.call(args); + } + }); + } + + /// A method of builtin type `owner`. The receiver is checked before the + /// implementation runs, so it may cast `args[0]` to the owner's class: + /// `bytes.hex(5)` is a TypeError here rather than an unchecked cast. + public static PyBuiltin method(final PyType owner, final String name, final Positional p) { + return new PyBuiltin(name, new Impl() { + @Override + public Object call(Object[] args, String[] kwNames) { + checkReceiver(owner, name, args); + if (kwNames.length != 0) { + throw Py.typeError(name + "() takes no keyword arguments"); + } + Object[] a = unwrap(owner, args); + Object r = p.call(a); + return a != args && r == a[0] && BuiltinSubclass.mutable(owner) ? args[0] : r; //NOPMD CompareObjectsWithEquals - identity is meant + } + }); + } + + /// [#method] for an implementation that takes keyword arguments. + public static PyBuiltin methodKw(final PyType owner, final String name, final Impl p) { + return new PyBuiltin(name, new Impl() { + @Override + public Object call(Object[] args, String[] kwNames) { + checkReceiver(owner, name, args); + Object[] a = unwrap(owner, args); + Object r = p.call(a, kwNames); + return a != args && r == a[0] && BuiltinSubclass.mutable(owner) ? args[0] : r; //NOPMD CompareObjectsWithEquals - identity is meant + } + }); + } + + /// The arguments with an instance of a subclass of builtin `owner` + /// replaced by the builtin value it extends, which is what the + /// implementation works on. A method returning that value (a chained + /// `__iadd__` of a list) gives the instance back. + static Object[] unwrap(PyType owner, Object[] args) { + Object self = args[0]; + if (owner != Py.OBJECT_TYPE && self instanceof PyInstance) { //NOPMD CompareObjectsWithEquals - identity is meant + Object b = ((PyInstance) self).base; + if (b != null) { + Object[] a = new Object[args.length]; + System.arraycopy(args, 0, a, 0, args.length); + a[0] = b; + return a; + } + } + return args; + } + + static void checkReceiver(PyType owner, String name, Object[] args) { + if (args.length == 0) { + throw Py.typeError("unbound method " + owner.name + "." + name + "() needs an argument"); + } + if (!Py.typeOf(args[0]).isSubtype(owner)) { + throw Py.typeError("descriptor '" + name + "' for '" + owner.name + "' objects doesn't apply to a '" + + Py.typeOf(args[0]).name + "' object"); + } + } + + @Override + public PyType type() { + return Py.BUILTIN_TYPE; + } + + @Override + public Object bind(Object self) { + return new PyMethod(self, this); + } + + @Override + public Object call(Object[] args, String[] kwNames) { + return impl.call(args, kwNames); + } + + @Override + public Object getAttrOrNull(String attr) { + if ("__name__".equals(attr)) { + return name; + } + return super.getAttrOrNull(attr); + } + + @Override + public String repr() { + return ""; + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyByteArray.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyByteArray.java new file mode 100644 index 00000000000..8e35cc9d276 --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyByteArray.java @@ -0,0 +1,397 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +/// `bytearray`: a growable buffer of bytes. +/// +/// The methods that read without changing it are `bytes`' own, run on a +/// snapshot; what they answer in bytes comes back as a bytearray, as +/// CPython's do. +public final class PyByteArray extends PyObject { + private static final String[] READERS = {"decode", "hex", "upper", "lower", "isdigit", "isalpha", + "isalnum", "isspace", "strip", "lstrip", "rstrip", "split", "rsplit", "startswith", "endswith", "find", + "rfind", "index", "rindex", "count", "replace", "join", "partition", "rpartition", "removeprefix", + "removesuffix", "splitlines", "center", "ljust", "rjust", "zfill", "expandtabs", "translate", "isascii", + "islower", "isupper", "istitle", "swapcase", "capitalize", "title"}; + + byte[] buf; + int size; + + public PyByteArray(byte[] value) { + this.buf = value; + this.size = value.length; + } + + @Override + public PyType type() { + return Py.BYTEARRAY_TYPE; + } + + /// The live contents, copied. + public byte[] bytes() { + byte[] r = new byte[size]; + System.arraycopy(buf, 0, r, 0, size); + return r; + } + + @Override + public Object toJava() { + return bytes(); + } + + @Override + public int len() { + return size; + } + + @Override + public boolean isTrue() { + return size != 0; + } + + @Override + public String repr() { + return "bytearray(" + new PyBytes(bytes()).repr() + ")"; + } + + @Override + public long hash() { + throw Py.typeError("unhashable type: 'bytearray'"); + } + + @Override + public Object eq(Object o) { + byte[] b = PyBytes.like(o); + return b == null ? null : Py.bool(PyBytes.same(bytes(), b)); + } + + @Override + public Object lt(Object o) { + byte[] b = PyBytes.like(o); + return b == null ? null : Py.bool(PyBytes.compare(bytes(), b) < 0); + } + + @Override + public Object le(Object o) { + byte[] b = PyBytes.like(o); + return b == null ? null : Py.bool(PyBytes.compare(bytes(), b) <= 0); + } + + @Override + public Object gt(Object o) { + byte[] b = PyBytes.like(o); + return b == null ? null : Py.bool(PyBytes.compare(bytes(), b) > 0); + } + + @Override + public Object ge(Object o) { + byte[] b = PyBytes.like(o); + return b == null ? null : Py.bool(PyBytes.compare(bytes(), b) >= 0); + } + + @Override + public Object add(Object o) { + byte[] b = PyBytes.like(o); + if (b == null) { + // Concatenation, not addition: only a class of the program's + // own may still answer through __radd__. + if (o instanceof PyInstance) { + return null; + } + throw Py.typeError("can't concat " + Py.typeOf(o).name + " to bytearray"); + } + byte[] r = new byte[size + b.length]; + System.arraycopy(buf, 0, r, 0, size); + System.arraycopy(b, 0, r, size, b.length); + return new PyByteArray(r); + } + + @Override + public Object iadd(Object o) { + byte[] b = PyBytes.like(o); + if (b == null) { + return null; + } + insertAt(size, b); + return this; + } + + @Override + public Object mul(Object o) { + Object r = new PyBytes(bytes()).mul(o); + return r instanceof PyBytes ? new PyByteArray(((PyBytes) r).value) : r; + } + + @Override + public Object rmul(Object o) { + return mul(o); + } + + @Override + public Object inplace(String dunder, Object o) { + if ("__imul__".equals(dunder) && Py.isInt(o)) { + Object r = mul(o); + byte[] b = ((PyByteArray) r).buf; + buf = b; + size = b.length; + return this; + } + return null; + } + + @Override + public boolean contains(Object x) { + return new PyBytes(bytes()).contains(x); + } + + @Override + public Object getItem(Object key) { + if (!(key instanceof PySlice)) { + return PyInt.of(buf[index(key)] & 0xff); + } + Object r = new PyBytes(bytes()).getItem(key); + return r instanceof PyBytes ? new PyByteArray(((PyBytes) r).value) : r; + } + + private static byte byteOf(Object v) { + if (!Py.isInt(v)) { + throw Py.typeError("'" + Py.typeOf(v).name + "' object cannot be interpreted as an integer"); + } + long n = Py.asLong(v); + if (n < 0 || n > 255) { + throw Py.valueError("byte must be in range(0, 256)"); + } + return (byte) n; + } + + private int index(Object key) { + long i = Py.index(key); + if (i < 0) { + i += size; + } + if (i < 0 || i >= size) { + throw Py.indexError("bytearray index out of range"); + } + return (int) i; + } + + private void insertAt(int at, byte[] b) { + if (size + b.length > buf.length) { + byte[] n = new byte[Math.max(size + b.length, buf.length * 2 + 8)]; + System.arraycopy(buf, 0, n, 0, size); + buf = n; + } + System.arraycopy(buf, at, buf, at + b.length, size - at); + System.arraycopy(b, 0, buf, at, b.length); + size += b.length; + } + + private void removeRange(int lo, int hi) { + System.arraycopy(buf, hi, buf, lo, size - hi); + size -= hi - lo; + } + + @Override + public void setItem(Object key, Object value) { + if (key instanceof PySlice) { + int[] r = ((PySlice) key).indices(size); + byte[] src = PyBytes.like(value); + if (src == null) { + if (Py.isInt(value)) { + throw Py.typeError("can assign only bytes, buffers, or iterables of ints in range(0, 256)"); + } + src = PyBytes.fromInts(value); + } + if (r[2] == 1) { + int lo = r[0]; + int hi = Math.max(r[0], r[1]); + removeRange(lo, hi); + insertAt(lo, src); + return; + } + int count = PySlice.length(r); + if (count != src.length) { + throw Py.valueError("attempt to assign bytes of size " + src.length + + " to extended slice of size " + count); + } + for (int i = 0, j = r[0]; i < count; i++, j += r[2]) { + buf[j] = src[i]; + } + return; + } + buf[index(key)] = byteOf(value); + } + + @Override + public void delItem(Object key) { + if (key instanceof PySlice) { + int[] r = ((PySlice) key).indices(size); + int count = PySlice.length(r); + boolean[] drop = new boolean[size]; + for (int i = 0, j = r[0]; i < count; i++, j += r[2]) { + drop[j] = true; + } + int w = 0; + for (int i = 0; i < size; i++) { + if (!drop[i]) { + buf[w++] = buf[i]; + } + } + size = w; + return; + } + int i = index(key); + removeRange(i, i + 1); + } + + @Override + public Object iter() { + return new PyIter(size) { + @Override + Object at(int i) { + return PyInt.of(buf[i] & 0xff); + } + + @Override + int liveLength() { + return size; + } + }; + } + + private static PyByteArray self(Object o) { + if (!(o instanceof PyByteArray)) { + throw Py.typeError("descriptor requires a 'bytearray' object but received a '" + + Py.typeOf(o).name + "'"); + } + return (PyByteArray) o; + } + + private static Object toArray(Object o) { + if (o instanceof PyBytes) { + return new PyByteArray(((PyBytes) o).value); + } + if (o instanceof PyList || o instanceof PyTuple) { + Object[] items = Py.toArray(o); + Object[] r = new Object[items.length]; + for (int i = 0; i < items.length; i++) { + r[i] = toArray(items[i]); + } + return o instanceof PyTuple ? new PyTuple(r) : new PyList(r); + } + return o; + } + + static void install() { + final PyType t = Py.BYTEARRAY_TYPE; + for (String e : READERS) { + final String name = e; + t.define(name, PyBuiltin.methodKw(t, name, (args, kw) -> { + Object[] mapped = new Object[args.length]; + mapped[0] = new PyBytes(self(args[0]).bytes()); + for (int k = 1; k < args.length; k++) { + mapped[k] = args[k] instanceof PyByteArray ? new PyBytes(((PyByteArray) args[k]).bytes()) + : args[k]; + } + return toArray(Py.obj(Py.BYTES_TYPE.lookup(name)).call(mapped, kw)); + })); + } + t.define("append", PyBuiltin.method(t, "append", a -> { + PyByteArray b = self(a[0]); + b.insertAt(b.size, new byte[] {byteOf(a[1])}); + return Py.NONE; + })); + t.define("extend", PyBuiltin.method(t, "extend", a -> { + PyByteArray b = self(a[0]); + byte[] src = PyBytes.like(a[1]); + b.insertAt(b.size, src != null ? src : PyBytes.fromInts(a[1])); + return Py.NONE; + })); + t.define("insert", PyBuiltin.method(t, "insert", a -> { + PyByteArray b = self(a[0]); + long i = Py.index(a[1]); + if (i < 0) { + i = Math.max(0, i + b.size); + } + b.insertAt((int) Math.min(i, b.size), new byte[] {byteOf(a[2])}); + return Py.NONE; + })); + t.define("pop", PyBuiltin.method(t, "pop", a -> { + PyByteArray b = self(a[0]); + if (b.size == 0) { + throw Py.indexError("pop from empty bytearray"); + } + int i = a.length > 1 ? b.index(a[1]) : b.size - 1; + int v = b.buf[i] & 0xff; + b.removeRange(i, i + 1); + return PyInt.of(v); + })); + t.define("remove", PyBuiltin.method(t, "remove", a -> { + PyByteArray b = self(a[0]); + byte v = byteOf(a[1]); + for (int i = 0; i < b.size; i++) { + if (b.buf[i] == v) { + b.removeRange(i, i + 1); + return Py.NONE; + } + } + throw Py.valueError("value not found in bytearray"); + })); + t.define("clear", PyBuiltin.method(t, "clear", a -> { + self(a[0]).size = 0; + return Py.NONE; + })); + t.define("reverse", PyBuiltin.method(t, "reverse", a -> { + PyByteArray b = self(a[0]); + for (int i = 0, j = b.size - 1; i < j; i++, j--) { + byte x = b.buf[i]; + b.buf[i] = b.buf[j]; + b.buf[j] = x; + } + return Py.NONE; + })); + t.define("copy", PyBuiltin.method(t, "copy", a -> new PyByteArray(self(a[0]).bytes()))); + t.define("maketrans", Py.BYTES_TYPE.lookup("maketrans")); + t.define("fromhex", new PyClassMethod(PyBuiltin.positional("fromhex", a -> { + Object b = Py.obj(Py.BYTES_TYPE.lookup("fromhex")).bind(Py.BYTES_TYPE); + return new PyByteArray(((PyBytes) Py.call(b, a[1])).value); + }))); + t.factory = (type, args, kw) -> { + if (args.length == 0) { + return new PyByteArray(new byte[0]); + } + Object x = args[0]; + if (x instanceof String || x instanceof PyStr) { + if (args.length < 2 && kw.length == 0) { + throw Py.typeError("string argument without an encoding"); + } + } + Object b = Py.BYTES_TYPE.factory.create(Py.BYTES_TYPE, args, kw); + if (!(b instanceof PyBytes)) { + throw Py.typeError("cannot convert '" + Py.typeOf(x).name + "' object to bytearray"); + } + return new PyByteArray(((PyBytes) b).bytes()); + }; + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyBytes.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyBytes.java new file mode 100644 index 00000000000..be27b9f28ea --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyBytes.java @@ -0,0 +1,643 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +/// Python's `bytes`, immutable. +public final class PyBytes extends PyObject { + final byte[] value; + + public PyBytes(byte[] value) { + this.value = value; + } + + @Override + public PyType type() { + return Py.BYTES_TYPE; + } + + public byte[] bytes() { + byte[] r = new byte[value.length]; + System.arraycopy(value, 0, r, 0, r.length); + return r; + } + + @Override + public Object toJava() { + return bytes(); + } + + @Override + public int len() { + return value.length; + } + + @Override + public boolean isTrue() { + return value.length != 0; + } + + @Override + public String repr() { + boolean sq = false; + boolean dq = false; + for (byte x : value) { + sq |= x == '\''; + dq |= x == '"'; + } + char q = sq && !dq ? '"' : '\''; + StringBuilder sb = new StringBuilder("b"); + sb.append(q); + for (byte e : value) { + int c = e & 0xff; + if (c == q || c == '\\') { + sb.append('\\').append((char) c); + } else if (c == '\n') { + sb.append("\\n"); + } else if (c == '\r') { + sb.append("\\r"); + } else if (c == '\t') { + sb.append("\\t"); + } else if (c < 0x20 || c >= 0x7f) { + sb.append("\\x"); + PyStr.hex(sb, c, 2); + } else { + sb.append((char) c); + } + } + return sb.append(q).toString(); + } + + @Override + public long hash() { + long h = 0; + for (byte e : value) { + h = h * 31 + e; + } + return h; + } + + /// The bytes of a bytes-like object (bytes, bytearray, memoryview), or + /// null for anything else. + static byte[] like(Object o) { + if (o instanceof PyBytes) { + return ((PyBytes) o).value; + } + if (o instanceof PyByteArray) { + return ((PyByteArray) o).bytes(); + } + if (o instanceof PyMemoryView) { + return ((PyMemoryView) o).bytes(); + } + return null; + } + + /// An iterable of ints in range(0, 256), as bytes. + static byte[] fromInts(Object o) { + Object[] items = Py.toArray(o); + byte[] r = new byte[items.length]; + for (int i = 0; i < items.length; i++) { + if (!Py.isInt(items[i])) { + throw Py.typeError("'" + Py.typeOf(items[i]).name + "' object cannot be interpreted as an integer"); + } + long v = Py.asLong(items[i]); + if (v < 0 || v > 255) { + throw Py.valueError("byte must be in range(0, 256)"); + } + r[i] = (byte) v; + } + return r; + } + + static boolean same(byte[] a, byte[] b) { + if (a.length != b.length) { + return false; + } + for (int i = 0; i < a.length; i++) { + if (a[i] != b[i]) { + return false; + } + } + return true; + } + + /// Unsigned lexicographic order. + static int compare(byte[] a, byte[] b) { + int n = Math.min(a.length, b.length); + for (int i = 0; i < n; i++) { + int d = (a[i] & 0xff) - (b[i] & 0xff); + if (d != 0) { + return d; + } + } + return a.length - b.length; + } + + static int indexOf(byte[] hay, byte[] needle) { + for (int i = 0; i + needle.length <= hay.length; i++) { + if (matchesAt(hay, i, needle)) { + return i; + } + } + return -1; + } + + private static boolean matchesAt(byte[] hay, int i, byte[] needle) { + for (int j = 0; j < needle.length; j++) { + if (hay[i + j] != needle[j]) { + return false; + } + } + return true; + } + + @Override + public boolean contains(Object x) { + byte[] b = like(x); + if (b != null) { + return indexOf(value, b) >= 0; + } + if (!Py.isInt(x)) { + throw Py.typeError("a bytes-like object is required, not '" + Py.typeOf(x).name + "'"); + } + long v = Py.asLong(x); + if (v < 0 || v > 255) { + throw Py.valueError("byte must be in range(0, 256)"); + } + for (byte e : value) { + if ((e & 0xff) == v) { + return true; + } + } + return false; + } + + @Override + public Object eq(Object o) { + byte[] b = like(o); + return b == null ? null : Py.bool(same(value, b)); + } + + @Override + public Object lt(Object o) { + byte[] b = like(o); + return b == null ? null : Py.bool(compare(value, b) < 0); + } + + @Override + public Object le(Object o) { + byte[] b = like(o); + return b == null ? null : Py.bool(compare(value, b) <= 0); + } + + @Override + public Object gt(Object o) { + byte[] b = like(o); + return b == null ? null : Py.bool(compare(value, b) > 0); + } + + @Override + public Object ge(Object o) { + byte[] b = like(o); + return b == null ? null : Py.bool(compare(value, b) >= 0); + } + + @Override + public Object mul(Object o) { + if (!(Py.isInt(o))) { + return null; + } + long n = Math.max(0, Py.asLong(o)); + byte[] r = new byte[(int) (value.length * n)]; + for (int i = 0; i < n; i++) { + System.arraycopy(value, 0, r, i * value.length, value.length); + } + return new PyBytes(r); + } + + @Override + public Object rmul(Object o) { + return mul(o); + } + + @Override + public Object add(Object o) { + byte[] b = like(o); + if (b == null) { + // Concatenation, not addition: only a class of the program's + // own may still answer through __radd__. + if (o instanceof PyInstance) { + return null; + } + throw Py.typeError("can't concat " + Py.typeOf(o).name + " to bytes"); + } + byte[] r = new byte[value.length + b.length]; + System.arraycopy(value, 0, r, 0, value.length); + System.arraycopy(b, 0, r, value.length, b.length); + return new PyBytes(r); + } + + @Override + public Object getItem(Object key) { + if (key instanceof PySlice) { + int[] r = ((PySlice) key).indices(value.length); + int n = PySlice.length(r); + byte[] out = new byte[n]; + for (int i = 0, j = r[0]; i < n; i++, j += r[2]) { + out[i] = value[j]; + } + return new PyBytes(out); + } + long i = Py.index(key); + if (i < 0) { + i += value.length; + } + if (i < 0 || i >= value.length) { + throw Py.indexError("index out of range"); + } + return PyInt.of(value[(int) i] & 0xff); + } + + @Override + public Object iter() { + return new PyIter(value.length) { + @Override + Object at(int i) { + return PyInt.of(value[i] & 0xff); + } + }; + } + + static void install() { + Py.BYTES_TYPE.define("decode", PyBuiltin.methodKw(Py.BYTES_TYPE, "decode", (a, kw) -> { + Object[] p = Args.bind("decode", a, kw, new String[] {"self", "encoding", "errors"}, + new Object[] {null, Py.str("utf-8"), Py.str("strict")}); + return Py.str(Codecs.decode(((PyBytes) p[0]).value, Py.toStr(p[1]), Py.toStr(p[2]))); + })); + Py.BYTES_TYPE.define("hex", PyBuiltin.methodKw(Py.BYTES_TYPE, "hex", (args, kw) -> { + Object[] p = Args.bind("hex", args, kw, new String[] {"self", "sep", "bytes_per_sep"}, + new Object[] {null, Py.NONE, PyInt.of(1)}); + byte[] v = self(p[0]).value; + String sep = null; + if (p[1] != Py.NONE) { //NOPMD CompareObjectsWithEquals - identity is meant + byte[] sb = like(p[1]); + sep = sb == null ? Py.toStr(p[1]) : sb.length == 1 ? String.valueOf((char) (sb[0] & 0xff)) : "--"; + if (sep.length() != 1) { + throw Py.valueError("sep must be length 1."); + } + if (sep.charAt(0) > 127) { + throw Py.valueError("sep must be ASCII."); + } + } + long per = Py.asLong(p[2]); + int group = (int) Math.min(Math.abs(per), Integer.MAX_VALUE); + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < v.length; i++) { + if (sep != null && group > 0 && i > 0) { + // A positive count groups from the right, a negative one from the left. + boolean boundary = per > 0 ? (v.length - i) % group == 0 : i % group == 0; + if (boundary) { + sb.append(sep); + } + } + PyStr.hex(sb, v[i] & 0xff, 2); + } + return Py.str(sb.toString()); + })); + installLatin1Delegates(); + Py.BYTES_TYPE.define("fromhex", new PyClassMethod(PyBuiltin.positional("fromhex", a -> { + String h = Py.toStr(a[1]); + java.io.ByteArrayOutputStream out = new java.io.ByteArrayOutputStream(); + int i = 0; + while (i < h.length()) { + char c = h.charAt(i); + if (c == ' ' || c == '\t' || c == '\n' || c == '\r') { + i++; + continue; + } + int hi = Character.digit(c, 16); + int lo = i + 1 < h.length() ? Character.digit(h.charAt(i + 1), 16) : -1; + if (hi < 0 || lo < 0) { + throw Py.valueError("non-hexadecimal number found in fromhex() arg at position " + + (hi < 0 ? i : i + 1)); + } + out.write(hi * 16 + lo); + i += 2; + } + return new PyBytes(out.toByteArray()); + }))); + Py.BYTES_TYPE.define("upper", PyBuiltin.method(Py.BYTES_TYPE, "upper", a -> asciiCase((PyBytes) a[0], true))); + Py.BYTES_TYPE.define("lower", PyBuiltin.method(Py.BYTES_TYPE, "lower", a -> asciiCase((PyBytes) a[0], false))); + Py.BYTES_TYPE.define("isdigit", PyBuiltin.method(Py.BYTES_TYPE, "isdigit", + a -> Py.bool(allAscii((PyBytes) a[0], "0123456789")))); + Py.BYTES_TYPE.define("isalpha", PyBuiltin.method(Py.BYTES_TYPE, "isalpha", + a -> Py.bool(allAscii((PyBytes) a[0], LETTERS)))); + Py.BYTES_TYPE.define("isalnum", PyBuiltin.method(Py.BYTES_TYPE, "isalnum", + a -> Py.bool(allAscii((PyBytes) a[0], LETTERS + "0123456789")))); + Py.BYTES_TYPE.define("isspace", PyBuiltin.method(Py.BYTES_TYPE, "isspace", + a -> Py.bool(allAscii((PyBytes) a[0], " \t\n\r\u000b\f")))); + installAsciiText(); + Py.BYTES_TYPE.factory = (t, args, kw) -> { + if (args.length == 0) { + return new PyBytes(new byte[0]); + } + Object x = args[0]; + if (x instanceof String) { + return new PyBytes(Utf8.encode(((String) x))); + } + if (Py.isInt(x)) { + return new PyBytes(new byte[(int) Py.asLong(x)]); + } + Object[] items = Py.toArray(x); + byte[] r = new byte[items.length]; + for (int i = 0; i < items.length; i++) { + long v = Py.asLong(items[i]); + if (v < 0 || v > 255) { + throw Py.valueError("bytes must be in range(0, 256)"); + } + r[i] = (byte) v; + } + return new PyBytes(r); + }; + } + + private static boolean isUpper(int c) { + return c >= 'A' && c <= 'Z'; + } + + private static boolean isLower(int c) { + return c >= 'a' && c <= 'z'; + } + + private static PyBytes self(Object o) { + if (!(o instanceof PyBytes)) { + throw Py.typeError("descriptor requires a 'bytes' object but received a '" + Py.typeOf(o).name + "'"); + } + return (PyBytes) o; + } + + /// The case and translation methods, which see only ASCII letters. + private static void installAsciiText() { + PyType t = Py.BYTES_TYPE; + t.define("maketrans", new PyStaticMethod(PyBuiltin.positional("maketrans", a -> { + if (a.length != 2) { + throw Py.typeError("maketrans expected 2 arguments, got " + a.length); + } + byte[] from = like(a[0]); + byte[] to = like(a[1]); + if (from == null || to == null) { + throw Py.typeError("a bytes-like object is required, not '" + + Py.typeOf(from == null ? a[0] : a[1]).name + "'"); + } + if (from.length != to.length) { + throw Py.valueError("maketrans arguments must have same length"); + } + byte[] table = new byte[256]; + for (int i = 0; i < 256; i++) { + table[i] = (byte) i; + } + for (int i = 0; i < from.length; i++) { + table[from[i] & 0xff] = to[i]; + } + return new PyBytes(table); + }))); + t.define("translate", PyBuiltin.methodKw(t, "translate", (args, kw) -> { + Object[] p = Args.bind("translate", args, kw, new String[] {"self", "table", "delete"}, + new Object[] {null, null, new PyBytes(new byte[0])}); + byte[] v = self(p[0]).value; + byte[] table = null; + if (p[1] != Py.NONE) { //NOPMD CompareObjectsWithEquals - identity is meant + table = like(p[1]); + if (table == null) { + throw Py.typeError("a bytes-like object is required, not '" + Py.typeOf(p[1]).name + "'"); + } + if (table.length != 256) { + throw Py.valueError("translation table must be 256 characters long"); + } + } + byte[] del = like(p[2]); + if (del == null) { + throw Py.typeError("a bytes-like object is required, not '" + Py.typeOf(p[2]).name + "'"); + } + boolean[] drop = new boolean[256]; + for (byte e : del) { + drop[e & 0xff] = true; + } + java.io.ByteArrayOutputStream out = new java.io.ByteArrayOutputStream(v.length); + for (byte el : v) { + int c = el & 0xff; + if (!drop[c]) { + out.write(table == null ? c : table[c] & 0xff); + } + } + return new PyBytes(out.toByteArray()); + })); + t.define("isascii", PyBuiltin.method(t, "isascii", a -> { + byte[] v = self(a[0]).value; + for (byte e : v) { + if (e < 0) { + return Py.FALSE; + } + } + return Py.TRUE; + })); + t.define("islower", PyBuiltin.method(t, "islower", a -> Py.bool(cased(self(a[0]).value, false)))); + t.define("isupper", PyBuiltin.method(t, "isupper", a -> Py.bool(cased(self(a[0]).value, true)))); + t.define("istitle", PyBuiltin.method(t, "istitle", a -> { + byte[] v = self(a[0]).value; + boolean cased = false; + boolean prevCased = false; + for (byte e : v) { + int c = e & 0xff; + if (isUpper(c)) { + if (prevCased) { + return Py.FALSE; + } + prevCased = true; + cased = true; + } else if (isLower(c)) { + if (!prevCased) { + return Py.FALSE; + } + prevCased = true; + cased = true; + } else { + prevCased = false; + } + } + return Py.bool(cased); + })); + t.define("swapcase", PyBuiltin.method(t, "swapcase", a -> { + byte[] v = self(a[0]).value; + byte[] r = new byte[v.length]; + for (int i = 0; i < v.length; i++) { + int c = v[i] & 0xff; + r[i] = (byte) (isUpper(c) ? c + 32 : isLower(c) ? c - 32 : c); + } + return new PyBytes(r); + })); + t.define("capitalize", PyBuiltin.method(t, "capitalize", a -> { + byte[] v = self(a[0]).value; + byte[] r = new byte[v.length]; + for (int i = 0; i < v.length; i++) { + int c = v[i] & 0xff; + if (i == 0) { + r[i] = (byte) (isLower(c) ? c - 32 : c); + } else { + r[i] = (byte) (isUpper(c) ? c + 32 : c); + } + } + return new PyBytes(r); + })); + t.define("title", PyBuiltin.method(t, "title", a -> { + byte[] v = self(a[0]).value; + byte[] r = new byte[v.length]; + boolean prevCased = false; + for (int i = 0; i < v.length; i++) { + int c = v[i] & 0xff; + if (isLower(c)) { + r[i] = (byte) (prevCased ? c : c - 32); + prevCased = true; + } else if (isUpper(c)) { + r[i] = (byte) (prevCased ? c + 32 : c); + prevCased = true; + } else { + r[i] = (byte) c; + prevCased = false; + } + } + return new PyBytes(r); + })); + } + + /// Whether `v` has a cased letter and every one is upper (or lower) case. + private static boolean cased(byte[] v, boolean upper) { + boolean any = false; + for (byte e : v) { + int c = e & 0xff; + if (upper ? isLower(c) : isUpper(c)) { + return false; + } + if (isUpper(c) || isLower(c)) { + any = true; + } + } + return any; + } + + private static final String LETTERS = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; + + /// The str methods whose bytes versions behave identically on a Latin-1 + /// view of the bytes (one char per byte); their results map back. + private static final String[] LATIN1_DELEGATES = {"strip", "lstrip", "rstrip", "split", "rsplit", "startswith", + "endswith", "find", "rfind", "index", "rindex", "count", "replace", "join", "partition", "rpartition", + "removeprefix", "removesuffix", "splitlines", "center", "ljust", "rjust", "zfill", "expandtabs"}; + + private static void installLatin1Delegates() { + for (String e : LATIN1_DELEGATES) { + final String name = e; + Py.BYTES_TYPE.define(name, PyBuiltin.methodKw(Py.BYTES_TYPE, name, (args, kw) -> { + Object[] mapped = new Object[args.length]; + for (int k = 0; k < args.length; k++) { + Object v = args[k]; + if ("join".equals(name) && k == 1 && !(v instanceof PyBytes)) { + // Any iterable of bytes: materialise it so each item maps. + v = new PyList(Py.toArray(v)); + } + mapped[k] = toLatin1(v); + } + Object m = Py.STR_TYPE.lookup(name); + return fromLatin1(Py.obj(m).call(mapped, kw)); + })); + } + } + + private static Object toLatin1(Object o) { + byte[] v = like(o); + if (v != null) { + char[] c = new char[v.length]; + for (int i = 0; i < v.length; i++) { + c[i] = (char) (v[i] & 0xff); + } + return new String(c); + } + if (o instanceof String) { + throw Py.typeError("a bytes-like object is required, not 'str'"); + } + if (o instanceof PyTuple || o instanceof PyList) { + Object[] items = Py.toArray(o); + Object[] r = new Object[items.length]; + for (int i = 0; i < items.length; i++) { + r[i] = toLatin1(items[i]); + } + return o instanceof PyTuple ? new PyTuple(r) : new PyList(r); + } + return o; + } + + private static Object fromLatin1(Object o) { + if (o instanceof String) { + String s = ((String) o); + byte[] b = new byte[s.length()]; + for (int i = 0; i < b.length; i++) { + b[i] = (byte) s.charAt(i); + } + return new PyBytes(b); + } + if (o instanceof PyTuple || o instanceof PyList) { + Object[] items = Py.toArray(o); + Object[] r = new Object[items.length]; + for (int i = 0; i < items.length; i++) { + r[i] = fromLatin1(items[i]); + } + return o instanceof PyTuple ? new PyTuple(r) : new PyList(r); + } + return o; + } + + private static Object asciiCase(PyBytes b, boolean upper) { + byte[] r = new byte[b.value.length]; + for (int i = 0; i < r.length; i++) { + byte c = b.value[i]; + if (upper && c >= 'a' && c <= 'z') { + c = (byte) (c - 32); + } else if (!upper && c >= 'A' && c <= 'Z') { + c = (byte) (c + 32); + } + r[i] = c; + } + return new PyBytes(r); + } + + private static boolean allAscii(PyBytes b, String set) { + if (b.value.length == 0) { + return false; + } + for (int i = 0; i < b.value.length; i++) { + if (set.indexOf((char) (b.value[i] & 0xff)) < 0) { + return false; + } + } + return true; + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyClassMethod.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyClassMethod.java new file mode 100644 index 00000000000..15330ec0fcd --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyClassMethod.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +/// `classmethod(func)`: binds to the class of whatever it is read through. +public final class PyClassMethod extends PyObject { + final Object func; + + public PyClassMethod(Object func) { + this.func = func; + } + + @Override + public PyType type() { + return Py.CLASSMETHOD_TYPE; + } + + @Override + public Object bind(Object self) { + return new PyMethod(self instanceof PyType ? self : Py.typeOf(self), func); + } + + @Override + public Object getAttrOrNull(String attr) { + if ("__func__".equals(attr) || "__wrapped__".equals(attr)) { + return func; + } + if ("__isabstractmethod__".equals(attr) || "__doc__".equals(attr) || "__name__".equals(attr)) { + Object v = Py.obj(func).getAttrOrNull(attr); + return v != null ? v : "__isabstractmethod__".equals(attr) ? Py.FALSE : Py.NONE; + } + return super.getAttrOrNull(attr); + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyComplex.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyComplex.java new file mode 100644 index 00000000000..fe11725a5e0 --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyComplex.java @@ -0,0 +1,362 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +/// `complex`: a pair of doubles with CPython's arithmetic, repr and hash. +public final class PyComplex extends PyObject { + /// CPython's `_PyHASH_IMAG`, the multiplier of the imaginary part's hash. + private static final long HASH_IMAG = 1000003L; + + final double real; + final double imag; + + public PyComplex(double real, double imag) { + this.real = real; + this.imag = imag; + } + + @Override + public PyType type() { + return Py.COMPLEX_TYPE; + } + + private static String part(double d) { + String s = PyFloat.repr(d); + return s.endsWith(".0") ? s.substring(0, s.length() - 2) : s; + } + + @Override + public String repr() { + if (real == 0 && 1 / real > 0) { + return part(imag) + "j"; + } + String im = part(imag); + if (!im.startsWith("-")) { + im = "+" + im; + } + return "(" + part(real) + im + "j)"; + } + + @Override + public boolean isTrue() { + return real != 0 || imag != 0; + } + + @Override + public long hash() { + long h = Hashing.ofDouble(real, this) + HASH_IMAG * Hashing.ofDouble(imag, this); + return h == -1 ? -2 : h; + } + + /// `o` as a complex, or null when it is not a number. + static PyComplex of(Object o) { + if (o instanceof PyComplex) { + return (PyComplex) o; + } + if (Py.isFloat(o) || Py.isInt(o)) { + return new PyComplex(Py.asDouble(o), 0); + } + return null; + } + + @Override + public Object add(Object o) { + PyComplex c = of(o); + return c == null ? null : new PyComplex(real + c.real, imag + c.imag); + } + + @Override + public Object radd(Object o) { + return add(o); + } + + @Override + public Object sub(Object o) { + PyComplex c = of(o); + return c == null ? null : new PyComplex(real - c.real, imag - c.imag); + } + + @Override + public Object rsub(Object o) { + PyComplex c = of(o); + return c == null ? null : new PyComplex(c.real - real, c.imag - imag); + } + + private static PyComplex mul(PyComplex a, PyComplex b) { + return new PyComplex(a.real * b.real - a.imag * b.imag, a.real * b.imag + a.imag * b.real); + } + + @Override + public Object mul(Object o) { + PyComplex c = of(o); + return c == null ? null : mul(this, c); + } + + @Override + public Object rmul(Object o) { + return mul(o); + } + + /// CPython's `_Py_c_quot`: Smith's algorithm, scaled by the larger part + /// of the divisor so that neither product overflows early. + private static PyComplex quot(PyComplex a, PyComplex b) { + double absBr = Math.abs(b.real); + double absBi = Math.abs(b.imag); + if (absBr == 0 && absBi == 0) { + throw Py.error(Py.ZERO_DIVISION_ERROR, "division by zero"); + } + if (absBr >= absBi) { + double ratio = b.imag / b.real; + double denom = b.real + b.imag * ratio; + return new PyComplex((a.real + a.imag * ratio) / denom, (a.imag - a.real * ratio) / denom); + } + if (absBi >= absBr) { + double ratio = b.real / b.imag; + double denom = b.real * ratio + b.imag; + return new PyComplex((a.real * ratio + a.imag) / denom, (a.imag * ratio - a.real) / denom); + } + return new PyComplex(Double.NaN, Double.NaN); + } + + @Override + public Object truediv(Object o) { + PyComplex c = of(o); + return c == null ? null : quot(this, c); + } + + @Override + public Object rtruediv(Object o) { + PyComplex c = of(o); + return c == null ? null : quot(c, this); + } + + private static PyComplex pow(PyComplex a, PyComplex b) { + if (b.real == 0 && b.imag == 0) { + return new PyComplex(1, 0); + } + if (a.real == 0 && a.imag == 0) { + if (b.imag != 0 || b.real < 0) { + throw Py.error(Py.ZERO_DIVISION_ERROR, "zero to a negative or complex power"); + } + return new PyComplex(0, 0); + } + if (b.imag == 0 && b.real == Math.floor(b.real) && Math.abs(b.real) <= 100) { + // CPython's c_powi: square-and-multiply for small integral powers. + long n = (long) b.real; + PyComplex r = new PyComplex(1, 0); + PyComplex x = a; + long m = Math.abs(n); + while (m > 0) { + if ((m & 1) != 0) { + r = mul(r, x); + } + x = mul(x, x); + m >>= 1; + } + return n < 0 ? quot(new PyComplex(1, 0), r) : r; + } + double vabs = Math.hypot(a.real, a.imag); + double len = Math.pow(vabs, b.real); + double at = Math.atan2(a.imag, a.real); + double phase = at * b.real; + if (b.imag != 0) { + len /= Math.exp(at * b.imag); + phase += b.imag * Math.log(vabs); + } + return new PyComplex(len * Math.cos(phase), len * Math.sin(phase)); + } + + @Override + public Object pow(Object o) { + PyComplex c = of(o); + return c == null ? null : pow(this, c); + } + + @Override + public Object rpow(Object o) { + PyComplex c = of(o); + return c == null ? null : pow(c, this); + } + + @Override + public Object neg() { + return new PyComplex(-real, -imag); + } + + @Override + public Object pos() { + return this; + } + + @Override + public Object eq(Object o) { + if (o instanceof PyComplex) { + PyComplex c = (PyComplex) o; + return Py.bool(real == c.real && imag == c.imag); + } + if (Py.isFloat(o) || Py.isInt(o)) { + return Py.bool(imag == 0 && Py.isTrue(Py.eq(Py.ofFloat(real), o))); + } + return null; + } + + @Override + public Object getAttrOrNull(String name) { + if ("real".equals(name)) { + return Py.ofFloat(real); + } + if ("imag".equals(name)) { + return Py.ofFloat(imag); + } + return super.getAttrOrNull(name); + } + + static double abs(PyComplex c) { + if (Double.isInfinite(c.real) || Double.isInfinite(c.imag)) { + return Double.POSITIVE_INFINITY; + } + double r = Math.hypot(c.real, c.imag); + if (Double.isInfinite(r)) { + throw Py.error(Py.OVERFLOW_ERROR, "absolute value too large"); + } + return r; + } + + static void install() { + PyType t = Py.COMPLEX_TYPE; + t.define("conjugate", PyBuiltin.method(t, "conjugate", a -> { + PyComplex c = self(a[0]); + return new PyComplex(c.real, -c.imag); + })); + t.define("__abs__", PyBuiltin.method(t, "__abs__", a -> Py.ofFloat(abs(self(a[0]))))); + t.define("__complex__", PyBuiltin.method(t, "__complex__", a -> self(a[0]))); + t.factory = (type, args, kw) -> { + Args p = new Args("complex", args, kw).check(2, "real", "imag"); + Object re = p.get(0, "real", null); + Object im = p.get(1, "imag", null); + if (re instanceof String || re instanceof PyStr) { + if (im != null) { + throw Py.typeError("complex() argument 'real' must be a real number, not str"); + } + return parse(Py.toStr(re)); + } + if (im instanceof String || im instanceof PyStr) { + throw Py.typeError("complex() argument 'imag' must be a real number, not str"); + } + PyComplex a = re == null ? new PyComplex(0, 0) : coerce(re, true); + PyComplex b = im == null ? null : coerce(im, false); + // CPython adds the other part only from an argument that is + // itself complex, so complex(0, -0.0) keeps its negative zero. + double r = a.real; + double i = re instanceof PyComplex ? a.imag : 0; + if (b != null) { + r = im instanceof PyComplex ? r - b.imag : r; + i = re instanceof PyComplex ? i + b.real : b.real; + } + return new PyComplex(r, i); + }; + } + + private static PyComplex self(Object o) { + if (!(o instanceof PyComplex)) { + throw Py.typeError("descriptor requires a 'complex' object but received a '" + Py.typeOf(o).name + "'"); + } + return (PyComplex) o; + } + + private static PyComplex coerce(Object o, boolean first) { + PyComplex c = of(o); + if (c != null) { + return c; + } + Object m = Py.typeOf(o).lookup("__complex__"); + if (m != null) { + Object r = Py.obj(m).call(new Object[] {o}, Py.NO_KW); + if (r instanceof PyComplex) { + return (PyComplex) r; + } + throw Py.typeError("__complex__ returned non-complex (type " + Py.typeOf(r).name + ")"); + } + m = Py.typeOf(o).lookup("__float__"); + if (m != null) { + return new PyComplex(Py.asDouble(Py.obj(m).call(new Object[] {o}, Py.NO_KW)), 0); + } + throw Py.typeError(first ? "complex() argument must be a string or a number, not " + Py.typeOf(o).name + : "complex() argument 'imag' must be a real number, not " + Py.typeOf(o).name); + } + + /// `complex("1+2j")`: an optional pair of parentheses around a real, + /// an imaginary, or a real followed by a signed imaginary part. + static PyComplex parse(String text) { + String s = text.trim(); + if (s.startsWith("(") && s.endsWith(")")) { + s = s.substring(1, s.length() - 1).trim(); + } + try { + if (s.isEmpty()) { + throw new NumberFormatException(); + } + char last = s.charAt(s.length() - 1); + if (last != 'j' && last != 'J') { + return new PyComplex(number(s), 0); + } + String body = s.substring(0, s.length() - 1); + int split = -1; + for (int i = body.length() - 1; i > 0; i--) { + char c = body.charAt(i); + char prev = body.charAt(i - 1); + if ((c == '+' || c == '-') && prev != 'e' && prev != 'E') { + split = i; + break; + } + } + if (split < 0) { + return new PyComplex(0, imagPart(body)); + } + return new PyComplex(number(body.substring(0, split)), imagPart(body.substring(split))); + } catch (NumberFormatException | PyException e) { + throw Py.valueError("complex() arg is a malformed string"); //NOPMD PreserveStackTrace - CPython's error has no cause + } + } + + private static double imagPart(String s) { + if (s.isEmpty() || "+".equals(s)) { + return 1; + } + if ("-".equals(s)) { + return -1; + } + return number(s); + } + + private static double number(String s) { + for (int i = 0; i < s.length(); i++) { + char c = s.charAt(i); + if (c == ' ' || c == '\t' || c == '\n' || c == '_') { + throw new NumberFormatException(); + } + } + return Py.asDouble(Builtins.toFloat(s)); + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyDict.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyDict.java new file mode 100644 index 00000000000..532c1984eb9 --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyDict.java @@ -0,0 +1,453 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +/// Python's `dict`: insertion ordered, laid out as CPython's compact dict. +/// +/// Entries are appended to parallel arrays in insertion order and a separate +/// open-addressed table of entry indices finds them by hash. Deleting leaves a +/// hole in the entry arrays, compacted on the next resize, so iteration order +/// is always insertion order and the table needs no linked list. +public class PyDict extends PyObject { + private static final int EMPTY = -1; + private static final int DUMMY = -2; + + Object[] keys; + Object[] values; + long[] hashes; + /// Number of entry slots used, live or deleted. + int used; + /// Number of live entries. + int size; + private int[] index; + + public PyDict() { + keys = new Object[8]; + values = new Object[8]; + hashes = new long[8]; + index = newIndex(16); + } + + private static int[] newIndex(int n) { + int[] ix = new int[n]; + for (int i = 0; i < n; i++) { + ix[i] = EMPTY; + } + return ix; + } + + @Override + public PyType type() { + return Py.DICT_TYPE; + } + + public int size() { + return size; + } + + private static boolean keyEq(Object a, long ha, Object b, long hb) { + if (a == b) { //NOPMD CompareObjectsWithEquals - identity is meant + return true; + } + if (ha != hb) { + return false; + } + if (a instanceof String && b instanceof String) { + return ((String) a).equals(((String) b)); + } + return Py.eqBool(a, b); + } + + /// [#find] for a str key: a stored str is compared with `equals` + /// directly, and only a key of another type with the same hash goes + /// through the general comparison. + private int findStr(String key, long h) { + int mask = index.length - 1; + int i = (int) (h & mask); + long perturb = h & Long.MAX_VALUE; + int freeSlot = -1; + while (true) { + int e = index[i]; + if (e == EMPTY) { + return -1 - (freeSlot >= 0 ? freeSlot : i); + } + if (e == DUMMY) { + if (freeSlot < 0) { + freeSlot = i; + } + } else if (hashes[e] == h) { + // equals tests identity first. A key of another type with an + // equal hash is left to its __eq__. + Object k = keys[e]; + if (k instanceof String ? key.equals(k) : keyEq(k, h, key, h)) { + return i; + } + } + perturb >>>= 5; + i = (int) ((i * 5L + perturb + 1) & mask); + } + } + + /// The index slot holding `key`, or the negative of (1 + the first free slot). + private int find(Object key, long h) { + int mask = index.length - 1; + int i = (int) (h & mask); + long perturb = h & Long.MAX_VALUE; + int freeSlot = -1; + while (true) { + int e = index[i]; + if (e == EMPTY) { + return -1 - (freeSlot >= 0 ? freeSlot : i); + } + if (e == DUMMY) { + if (freeSlot < 0) { + freeSlot = i; + } + } else if (keyEq(keys[e], hashes[e], key, h)) { + return i; + } + perturb >>>= 5; + i = (int) ((i * 5L + perturb + 1) & mask); + } + } + + /// `self[key] = self.get(key, dflt) + inc` with one probe: the slot found + /// for the read is the one written, unless the addition grew the dict + /// (it cannot, but a user `__add__` could), in which case it probes again. + Object count(Object key, Object dflt, Object inc) { + long h = Py.keyHash(key, "a dict key"); + int s = key instanceof String ? findStr((String) key, h) : find(key, h); + if (s >= 0) { + int e = index[s]; + Object stored = keys[e]; + Object r = Py.add(values[e], inc); + if (index[s] != e || keys[e] != stored) { //NOPMD CompareObjectsWithEquals - identity is meant + // The addition ran Python code that changed the dict. + s = find(key, h); + } + if (s >= 0) { + values[index[s]] = r; + return r; + } + setItem(key, r); + return r; + } + Object r = Py.add(dflt, inc); + setItem(key, r); + return r; + } + + /// `self[key]`, or `null` when absent. + public Object getOrNull(Object key) { + int s = find(key, Py.keyHash(key, "a dict key")); + return s >= 0 ? values[index[s]] : null; + } + + public Object get(String key) { + return getOrNull(key); + } + + @Override + public Object getItem(Object key) { + Object v = getOrNull(key); + if (v == null) { + Object missing = type() == Py.DICT_TYPE ? null : type().lookup("__missing__"); //NOPMD CompareObjectsWithEquals - identity is meant + if (missing != null) { + return Py.obj(missing).call(new Object[] {this, key}, Py.NO_KW); + } + throw Py.keyError(key); + } + return v; + } + + @Override + public void setItem(Object key, Object value) { + long h = Py.keyHash(key, "a dict key"); + int s = find(key, h); + if (s >= 0) { + values[index[s]] = value; + return; + } + if (used == keys.length) { + grow(); + s = find(key, h); + } + index[-1 - s] = used; + keys[used] = key; + values[used] = value; + hashes[used] = h; + used++; + size++; + if (used * 3 >= index.length * 2) { + rebuild(index.length * 2); + } + } + + private void grow() { + int cap = Math.max(8, size * 2); + if (cap < keys.length && size < used) { + cap = keys.length; + } + Object[] nk = new Object[cap]; + Object[] nv = new Object[cap]; + long[] nh = new long[cap]; + int w = 0; + for (int i = 0; i < used; i++) { + if (keys[i] != null) { + nk[w] = keys[i]; + nv[w] = values[i]; + nh[w] = hashes[i]; + w++; + } + } + keys = nk; + values = nv; + hashes = nh; + used = w; + int n = index.length; + while (cap * 3 >= n * 2) { + n *= 2; + } + rebuild(n); + } + + private void rebuild(int n) { + index = newIndex(n); + int mask = n - 1; + for (int e = 0; e < used; e++) { + if (keys[e] == null) { + continue; + } + long h = hashes[e]; + int i = (int) (h & mask); + long perturb = h & Long.MAX_VALUE; + while (index[i] != EMPTY) { + perturb >>>= 5; + i = (int) ((i * 5L + perturb + 1) & mask); + } + index[i] = e; + } + } + + /// Removes `key`, returning its value or `null` when absent. + public Object remove(Object key) { + int s = find(key, Py.keyHash(key, "a dict key")); + if (s < 0) { + return null; + } + int e = index[s]; + Object v = values[e]; + index[s] = DUMMY; + keys[e] = null; + values[e] = null; + size--; + return v; + } + + @Override + public void delItem(Object key) { + if (remove(key) == null) { + throw Py.keyError(key); + } + } + + @Override + public boolean contains(Object x) { + return find(x, Py.keyHash(x, "a dict key")) >= 0; + } + + @Override + public int len() { + return size; + } + + @Override + public boolean isTrue() { + return size != 0; + } + + @Override + public long hash() { + throw Py.typeError("unhashable type: '" + type().name + "'"); + } + + public void clear() { + keys = new Object[8]; + values = new Object[8]; + hashes = new long[8]; + index = newIndex(16); + used = 0; + size = 0; + } + + /// `dict.update(other)` for a mapping or an iterable of pairs. + public void update(Object other) { + if (other instanceof PyDict) { + PyDict d = (PyDict) other; + for (int i = 0; i < d.used; i++) { + if (d.keys[i] != null) { + setItem(d.keys[i], d.values[i]); + } + } + return; + } + Object keysFn = Py.obj(other).getAttrOrNull("keys"); + if (keysFn != null) { + Object it = Py.iter(Py.obj(keysFn).call(Py.NO_ARGS, Py.NO_KW)); + for (Object k = Py.next(it); k != null; k = Py.next(it)) { + setItem(k, Py.obj(other).getItem(k)); + } + return; + } + Object it = Py.iter(other); + int n = 0; + for (Object pair = Py.next(it); pair != null; pair = Py.next(it), n++) { + Object[] kv = Py.toArray(pair); + if (kv.length != 2) { + throw Py.valueError("dictionary update sequence element #" + n + " has length " + kv.length + + "; 2 is required"); + } + setItem(kv[0], kv[1]); + } + } + + /// The live keys in order. + public Object[] keyArray() { + Object[] r = new Object[size]; + int w = 0; + for (int i = 0; i < used; i++) { + if (keys[i] != null) { + r[w++] = keys[i]; + } + } + return r; + } + + /// Iterates over entries; `kind` 0 = keys, 1 = values, 2 = items. + Object iterate(final int kind) { + final int startSize = size; + return new PyJavaIter(new PyJavaIter.Source() { + int i; + + @Override + public Object next() { + if (size != startSize) { + throw Py.runtimeError("dictionary changed size during iteration"); + } + while (i < used && keys[i] == null) { + i++; + } + if (i >= used) { + return null; + } + int e = i++; + if (kind == 0) { + return keys[e]; + } + if (kind == 1) { + return values[e]; + } + return new PyTuple(new Object[] {keys[e], values[e]}); + } + }); + } + + @Override + public Object iter() { + return iterate(0); + } + + @Override + public String repr() { + if (size == 0) { + return "{}"; + } + if (Py.enterRepr(this)) { + return "{...}"; + } + try { + StringBuilder sb = new StringBuilder("{"); + boolean first = true; + for (int i = 0; i < used; i++) { + if (keys[i] == null) { + continue; + } + if (!first) { + sb.append(", "); + } + first = false; + sb.append(Py.toRepr(keys[i])).append(": ").append(Py.toRepr(values[i])); + } + return sb.append('}').toString(); + } finally { + Py.exitRepr(this); + } + } + + @Override + public Object eq(Object o) { + if (!(o instanceof PyDict)) { + return null; + } + PyDict b = (PyDict) o; + if (b.size != size) { + return Py.FALSE; + } + for (int i = 0; i < used; i++) { + if (keys[i] == null) { + continue; + } + Object v = b.getOrNull(keys[i]); + if (v == null || !Py.eqBool(values[i], v)) { + return Py.FALSE; + } + } + return Py.TRUE; + } + + @Override + public Object or(Object o) { + if (!(o instanceof PyDict)) { + return null; + } + PyDict r = copy(); + r.update(o); + return r; + } + + @Override + public Object inplace(String name, Object o) { + if ("__ior__".equals(name)) { + update(o); + return this; + } + return null; + } + + public PyDict copy() { + PyDict r = new PyDict(); + r.update(this); + return r; + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyDictView.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyDictView.java new file mode 100644 index 00000000000..05dd13c96b7 --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyDictView.java @@ -0,0 +1,120 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +/// `dict.keys()`, `dict.values()` and `dict.items()`: live views. +final class PyDictView extends PyObject { + private static final String[] NAMES = {"dict_keys", "dict_values", "dict_items"}; + private final PyDict d; + private final int kind; + + PyDictView(PyDict d, int kind) { + this.d = d; + this.kind = kind; + } + + @Override + public PyType type() { + return Py.ITERATOR_TYPE; + } + + @Override + public Object iter() { + return d.iterate(kind); + } + + @Override + public int len() { + return d.size; + } + + @Override + public boolean isTrue() { + return d.size != 0; + } + + @Override + public boolean contains(Object x) { + if (kind == 0) { + return d.contains(x); + } + if (kind == 2) { + Object[] kv = Py.unpack(x, 2); + Object v = d.getOrNull(kv[0]); + return v != null && Py.eqBool(v, kv[1]); + } + return super.contains(x); + } + + @Override + public String repr() { + Object[] items = Py.toArray(this); + return NAMES[kind] + Py.joinRepr("([", items, items.length, "])"); + } + + @Override + public Object eq(Object o) { + if (kind != 1 && (o instanceof PyDictView || o instanceof PySet)) { + return Py.eq(PySet.of(this, true), PySet.of(o, true)); + } + return null; + } + + private Object setOp(Object o, int op) { + if (kind == 1) { + return null; + } + PySet a = PySet.of(this, false); + switch (op) { + case 0: + return a.union(o); + case 1: + return a.intersection(o); + case 2: + return a.difference(o); + default: + return a.symmetricDifference(o); + } + } + + @Override + public Object or(Object o) { + return setOp(o, 0); + } + + @Override + public Object and(Object o) { + return setOp(o, 1); + } + + @Override + public Object sub(Object o) { + return setOp(o, 2); + } + + @Override + public Object xor(Object o) { + return setOp(o, 3); + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyException.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyException.java new file mode 100644 index 00000000000..61d5f217ebc --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyException.java @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +/// The Java carrier of a raised Python exception. `value` is the exception +/// instance (`ValueError("x")`); `except` clauses test it with [#matches]. +public final class PyException extends RuntimeException { + private static final long serialVersionUID = 1L; + + public final transient Object value; + + public PyException(Object value) { + super(); + this.value = value; + // Raised while an except clause runs: the exception it handles is + // this one's __context__. A bare `raise` hands the same one back. + Object h = Py.handling(); + if (h != Py.NONE && h != value && value instanceof PyInstance) { //NOPMD CompareObjectsWithEquals - identity is meant + ((PyInstance) value).setAttrDirect("__context__", h); + } + } + + /// `isinstance(value, type)`, where `type` may be a tuple of types. + public boolean matches(Object type) { + if (type instanceof PyTuple) { + Object[] items = ((PyTuple) type).items; + for (Object item : items) { + if (matches(item)) { + return true; + } + } + return false; + } + if (!(type instanceof PyType)) { + throw Py.typeError("catching classes that do not inherit from BaseException is not allowed"); + } + return Py.typeOf(value).isSubtype((PyType) type); + } + + @Override + public String getMessage() { + String s = Py.toStr(value); + return s.length() == 0 ? Py.typeOf(value).name : Py.typeOf(value).name + ": " + s; + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyFileSystem.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyFileSystem.java new file mode 100644 index 00000000000..07206ad3efe --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyFileSystem.java @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +/// The file system `open()`, `os` and `os.path` see. +/// +/// Paths are POSIX-style absolute paths (`/a/b`); the caller has already +/// joined a relative path with the working directory. An application runs +/// on Codename One's `FileSystemStorage` unless something else is +/// installed, and the working directory starts at the application's home. +public abstract class PyFileSystem { + private static PyFileSystem current = new Cn1FileSystem(); + + /// The file system in use. + public static PyFileSystem get() { + return current; + } + + /// Replaces the file system, for a host that is not a Codename One + /// application (a test harness, a command line tool). + public static void install(PyFileSystem fs) { + current = fs; + } + + /// The directory a relative path starts from when the program begins. + public abstract String home(); + + public abstract InputStream openInput(String path) throws IOException; + + public abstract OutputStream openOutput(String path, boolean append) throws IOException; + + public abstract boolean exists(String path); + + public abstract boolean isDirectory(String path); + + /// The names (not paths) of the entries of directory `path`. + public abstract String[] list(String path) throws IOException; + + public abstract void mkdir(String path) throws IOException; + + /// Deletes a file or an empty directory. + public abstract void delete(String path) throws IOException; + + public abstract void rename(String from, String to) throws IOException; + + public abstract long size(String path); + + /// Milliseconds since the epoch. + public abstract long lastModified(String path); +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyFloat.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyFloat.java new file mode 100644 index 00000000000..1dacf171a77 --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyFloat.java @@ -0,0 +1,390 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +/// Python's `float`, an IEEE double. +public final class PyFloat extends PyObject { + public final double value; + + public PyFloat(double value) { + this.value = value; + } + + @Override + public PyType type() { + return Py.FLOAT_TYPE; + } + + @Override + public String repr() { + return repr(value); + } + + /// CPython's `repr(float)`: the shortest text that round-trips, which Java's + /// `Double.toString` also produces, re-spelled in Python's notation + /// (`1e+16`, `1e-05`, `100.0`). + public static String repr(double d) { + if (Double.isNaN(d)) { + return "nan"; + } + if (Double.isInfinite(d)) { + return d > 0 ? "inf" : "-inf"; + } + if (d == 0) { + return 1 / d < 0 ? "-0.0" : "0.0"; + } + String j = Double.toString(Math.abs(d)); + String mant; + int exp; + int e = j.indexOf('E'); + if (e >= 0) { + mant = j.substring(0, e); + exp = Integer.parseInt(j.substring(e + 1)); + } else { + mant = j; + exp = 0; + } + int dot = mant.indexOf('.'); + String digits = mant.substring(0, dot) + mant.substring(dot + 1); + int pointPos = dot + exp; + int lead = 0; + while (lead < digits.length() - 1 && digits.charAt(lead) == '0') { + lead++; + pointPos--; + } + digits = digits.substring(lead); + int end = digits.length(); + while (end > 1 && digits.charAt(end - 1) == '0') { + end--; + } + digits = digits.substring(0, end); + // Double.toString is not always the shortest round trip before JDK 19 + // (JDK-4511638: 6.9175290276410819E18 for 6.917529027641082e+18), so a + // long answer is shortened to the fewest digits that still read back + // as the same double, which is what CPython prints. Subnormals are + // always checked: the smallest one is 4.9E-324 there, 5e-324 here. + if (digits.length() >= 16 || digits.length() > 1 && Math.abs(d) < 2.2250738585072014E-308) { + double a = Math.abs(d); + for (int k = 1; k < digits.length(); k++) { + String r = roundDigits(digits, k); + int shift = r.length() > k ? 1 : 0; + r = r.substring(0, k); + if (Double.parseDouble(r + "E" + (pointPos + shift - k)) == a) { + end = r.length(); + while (end > 1 && r.charAt(end - 1) == '0') { + end--; + } + digits = r.substring(0, end); + pointPos += shift; + break; + } + } + } + int sciExp = pointPos - 1; + StringBuilder sb = new StringBuilder(); + if (d < 0) { + sb.append('-'); + } + if (sciExp < -4 || sciExp >= 16) { + sb.append(digits.charAt(0)); + if (digits.length() > 1) { + sb.append('.').append(digits, 1, digits.length()); + } + sb.append('e').append(sciExp < 0 ? '-' : '+'); + int a = Math.abs(sciExp); + if (a < 10) { + sb.append('0'); + } + sb.append(a); + } else if (pointPos <= 0) { + sb.append("0."); + for (int i = 0; i < -pointPos; i++) { + sb.append('0'); + } + sb.append(digits); + } else if (pointPos >= digits.length()) { + sb.append(digits); + for (int i = digits.length(); i < pointPos; i++) { + sb.append('0'); + } + sb.append(".0"); + } else { + sb.append(digits, 0, pointPos).append('.').append(digits, pointPos, digits.length()); + } + return sb.toString(); + } + + /// The first `k` digits of `digits`, rounded half up on the next one; a + /// carry out of the first digit makes the answer one digit longer. + private static String roundDigits(String digits, int k) { + char[] c = digits.substring(0, k).toCharArray(); + if (digits.charAt(k) >= '5') { + int i = k - 1; + while (i >= 0 && c[i] == '9') { + c[i] = '0'; + i--; + } + if (i < 0) { + return "1" + new String(c); + } + c[i]++; + } + return new String(c); + } + + @Override + public boolean isTrue() { + return value != 0; + } + + @Override + public double asDouble() { + return value; + } + + @Override + public Object toJava() { + return Double.valueOf(value); + } + + @Override + public long hash() { + return Hashing.ofDouble(value, this); + } + + private static double num(Object o) { + if (Py.isFloat(o)) { + return Py.asDouble(o); + } + if (Py.isInt(o)) { + return Py.asDouble(o); + } + return Double.NaN; + } + + /// -1, 0 or 1 as this float is below, equal to or above number `o`, and + /// 2 when either is NaN. An int is compared exactly, as CPython does: + /// `2**53 + 1 == float(2**53 + 1)` is false although the conversion of + /// the int to a float would round it to equal. + private int cmpTo(Object o) { + if (o instanceof Long || o instanceof PyInt && !((PyInt) o).isBig()) { + return compareLong(value, Py.asLong(o)); + } + double d = num(o); + if (Double.isNaN(value) || Double.isNaN(d)) { + return 2; + } + return value < d ? -1 : value > d ? 1 : 0; + } + + /// The exact comparison of `d` with `v`: -1, 0, 1, or 2 for NaN. + static int compareLong(double d, long v) { + if (Double.isNaN(d)) { + return 2; + } + if (d >= 9.223372036854775807E18) { + return 1; + } + if (d < -9.223372036854775808E18) { + return -1; + } + long f = (long) Math.floor(d); + if (f < v) { + return -1; + } + if (f > v) { + return 1; + } + return d > f ? 1 : 0; + } + + private static boolean isNum(Object o) { + return Py.isFloat(o) || Py.isInt(o); + } + + @Override + public Object add(Object o) { + return isNum(o) ? Py.ofFloat(value + num(o)) : null; + } + + @Override + public Object radd(Object o) { + return add(o); + } + + @Override + public Object sub(Object o) { + return isNum(o) ? Py.ofFloat(value - num(o)) : null; + } + + @Override + public Object rsub(Object o) { + return isNum(o) ? Py.ofFloat(num(o) - value) : null; + } + + @Override + public Object mul(Object o) { + return isNum(o) ? Py.ofFloat(value * num(o)) : null; + } + + @Override + public Object rmul(Object o) { + return mul(o); + } + + @Override + public Object truediv(Object o) { + if (!isNum(o)) { + return null; + } + double d = num(o); + if (d == 0) { + throw Py.zeroDivision("division by zero"); + } + return Py.ofFloat(value / d); + } + + @Override + public Object rtruediv(Object o) { + return isNum(o) ? new PyFloat(num(o)).truediv(this) : null; + } + + @Override + public Object floordiv(Object o) { + if (!isNum(o)) { + return null; + } + double d = num(o); + if (d == 0) { + throw Py.zeroDivision("division by zero"); + } + return Py.ofFloat(Math.floor(value / d)); + } + + @Override + public Object rfloordiv(Object o) { + return isNum(o) ? new PyFloat(num(o)).floordiv(this) : null; + } + + @Override + public Object mod(Object o) { + if (!isNum(o)) { + return null; + } + double d = num(o); + if (d == 0) { + throw Py.zeroDivision("division by zero"); + } + double r = value % d; + if (r != 0 && (r < 0) != (d < 0)) { + r += d; + } + return Py.ofFloat(r); + } + + @Override + public Object rmod(Object o) { + return isNum(o) ? new PyFloat(num(o)).mod(this) : null; + } + + @Override + public Object pow(Object o) { + if (!isNum(o)) { + return null; + } + double e = num(o); + if (value == 0 && e < 0) { + throw Py.zeroDivision("zero to a negative power"); + } + double r = Math.pow(value, e); + if (Double.isNaN(r) && !Double.isNaN(value) && !Double.isNaN(e)) { + // CPython answers a complex number here, which is not supported. + throw Py.valueError("complex numbers are not supported: negative number raised to a fractional power"); + } + if (Double.isInfinite(r) && !Double.isInfinite(value) && !Double.isInfinite(e)) { + throw Py.overflowError("(34, 'Result too large')"); + } + return Py.ofFloat(r); + } + + @Override + public Object rpow(Object o) { + return isNum(o) ? new PyFloat(num(o)).pow(this) : null; + } + + @Override + public Object neg() { + return Py.ofFloat(-value); + } + + @Override + public Object pos() { + return Double.valueOf(value); + } + + @Override + public Object eq(Object o) { + if (!isNum(o)) { + return null; + } + int c = cmpTo(o); + return Py.bool(c == 0); + } + + @Override + public Object lt(Object o) { + if (!isNum(o)) { + return null; + } + int c = cmpTo(o); + return Py.bool(c == -1); + } + + @Override + public Object le(Object o) { + if (!isNum(o)) { + return null; + } + int c = cmpTo(o); + return Py.bool(c == -1 || c == 0); + } + + @Override + public Object gt(Object o) { + if (!isNum(o)) { + return null; + } + int c = cmpTo(o); + return Py.bool(c == 1); + } + + @Override + public Object ge(Object o) { + if (!isNum(o)) { + return null; + } + int c = cmpTo(o); + return Py.bool(c == 1 || c == 0); + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyFunction.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyFunction.java new file mode 100644 index 00000000000..94807bf683f --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyFunction.java @@ -0,0 +1,478 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +import java.util.ArrayList; +import java.util.List; + +/// A function defined in Python source. +/// +/// The transpiler emits the body as a [Body] lambda that receives its +/// parameters already bound into a flat array, in declaration order: +/// positional parameters, then `*args` (a tuple), keyword-only parameters, +/// then `**kwargs` (a dict). Binding follows CPython's rules and messages. +public final class PyFunction extends PyObject { + + /// The compiled body of a function. + public interface Body { + Object run(Object[] a); + } + + // A function with only positional parameters (at most four, defaults + // allowed) is also compiled to take them directly: the transpiler picks + // the matching constructor, and a call with exactly that many positional + // arguments then allocates no argument array. + + public interface Body0 { + Object run(); + } + + public interface Body1 { + Object run(Object a); + } + + public interface Body2 { + Object run(Object a, Object b); + } + + public interface Body3 { + Object run(Object a, Object b, Object c); + } + + public interface Body4 { + Object run(Object a, Object b, Object c, Object d); + } + + /// A function whose body is one float expression of its parameters + /// also has its value as a raw double: the arithmetic alone, for the + /// parameter types the expression implies. It answers NaN for "not + /// computed here" (other argument types, an int overflow, a zero + /// divisor, or a NaN result), and the caller then makes the ordinary + /// call. It reads nothing but its arguments and has no side effects, so + /// trying it first is never observable. + public interface PureD1 { + double run(Object a); + } + + public interface PureD2 { + double run(Object a, Object b); + } + + public interface PureD3 { + double run(Object a, Object b, Object c); + } + + public interface PureD4 { + double run(Object a, Object b, Object c, Object d); + } + + /// The same twin for int arguments a caller holds unboxed (a `range` + /// loop index), so no argument is boxed to make the call. Same NaN + /// contract. + public interface PureL1 { + double run(long a); + } + + public interface PureL2 { + double run(long a, long b); + } + + public final String name; + private String qualname; + final String[] params; + final int positional; + final boolean varArgs; + final int kwOnly; + final boolean varKw; + final Object[] defaults; + final Object[] kwDefaults; + final Body body; + private Body0 body0; + private Body1 body1; + private Body2 body2; + private Body3 body3; + private Body4 body4; + private PureD1 pure1; + private PureD2 pure2; + private PureD3 pure3; + private PureD4 pure4; + private PureL1 pureL1; + private PureL2 pureL2; + /// The `__dict__` of the function object, created on first write. + PyDict attrs; + + /// @param params every parameter name in binding order + /// @param positional how many leading names are positional (or positional-or-keyword) + /// @param varArgs whether `*args` follows the positional names + /// @param kwOnly how many keyword-only names follow + /// @param varKw whether `**kwargs` is last + /// @param defaults defaults for the LAST `defaults.length` positional parameters + /// @param kwDefaults defaults for keyword-only parameters, `null` entries where required + public PyFunction(String name, String[] params, int positional, boolean varArgs, int kwOnly, + boolean varKw, Object[] defaults, Object[] kwDefaults, Body body) { + this.name = name; + this.params = params; + this.positional = positional; + this.varArgs = varArgs; + this.kwOnly = kwOnly; + this.varKw = varKw; + this.defaults = defaults; + this.kwDefaults = kwDefaults; + this.body = body; + } + + public PyFunction(String name, String[] params, int positional, boolean varArgs, int kwOnly, + boolean varKw, Object[] defaults, Object[] kwDefaults, Body0 b) { + this(name, params, positional, varArgs, kwOnly, varKw, defaults, kwDefaults, (Object[] a) -> b.run()); + body0 = b; + } + + public PyFunction(String name, String[] params, int positional, boolean varArgs, int kwOnly, + boolean varKw, Object[] defaults, Object[] kwDefaults, Body1 b) { + this(name, params, positional, varArgs, kwOnly, varKw, defaults, kwDefaults, (Object[] a) -> b.run(a[0])); + body1 = b; + } + + public PyFunction(String name, String[] params, int positional, boolean varArgs, int kwOnly, + boolean varKw, Object[] defaults, Object[] kwDefaults, Body2 b) { + this(name, params, positional, varArgs, kwOnly, varKw, defaults, kwDefaults, (Object[] a) -> b.run(a[0], a[1])); + body2 = b; + } + + public PyFunction(String name, String[] params, int positional, boolean varArgs, int kwOnly, + boolean varKw, Object[] defaults, Object[] kwDefaults, Body3 b) { + this(name, params, positional, varArgs, kwOnly, varKw, defaults, kwDefaults, + (Object[] a) -> b.run(a[0], a[1], a[2])); + body3 = b; + } + + public PyFunction(String name, String[] params, int positional, boolean varArgs, int kwOnly, + boolean varKw, Object[] defaults, Object[] kwDefaults, Body4 b) { + this(name, params, positional, varArgs, kwOnly, varKw, defaults, kwDefaults, + (Object[] a) -> b.run(a[0], a[1], a[2], a[3])); + body4 = b; + } + + @Override + public Object call0() { + return body0 != null ? body0.run() : call(Py.NO_ARGS, Py.NO_KW); + } + + @Override + public Object call1(Object a) { + return body1 != null ? body1.run(a) : call(new Object[] {a}, Py.NO_KW); + } + + @Override + public Object call2(Object a, Object b) { + return body2 != null ? body2.run(a, b) : call(new Object[] {a, b}, Py.NO_KW); + } + + @Override + public Object call3(Object a, Object b, Object c) { + return body3 != null ? body3.run(a, b, c) : call(new Object[] {a, b, c}, Py.NO_KW); + } + + @Override + public Object call4(Object a, Object b, Object c, Object d) { + return body4 != null ? body4.run(a, b, c, d) : call(new Object[] {a, b, c, d}, Py.NO_KW); + } + + public PyFunction pure(PureD1 p) { + pure1 = p; + return this; + } + + public PyFunction pure(PureD2 p) { + pure2 = p; + return this; + } + + public PyFunction pure(PureD3 p) { + pure3 = p; + return this; + } + + public PyFunction pure(PureD4 p) { + pure4 = p; + return this; + } + + public PyFunction pure(PureL1 p) { + pureL1 = p; + return this; + } + + public PyFunction pure(PureL2 p) { + pureL2 = p; + return this; + } + + double callL1(long a) { + if (pureL1 != null) { + return pureL1.run(a); + } + return pure1 != null ? pure1.run(Long.valueOf(a)) : Double.NaN; + } + + double callL2(long a, long b) { + if (pureL2 != null) { + return pureL2.run(a, b); + } + return pure2 != null ? pure2.run(Long.valueOf(a), Long.valueOf(b)) : Double.NaN; + } + + double callD1(Object a) { + return pure1 != null ? pure1.run(a) : Double.NaN; + } + + double callD2(Object a, Object b) { + return pure2 != null ? pure2.run(a, b) : Double.NaN; + } + + double callD3(Object a, Object b, Object c) { + return pure3 != null ? pure3.run(a, b, c) : Double.NaN; + } + + double callD4(Object a, Object b, Object c, Object d) { + return pure4 != null ? pure4.run(a, b, c, d) : Double.NaN; + } + + /// The common shape: positional parameters only, no defaults. + public PyFunction(String name, String[] params, Body body) { + this(name, params, params.length, false, 0, false, Py.NO_ARGS, null, body); + } + + @Override + public PyType type() { + return Py.FUNCTION_TYPE; + } + + @Override + public Object bind(Object self) { + return new PyMethod(self, this); + } + + @Override + public String repr() { + return ""; + } + + /// The docstring, or null for none. + private Object doc; + /// Builds the annotations as name/value pairs; run once, on first read. + private Body0 annotationThunk; + + /// Attaches what `def` knows besides the code: the docstring and the + /// annotations, which are built on first read of `__annotations__`. + public static Object meta(Object fn, Object doc, Body0 annotations) { + if (fn instanceof PyFunction) { + PyFunction f = (PyFunction) fn; + f.doc = doc; + f.annotationThunk = annotations; + } + return fn; + } + + /// One annotation's value: `value`'s result, or `fallback` when + /// evaluating it raises (an annotation the runtime cannot express, such + /// as a generic alias it does not model, never fails the definition). + public static Object annotation(Body0 value, Object fallback) { + try { + return value.run(); + } catch (PyException e) { + return fallback; + } + } + + /// Sets `__qualname__` where it is not just the name. + public PyFunction qual(String q) { + qualname = q; + return this; + } + + private String qn() { + return qualname != null ? qualname : name; + } + + @Override + public Object getAttrOrNull(String attr) { + if (attrs != null) { + Object v = attrs.getOrNull(attr); + if (v != null) { + return v; + } + } + switch (attr) { + case "__name__": + return name; + case "__qualname__": + return qn(); + case "__doc__": + return doc != null ? doc : Py.NONE; + case "__module__": + return Py.NONE; + case "__dict__": + if (attrs == null) { + attrs = new PyDict(); + } + return attrs; + case "__defaults__": + return defaults.length == 0 ? Py.NONE : PyTuple.of(defaults); + case "__annotations__": + PyDict d = new PyDict(); + if (annotationThunk != null) { + Object made = annotationThunk.run(); + Object[] kv = made instanceof Object[] ? (Object[]) made : Py.NO_ARGS; + for (int i = 0; i < kv.length; i += 2) { + d.setItem(kv[i], kv[i + 1]); + } + } + setAttr("__annotations__", d); + return d; + default: + return super.getAttrOrNull(attr); + } + } + + @Override + public void setAttr(String attr, Object value) { + if (attrs == null) { + attrs = new PyDict(); + } + attrs.setItem(attr, value); + } + + @Override + public Object call(Object[] args, String[] kwNames) { + if (kwNames.length == 0 && !varArgs && kwOnly == 0 && !varKw && args.length == positional) { + return body.run(args); + } + return body.run(bindArgs(args, kwNames)); + } + + private static List add(List l, String n) { + List r = l == null ? new ArrayList() : l; + r.add(n); + return r; + } + + /// CPython's "missing 2 required positional arguments: 'a' and 'b'". + private PyException missing(String kind, List names) { + StringBuilder sb = new StringBuilder(qn()).append("() missing ").append(names.size()).append(" required ") + .append(kind).append(" argument").append(names.size() == 1 ? "" : "s").append(": "); + for (int i = 0; i < names.size(); i++) { + if (i > 0) { + sb.append(names.size() == 2 ? " and " : i == names.size() - 1 ? ", and " : ", "); + } + sb.append('\'').append(names.get(i)).append('\''); + } + return Py.typeError(sb.toString()); + } + + Object[] bindArgs(Object[] args, String[] kwNames) { + int slots = positional + (varArgs ? 1 : 0) + kwOnly + (varKw ? 1 : 0); + Object[] out = new Object[slots]; + int nPos = args.length - kwNames.length; + int copy = Math.min(nPos, positional); + System.arraycopy(args, 0, out, 0, copy); + if (nPos > positional) { + if (!varArgs) { + int min = positional - defaults.length; + String takes = min == positional ? String.valueOf(positional) : "from " + min + " to " + positional; + throw Py.typeError(qn() + "() takes " + takes + " positional argument" + + (positional == 1 && min == positional ? "" : "s") + " but " + nPos + + (nPos == 1 ? " was" : " were") + " given"); + } + Object[] rest = new Object[nPos - positional]; + System.arraycopy(args, positional, rest, 0, rest.length); + out[positional] = new PyTuple(rest); + } else if (varArgs) { + out[positional] = PyTuple.EMPTY; + } + PyDict kw = varKw ? new PyDict() : null; + int kwBase = positional + (varArgs ? 1 : 0); + for (int i = 0; i < kwNames.length; i++) { + String k = kwNames[i]; + Object v = args[nPos + i]; + int slot = -1; + for (int p = 0; p < positional; p++) { + if (params[p].equals(k)) { + slot = p; + break; + } + } + if (slot < 0) { + for (int p = 0; p < kwOnly; p++) { + if (params[kwBase + p].equals(k)) { + slot = kwBase + p; + break; + } + } + } + if (slot < 0) { + if (kw == null) { + throw Py.typeError(qn() + "() got an unexpected keyword argument '" + k + "'"); + } + kw.setItem(k, v); + continue; + } + if (out[slot] != null) { + throw Py.typeError(qn() + "() got multiple values for argument '" + k + "'"); + } + out[slot] = v; + } + int firstDefault = positional - defaults.length; + List missing = null; + for (int p = 0; p < positional; p++) { + if (out[p] == null) { + if (p >= firstDefault) { + out[p] = defaults[p - firstDefault]; + } else { + missing = add(missing, params[p]); + } + } + } + if (missing != null) { + throw missing("positional", missing); + } + for (int p = 0; p < kwOnly; p++) { + if (out[kwBase + p] == null) { + Object d = kwDefaults == null ? null : kwDefaults[p]; + if (d == null) { + missing = add(missing, params[kwBase + p]); + } else { + out[kwBase + p] = d; + } + } + } + if (missing != null) { + throw missing("keyword-only", missing); + } + if (varKw) { + out[slots - 1] = kw; + } + return out; + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyGenerator.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyGenerator.java new file mode 100644 index 00000000000..ccb14463d49 --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyGenerator.java @@ -0,0 +1,462 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +/// A generator or coroutine object: the state of a suspended function body. +/// +/// The transpiler lowers the body of a function that contains `yield` (or an +/// `async def`) into [#step], a `switch` over the suspension points, and the +/// function's locals into fields of an anonymous subclass. `pc` is the point +/// to resume at: 0 before the first resumption, a positive state while +/// suspended, -1 once finished. A step that returns with `pc` positive has +/// yielded the value it returns; one that returns with `pc == -1` has finished, +/// and the value is the function's return value. +/// +/// Nothing here relies on a thread: suspension is a return from `step`, so a +/// generator costs one object and its fields, and an abandoned one is simply +/// garbage. Unlike CPython, an abandoned generator is not closed, so the +/// `finally` blocks of a generator never resumed to completion do not run. +public abstract class PyGenerator extends PyObject { + /// The resumption point, see the class comment. + protected int pc; + /// The exception an `except` or `finally` state was entered with. + protected PyException exc; + /// The iterator a `yield from` or `await` is delegating to. + protected PyObject delegate; + /// What the finished delegate returned, the value of `yield from`. + protected Object delegateResult; + + private final String name; + private final boolean coroutine; + private boolean running; + private Object sent = Py.NONE; + private PyException thrown; + private Object returned = Py.NONE; + + protected PyGenerator(String name, boolean coroutine) { + this.name = name; + this.coroutine = coroutine; + } + + /// Runs the body from `pc` to its next suspension or its end. + protected abstract Object step(); + + /// The value of the `yield` the body resumed at, or the exception thrown + /// into it, raised. + protected final Object resumed() { + PyException t = thrown; + Object s = sent; + thrown = null; + sent = Py.NONE; + if (t != null) { + throw t; + } + return s; + } + + /// Resumes the body: `v` is sent in, or `t` raised at the suspension + /// point. Returns the next value yielded, or null once the body finished + /// (its return value is then [#returned]). + final Object resume(Object v, PyException t) { + if (running) { + throw Py.valueError((coroutine ? "coroutine" : "generator") + " already executing"); + } + if (pc == -1) { + if (t != null) { + throw t; + } + if (coroutine) { + throw Py.runtimeError("cannot reuse already awaited coroutine"); + } + return null; + } + if (pc == 0) { + if (t != null) { + pc = -1; + throw t; + } + if (v != Py.NONE) { //NOPMD CompareObjectsWithEquals - identity is meant + throw Py.typeError("can't send non-None value to a just-started " + + (coroutine ? "coroutine" : "generator")); + } + } + sent = v; + thrown = t; + running = true; + try { + Object r = step(); + if (pc == -1) { + returned = r; + return null; + } + return r; + } catch (PyException e) { + pc = -1; + if (e.matches(Py.STOP_ITERATION)) { + // PEP 479: a StopIteration escaping the body would end the + // caller's loop silently. + Object err = Py.runtimeError((coroutine ? "coroutine" : "generator") + " raised StopIteration").value; + PyException re = Py.raise(err, e.value); + if (err instanceof PyInstance) { + ((PyInstance) err).setAttrDirect("__context__", e.value); + } + throw re; //NOPMD PreserveStackTrace - the Python cause is chained + } + throw e; + } catch (RuntimeException e) { + pc = -1; + throw e; + } catch (Error e) { + pc = -1; + throw e; + } finally { + running = false; + } + } + + /// The finished body's return value. + final Object returned() { + return returned; + } + + /// One step of a `yield from` (or `await`) over [#delegate]: the value + /// to pass out, or null when the delegate is exhausted, with its result + /// in [#delegateResult]. A value sent in or an exception thrown in is + /// forwarded to the delegate, as PEP 380 specifies. + protected final Object delegateStep() { + PyException t = thrown; + Object s = sent; + thrown = null; + sent = Py.NONE; + PyObject d = delegate; + Object r; + if (d instanceof PyGenerator) { + PyGenerator g = (PyGenerator) d; + if (t != null && t.matches(Py.GENERATOR_EXIT)) { + g.close(); + throw t; + } + r = g.resume(s, t); + if (r == null) { + delegate = null; + delegateResult = g.returned; + return null; + } + return r; + } + try { + if (t != null) { + if (t.matches(Py.GENERATOR_EXIT)) { + Object close = d.getAttrOrNull("close"); + if (close != null) { + Py.call(close); + } + throw t; + } + Object m = d.getAttrOrNull("throw"); + if (m == null) { + throw t; + } + r = Py.call(m, t.value); + } else if (s == Py.NONE) { //NOPMD CompareObjectsWithEquals - identity is meant + r = d.next(); + if (r == null) { + delegate = null; + delegateResult = Py.NONE; + return null; + } + } else { + r = Py.callMethod(d, "send", s); + } + return r; + } catch (PyException e) { + if (e.matches(Py.STOP_ITERATION)) { + delegate = null; + delegateResult = Py.getattr(e.value, "value"); + return null; + } + throw e; + } + } + + // ---- the Python-visible protocol -------------------------------------- + + /// `gen.send(v)`. + public Object send(Object v) { + Object r = resume(v, null); + if (r == null) { + throw stopIteration(returned); + } + return r; + } + + /// `gen.throw(e)`. + public Object throwIn(Object e) { + Object r = resume(Py.NONE, Py.raise(e)); + if (r == null) { + throw stopIteration(returned); + } + return r; + } + + /// `gen.close()`. + public void close() { + if (pc <= 0) { + pc = -1; + return; + } + Object r; + try { + r = resume(Py.NONE, Py.raise(Py.GENERATOR_EXIT)); + } catch (PyException e) { + if (e.matches(Py.GENERATOR_EXIT) || e.matches(Py.STOP_ITERATION)) { + return; + } + throw e; + } + if (r != null) { + throw Py.runtimeError((coroutine ? "coroutine" : "generator") + " ignored GeneratorExit"); + } + } + + static PyException stopIteration(Object value) { + Object[] args = value == Py.NONE ? Py.NO_ARGS : new Object[] {value}; //NOPMD CompareObjectsWithEquals - identity is meant + return new PyException(Py.STOP_ITERATION.call(args, Py.NO_KW)); + } + + public boolean isCoroutine() { + return coroutine; + } + + @Override + public PyType type() { + return coroutine ? Py.COROUTINE_TYPE : Py.GENERATOR_TYPE; + } + + @Override + public String repr() { + return "<" + (coroutine ? "coroutine" : "generator") + " object " + name + ">"; + } + + @Override + public Object iter() { + if (coroutine) { + return super.iter(); + } + return this; + } + + @Override + public Object next() { + if (coroutine) { + return super.next(); + } + return resume(Py.NONE, null); + } + + /// The iterator `await x` delegates to. + public static PyObject awaitable(Object x) { + if (x instanceof PyGenerator && ((PyGenerator) x).coroutine) { + // delegateStep drives a coroutine directly. + return (PyGenerator) x; + } + Object m = Py.obj(x).getAttrOrNull("__await__"); + if (m == null) { + throw Py.typeError("object " + Py.typeOf(x).name + " can't be used in 'await' expression"); + } + return Py.iter(Py.call(m)); + } + + /// `coro.__await__()`: the coroutine driven as an iterator. + static final class Awaiter extends PyGenerator { + private final PyGenerator coro; + + Awaiter(PyGenerator coro) { + super("__await__", false); + this.coro = coro; + } + + @Override + protected Object step() { + if (pc == 0) { + delegate = coro; + pc = 1; + } + Object r = delegateStep(); + if (r == null) { + pc = -1; + return delegateResult; + } + return r; + } + } + + /// An async generator: `async def` with `yield`. Its body yields a + /// [Yielded] for each `yield` and passes anything else -- what an + /// `await` inside it suspends on -- out to the event loop unchanged. + public abstract static class Async extends PyGenerator { + protected Async(String name) { + super(name, false); + } + + @Override + public PyType type() { + return Py.ASYNC_GENERATOR_TYPE; + } + + @Override + public String repr() { + return " a[0])); + ag.define("__anext__", PyBuiltin.method(ag, "__anext__", a -> new ASend(gen(a[0]), ASEND, Py.NONE))); + ag.define("asend", PyBuiltin.method(ag, "asend", a -> new ASend(gen(a[0]), ASEND, a[1]))); + ag.define("athrow", PyBuiltin.method(ag, "athrow", a -> new ASend(gen(a[0]), ATHROW, + a.length > 2 && a[2] != Py.NONE ? a[2] : a[1]))); + ag.define("aclose", PyBuiltin.method(ag, "aclose", a -> new ASend(gen(a[0]), ACLOSE, Py.NONE))); + for (int i = 0; i < 2; i++) { + PyType t = i == 0 ? Py.GENERATOR_TYPE : Py.COROUTINE_TYPE; + t.define("send", PyBuiltin.method(t, "send", a -> gen(a[0]).send(a.length > 1 ? a[1] : Py.NONE))); + t.define("throw", PyBuiltin.method(t, "throw", a -> gen(a[0]).throwIn(a.length > 2 && a[2] != Py.NONE + ? a[2] : a[1]))); + t.define("close", PyBuiltin.method(t, "close", a -> { + gen(a[0]).close(); + return Py.NONE; + })); + } + Py.GENERATOR_TYPE.define("__iter__", PyBuiltin.method(Py.GENERATOR_TYPE, "__iter__", a -> a[0])); + Py.GENERATOR_TYPE.define("__next__", PyBuiltin.method(Py.GENERATOR_TYPE, "__next__", a -> { + Object r = gen(a[0]).resume(Py.NONE, null); + if (r == null) { + throw stopIteration(gen(a[0]).returned); + } + return r; + })); + Py.COROUTINE_TYPE.define("__await__", PyBuiltin.method(Py.COROUTINE_TYPE, "__await__", + a -> new Awaiter(gen(a[0])))); + } + + private static PyGenerator gen(Object o) { + if (o instanceof PyGenerator) { + return (PyGenerator) o; + } + throw Py.typeError("descriptor requires a generator but received a '" + Py.typeOf(o).name + "'"); + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyInstance.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyInstance.java new file mode 100644 index 00000000000..09fbe76add3 --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyInstance.java @@ -0,0 +1,621 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + + +/// An instance of a class the application defines (and of the builtin +/// exception types, which are defined the same way). +/// +/// Every operator is routed to the matching dunder method of the class, so a +/// class that defines `__add__` or `__getitem__` behaves under `+` and `[]` +/// exactly as in CPython. +public class PyInstance extends PyObject { + public final PyType cls; + /// Set once the instance holds an attribute outside its compiled fields, + /// which might shadow a method the compiled dispatch would call. + public boolean extra; + /// The attribute layout and the values it describes; see `Shape`. An + /// array shared by every instance of a class instead of a hash table + /// per instance, which is both the lookup cost and most of the memory + /// a small object would otherwise take. + Shape shape = Shape.EMPTY; + Object[] slots = NO_SLOTS; + private static final Object[] NO_SLOTS = new Object[0]; + /// For an instance of a subclass of a builtin type, the builtin value it + /// extends (see `BuiltinSubclass`); null otherwise. + public Object base; + + public PyInstance(PyType cls) { + this.cls = cls; + } + + @Override + public PyType type() { + return cls; + } + + /// The compiled field holding attribute `name`, or -1. + /// + /// A class the transpiler can see whole gets a Java subclass whose + /// attributes are fields (`PyType.layout`): generated code reads them + /// straight after an `instanceof`, and these hooks let every generic path + /// see them too. An unset field is null, as an absent attribute is. + protected int fieldIndex(String name) { + return -1; + } + + /// Field `i` (see [#fieldIndex]). + protected Object getField(int i) { + return null; + } + + /// Stores field `i`; null unsets it. + protected void setField(int i, Object value) { + } + + /// The names of the fields, in declaration order. + protected String[] fieldNames() { + return NO_NAMES; + } + + private static final String[] NO_NAMES = new String[0]; + + /// The instance attribute `name`, or null. + Object own(String name) { + int f = fieldIndex(name); + if (f >= 0) { + Object v = getField(f); + if (v != null || !extra) { + return v; + } + } + int i = shape.indexOf(name); + return i < 0 ? null : slots[i]; + } + + /// Stores `name`, adding it to the shape if the instance lacks it. + void store(String name, Object value) { + int f = fieldIndex(name); + // A field first set after some other attribute would list before + // it; such a late attribute is kept in the shape instead, which + // remembers the order. + if (f >= 0 && (!extra || getField(f) != null)) { + setField(f, value); + return; + } + int i = shape.indexOf(name); + if (i >= 0) { + slots[i] = value; + } else { + Set allowed = cls.slotNames(); + if (allowed != null && !allowed.contains(name)) { + throw Py.attributeError("'" + cls.name + "' object has no attribute '" + name + + "' and no __dict__ for setting new attributes"); + } + append(shape.with(name), value); + } + } + + /// Moves to `next`, which is this instance's shape plus one name, and + /// stores that name's value. + void append(Shape next, Object value) { + int n = shape.size(); + if (n == slots.length) { + Object[] grown = new Object[n < 4 ? 4 : n * 2]; + System.arraycopy(slots, 0, grown, 0, n); + slots = grown; + } + slots[n] = value; + shape = next; + extra = true; + } + + /// The attribute names: the set fields, then the rest in the order they + /// were first assigned. + String[] attributeNames() { + String[] f = fieldNames(); + if (f.length == 0) { + return shape.names; + } + List out = new ArrayList(); + for (int i = 0; i < f.length; i++) { + if (getField(i) != null) { + out.add(f[i]); + } + } + for (int i = 0; i < shape.names.length; i++) { + out.add(shape.names[i]); + } + return out.toArray(new String[out.size()]); + } + + @Override + public Object getAttrOrNull(String name) { + Object clsAttr = cls.lookup(name); + if (clsAttr instanceof PyProperty) { + return ((PyProperty) clsAttr).get(this); + } + Object get = null; + if (clsAttr instanceof PyInstance) { + // A descriptor: one with __set__ or __delete__ (a data + // descriptor) wins over the instance's own attribute. + PyType dt = ((PyInstance) clsAttr).cls; + get = dt.lookup("__get__"); + if (get != null && (dt.lookup("__set__") != null || dt.lookup("__delete__") != null)) { + return Py.obj(get).call(new Object[] {clsAttr, this, cls}, Py.NO_KW); + } + } + Object v = own(name); + if (v != null) { + return v; + } + if (get != null) { + return Py.obj(get).call(new Object[] {clsAttr, this, cls}, Py.NO_KW); + } + if ("__dict__".equals(name)) { + // A copy of the attributes whose stores and deletes reach the + // instance too; a change made the other way after it was read + // is not seen through it. + PyDict d = new InstanceDict(this); + String[] names = attributeNames(); + for (String e : names) { + d.setItem(e, own(e)); + } + return d; + } + if ("__class__".equals(name)) { + return cls; + } + if (clsAttr != null) { + return Py.bindAttr(clsAttr, this); + } + Object ga = cls.lookup("__getattr__"); + if (ga != null) { + return Py.obj(ga).call(new Object[] {this, name}, Py.NO_KW); + } + return null; + } + + @Override + public void setAttr(String name, Object value) { + Object clsAttr = cls.lookup(name); + if (clsAttr instanceof PyProperty) { + ((PyProperty) clsAttr).set(this, value); + return; + } + if (clsAttr instanceof PyInstance) { + Object set = ((PyInstance) clsAttr).cls.lookup("__set__"); + if (set != null) { + Py.obj(set).call(new Object[] {clsAttr, this, value}, Py.NO_KW); + return; + } + } + Object sa = cls.lookup("__setattr__"); + if (sa != null && !(sa instanceof PyBuiltin)) { + Py.obj(sa).call(new Object[] {this, name, value}, Py.NO_KW); + return; + } + store(name, value); + } + + /// `object.__setattr__(self, name, value)`: the plain store, bypassing an + /// overriding `__setattr__`. + public void setAttrDirect(String name, Object value) { + store(name, value); + } + + @Override + public void delAttr(String name) { + Object clsAttr = cls.lookup(name); + if (clsAttr instanceof PyInstance) { + Object del = ((PyInstance) clsAttr).cls.lookup("__delete__"); + if (del != null) { + Py.obj(del).call(new Object[] {clsAttr, this}, Py.NO_KW); + return; + } + } + delAttrDirect(name); + } + + /// Removes the instance attribute `name`, bypassing descriptors. + void delAttrDirect(String name) { + int f = fieldIndex(name); + if (f >= 0) { + if (getField(f) != null) { + setField(f, null); + // Set again, it is the newest attribute: see store. + extra = true; + return; + } + } + int at = shape.indexOf(name); + if (at < 0) { + throw Py.attributeError("'" + cls.name + "' object has no attribute '" + name + "'"); + } + // Rebuilt from the empty shape, so the instance meets every other + // instance that holds the same names in the same order. + String[] names = shape.names; + Object[] old = slots; + shape = Shape.EMPTY; + slots = NO_SLOTS; + for (int i = 0; i < names.length; i++) { + if (i != at) { + append(shape.with(names[i]), old[i]); + } + } + } + + /// Calls dunder `name` if the class defines it, else returns `null`. + protected final Object dunder(String name, Object arg) { + Object m = cls.lookup(name); + if (m == null) { + return null; + } + Object r = Py.obj(m).call(new Object[] {this, arg}, Py.NO_KW); + return r == Py.NOT_IMPLEMENTED ? null : r; //NOPMD CompareObjectsWithEquals - identity is meant + } + + private Object dunder0(String name) { + Object m = cls.lookup(name); + return m == null ? null : Py.obj(m).call(new Object[] {this}, Py.NO_KW); + } + + @Override + public String repr() { + Object r = dunder0("__repr__"); + if (r != null) { + return Py.strValue(r, "__repr__"); + } + return "<" + cls.name + " object at 0x" + Integer.toHexString(System.identityHashCode(this)) + ">"; + } + + @Override + public String str() { + Object r = dunder0("__str__"); + if (r != null) { + return Py.strValue(r, "__str__"); + } + return repr(); + } + + @Override + public boolean isTrue() { + Object m = cls.lookup("__bool__"); + if (m != null) { + Object r = Py.obj(m).call(new Object[] {this}, Py.NO_KW); + if (!(r instanceof PyBool)) { + throw Py.typeError("__bool__ should return bool, returned " + Py.typeOf(r).name); + } + return r == Py.TRUE; //NOPMD CompareObjectsWithEquals - identity is meant + } + m = cls.lookup("__len__"); + if (m != null) { + return len() != 0; + } + return true; + } + + @Override + public Object call(Object[] args, String[] kwNames) { + Object m = cls.lookup("__call__"); + if (m == null) { + return super.call(args, kwNames); + } + return Py.obj(m).call(Py.prepend(this, args), kwNames); + } + + @Override + public Object add(Object o) { + return dunder("__add__", o); + } + + @Override + public Object radd(Object o) { + return dunder("__radd__", o); + } + + @Override + public Object iadd(Object o) { + return dunder("__iadd__", o); + } + + @Override + public Object inplace(String name, Object o) { + return dunder(name, o); + } + + @Override + public Object sub(Object o) { + return dunder("__sub__", o); + } + + @Override + public Object rsub(Object o) { + return dunder("__rsub__", o); + } + + @Override + public Object mul(Object o) { + return dunder("__mul__", o); + } + + @Override + public Object rmul(Object o) { + return dunder("__rmul__", o); + } + + @Override + public Object truediv(Object o) { + return dunder("__truediv__", o); + } + + @Override + public Object rtruediv(Object o) { + return dunder("__rtruediv__", o); + } + + @Override + public Object floordiv(Object o) { + return dunder("__floordiv__", o); + } + + @Override + public Object rfloordiv(Object o) { + return dunder("__rfloordiv__", o); + } + + @Override + public Object mod(Object o) { + return dunder("__mod__", o); + } + + @Override + public Object rmod(Object o) { + return dunder("__rmod__", o); + } + + @Override + public Object pow(Object o) { + return dunder("__pow__", o); + } + + @Override + public Object rpow(Object o) { + return dunder("__rpow__", o); + } + + @Override + public Object and(Object o) { + return dunder("__and__", o); + } + + @Override + public Object or(Object o) { + return dunder("__or__", o); + } + + @Override + public Object xor(Object o) { + return dunder("__xor__", o); + } + + @Override + public Object lshift(Object o) { + return dunder("__lshift__", o); + } + + @Override + public Object rshift(Object o) { + return dunder("__rshift__", o); + } + + @Override + public Object neg() { + Object r = dunder0("__neg__"); + return r != null ? r : super.neg(); + } + + @Override + public Object pos() { + Object r = dunder0("__pos__"); + return r != null ? r : super.pos(); + } + + @Override + public Object invert() { + Object r = dunder0("__invert__"); + return r != null ? r : super.invert(); + } + + @Override + public Object eq(Object o) { + Object r = dunder("__eq__", o); + return r != null ? r : super.eq(o); + } + + @Override + public Object lt(Object o) { + return dunder("__lt__", o); + } + + @Override + public Object le(Object o) { + return dunder("__le__", o); + } + + @Override + public Object gt(Object o) { + return dunder("__gt__", o); + } + + @Override + public Object ge(Object o) { + return dunder("__ge__", o); + } + + @Override + public long hash() { + Object m = cls.lookup("__hash__"); + if (m == Py.NONE) { //NOPMD CompareObjectsWithEquals - identity is meant + throw Py.typeError("unhashable type: '" + cls.name + "'"); + } + if (m == null) { + if (cls.lookup("__eq__") != null) { + throw Py.typeError("unhashable type: '" + cls.name + "'"); + } + return System.identityHashCode(this); + } + return Py.asLong(Py.obj(m).call(new Object[] {this}, Py.NO_KW)); + } + + @Override + public Object getItem(Object key) { + if (base instanceof PyDict) { + Object m = cls.lookup("__getitem__"); + if (m instanceof PyBuiltin) { + Object v = ((PyDict) base).getOrNull(key); + if (v != null) { + return v; + } + Object missing = cls.lookup("__missing__"); + if (missing != null) { + return Py.obj(missing).call(new Object[] {this, key}, Py.NO_KW); + } + } + } + Object r = dunder("__getitem__", key); + return r != null ? r : super.getItem(key); + } + + @Override + public void setItem(Object key, Object value) { + Object m = cls.lookup("__setitem__"); + if (m == null) { + super.setItem(key, value); + return; + } + Py.obj(m).call(new Object[] {this, key, value}, Py.NO_KW); + } + + @Override + public void delItem(Object key) { + Object m = cls.lookup("__delitem__"); + if (m == null) { + super.delItem(key); + return; + } + Py.obj(m).call(new Object[] {this, key}, Py.NO_KW); + } + + @Override + public int len() { + Object r = dunder0("__len__"); + if (r == null) { + return super.len(); + } + long n = Py.asLong(r); + if (n < 0) { + throw Py.valueError("__len__() should return >= 0"); + } + return (int) n; + } + + @Override + public boolean contains(Object x) { + Object m = cls.lookup("__contains__"); + if (m != null) { + return Py.isTrue(Py.obj(m).call(new Object[] {this, x}, Py.NO_KW)); + } + return super.contains(x); + } + + @Override + public Object iter() { + Object r = dunder0("__iter__"); + if (r != null) { + return r; + } + if (cls.lookup("__getitem__") != null) { + return new PySeqIter(this); + } + return super.iter(); + } + + @Override + public Object next() { + Object m = cls.lookup("__next__"); + if (m == null) { + return super.next(); + } + try { + return Py.obj(m).call(new Object[] {this}, Py.NO_KW); + } catch (PyException e) { + if (e.matches(Py.STOP_ITERATION)) { + return null; + } + throw e; + } + } + + @Override + public long asLong() { + Object r = dunder0("__index__"); + if (r == null) { + r = dunder0("__int__"); + } + return r != null ? Py.asLong(r) : super.asLong(); + } + + @Override + public double asDouble() { + Object r = dunder0("__float__"); + return r != null ? Py.asDouble(r) : super.asDouble(); + } + + /// `__dict__` of an instance: its stores and deletes reach the instance. + private static final class InstanceDict extends PyDict { + private final PyInstance self; + + InstanceDict(PyInstance self) { + this.self = self; + } + + @Override + public void setItem(Object key, Object value) { + super.setItem(key, value); + self.store(Py.toStr(key), value); + } + + @Override + public void delItem(Object key) { + super.delItem(key); + self.delAttrDirect(Py.toStr(key)); + } + } + +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyInt.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyInt.java new file mode 100644 index 00000000000..a09b3b3c973 --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyInt.java @@ -0,0 +1,610 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +import com.codename1.util.BigInteger; + +/// Python's `int`: arbitrary precision. +/// +/// The value lives in a `long` and the arithmetic stays there until a result +/// overflows, when it moves to a [BigInteger]. A big value that comes back into +/// `long` range is normalised back, so `big == null` always means "fits". +public class PyInt extends PyObject { + private static final int CACHE_LOW = -5; + private static final int CACHE_HIGH = 256; + private static final PyInt[] CACHE = new PyInt[CACHE_HIGH - CACHE_LOW + 1]; + private static final BigInteger LONG_MIN = BigInteger.valueOf(Long.MIN_VALUE); + private static final BigInteger LONG_MAX = BigInteger.valueOf(Long.MAX_VALUE); + + static { + for (int i = 0; i < CACHE.length; i++) { + CACHE[i] = new PyInt(i + CACHE_LOW); + } + } + + public final long value; + final BigInteger big; + + PyInt(long value) { + this.value = value; + this.big = null; + } + + private PyInt(BigInteger big) { + this.value = 0; + this.big = big; + } + + public static Object of(long v) { + return Long.valueOf(v); + } + + public static Object of(BigInteger b) { + if (b.compareTo(LONG_MIN) >= 0 && b.compareTo(LONG_MAX) <= 0) { + return of(b.longValue()); + } + return new PyInt(b); + } + + /// A PyInt for the PyInt behaviour of a value already known to be an int. + static PyInt view(long v) { + if (v >= CACHE_LOW && v <= CACHE_HIGH) { + return CACHE[(int) v - CACHE_LOW]; + } + return new PyInt(v); + } + + /// Parses an integer literal or `int(str, base)` text. + public static Object parse(String s, int base) { + String t = s.trim().replace("_", ""); + boolean neg = false; + if (t.startsWith("-") || t.startsWith("+")) { + neg = t.charAt(0) == '-'; + t = t.substring(1); + } + if (base == 0 || base == 16 || base == 8 || base == 2) { + String lower = t.length() > 1 ? t.substring(0, 2) : ""; + if ("0x".equals(lower) || "0X".equals(lower)) { + base = 16; + t = t.substring(2); + } else if ("0o".equals(lower) || "0O".equals(lower)) { + base = 8; + t = t.substring(2); + } else if ("0b".equals(lower) || "0B".equals(lower)) { + base = 2; + t = t.substring(2); + } else if (base == 0) { + base = 10; + } + } + if (t.length() == 0) { + throw Py.valueError("invalid literal for int() with base " + base + ": '" + s + "'"); + } + for (int i = 0; i < t.length(); i++) { + if (Character.digit(t.charAt(i), base) < 0) { + throw Py.valueError("invalid literal for int() with base " + base + ": '" + s + "'"); + } + } + if (t.length() <= 12 || base == 10 && t.length() <= 18) { + long v = Long.parseLong(t, base); + return of(neg ? -v : v); + } + BigInteger b = new BigInteger(t, base); + return of(neg ? b.negate() : b); + } + + @Override + public PyType type() { + return Py.INT_TYPE; + } + + public boolean isBig() { + return big != null; + } + + public BigInteger toBig() { + return big != null ? big : BigInteger.valueOf(value); + } + + @Override + public String repr() { + return big != null ? big.toString() : Long.toString(value); + } + + @Override + public boolean isTrue() { + return big != null || value != 0; + } + + @Override + public long asLong() { + if (big != null) { + throw Py.overflowError("Python int too large to convert to C long"); + } + return value; + } + + @Override + public double asDouble() { + if (big == null) { + return value; + } + double d = Double.parseDouble(big.toString()); + if (Double.isInfinite(d)) { + throw Py.overflowError("int too large to convert to float"); + } + return d; + } + + @Override + public Object toJava() { + return big != null ? (Object) big : (Object) Long.valueOf(value); + } + + @Override + public long hash() { + if (big != null) { + return Hashing.ofBig(big); + } + return Hashing.ofLong(value); + } + + @Override + public Object add(Object o) { + if (Py.isInt(o)) { + PyInt b = Py.intView(o); + if (big == null && b.big == null) { + long r = value + b.value; + if (((value ^ r) & (b.value ^ r)) < 0) { + return of(toBig().add(b.toBig())); + } + return of(r); + } + return of(toBig().add(b.toBig())); + } + if (Py.isFloat(o)) { + return Py.ofFloat(asDouble() + Py.asDouble(o)); + } + return null; + } + + @Override + public Object radd(Object o) { + return add(o); + } + + @Override + public Object sub(Object o) { + if (Py.isInt(o)) { + PyInt b = Py.intView(o); + if (big == null && b.big == null) { + long r = value - b.value; + if (((value ^ b.value) & (value ^ r)) < 0) { + return of(toBig().subtract(b.toBig())); + } + return of(r); + } + return of(toBig().subtract(b.toBig())); + } + if (Py.isFloat(o)) { + return Py.ofFloat(asDouble() - Py.asDouble(o)); + } + return null; + } + + @Override + public Object rsub(Object o) { + if (Py.isFloat(o)) { + return Py.ofFloat(Py.asDouble(o) - asDouble()); + } + return Py.isInt(o) ? Py.intView(o).sub(this) : null; + } + + @Override + public Object mul(Object o) { + if (Py.isInt(o)) { + PyInt b = Py.intView(o); + if (big == null && b.big == null) { + long a = value; + long c = b.value; + long hi = Math.abs(a) | Math.abs(c); + if ((hi >>> 31) == 0) { + return of(a * c); + } + long r = a * c; + if (a != 0 && (r / a != c || a == -1 && c == Long.MIN_VALUE)) { + return of(toBig().multiply(b.toBig())); + } + return of(r); + } + return of(toBig().multiply(b.toBig())); + } + if (Py.isFloat(o)) { + return Py.ofFloat(asDouble() * Py.asDouble(o)); + } + if (o instanceof String || o instanceof PyList || o instanceof PyTuple) { + return Py.obj(o).mul(this); + } + return null; + } + + @Override + public Object rmul(Object o) { + return mul(o); + } + + @Override + public Object truediv(Object o) { + if (Py.isInt(o)) { + return Py.ofFloat(divide(toBig(), Py.intView(o).toBig())); + } + if (Py.isFloat(o)) { + double d = Py.asDouble(o); + if (d == 0) { + throw Py.zeroDivision("division by zero"); + } + return Py.ofFloat(asDouble() / d); + } + return null; + } + + /// `a / b` correctly rounded, as CPython divides two ints: converting + /// either to a double first rounds twice once it passes 2**53. + static double divide(BigInteger a, BigInteger b) { + if (b.signum() == 0) { + throw Py.zeroDivision("division by zero"); + } + boolean negative = a.signum() * b.signum() < 0; + a = a.abs(); + b = b.abs(); + if (a.signum() == 0) { + return negative ? -0.0 : 0.0; + } + // A quotient of 55 or 56 bits, with the remainder folded into its + // lowest bit, rounds to 53 bits exactly as the full quotient would. + int shift = 55 - (a.bitLength() - b.bitLength()); + BigInteger n = shift >= 0 ? a.shiftLeft(shift) : a; + BigInteger d = shift >= 0 ? b : b.shiftLeft(-shift); + BigInteger[] qr = n.divideAndRemainder(d); + long q = qr[0].longValue(); + if (qr[1].signum() != 0) { + q |= 1; + } + double r = (double) q; + // Scale by 2**-shift in steps that stay exact. + int e = -shift; + while (e > 0) { + int k = Math.min(e, 1000); + r *= Double.longBitsToDouble((long) (1023 + k) << 52); + e -= k; + if (Double.isInfinite(r)) { + throw Py.overflowError("integer division result too large for a float"); + } + } + while (e < 0) { + int k = Math.min(-e, 1000); + r *= Double.longBitsToDouble((long) (1023 - k) << 52); + e += k; + } + return negative ? -r : r; + } + + @Override + public Object rtruediv(Object o) { + if (Py.isFloat(o)) { + return Py.obj(o).truediv(this); + } + return Py.isInt(o) ? Py.intView(o).truediv(this) : null; + } + + @Override + public Object floordiv(Object o) { + if (Py.isInt(o)) { + PyInt b = Py.intView(o); + if (!b.isTrue()) { + throw Py.zeroDivision("division by zero"); + } + if (big == null && b.big == null && !(value == Long.MIN_VALUE && b.value == -1)) { + return of(Math.floorDiv(value, b.value)); + } + BigInteger[] qr = toBig().divideAndRemainder(b.toBig()); + BigInteger q = qr[0]; + if (qr[1].signum() != 0 && qr[1].signum() != b.toBig().signum()) { + q = q.subtract(BigInteger.ONE); + } + return of(q); + } + if (Py.isFloat(o)) { + return new PyFloat(asDouble()).floordiv(o); + } + return null; + } + + @Override + public Object rfloordiv(Object o) { + if (Py.isFloat(o)) { + return Py.obj(o).floordiv(Py.ofFloat(asDouble())); + } + return Py.isInt(o) ? Py.intView(o).floordiv(this) : null; + } + + @Override + public Object mod(Object o) { + if (Py.isInt(o)) { + PyInt b = Py.intView(o); + if (!b.isTrue()) { + throw Py.zeroDivision("division by zero"); + } + if (big == null && b.big == null) { + return of(Math.floorMod(value, b.value)); + } + BigInteger r = toBig().remainder(b.toBig()); + if (r.signum() != 0 && r.signum() != b.toBig().signum()) { + r = r.add(b.toBig()); + } + return of(r); + } + if (Py.isFloat(o)) { + return new PyFloat(asDouble()).mod(o); + } + return null; + } + + @Override + public Object rmod(Object o) { + if (Py.isFloat(o)) { + return Py.obj(o).mod(Py.ofFloat(asDouble())); + } + return Py.isInt(o) ? Py.intView(o).mod(this) : null; + } + + @Override + public Object pow(Object o) { + if (Py.isInt(o)) { + PyInt e = Py.intView(o); + if (e.big != null) { + throw Py.overflowError("exponent too large"); + } + if (e.value < 0) { + return new PyFloat(asDouble()).pow(Py.ofFloat(e.value)); + } + if (e.value > Integer.MAX_VALUE) { + throw Py.overflowError("exponent too large"); + } + if (big == null) { + try { + long result = 1; + long base = value; + long n = e.value; + while (n > 0) { + if ((n & 1) != 0) { + result = Math.multiplyExact(result, base); + } + n >>= 1; + if (n > 0) { + base = Math.multiplyExact(base, base); + } + } + return of(result); + } catch (ArithmeticException overflow) { + // falls through to the arbitrary-precision path + } + } + return of(toBig().pow((int) e.value)); + } + if (Py.isFloat(o)) { + return new PyFloat(asDouble()).pow(o); + } + return null; + } + + @Override + public Object rpow(Object o) { + if (Py.isFloat(o)) { + return Py.obj(o).pow(Py.ofFloat(asDouble())); + } + return Py.isInt(o) ? Py.intView(o).pow(this) : null; + } + + @Override + public Object lshift(Object o) { + if (!(Py.isInt(o))) { + return null; + } + long n = Py.asLong(o); + if (n < 0) { + throw Py.valueError("negative shift count"); + } + if (big == null && n < 63 && (value == 0 || Math.abs(value) < (1L << (62 - n)))) { + return of(value << n); + } + return of(toBig().shiftLeft((int) n)); + } + + @Override + public Object rshift(Object o) { + if (!(Py.isInt(o))) { + return null; + } + long n = Py.asLong(o); + if (n < 0) { + throw Py.valueError("negative shift count"); + } + if (big == null) { + return of(n >= 64 ? (value < 0 ? -1 : 0) : value >> n); + } + return of(big.shiftRight((int) n)); + } + + @Override + public Object and(Object o) { + if (!(Py.isInt(o))) { + return null; + } + PyInt b = Py.intView(o); + if (big == null && b.big == null) { + return Py.wrapIntResult(this, b, value & b.value); + } + return of(toBig().and(b.toBig())); + } + + @Override + public Object rand(Object o) { + return and(o); + } + + @Override + public Object or(Object o) { + if (!(Py.isInt(o))) { + return null; + } + PyInt b = Py.intView(o); + if (big == null && b.big == null) { + return Py.wrapIntResult(this, b, value | b.value); + } + return of(toBig().or(b.toBig())); + } + + @Override + public Object ror(Object o) { + return or(o); + } + + @Override + public Object xor(Object o) { + if (!(Py.isInt(o))) { + return null; + } + PyInt b = Py.intView(o); + if (big == null && b.big == null) { + return Py.wrapIntResult(this, b, value ^ b.value); + } + return of(toBig().xor(b.toBig())); + } + + @Override + public Object rxor(Object o) { + return xor(o); + } + + @Override + public Object neg() { + if (big == null && value != Long.MIN_VALUE) { + return of(-value); + } + return of(toBig().negate()); + } + + @Override + public Object pos() { + return big == null ? of(value) : this; + } + + @Override + public Object invert() { + return big == null ? of(~value) : of(big.not()); + } + + private int compare(PyInt b) { + if (big == null && b.big == null) { + return value < b.value ? -1 : (value == b.value ? 0 : 1); + } + return toBig().compareTo(b.toBig()); + } + + @Override + public Object eq(Object o) { + if (Py.isInt(o)) { + return Py.bool(compare(Py.intView(o)) == 0); + } + if (Py.isFloat(o)) { + if (big == null) { + int c = PyFloat.compareLong(Py.asDouble(o), value); + return Py.bool(c == 0); + } + return Py.bool(asDouble() == Py.asDouble(o)); + } + return null; + } + + @Override + public Object lt(Object o) { + if (Py.isInt(o)) { + return Py.bool(compare(Py.intView(o)) < 0); + } + if (Py.isFloat(o)) { + if (big == null) { + int c = PyFloat.compareLong(Py.asDouble(o), value); + return Py.bool(c == 1); + } + return Py.bool(asDouble() < Py.asDouble(o)); + } + return null; + } + + @Override + public Object le(Object o) { + if (Py.isInt(o)) { + return Py.bool(compare(Py.intView(o)) <= 0); + } + if (Py.isFloat(o)) { + if (big == null) { + int c = PyFloat.compareLong(Py.asDouble(o), value); + return Py.bool(c == 1 || c == 0); + } + return Py.bool(asDouble() <= Py.asDouble(o)); + } + return null; + } + + @Override + public Object gt(Object o) { + if (Py.isInt(o)) { + return Py.bool(compare(Py.intView(o)) > 0); + } + if (Py.isFloat(o)) { + if (big == null) { + int c = PyFloat.compareLong(Py.asDouble(o), value); + return Py.bool(c == -1); + } + return Py.bool(asDouble() > Py.asDouble(o)); + } + return null; + } + + @Override + public Object ge(Object o) { + if (Py.isInt(o)) { + return Py.bool(compare(Py.intView(o)) >= 0); + } + if (Py.isFloat(o)) { + if (big == null) { + int c = PyFloat.compareLong(Py.asDouble(o), value); + return Py.bool(c == -1 || c == 0); + } + return Py.bool(asDouble() >= Py.asDouble(o)); + } + return null; + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyIter.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyIter.java new file mode 100644 index 00000000000..45bf1f64719 --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyIter.java @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +/// An iterator over an indexed source of known length. +abstract class PyIter extends PyObject { + final int length; + int pos; + + PyIter(int length) { + this.length = length; + } + + abstract Object at(int i); + + /// The live length, for sources that may change while iterated. + int liveLength() { + return length; + } + + @Override + public PyType type() { + return Py.ITERATOR_TYPE; + } + + @Override + public Object iter() { + return this; + } + + @Override + public Object next() { + if (pos >= liveLength()) { + return null; + } + return at(pos++); + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyJavaIter.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyJavaIter.java new file mode 100644 index 00000000000..387608edcb1 --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyJavaIter.java @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +/// An iterator implemented as a Java "next or null" function: map, filter, +/// zip, enumerate and the other lazy builtins. +public final class PyJavaIter extends PyObject { + + /// Produces the next value, or `null` when done. + public interface Source { + Object next(); + } + + private final Source source; + private boolean done; + + public PyJavaIter(Source source) { + this.source = source; + } + + @Override + public PyType type() { + return Py.ITERATOR_TYPE; + } + + @Override + public Object iter() { + return this; + } + + @Override + public Object next() { + if (done) { + return null; + } + Object v = source.next(); + if (v == null) { + done = true; + } + return v; + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyJavaObject.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyJavaObject.java new file mode 100644 index 00000000000..9b15a115758 --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyJavaObject.java @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +/// A Java object handed to Python code that has no Python counterpart. +/// +/// Python sees it as an opaque value: it can be stored, compared by identity, +/// passed back to Java and printed. Calling its methods by name would need +/// reflection, which ParparVM does not keep; the transpiler instead binds +/// Java calls statically (see the interop contract in the developer guide). +public final class PyJavaObject extends PyObject { + public final Object value; + + public PyJavaObject(Object value) { + this.value = value; + } + + @Override + public PyType type() { + return Py.OBJECT_TYPE; + } + + @Override + public Object toJava() { + return value; + } + + @Override + public String repr() { + return ""; + } + + @Override + public Object eq(Object o) { + return o instanceof PyJavaObject ? Py.bool(value.equals(((PyJavaObject) o).value)) : null; + } + + @Override + public long hash() { + return value.hashCode(); + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyList.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyList.java new file mode 100644 index 00000000000..eb46b173b1b --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyList.java @@ -0,0 +1,920 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +/// Python's `list`, a growable array. +public final class PyList extends PyObject { + // The storage is public for the typed code the transpiler generates + // for a closed program: ParparVM does not inline a call into another + // class, so an element access through a method here cost several times + // the access itself. Nothing else may touch these fields. + public Object[] items; + public int size; + + /// Shared by every list that has never held an item; the first append + /// replaces it, so an empty list costs one object. + private static final Object[] NO_ITEMS = new Object[0]; + + /// The items of a list that has only ever held floats, unboxed; `items` + /// is then unused. A float region (generated code) reads and writes + /// them without allocating. Anything else turns the list back into + /// objects first ([#objects()]), once. + public double[] floats; + + /// The items of a list that has only ever held ints, unboxed, as + /// `floats` is for floats; never both. A Long that fits 61 bits is an + /// immediate under ParparVM, so reading one back allocates nothing, and + /// storing one needs no GC write barrier. + public long[] ints; + + /// Set on a list Py.recycleSlice or Py.recycleList made for a local the + /// transpiler proved never shares it; only such a list is reused. A list + /// a user `__getitem__` returned may be held elsewhere and never is. + boolean owned; + + public PyList() { + items = NO_ITEMS; + } + + /// A list of `items`, unboxed when every one of them is a float. + static PyList of(Object[] items) { + int n = items.length; + if (n == 0) { + return new PyList(items); + } + for (int i = 0; i < n; i++) { + if (!(items[i] instanceof Double)) { + return new PyList(items); + } + } + PyList r = new PyList(); + double[] f = new double[n]; + for (int i = 0; i < n; i++) { + f[i] = ((Double) items[i]).doubleValue(); + } + r.floats = f; + r.size = n; + return r; + } + + /// The items as objects, boxing an unboxed list for good. + Object[] objects() { + long[] l = ints; + if (l != null) { + Object[] o = new Object[Math.max(4, l.length)]; + for (int i = 0; i < size; i++) { + o[i] = Long.valueOf(l[i]); + } + items = o; + ints = null; + return o; + } + double[] f = floats; + if (f != null) { + Object[] o = new Object[Math.max(4, f.length)]; + for (int i = 0; i < size; i++) { + o[i] = Double.valueOf(f[i]); + } + items = o; + floats = null; + } + return items; + } + + /// Whether item `i` is a float a region can read unboxed. + boolean isFloatAt(int i) { + return floats != null || ints == null && items[i] instanceof Double; + } + + double floatAt(int i) { + return floats != null ? floats[i] : ((Double) items[i]).doubleValue(); + } + + /// Stores a float a region computed; an unboxed list keeps it unboxed. + void setFloat(int i, double v) { + if (floats != null) { + floats[i] = v; + } else { + objects()[i] = Double.valueOf(v); + } + } + + /// Takes ownership of `items`. + public PyList(Object[] items) { + this.items = items; + this.size = items.length; + } + + public PyList(int capacity) { + items = new Object[Math.max(4, capacity)]; + } + + @Override + public PyType type() { + return Py.LIST_TYPE; + } + + public int size() { + return size; + } + + public Object get(int i) { + long[] l = ints; + if (l != null) { + return Long.valueOf(l[i]); + } + return floats != null ? Double.valueOf(floats[i]) : items[i]; + } + + public void append(Object v) { + if (ints != null || floats != null || size == 0 && items == NO_ITEMS) { + if (v instanceof Double && ints == null) { + appendFloat(((Double) v).doubleValue()); + return; + } + if (v instanceof Long && floats == null) { + appendInt(((Long) v).longValue()); + return; + } + objects(); + } + appendObject(v); + } + + private void appendInt(long v) { + long[] l = ints; + if (l == null) { + l = new long[4]; + ints = l; + } else if (size == l.length) { + long[] n = new long[size * 2 + 4]; + System.arraycopy(l, 0, n, 0, size); + ints = n; + l = n; + } + l[size++] = v; + } + + private void appendObject(Object v) { + if (size == items.length) { + Object[] n = new Object[size * 2 + 4]; + System.arraycopy(items, 0, n, 0, size); + items = n; + } + items[size++] = v; + } + + private void appendFloat(double v) { + double[] f = floats; + if (f == null) { + f = new double[4]; + floats = f; + } else if (size == f.length) { + double[] n = new double[size * 2 + 4]; + System.arraycopy(f, 0, n, 0, size); + floats = n; + f = n; + } + f[size++] = v; + } + + public void extend(Object iterable) { + if (iterable instanceof PyList) { + PyList o = (PyList) iterable; + int n = o.size; + for (int i = 0; i < n; i++) { + append(o.get(i)); + } + return; + } + Object it = Py.iter(iterable); + for (Object v = Py.next(it); v != null; v = Py.next(it)) { + append(v); + } + } + + public void insert(long index, Object v) { + objects(); + if (index < 0) { + index = Math.max(0, index + size); + } + int i = (int) Math.min(index, size); + appendObject(v); + System.arraycopy(items, i, items, i + 1, size - 1 - i); + items[i] = v; + } + + public Object pop(long index) { + if (ints != null && size > 0 && (index == -1 || index == size - 1)) { + return Long.valueOf(ints[--size]); + } + objects(); + if (size == 0) { + throw Py.indexError("pop from empty list"); + } + if (index < 0) { + index += size; + } + if (index < 0 || index >= size) { + throw Py.indexError("pop index out of range"); + } + int i = (int) index; + Object v = items[i]; + System.arraycopy(items, i + 1, items, i, size - i - 1); + items[--size] = null; + return v; + } + + public Object[] toArray() { + Object[] r = new Object[size]; + if (ints != null) { + for (int i = 0; i < size; i++) { + r[i] = Long.valueOf(ints[i]); + } + } else if (floats != null) { + for (int i = 0; i < size; i++) { + r[i] = Double.valueOf(floats[i]); + } + } else { + System.arraycopy(items, 0, r, 0, size); + } + return r; + } + + @Override + public int len() { + return size; + } + + @Override + public boolean isTrue() { + return size != 0; + } + + @Override + public String repr() { + if (Py.enterRepr(this)) { + return "[...]"; + } + try { + return Py.joinRepr("[", floats != null || ints != null ? toArray() : items, size, "]"); + } finally { + Py.exitRepr(this); + } + } + + @Override + public long hash() { + throw Py.typeError("unhashable type: 'list'"); + } + + @Override + public Object eq(Object o) { + if (!(o instanceof PyList)) { + return null; + } + PyList b = (PyList) o; + return Py.bool(Py.seqEquals(objects(), size, b.objects(), b.size)); + } + + @Override + public Object lt(Object o) { + return o instanceof PyList ? Py.bool(cmp(o) < 0) : null; + } + + @Override + public Object le(Object o) { + return o instanceof PyList ? Py.bool(cmp(o) <= 0) : null; + } + + @Override + public Object gt(Object o) { + return o instanceof PyList ? Py.bool(cmp(o) > 0) : null; + } + + @Override + public Object ge(Object o) { + return o instanceof PyList ? Py.bool(cmp(o) >= 0) : null; + } + + private int cmp(Object o) { + PyList b = (PyList) o; + return Py.seqCompare(objects(), size, b.objects(), b.size); + } + + @Override + public Object add(Object o) { + if (!(o instanceof PyList)) { + return null; + } + PyList b = (PyList) o; + Object[] r = new Object[size + b.size]; + System.arraycopy(objects(), 0, r, 0, size); + System.arraycopy(b.objects(), 0, r, size, b.size); + return new PyList(r); + } + + @Override + public Object iadd(Object o) { + extend(o); + return this; + } + + @Override + public Object inplace(String name, Object o) { + objects(); + if ("__imul__".equals(name) && Py.isInt(o)) { + Object[] items = toArray(); + long n = Py.asLong(o); + clear(); + for (long i = 0; i < n; i++) { + for (Object item : items) { + append(item); + } + } + return this; + } + return null; + } + + @Override + public Object mul(Object o) { + if (!(Py.isInt(o))) { + return null; + } + long n = Math.max(0, Py.asLong(o)); + if (ints == null && floats == null && size > 0) { + boolean all = true; + for (int i = 0; i < size && all; i++) { + all = items[i] instanceof Long; + } + if (all) { + // `[0] * n` is a list of ints: unboxed. + long[] one = new long[size]; + for (int i = 0; i < size; i++) { + one[i] = ((Long) items[i]).longValue(); + } + return repeatInts(one, n); + } + } + if (ints != null) { + return repeatInts(ints, n); + } + if (floats != null) { + // `[0.0] * n` stays unboxed. + double[] f = new double[Math.max(4, (int) (size * n))]; + for (int i = 0; i < n; i++) { + System.arraycopy(floats, 0, f, i * size, size); + } + PyList r = new PyList(); + r.floats = f; + r.size = (int) (size * n); + return r; + } + Object[] r = new Object[(int) (size * n)]; + for (int i = 0; i < n; i++) { + System.arraycopy(items, 0, r, i * size, size); + } + return new PyList(r); + } + + private PyList repeatInts(long[] one, long n) { + int k = size; + long[] l = new long[Math.max(4, (int) (k * n))]; + for (int i = 0; i < n; i++) { + System.arraycopy(one, 0, l, i * k, k); + } + PyList r = new PyList(); + r.ints = l; + r.size = (int) (k * n); + return r; + } + + @Override + public Object rmul(Object o) { + return mul(o); + } + + private int checkIndex(Object key) { + long i = Py.index(key); + if (i < 0) { + i += size; + } + if (i < 0 || i >= size) { + throw Py.indexError("list index out of range"); + } + return (int) i; + } + + @Override + public Object getItem(Object key) { + if (key instanceof Long) { + long i = ((Long) key).longValue(); + if (i >= 0 && i < size) { + return get((int) i); + } + } + if (floats != null || ints != null) { + if (!(key instanceof PySlice)) { + return get(checkIndex(key)); + } + objects(); + } + if (key instanceof PySlice) { + int[] r = ((PySlice) key).indices(size); + int n = PySlice.length(r); + Object[] out = new Object[n]; + for (int i = 0, j = r[0]; i < n; i++, j += r[2]) { + out[i] = items[j]; + } + return new PyList(out); + } + return items[checkIndex(key)]; + } + + /// `self[start:stop:step]` without a slice object; NONE for a missing bound. + public PyList getSlice(Object start, Object stop, Object step) { + if (ints != null) { + int st = step == Py.NONE ? 1 : PySlice.step(step); //NOPMD CompareObjectsWithEquals - identity is meant + int lo = PySlice.lower(start, stop, st, size); + int hi = PySlice.upper(stop, st, size); + int n = PySlice.count(lo, hi, st); + long[] out = new long[Math.max(4, n)]; + if (st == 1) { + System.arraycopy(ints, lo, out, 0, n); + } else { + for (int i = 0, j = lo; i < n; i++, j += st) { + out[i] = ints[j]; + } + } + PyList r = new PyList(); + r.ints = out; + r.size = n; + return r; + } + objects(); + int st = step == Py.NONE ? 1 : PySlice.step(step); //NOPMD CompareObjectsWithEquals - identity is meant + int lo = PySlice.lower(start, stop, st, size); + int hi = PySlice.upper(stop, st, size); + int n = PySlice.count(lo, hi, st); + Object[] out = new Object[Math.max(4, n)]; + if (st == 1) { + System.arraycopy(items, lo, out, 0, n); + } else { + for (int i = 0, j = lo; i < n; i++, j += st) { + out[i] = items[j]; + } + } + PyList r = new PyList(out); + r.size = n; + return r; + } + + /// `self[start:stop:step]` copied into `dst`, whose items array is + /// reused when it is large enough. Only for a list nothing else can see: + /// the generated code calls it for a local the transpiler proved holds + /// the sole reference to its list. + PyList sliceInto(PyList dst, Object start, Object stop, Object step) { + if (ints != null) { + int st = step == Py.NONE ? 1 : PySlice.step(step); //NOPMD CompareObjectsWithEquals - identity is meant + int lo = PySlice.lower(start, stop, st, size); + int hi = PySlice.upper(stop, st, size); + int n = PySlice.count(lo, hi, st); + long[] out = dst.ints; + if (out == null || out.length < n) { + out = new long[Math.max(4, n)]; + } + if (st == 1) { + System.arraycopy(ints, lo, out, 0, n); + } else { + for (int i = 0, j = lo; i < n; i++, j += st) { + out[i] = ints[j]; + } + } + dst.floats = null; + dst.items = NO_ITEMS; + dst.ints = out; + dst.size = n; + return dst; + } + objects(); + int st = step == Py.NONE ? 1 : PySlice.step(step); //NOPMD CompareObjectsWithEquals - identity is meant + int lo = PySlice.lower(start, stop, st, size); + int hi = PySlice.upper(stop, st, size); + int n = PySlice.count(lo, hi, st); + Object[] out = dst.items; + if (dst.floats != null || dst.ints != null || out.length < n || out == NO_ITEMS) { + out = new Object[Math.max(4, n)]; + } + if (st == 1) { + System.arraycopy(items, lo, out, 0, n); + } else { + for (int i = 0, j = lo; i < n; i++, j += st) { + out[i] = items[j]; + } + } + for (int i = n; i < dst.size && i < out.length; i++) { + out[i] = null; + } + dst.floats = null; + dst.ints = null; + dst.items = out; + dst.size = n; + return dst; + } + + /// `self[start:stop:step] = value` without a slice object. A contiguous + /// assignment that keeps the length (the common `a[:k] = b[k::-1]`) + /// writes in place and allocates nothing. + public void setSlice(Object start, Object stop, Object step, Object value) { + objects(); + int st = step == Py.NONE ? 1 : PySlice.step(step); //NOPMD CompareObjectsWithEquals - identity is meant + int lo = PySlice.lower(start, stop, st, size); + int hi = PySlice.upper(stop, st, size); + Object[] src; + int srcLen; + if (value instanceof PyList && value != this) { //NOPMD CompareObjectsWithEquals - identity is meant + src = ((PyList) value).objects(); + srcLen = ((PyList) value).size; + } else { + src = Py.toArray(value); + srcLen = src.length; + } + if (st == 1) { + if (hi < lo) { + hi = lo; + } + int newSize = size - (hi - lo) + srcLen; + if (newSize <= items.length) { + System.arraycopy(items, hi, items, lo + srcLen, size - hi); + System.arraycopy(src, 0, items, lo, srcLen); + for (int i = newSize; i < size; i++) { + items[i] = null; + } + } else { + Object[] n = new Object[Math.max(4, newSize + (newSize >> 3))]; + System.arraycopy(items, 0, n, 0, lo); + System.arraycopy(src, 0, n, lo, srcLen); + System.arraycopy(items, hi, n, lo + srcLen, size - hi); + items = n; + } + size = newSize; + return; + } + int count = PySlice.count(lo, hi, st); + if (count != srcLen) { + throw Py.valueError("attempt to assign sequence of size " + srcLen + + " to extended slice of size " + count); + } + for (int i = 0, j = lo; i < count; i++, j += st) { + items[j] = src[i]; + } + } + + /// `self[start:stop:step] = src[sStart:sStop:sStep]`. When the target is + /// contiguous and as long as the source, the items are copied directly; + /// a list read into itself is only done in place when that is exact (a + /// forward copy, or reversing the very range it writes). Everything + /// else takes the source slice as a list first. + void setSliceFrom(Object start, Object stop, Object step, PyList src, Object sStart, Object sStop, Object sStep) { + boolean li = ints != null && src.ints != null; + if ((li || floats == null && ints == null && src.floats == null && src.ints == null) && step == Py.NONE //NOPMD CompareObjectsWithEquals - identity is meant + && (sStep == Py.NONE || sStep instanceof Long)) { //NOPMD CompareObjectsWithEquals - identity is meant + // Plain bounds, no objects to box: the common shapes, without + // a call per bound. + int lo = bound(start, 0, size); + int hr = bound(stop, size, size); + int hi = Math.max(lo, hr); + long ss = sStep == Py.NONE ? 1 : ((Long) sStep).longValue(); //NOPMD CompareObjectsWithEquals - identity is meant + if (ss == 1) { + int slo = bound(sStart, 0, src.size); + int shr = bound(sStop, src.size, src.size); + int shi = Math.max(slo, shr); + if (hi - lo == shi - slo && lo >= 0 && hr >= 0 && slo >= 0 && shr >= 0) { + if (li) { + System.arraycopy(src.ints, slo, ints, lo, hi - lo); + } else { + System.arraycopy(src.items, slo, items, lo, hi - lo); + } + return; + } + } else if (ss == -1 && src == this && sStop == Py.NONE && sStart instanceof Long) { //NOPMD CompareObjectsWithEquals - identity is meant + long s0 = ((Long) sStart).longValue(); + if (s0 < 0) { + s0 += size; + } + if (s0 >= size) { + s0 = size - 1; + } + if (lo == 0 && hr >= 0 && hi == s0 + 1 && li) { + long[] it = ints; + for (int i = 0, j = (int) s0; i < j; i++, j--) { + long t = it[i]; + it[i] = it[j]; + it[j] = t; + } + return; + } + if (lo == 0 && hr >= 0 && hi == s0 + 1) { + Object[] it = items; + for (int i = 0, j = (int) s0; i < j; i++, j--) { + Object t = it[i]; + it[i] = it[j]; + it[j] = t; + } + return; + } + } + } + objects(); + src.objects(); + int sst = sStep == Py.NONE ? 1 : PySlice.step(sStep); //NOPMD CompareObjectsWithEquals - identity is meant + int slo = PySlice.lower(sStart, sStop, sst, src.size); + int shi = PySlice.upper(sStop, sst, src.size); + int n = PySlice.count(slo, shi, sst); + int st = step == Py.NONE ? 1 : PySlice.step(step); //NOPMD CompareObjectsWithEquals - identity is meant + if (st == 1) { + int lo = PySlice.lower(start, stop, 1, size); + int hi = Math.max(lo, PySlice.upper(stop, 1, size)); + if (hi - lo == n) { + if (sst == 1) { + System.arraycopy(src.items, slo, items, lo, n); + return; + } + if (src != this) { //NOPMD CompareObjectsWithEquals - identity is meant + for (int i = 0, j = slo; i < n; i++, j += sst) { + items[lo + i] = src.items[j]; + } + return; + } + if (sst == -1 && slo == hi - 1) { + for (int i = lo, j = hi - 1; i < j; i++, j--) { + Object t = items[i]; + items[i] = items[j]; + items[j] = t; + } + return; + } + } + } + setSlice(start, stop, step, src.getSlice(sStart, sStop, sStep)); + } + + /// A bound of a step-1 slice over `n` items, `dflt` when absent; + /// -1 when it is not a plain int (the caller then takes the general path). + private static int bound(Object b, int dflt, int n) { + if (b == Py.NONE) { //NOPMD CompareObjectsWithEquals - identity is meant + return dflt; + } + if (!(b instanceof Long)) { + return -1; + } + long v = ((Long) b).longValue(); + if (v < 0) { + v += n; + if (v < 0) { + v = 0; + } + } + return v > n ? n : (int) v; + } + + @Override + public void setItem(Object key, Object value) { + if (ints != null && value instanceof Long && !(key instanceof PySlice)) { + ints[checkIndex(key)] = ((Long) value).longValue(); + return; + } + if (floats != null && value instanceof Double && !(key instanceof PySlice)) { + floats[checkIndex(key)] = ((Double) value).doubleValue(); + return; + } + objects(); + if (key instanceof Long) { + long i = ((Long) key).longValue(); + if (i >= 0 && i < size) { + items[(int) i] = value; + return; + } + } + if (key instanceof PySlice) { + int[] r = ((PySlice) key).indices(size); + Object[] src = Py.toArray(value); + if (r[2] == 1) { + int lo = r[0]; + int hi = Math.max(r[0], r[1]); + int newSize = size - (hi - lo) + src.length; + Object[] n = new Object[Math.max(4, newSize)]; + System.arraycopy(items, 0, n, 0, lo); + System.arraycopy(src, 0, n, lo, src.length); + System.arraycopy(items, hi, n, lo + src.length, size - hi); + items = n; + size = newSize; + return; + } + int count = PySlice.length(r); + if (count != src.length) { + throw Py.valueError("attempt to assign sequence of size " + src.length + + " to extended slice of size " + count); + } + for (int i = 0, j = r[0]; i < count; i++, j += r[2]) { + items[j] = src[i]; + } + return; + } + items[checkIndex(key)] = value; + } + + @Override + public void delItem(Object key) { + objects(); + if (key instanceof PySlice) { + int[] r = ((PySlice) key).indices(size); + int count = PySlice.length(r); + if (count == 0) { + return; + } + boolean[] drop = new boolean[size]; + for (int i = 0, j = r[0]; i < count; i++, j += r[2]) { + drop[j] = true; + } + int w = 0; + for (int i = 0; i < size; i++) { + if (!drop[i]) { + items[w++] = items[i]; + } + } + for (int i = w; i < size; i++) { + items[i] = null; + } + size = w; + return; + } + pop(checkIndex(key)); + } + + @Override + public boolean contains(Object x) { + objects(); + for (int i = 0; i < size; i++) { + if (items[i] == x || Py.eqBool(items[i], x)) { //NOPMD CompareObjectsWithEquals - identity is meant + return true; + } + } + return false; + } + + @Override + public Object iter() { + return new PyIter(size) { + @Override + int liveLength() { + return size; + } + + @Override + Object at(int i) { + return get(i); + } + }; + } + + /// `list.sort(key=None, reverse=False)`: a stable merge sort. + public void sort(Object key, boolean reverse) { + objects(); + Object[] keys = new Object[size]; + for (int i = 0; i < size; i++) { + keys[i] = key == null || key == Py.NONE ? items[i] : Py.obj(key).call(new Object[] {items[i]}, Py.NO_KW); //NOPMD CompareObjectsWithEquals - identity is meant + } + Integer[] order = new Integer[size]; + for (int i = 0; i < size; i++) { + order[i] = Integer.valueOf(i); + } + Integer[] tmp = new Integer[size]; + mergeSort(order, tmp, keys, 0, size, reverse); + Object[] out = new Object[Math.max(4, size)]; + for (int i = 0; i < size; i++) { + out[i] = items[order[i].intValue()]; + } + items = out; + } + + private static void mergeSort(Integer[] a, Integer[] tmp, Object[] keys, int lo, int hi, boolean rev) { + if (hi - lo < 2) { + return; + } + if (hi - lo <= 8) { + for (int i = lo + 1; i < hi; i++) { + Integer v = a[i]; + int j = i - 1; + while (j >= lo && before(keys[v.intValue()], keys[a[j].intValue()], rev)) { + a[j + 1] = a[j]; + j--; + } + a[j + 1] = v; + } + return; + } + int mid = (lo + hi) >>> 1; + mergeSort(a, tmp, keys, lo, mid, rev); + mergeSort(a, tmp, keys, mid, hi, rev); + int i = lo; + int j = mid; + int k = lo; + while (i < mid && j < hi) { + if (before(keys[a[j].intValue()], keys[a[i].intValue()], rev)) { + tmp[k++] = a[j++]; + } else { + tmp[k++] = a[i++]; + } + } + while (i < mid) { + tmp[k++] = a[i++]; + } + while (j < hi) { + tmp[k++] = a[j++]; + } + System.arraycopy(tmp, lo, a, lo, hi - lo); + } + + /// Whether `x` must move before `y`: `x < y`, or `y < x` reversed. Equal + /// keys never move, which keeps the sort stable in both directions. + private static boolean before(Object x, Object y, boolean rev) { + return rev ? Py.ltBool(y, x) : Py.ltBool(x, y); + } + + public void reverse() { + if (ints != null) { + long[] l = ints; + for (int i = 0, j = size - 1; i < j; i++, j--) { + long t = l[i]; + l[i] = l[j]; + l[j] = t; + } + return; + } + objects(); + for (int i = 0, j = size - 1; i < j; i++, j--) { + Object t = items[i]; + items[i] = items[j]; + items[j] = t; + } + } + + public long index(Object v) { + objects(); + for (int i = 0; i < size; i++) { + if (Py.eqBool(items[i], v)) { + return i; + } + } + throw Py.valueError("list.index(x): x not in list"); + } + + public void remove(Object v) { + pop(index(v)); + } + + /// Empties the list but keeps its items array for the next fill; for a + /// list nothing else can see (see Py.recycleList). + void reset() { + if (floats != null || ints != null) { + floats = null; + ints = null; + items = NO_ITEMS; + } else { + for (int i = 0; i < size; i++) { + items[i] = null; + } + } + size = 0; + } + + public void clear() { + items = NO_ITEMS; + floats = null; + ints = null; + size = 0; + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyMatch.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyMatch.java new file mode 100644 index 00000000000..2ea7dadd35b --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyMatch.java @@ -0,0 +1,231 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +/// Structural pattern matching: the `match` statement. +/// +/// The transpiler rewrites each `case` into a call of [#match] with a +/// description of the pattern built of tuples, whose first item says what +/// the pattern is: +/// +/// - `("_",)` matches anything; `("c", name)` captures the subject; +/// - `("v", value)` compares with `==`, `("s", value)` with `is`; +/// - `("q", p...)` a sequence, where `("*", name or None)` is a star; +/// - `("m", keys, patterns, rest name or None)` a mapping; +/// - `("k", cls, positional, keyword names, keyword patterns)` a class; +/// - `("|", alternative...)` and `("as", pattern, name)`. +/// +/// The result is the dict of captured names, or None for no match. +public final class PyMatch { + private PyMatch() { + } + + public static Object match(Object pattern, Object subject) { + PyDict out = new PyDict(); + return match(pattern, subject, out) ? out : Py.NONE; + } + + private static Object[] parts(Object pattern) { + if (pattern instanceof PyTuple) { + return ((PyTuple) pattern).items; + } + throw Py.runtimeError("internal error: bad pattern"); + } + + private static boolean match(Object pattern, Object s, PyDict out) { + Object[] p = parts(pattern); + String tag = (String) p[0]; + if ("_".equals(tag)) { + return true; + } + if ("c".equals(tag)) { + out.setItem(p[1], s); + return true; + } + if ("v".equals(tag)) { + return Py.eqBool(s, p[1]); + } + if ("s".equals(tag)) { + return s == p[1]; //NOPMD CompareObjectsWithEquals - identity is meant + } + if ("as".equals(tag)) { + if (!match(p[1], s, out)) { + return false; + } + out.setItem(p[2], s); + return true; + } + if ("|".equals(tag)) { + for (int i = 1; i < p.length; i++) { + PyDict alt = new PyDict(); + if (match(p[i], s, alt)) { + out.update(alt); + return true; + } + } + return false; + } + if ("q".equals(tag)) { + return sequence(p, s, out); + } + if ("m".equals(tag)) { + return mapping(p, s, out); + } + if ("k".equals(tag)) { + return cls(p, s, out); + } + throw Py.runtimeError("internal error: bad pattern '" + tag + "'"); + } + + private static Object storage(Object s) { + return s instanceof PyInstance && ((PyInstance) s).base != null ? ((PyInstance) s).base : s; + } + + private static boolean sequence(Object[] p, Object s, PyDict out) { + Object v = storage(s); + if (!(v instanceof PyList || v instanceof PyTuple || v instanceof PyRange)) { + return false; + } + Object[] items = Py.toArray(v); + int n = p.length - 1; + int star = -1; + for (int i = 0; i < n; i++) { + if ("*".equals(parts(p[i + 1])[0])) { + star = i; + } + } + if (star < 0 ? items.length != n : items.length < n - 1) { + return false; + } + if (star < 0) { + for (int i = 0; i < n; i++) { + if (!match(p[i + 1], items[i], out)) { + return false; + } + } + return true; + } + int after = n - star - 1; + for (int i = 0; i < star; i++) { + if (!match(p[i + 1], items[i], out)) { + return false; + } + } + for (int i = 0; i < after; i++) { + if (!match(p[star + 2 + i], items[items.length - after + i], out)) { + return false; + } + } + Object name = parts(p[star + 1])[1]; + if (name != Py.NONE) { //NOPMD CompareObjectsWithEquals - identity is meant + PyList rest = new PyList(); + for (int i = star; i < items.length - after; i++) { + rest.append(items[i]); + } + out.setItem(name, rest); + } + return true; + } + + private static boolean mapping(Object[] p, Object s, PyDict out) { + Object v = storage(s); + if (!(v instanceof PyDict)) { + return false; + } + PyDict d = (PyDict) v; + Object[] keys = parts(p[1]); + Object[] pats = parts(p[2]); + for (int i = 0; i < keys.length; i++) { + Object x = d.getOrNull(keys[i]); + if (x == null || !match(pats[i], x, out)) { + return false; + } + } + if (p[3] != Py.NONE) { //NOPMD CompareObjectsWithEquals - identity is meant + PyDict rest = new PyDict(); + Object[] all = d.keyArray(); + for (Object e : all) { + boolean named = false; + for (int j = 0; j < keys.length && !named; j++) { + named = Py.eqBool(e, keys[j]); + } + if (!named) { + rest.setItem(e, d.getItem(e)); + } + } + out.setItem(p[3], rest); + } + return true; + } + + /// The builtins whose one positional sub-pattern matches the subject itself. + private static boolean selfMatching(PyType t) { + return t == Py.BOOL_TYPE || t == Py.BYTES_TYPE || t == Py.DICT_TYPE || t == Py.FLOAT_TYPE //NOPMD CompareObjectsWithEquals - identity is meant + || t == Py.FROZENSET_TYPE || t == Py.INT_TYPE || t == Py.LIST_TYPE || t == Py.SET_TYPE //NOPMD CompareObjectsWithEquals - identity is meant + || t == Py.STR_TYPE || t == Py.TUPLE_TYPE; //NOPMD CompareObjectsWithEquals - identity is meant + } + + private static boolean cls(Object[] p, Object s, PyDict out) { + if (!(p[1] instanceof PyType)) { + throw Py.typeError("called match pattern must be a class"); + } + PyType t = (PyType) p[1]; + if (!Py.isinstance(s, t)) { + return false; + } + Object[] pos = parts(p[2]); + Object[] kwNames = parts(p[3]); + Object[] kwPats = parts(p[4]); + if (pos.length > 0) { + if (selfMatching(t)) { + if (pos.length > 1) { + throw Py.typeError(t.name + "() accepts 1 positional sub-pattern (" + pos.length + " given)"); + } + if (!match(pos[0], s, out)) { + return false; + } + } else { + Object ma = t.getAttrOrNull("__match_args__"); + Object[] names = ma == null ? new Object[0] : Py.toArray(ma); + if (pos.length > names.length) { + throw Py.typeError(t.name + "() accepts " + names.length + " positional sub-pattern" + + (names.length == 1 ? "" : "s") + " (" + pos.length + " given)"); + } + for (int i = 0; i < pos.length; i++) { + Object a = Py.obj(s).getAttrOrNull(Py.toStr(names[i])); + if (a == null || !match(pos[i], a, out)) { + return false; + } + } + } + } + for (int i = 0; i < kwNames.length; i++) { + Object a = Py.obj(s).getAttrOrNull(Py.toStr(kwNames[i])); + if (a == null || !match(kwPats[i], a, out)) { + return false; + } + } + return true; + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyMemoryView.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyMemoryView.java new file mode 100644 index 00000000000..34d59a2e77e --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyMemoryView.java @@ -0,0 +1,245 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +/// `memoryview` over bytes or a bytearray: a window of unsigned bytes +/// (format `B`) that shares the bytearray's storage, so writes through the +/// view land in it. Only one-dimensional byte views are supported; `cast` +/// to another format is not. +public final class PyMemoryView extends PyObject { + private final Object obj; + private final int start; + private final int step; + private final int length; + private boolean released; + + private PyMemoryView(Object obj, int start, int step, int length) { + this.obj = obj; + this.start = start; + this.step = step; + this.length = length; + } + + @Override + public PyType type() { + return Py.MEMORYVIEW_TYPE; + } + + private byte[] store() { + if (released) { + throw Py.valueError("operation forbidden on released memoryview object"); + } + if (obj instanceof PyByteArray) { + return ((PyByteArray) obj).buf; + } + return ((PyBytes) obj).value; + } + + byte[] bytes() { + byte[] s = store(); + byte[] r = new byte[length]; + for (int i = 0; i < length; i++) { + r[i] = s[start + i * step]; + } + return r; + } + + @Override + public int len() { + store(); + return length; + } + + @Override + public String repr() { + return released ? "" + : ""; + } + + @Override + public long hash() { + if (obj instanceof PyByteArray) { + throw Py.valueError("cannot hash writable memoryview object"); + } + return new PyBytes(bytes()).hash(); + } + + @Override + public Object eq(Object o) { + if (o == this) { //NOPMD CompareObjectsWithEquals - identity is meant + return Py.TRUE; + } + byte[] b = PyBytes.like(o); + if (b == null) { + return Py.FALSE; + } + return Py.bool(PyBytes.same(bytes(), b)); + } + + @Override + public Object getItem(Object key) { + store(); + if (key instanceof PySlice) { + int[] r = ((PySlice) key).indices(length); + return new PyMemoryView(obj, start + r[0] * step, step * r[2], PySlice.length(r)); + } + return PyInt.of(store()[start + index(key) * step] & 0xff); + } + + private int index(Object key) { + long i = Py.index(key); + if (i < 0) { + i += length; + } + if (i < 0 || i >= length) { + throw Py.indexError("index out of bounds on dimension 1"); + } + return (int) i; + } + + @Override + public void setItem(Object key, Object value) { + if (!(obj instanceof PyByteArray)) { + throw Py.typeError("cannot modify read-only memory"); + } + byte[] s = store(); + if (key instanceof PySlice) { + int[] r = ((PySlice) key).indices(length); + int n = PySlice.length(r); + byte[] src = PyBytes.like(value); + if (src == null) { + throw Py.typeError("a bytes-like object is required, not '" + Py.typeOf(value).name + "'"); + } + if (src.length != n) { + throw Py.valueError("memoryview assignment: lvalue and rvalue have different structures"); + } + for (int i = 0; i < n; i++) { + s[start + (r[0] + i * r[2]) * step] = src[i]; + } + return; + } + long v = Py.asLong(value); + if (v < 0 || v > 255) { + throw Py.valueError("memoryview: invalid value for format 'B'"); + } + s[start + index(key) * step] = (byte) v; + } + + @Override + public Object iter() { + return byteIter(bytes()); + } + + private static PyIter byteIter(final byte[] b) { + return new PyIter(b.length) { + @Override + Object at(int i) { + return PyInt.of(b[i] & 0xff); + } + }; + } + + @Override + public Object getAttrOrNull(String name) { + if ("obj".equals(name)) { + store(); + return obj; + } + if ("nbytes".equals(name)) { + store(); + return PyInt.of(length); + } + if ("readonly".equals(name)) { + store(); + return Py.bool(!(obj instanceof PyByteArray)); + } + if ("format".equals(name)) { + return Py.str("B"); + } + if ("itemsize".equals(name) || "ndim".equals(name)) { + return PyInt.of(1); + } + if ("shape".equals(name)) { + return new PyTuple(new Object[] {PyInt.of(length)}); + } + if ("strides".equals(name)) { + return new PyTuple(new Object[] {PyInt.of(step)}); + } + if ("contiguous".equals(name) || "c_contiguous".equals(name) || "f_contiguous".equals(name)) { + return Py.bool(step == 1); + } + return super.getAttrOrNull(name); + } + + private static PyMemoryView self(Object o) { + if (!(o instanceof PyMemoryView)) { + throw Py.typeError("descriptor requires a 'memoryview' object but received a '" + + Py.typeOf(o).name + "'"); + } + return (PyMemoryView) o; + } + + static void install() { + PyType t = Py.MEMORYVIEW_TYPE; + t.define("tobytes", PyBuiltin.method(t, "tobytes", a -> new PyBytes(self(a[0]).bytes()))); + t.define("tolist", PyBuiltin.method(t, "tolist", a -> { + byte[] b = self(a[0]).bytes(); + Object[] r = new Object[b.length]; + for (int i = 0; i < b.length; i++) { + r[i] = PyInt.of(b[i] & 0xff); + } + return new PyList(r); + })); + t.define("hex", PyBuiltin.method(t, "hex", a -> Py.call(Py.BYTES_TYPE.lookup("hex"), + new PyBytes(self(a[0]).bytes())))); + t.define("release", PyBuiltin.method(t, "release", a -> { + self(a[0]).released = true; + return Py.NONE; + })); + t.define("toreadonly", PyBuiltin.method(t, "toreadonly", a -> { + PyMemoryView m = self(a[0]); + return new PyMemoryView(new PyBytes(m.bytes()), 0, 1, m.length); + })); + t.define("__enter__", PyBuiltin.method(t, "__enter__", a -> self(a[0]))); + t.define("__exit__", PyBuiltin.method(t, "__exit__", a -> { + self(a[0]).released = true; + return Py.FALSE; + })); + t.factory = (type, args, kw) -> { + Object o = args.length > 0 ? args[0] : null; + if (o instanceof PyBytes) { + return new PyMemoryView(o, 0, 1, ((PyBytes) o).value.length); + } + if (o instanceof PyByteArray) { + return new PyMemoryView(o, 0, 1, ((PyByteArray) o).size); + } + if (o instanceof PyMemoryView) { + PyMemoryView m = (PyMemoryView) o; + return new PyMemoryView(m.obj, m.start, m.step, m.length); + } + throw Py.typeError("memoryview: a bytes-like object is required, not '" + + (o == null ? "NoneType" : Py.typeOf(o).name) + "'"); + }; + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyMethod.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyMethod.java new file mode 100644 index 00000000000..43926ea4789 --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyMethod.java @@ -0,0 +1,95 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +/// A function bound to the object it was read from (`obj.method`). +public final class PyMethod extends PyObject { + public final Object self; + public final Object func; + + public PyMethod(Object self, Object func) { + this.self = self; + this.func = func; + } + + @Override + public PyType type() { + return Py.METHOD_TYPE; + } + + @Override + public Object call(Object[] args, String[] kwNames) { + return Py.obj(func).call(Py.prepend(self, args), kwNames); + } + + @Override + public Object call0() { + return Py.obj(func).call1(self); + } + + @Override + public Object call1(Object a) { + return Py.obj(func).call2(self, a); + } + + @Override + public Object call2(Object a, Object b) { + return Py.obj(func).call3(self, a, b); + } + + @Override + public Object call3(Object a, Object b, Object c) { + return Py.obj(func).call4(self, a, b, c); + } + + @Override + public Object getAttrOrNull(String name) { + if ("__self__".equals(name)) { + return self; + } + if ("__func__".equals(name)) { + return func; + } + return Py.obj(func).getAttrOrNull(name); + } + + @Override + public Object eq(Object o) { + if (o instanceof PyMethod) { + PyMethod m = (PyMethod) o; + return Py.bool(m.self == self && m.func == func); //NOPMD CompareObjectsWithEquals - identity is meant + } + return null; + } + + @Override + public long hash() { + return System.identityHashCode(self) * 31L + System.identityHashCode(func); + } + + @Override + public String repr() { + return ""; + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyModule.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyModule.java new file mode 100644 index 00000000000..6844f2fda8f --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyModule.java @@ -0,0 +1,245 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; + +/// A Python module. +/// +/// Every global the transpiler can see being assigned gets a fixed slot, and +/// generated code reads and writes `slots[i]` directly instead of hashing the +/// name. Names created at run time (`setattr(module, ...)`, `globals()[k]`) +/// go to an overflow map. Modules register a loader by name, so `import` +/// never looks a class up by reflection. +public final class PyModule extends PyObject { + + /// Runs a module body, filling its globals. + public interface Loader { + void exec(PyModule m); + } + + private static final Map LOADERS = new HashMap(); + private static final Map LOADED = new HashMap(); + /// `sys.modules`: every module as it starts loading. A program may read + /// and change it, but importing goes by the loaded table alone. + static final PyDict SYS_MODULES = new PyDict(); + + public final String name; + public final Object[] slots; + private final String[] slotNames; + private Map extra; + + public PyModule(String name, String[] slotNames) { + this.name = name; + this.slotNames = slotNames; + this.slots = new Object[slotNames.length]; + } + + /// Registers `loader` as the implementation of module `name`. + public static void register(String name, Loader loader) { + LOADERS.put(name, loader); + } + + /// Registers an already populated module (runtime-provided modules). + public static void registerModule(PyModule m) { + LOADED.put(m.name, m); + SYS_MODULES.setItem(m.name, m); + } + + static PyModule find(String name) { + PyModule m = LOADED.get(name); + if (m == null && LOADERS.containsKey(name)) { + m = load(name); + } + return m; + } + + /// `import name`: runs the module body once, parents first. + public static PyModule load(String name) { + PyModule m = LOADED.get(name); + if (m != null) { + return m; + } + int dot = name.lastIndexOf('.'); + if (dot > 0) { + load(name.substring(0, dot)); + } + m = LOADED.get(name); + if (m != null) { + return m; + } + Loader l = LOADERS.get(name); + if (l == null) { + if (StdModules.provide(name)) { + m = LOADED.get(name); + if (m != null) { + return m; + } + } + throw Py.error(Py.MODULE_NOT_FOUND_ERROR, "No module named '" + name + "'"); + } + l.exec(null); + m = LOADED.get(name); + if (m == null) { + throw Py.error(Py.IMPORT_ERROR, "module '" + name + "' did not initialise"); + } + return m; + } + + /// Called by a generated module body before it runs, so a circular import + /// sees the partially initialised module as CPython does. + public static PyModule begin(String name, String[] slotNames) { + PyModule m = new PyModule(name, slotNames); + LOADED.put(name, m); + SYS_MODULES.setItem(name, m); + int dot = name.lastIndexOf('.'); + if (dot > 0) { + PyModule parent = LOADED.get(name.substring(0, dot)); + if (parent != null) { + parent.setAttr(name.substring(dot + 1), m); + } + } + return m; + } + + @Override + public PyType type() { + return Py.MODULE_TYPE; + } + + private int slotOf(String n) { + for (int i = 0; i < slotNames.length; i++) { + if (slotNames[i].equals(n)) { + return i; + } + } + return -1; + } + + /// A global the generated code reads that was never assigned: the + /// builtin of that name when there is one (a module may shadow `property` + /// and still read the builtin before its own assignment), else NameError. + public Object unbound(int slot) { + Object v = Builtins.lookup(slotNames[slot]); + if (v == null) { + throw Py.nameError(slotNames[slot]); + } + return v; + } + + @Override + public Object getAttrOrNull(String n) { + int s = slotOf(n); + if (s >= 0 && slots[s] != null) { + return slots[s]; + } + if (extra != null) { + Object v = extra.get(n); + if (v != null) { + return v; + } + } + if ("__name__".equals(n)) { + return Py.str(name); + } + return null; + } + + @Override + public void setAttr(String n, Object value) { + int s = slotOf(n); + if (s >= 0) { + slots[s] = value; + return; + } + if (extra == null) { + extra = new LinkedHashMap(); + } + extra.put(n, value); + } + + @Override + public void delAttr(String n) { + int s = slotOf(n); + if (s >= 0 && slots[s] != null) { + slots[s] = null; + return; + } + if (extra == null || extra.remove(n) == null) { + throw Py.attributeError("module '" + name + "' has no attribute '" + n + "'"); + } + } + + /// `globals()`: a snapshot, since the slots are not a dict. + public PyDict globals() { + PyDict d = new PyDict(); + d.setItem(Py.str("__name__"), Py.str(name)); + for (int i = 0; i < slots.length; i++) { + if (slots[i] != null) { + d.setItem(Py.str(slotNames[i]), slots[i]); + } + } + if (extra != null) { + for (Map.Entry e : extra.entrySet()) { + d.setItem(Py.str(e.getKey()), e.getValue()); + } + } + return d; + } + + /// The names `from m import *` copies: `__all__` when defined, else every + /// public global. + public String[] exportedNames() { + Object all = getAttrOrNull("__all__"); + if (all != null) { + Object[] items = Py.toArray(all); + String[] r = new String[items.length]; + for (int i = 0; i < items.length; i++) { + r[i] = Py.toStr(items[i]); + } + return r; + } + java.util.List r = new java.util.ArrayList(); + for (int i = 0; i < slots.length; i++) { + if (slots[i] != null && !slotNames[i].startsWith("_")) { + r.add(slotNames[i]); + } + } + if (extra != null) { + for (String k : extra.keySet()) { + if (!k.startsWith("_")) { + r.add(k); + } + } + } + return r.toArray(new String[r.size()]); + } + + @Override + public String repr() { + return ""; + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyNone.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyNone.java new file mode 100644 index 00000000000..7e7eb44e82c --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyNone.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +/// The singletons `None`, `NotImplemented` and `Ellipsis`. +final class PyNone extends PyObject { + private final String name; + private final PyType type; + + PyNone(String name, PyType type) { + this.name = name; + this.type = type; + } + + @Override + public PyType type() { + return type == null ? Py.NONE_TYPE : type; + } + + @Override + public String repr() { + return name; + } + + @Override + public boolean isTrue() { + return this != Py.NONE; //NOPMD CompareObjectsWithEquals - identity is meant + } + + @Override + public Object toJava() { + return this == Py.NONE ? null : this; //NOPMD CompareObjectsWithEquals - identity is meant + } + + @Override + public long hash() { + return name.hashCode(); + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyObject.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyObject.java new file mode 100644 index 00000000000..af290e85f65 --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyObject.java @@ -0,0 +1,344 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +/// Root of the Python object model that transpiled code runs against. +/// +/// Every Python value is a `PyObject`. The operations Python spells with +/// operators and dunder methods are virtual methods here; a type overrides the +/// ones it supports and the rest raise the `TypeError` CPython would. Binary +/// operators return `null` for "not implemented" so that [Py] can try the +/// reflected operation on the right operand, exactly as CPython's +/// `NotImplemented` protocol does. +public abstract class PyObject { + + /// The Python type of this value. + public abstract PyType type(); + + // ---- conversion ------------------------------------------------------ + + /// `repr(self)`. + public String repr() { + return "<" + type().name + " object>"; + } + + /// `str(self)`; defaults to [#repr()] as in CPython. + public String str() { + return repr(); + } + + /// `bool(self)`. + public boolean isTrue() { + return true; + } + + // ---- attributes ------------------------------------------------------ + + /// `self.name`, or `null` when the attribute does not exist. + public Object getAttrOrNull(String name) { + Object attr = type().lookup(name); + if (attr != null) { + return Py.bindAttr(attr, this); + } + if ("__class__".equals(name)) { + return type(); + } + return null; + } + + /// `self.name`, raising `AttributeError` when it does not exist. + public final Object getAttr(String name) { + Object v = getAttrOrNull(name); + if (v == null) { + if (this instanceof PyType) { + throw Py.attributeError("type object '" + ((PyType) this).name + "' has no attribute '" + name + "'"); + } + throw Py.attributeError("'" + type().name + "' object has no attribute '" + name + "'"); + } + return v; + } + + /// `self.name = value`. + public void setAttr(String name, Object value) { + throw Py.attributeError("'" + type().name + "' object attribute '" + name + "' is read-only"); + } + + /// `del self.name`. + public void delAttr(String name) { + throw Py.attributeError("'" + type().name + "' object attribute '" + name + "' is read-only"); + } + + /// Descriptor protocol: what an attribute found on a type becomes when read + /// through an instance. Plain values bind to themselves. + public Object bind(Object self) { + return this; + } + + // ---- calls ----------------------------------------------------------- + + /// `self(*args, **kwargs)`. `kwNames` names the trailing `kwNames.length` + /// entries of `args`; it is never `null`. + public Object call(Object[] args, String[] kwNames) { + throw Py.typeError("'" + type().name + "' object is not callable"); + } + + // Positional calls of a fixed arity. A function compiled with that many + // parameters takes them directly; everything else gets the argument array + // it would have been handed anyway. + + public Object call0() { + return call(Py.NO_ARGS, Py.NO_KW); + } + + public Object call1(Object a) { + return call(new Object[] {a}, Py.NO_KW); + } + + public Object call2(Object a, Object b) { + return call(new Object[] {a, b}, Py.NO_KW); + } + + public Object call3(Object a, Object b, Object c) { + return call(new Object[] {a, b, c}, Py.NO_KW); + } + + public Object call4(Object a, Object b, Object c, Object d) { + return call(new Object[] {a, b, c, d}, Py.NO_KW); + } + + // ---- numbers --------------------------------------------------------- + + public Object add(Object o) { + return null; + } + + public Object radd(Object o) { + return null; + } + + public Object sub(Object o) { + return null; + } + + public Object rsub(Object o) { + return null; + } + + public Object mul(Object o) { + return null; + } + + public Object rmul(Object o) { + return null; + } + + public Object truediv(Object o) { + return null; + } + + public Object rtruediv(Object o) { + return null; + } + + public Object floordiv(Object o) { + return null; + } + + public Object rfloordiv(Object o) { + return null; + } + + public Object mod(Object o) { + return null; + } + + public Object rmod(Object o) { + return null; + } + + public Object pow(Object o) { + return null; + } + + public Object rpow(Object o) { + return null; + } + + public Object lshift(Object o) { + return null; + } + + public Object rshift(Object o) { + return null; + } + + public Object and(Object o) { + return null; + } + + public Object rand(Object o) { + return null; + } + + public Object or(Object o) { + return null; + } + + public Object ror(Object o) { + return null; + } + + public Object xor(Object o) { + return null; + } + + public Object rxor(Object o) { + return null; + } + + public Object neg() { + throw Py.typeError("bad operand type for unary -: '" + type().name + "'"); + } + + public Object pos() { + throw Py.typeError("bad operand type for unary +: '" + type().name + "'"); + } + + public Object invert() { + throw Py.typeError("bad operand type for unary ~: '" + type().name + "'"); + } + + /// In-place variants default to the plain operator; mutable types override. + public Object iadd(Object o) { + return null; + } + + /// The other in-place operators: `dunder` names one (`__isub__`); null + /// means "not supported in place", and the plain operator runs instead. + public Object inplace(String dunder, Object o) { + return null; + } + + // ---- comparison ------------------------------------------------------ + + /// `self == o`, or `null` for not implemented. + public Object eq(Object o) { + return this == o ? Py.TRUE : null; //NOPMD CompareObjectsWithEquals - identity is meant + } + + public Object lt(Object o) { + return null; + } + + public Object le(Object o) { + return null; + } + + public Object gt(Object o) { + return null; + } + + public Object ge(Object o) { + return null; + } + + /// `hash(self)`. + public long hash() { + return System.identityHashCode(this); + } + + // ---- containers ------------------------------------------------------ + + public Object getItem(Object key) { + throw Py.typeError("'" + type().name + "' object is not subscriptable"); + } + + public void setItem(Object key, Object value) { + throw Py.typeError("'" + type().name + "' object does not support item assignment"); + } + + public void delItem(Object key) { + throw Py.typeError("'" + type().name + "' object does not support item deletion"); + } + + public int len() { + throw Py.typeError("object of type '" + type().name + "' has no len()"); + } + + /// `x in self`. + public boolean contains(Object x) { + Object it = iter(); + for (Object v = Py.next(it); v != null; v = Py.next(it)) { + if (Py.eqBool(v, x)) { + return true; + } + } + return false; + } + + /// `iter(self)`. + public Object iter() { + throw Py.typeError("'" + type().name + "' object is not iterable"); + } + + /// `next(self)`, returning `null` at exhaustion instead of raising + /// `StopIteration`, so loops need no exception. + public Object next() { + throw Py.typeError("'" + type().name + "' object is not an iterator"); + } + + // ---- Java interop ---------------------------------------------------- + + /// The natural Java value: `Long`, `Double`, `String`, `Boolean`, `null`, + /// or the object itself when there is none. + public Object toJava() { + return this; + } + + /// `int(self)` as a Java index; raises `TypeError` for non-integers. + public long asLong() { + throw Py.typeError("'" + type().name + "' object cannot be interpreted as an integer"); + } + + /// `float(self)`. + public double asDouble() { + throw Py.typeError("must be real number, not " + type().name); + } + + @Override + public String toString() { + return str(); + } + + @Override + public boolean equals(Object o) { + return o instanceof PyObject && Py.eqBool(this, (PyObject) o); + } + + @Override + public int hashCode() { + long h = hash(); + return (int) (h ^ (h >>> 32)); + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyProperty.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyProperty.java new file mode 100644 index 00000000000..584dd8b1c4c --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyProperty.java @@ -0,0 +1,108 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +/// `property(fget, fset, fdel)`, the descriptor `@property` produces. +public final class PyProperty extends PyObject { + // Not final: a property subclass's instance holds an empty one that its + // __init__ fills (see BuiltinSubclass). + Object fget; + Object fset; + Object fdel; + + public PyProperty(Object fget, Object fset, Object fdel) { + this.fget = fget; + this.fset = fset; + this.fdel = fdel; + } + + @Override + public PyType type() { + return Py.PROPERTY_TYPE; + } + + void fill(PyProperty from) { + fget = from.fget; + fset = from.fset; + fdel = from.fdel; + } + + Object get(Object self) { + if (fget == null || fget == Py.NONE) { //NOPMD CompareObjectsWithEquals - identity is meant + throw Py.attributeError("property has no getter"); + } + return Py.obj(fget).call(new Object[] {self}, Py.NO_KW); + } + + void delete(Object self) { + if (fdel == null || fdel == Py.NONE) { //NOPMD CompareObjectsWithEquals - identity is meant + throw Py.attributeError("property has no deleter"); + } + Py.obj(fdel).call(new Object[] {self}, Py.NO_KW); + } + + void set(Object self, Object value) { + if (fset == null || fset == Py.NONE) { //NOPMD CompareObjectsWithEquals - identity is meant + throw Py.attributeError("property has no setter"); + } + Py.obj(fset).call(new Object[] {self, value}, Py.NO_KW); + } + + @Override + public Object getAttrOrNull(String name) { + if ("setter".equals(name)) { + return PyBuiltin.positional("setter", args -> new PyProperty(fget, args[0], fdel)); + } + if ("getter".equals(name)) { + return PyBuiltin.positional("getter", args -> new PyProperty(args[0], fset, fdel)); + } + if ("deleter".equals(name)) { + return PyBuiltin.positional("deleter", args -> new PyProperty(fget, fset, args[0])); + } + if ("fget".equals(name)) { + return fget == null ? Py.NONE : fget; + } + if ("fset".equals(name)) { + return fset == null ? Py.NONE : fset; + } + if ("fdel".equals(name)) { + return fdel == null ? Py.NONE : fdel; + } + if ("__isabstractmethod__".equals(name)) { + Object[] parts = {fget, fset, fdel}; + for (Object part : parts) { + Object v = part == null ? null : Py.obj(part).getAttrOrNull(name); + if (v != null && Py.isTrue(v)) { + return Py.TRUE; + } + } + return Py.FALSE; + } + if ("__doc__".equals(name)) { + Object v = fget == null ? null : Py.obj(fget).getAttrOrNull(name); + return v != null ? v : Py.NONE; + } + return super.getAttrOrNull(name); + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyRange.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyRange.java new file mode 100644 index 00000000000..37c37402619 --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyRange.java @@ -0,0 +1,112 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +/// `range(start, stop, step)`, lazily. +public final class PyRange extends PyObject { + final long start; + final long stop; + final long step; + + public PyRange(long start, long stop, long step) { + if (step == 0) { + throw Py.valueError("range() arg 3 must not be zero"); + } + this.start = start; + this.stop = stop; + this.step = step; + } + + @Override + public PyType type() { + return Py.RANGE_TYPE; + } + + public long length() { + if (step > 0) { + return stop > start ? (stop - start + step - 1) / step : 0; + } + return start > stop ? (start - stop - step - 1) / -step : 0; + } + + @Override + public int len() { + return (int) length(); + } + + @Override + public boolean isTrue() { + return length() != 0; + } + + @Override + public Object iter() { + final long n = length(); + return new PyJavaIter(new PyJavaIter.Source() { + long i; + + @Override + public Object next() { + if (i >= n) { + return null; + } + return PyInt.of(start + step * i++); + } + }); + } + + @Override + public Object getItem(Object key) { + long n = length(); + if (key instanceof PySlice) { + int[] r = ((PySlice) key).indices((int) n); + return new PyRange(start + r[0] * step, start + r[1] * step, step * r[2]); + } + long i = Py.index(key); + if (i < 0) { + i += n; + } + if (i < 0 || i >= n) { + throw Py.indexError("range object index out of range"); + } + return PyInt.of(start + i * step); + } + + @Override + public boolean contains(Object x) { + if (!(Py.isInt(x)) || Py.intView(x).isBig()) { + return super.contains(x); + } + long v = Py.intView(x).value; + if (step > 0 ? v < start || v >= stop : v > start || v <= stop) { + return false; + } + return (v - start) % step == 0; + } + + @Override + public String repr() { + return step == 1 ? "range(" + start + ", " + stop + ")" : "range(" + start + ", " + stop + ", " + step + ")"; + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PySeqIter.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PySeqIter.java new file mode 100644 index 00000000000..e7775ce33b0 --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PySeqIter.java @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +/// The legacy iteration protocol: `__getitem__(0)`, `(1)`, ... until +/// `IndexError`. +final class PySeqIter extends PyObject { + private final Object seq; + private long i; + + PySeqIter(Object seq) { + this.seq = seq; + } + + @Override + public PyType type() { + return Py.ITERATOR_TYPE; + } + + @Override + public Object iter() { + return this; + } + + @Override + public Object next() { + try { + return Py.obj(seq).getItem(PyInt.of(i++)); + } catch (PyException e) { + if (e.matches(Py.INDEX_ERROR) || e.matches(Py.STOP_ITERATION)) { + return null; + } + throw e; + } + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PySet.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PySet.java new file mode 100644 index 00000000000..e23e4743808 --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PySet.java @@ -0,0 +1,587 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +/// Python's `set` and `frozenset`. +/// +/// The table follows CPython's `setobject.c` -- open addressing with the +/// same size policy, linear probes, perturbation and dummy entries -- so a +/// set iterates in the same order as on CPython wherever CPython's own +/// order is deterministic (numbers and tuples; str hashes are randomized +/// per process there, so no program can depend on their order). +public final class PySet extends PyObject { + private static final int MINSIZE = 8; + private static final int LINEAR_PROBES = 9; + private static final int PERTURB_SHIFT = 5; + /// Marks a deleted entry; its hash slot holds -1. + private static final Object DUMMY = new PyObject() { + @Override + public PyType type() { + return Py.OBJECT_TYPE; + } + }; + + final boolean frozen; + private Object[] keys = new Object[MINSIZE]; + private long[] hashes = new long[MINSIZE]; + private int mask = MINSIZE - 1; + /// Active plus dummy entries. + private int fill; + /// Active entries. + private int used; + /// Where pop() resumes its search. + private int finger; + + public PySet(boolean frozen) { + this.frozen = frozen; + } + + public PySet() { + this(false); + } + + public static PySet of(Object iterable, boolean frozen) { + PySet s = new PySet(frozen); + s.update(iterable); + return s; + } + + @Override + public PyType type() { + return frozen ? Py.FROZENSET_TYPE : Py.SET_TYPE; + } + + private void checkMutable() { + if (frozen) { + throw Py.attributeError("'frozenset' object is immutable"); + } + } + + // ---- the table ------------------------------------------------------- + + private static boolean same(Object a, Object b) { + return a == b || Py.eqBool(a, b); //NOPMD CompareObjectsWithEquals - identity is meant + } + + private void insert(Object key, long hash) { + long perturb = hash; + int i = (int) (hash & mask); + int freeslot = -1; + while (true) { + int probes = i + LINEAR_PROBES <= mask ? LINEAR_PROBES : 0; + int j = i; + while (true) { + Object k = keys[j]; + if (k == null) { + if (freeslot >= 0) { + keys[freeslot] = key; + hashes[freeslot] = hash; + used++; + return; + } + keys[j] = key; + hashes[j] = hash; + fill++; + used++; + if ((long) fill * 5 >= (long) mask * 3) { + resize(used > 50000 ? used * 2 : used * 4); + } + return; + } + if (k == DUMMY) { + if (freeslot < 0) { + freeslot = j; + } + } else if (hashes[j] == hash && same(k, key)) { + return; + } + if (probes == 0) { + break; + } + probes--; + j++; + } + perturb >>>= PERTURB_SHIFT; + i = (int) ((i * 5L + 1 + perturb) & mask); + } + } + + private int find(Object key, long hash) { + long perturb = hash; + int i = (int) (hash & mask); + while (true) { + int probes = i + LINEAR_PROBES <= mask ? LINEAR_PROBES : 0; + int j = i; + while (true) { + Object k = keys[j]; + if (k == null) { + return -1; + } + if (k != DUMMY && hashes[j] == hash && same(k, key)) { + return j; + } + if (probes == 0) { + break; + } + probes--; + j++; + } + perturb >>>= PERTURB_SHIFT; + i = (int) ((i * 5L + 1 + perturb) & mask); + } + } + + /// `set_insert_clean`: into a table with no dummies and no equal key. + private static void insertClean(Object[] ks, long[] hs, int m, Object key, long hash) { + long perturb = hash; + int i = (int) (hash & m); + while (true) { + if (ks[i] == null) { + ks[i] = key; + hs[i] = hash; + return; + } + if (i + LINEAR_PROBES <= m) { + for (int j = 1; j <= LINEAR_PROBES; j++) { + if (ks[i + j] == null) { + ks[i + j] = key; + hs[i + j] = hash; + return; + } + } + } + perturb >>>= PERTURB_SHIFT; + i = (int) ((i * 5L + 1 + perturb) & m); + } + } + + private void resize(int minused) { + int size = MINSIZE; + while (size <= minused) { + size <<= 1; + } + Object[] oldKeys = keys; + long[] oldHashes = hashes; + keys = new Object[size]; + hashes = new long[size]; + mask = size - 1; + for (int i = 0; i < oldKeys.length; i++) { + Object k = oldKeys[i]; + if (k != null && k != DUMMY) { + insertClean(keys, hashes, mask, k, oldHashes[i]); + } + } + fill = used; + } + + private static long hashOf(Object key) { + return Py.keyHash(key, "a set element"); + } + + // ---- element operations ---------------------------------------------- + + public void addItem(Object v) { + checkMutable(); + insert(v, hashOf(v)); + } + + /// Adds without the frozen check, for building a new set. + void addRaw(Object v) { + insert(v, hashOf(v)); + } + + public boolean discard(Object v) { + checkMutable(); + int i = find(v, hashOf(v)); + if (i < 0) { + return false; + } + keys[i] = DUMMY; + hashes[i] = -1; + used--; + return true; + } + + public void clear() { + checkMutable(); + keys = new Object[MINSIZE]; + hashes = new long[MINSIZE]; + mask = MINSIZE - 1; + fill = 0; + used = 0; + finger = 0; + } + + /// `set.pop()`: CPython's finger walk, so successive pops come out in + /// table order. + public Object pop() { + checkMutable(); + if (used == 0) { + throw Py.error(Py.KEY_ERROR, "pop from an empty set"); + } + int i = finger & mask; + while (keys[i] == null || keys[i] == DUMMY) { + i++; + if (i > mask) { + i = 0; + } + } + Object k = keys[i]; + keys[i] = DUMMY; + hashes[i] = -1; + used--; + finger = i + 1; + return k; + } + + /// The members in iteration order. + public Object[] toArray() { + Object[] r = new Object[used]; + int n = 0; + for (int i = 0; i <= mask; i++) { + Object k = keys[i]; + if (k != null && k != DUMMY) { + r[n++] = k; + } + } + return r; + } + + /// `set_update_internal`: a set or dict merges in table/insertion order + /// after sizing the table once, as CPython does. + void update(Object iterable) { + if (iterable instanceof PySet) { + merge((PySet) iterable); + return; + } + if (iterable instanceof PyDict) { + PyDict d = (PyDict) iterable; + int n = d.len(); + if ((long) (fill + n) * 5 >= (long) mask * 3) { + resize((used + n) * 2); + } + Object[] ks = d.keyArray(); + for (Object e : ks) { + insert(e, hashOf(e)); + } + return; + } + Object it = Py.iter(iterable); + for (Object v = Py.next(it); v != null; v = Py.next(it)) { + insert(v, hashOf(v)); + } + } + + /// `set_merge`. + /// A set display whose elements are all constants: CPython compiles it + /// to a frozenset constant merged into an empty set, which sizes the + /// table differently from adding the elements one by one, and so + /// iterates in a different order. + public static PySet ofConstants(Object... items) { + PySet constant = new PySet(true); + for (Object item : items) { + constant.addRaw(item); + } + PySet s = new PySet(); + s.merge(constant); + return s; + } + + private void merge(PySet other) { + if (other == this || other.used == 0) { //NOPMD CompareObjectsWithEquals - identity is meant + return; + } + if ((long) (fill + other.used) * 5 >= (long) mask * 3) { + resize((used + other.used) * 2); + } + if (fill == 0 && mask == other.mask && other.fill == other.used) { + for (int i = 0; i <= other.mask; i++) { + if (other.keys[i] != null) { + keys[i] = other.keys[i]; + hashes[i] = other.hashes[i]; + } + } + fill = other.used; + used = other.used; + return; + } + if (fill == 0) { + for (int i = 0; i <= other.mask; i++) { + Object k = other.keys[i]; + if (k != null && k != DUMMY) { + insertClean(keys, hashes, mask, k, other.hashes[i]); + fill++; + used++; + } + } + return; + } + for (int i = 0; i <= other.mask; i++) { + Object k = other.keys[i]; + if (k != null && k != DUMMY) { + insert(k, other.hashes[i]); + } + } + } + + PySet copyAs(boolean asFrozen) { + PySet r = new PySet(asFrozen); + r.merge(this); + return r; + } + + @Override + public boolean contains(Object x) { + return find(x, hashOf(x)) >= 0; + } + + @Override + public int len() { + return used; + } + + @Override + public boolean isTrue() { + return used != 0; + } + + @Override + public Object iter() { + final int expected = used; + return new PyJavaIter(new PyJavaIter.Source() { + int pos; + + @Override + public Object next() { + if (used != expected) { + throw Py.runtimeError("Set changed size during iteration"); + } + while (pos <= mask) { + Object k = keys[pos++]; + if (k != null && k != DUMMY) { + return k; + } + } + return null; + } + }); + } + + /// CPython's `frozenset_hash`, so equal frozensets hash alike regardless + /// of insertion history. + @Override + public long hash() { + if (!frozen) { + throw Py.typeError("unhashable type: 'set'"); + } + long hash = 0; + for (int i = 0; i <= mask; i++) { + Object k = keys[i]; + if (k != null && k != DUMMY) { + long h = hashes[i]; + hash ^= ((h ^ 89869747L) ^ (h << 16)) * 3644798167L; + } + } + hash ^= ((long) used + 1) * 1927868237L; + hash ^= (hash >>> 11) ^ (hash >>> 25); + hash = hash * 69069L + 907133923L; + return hash == -1 ? 590923713L : hash; + } + + @Override + public String repr() { + if (used == 0) { + return frozen ? "frozenset()" : "set()"; + } + Object[] k = toArray(); + String body = Py.joinRepr("{", k, k.length, "}"); + return frozen ? "frozenset(" + body + ")" : body; + } + + private static PySet asSet(Object o) { + return o instanceof PySet ? (PySet) o : null; + } + + private static PySet setOf(Object o) { + PySet s = asSet(o); + return s != null ? s : of(o, true); + } + + // ---- set algebra, in CPython's iteration orders --------------------- + + public PySet union(Object o) { + PySet r = copyAs(frozen); + r.update(o); + return r; + } + + public PySet intersection(Object o) { + PySet other = setOf(o); + PySet big = this; + PySet small = other; + if (other.used > used) { + big = other; + small = this; + } + PySet r = new PySet(frozen); + for (int i = 0; i <= small.mask; i++) { + Object k = small.keys[i]; + if (k != null && k != DUMMY && big.find(k, small.hashes[i]) >= 0) { + r.insert(k, small.hashes[i]); + } + } + return r; + } + + public PySet difference(Object o) { + PySet other = setOf(o); + PySet r = new PySet(frozen); + for (int i = 0; i <= mask; i++) { + Object k = keys[i]; + if (k != null && k != DUMMY && other.find(k, hashes[i]) < 0) { + r.insert(k, hashes[i]); + } + } + return r; + } + + public PySet symmetricDifference(Object o) { + PySet other = setOf(o); + PySet r = copyAs(frozen); + for (int i = 0; i <= other.mask; i++) { + Object k = other.keys[i]; + if (k == null || k == DUMMY) { + continue; + } + int at = r.find(k, other.hashes[i]); + if (at >= 0) { + r.keys[at] = DUMMY; + r.hashes[at] = -1; + r.used--; + } else { + r.insert(k, other.hashes[i]); + } + } + return r; + } + + public boolean isSubset(Object o) { + PySet other = setOf(o); + if (used > other.used) { + return false; + } + for (int i = 0; i <= mask; i++) { + Object k = keys[i]; + if (k != null && k != DUMMY && other.find(k, hashes[i]) < 0) { + return false; + } + } + return true; + } + + /// In-place forms: `|=`, `&=`, `-=`, `^=` keep the object's identity. + void replaceWith(PySet s) { + keys = s.keys; + hashes = s.hashes; + mask = s.mask; + fill = s.fill; + used = s.used; + finger = 0; + } + + @Override + public Object inplace(String name, Object o) { + if (frozen || asSet(o) == null) { + return null; + } + switch (name) { + case "__ior__": + update(o); + return this; + case "__iand__": + replaceWith(intersection(o)); + return this; + case "__isub__": + replaceWith(difference(o)); + return this; + case "__ixor__": + replaceWith(symmetricDifference(o)); + return this; + default: + return null; + } + } + + @Override + public Object or(Object o) { + return asSet(o) == null ? null : union(o); + } + + @Override + public Object and(Object o) { + return asSet(o) == null ? null : intersection(o); + } + + @Override + public Object sub(Object o) { + return asSet(o) == null ? null : difference(o); + } + + @Override + public Object xor(Object o) { + return asSet(o) == null ? null : symmetricDifference(o); + } + + @Override + public Object eq(Object o) { + PySet b = asSet(o); + if (b == null) { + return null; + } + return Py.bool(b.used == used && isSubset(b)); + } + + @Override + public Object le(Object o) { + return asSet(o) == null ? null : Py.bool(isSubset(o)); + } + + @Override + public Object lt(Object o) { + PySet b = asSet(o); + return b == null ? null : Py.bool(used < b.used && isSubset(b)); + } + + @Override + public Object ge(Object o) { + PySet b = asSet(o); + return b == null ? null : Py.bool(b.isSubset(this)); + } + + @Override + public Object gt(Object o) { + PySet b = asSet(o); + return b == null ? null : Py.bool(used > b.used && b.isSubset(this)); + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PySlice.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PySlice.java new file mode 100644 index 00000000000..144b51f785c --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PySlice.java @@ -0,0 +1,133 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +/// `slice(start, stop, step)`; any part may be `None`. +public final class PySlice extends PyObject { + final Object start; + final Object stop; + final Object step; + + public PySlice(Object start, Object stop, Object step) { + this.start = start == null ? Py.NONE : start; + this.stop = stop == null ? Py.NONE : stop; + this.step = step == null ? Py.NONE : step; + } + + @Override + public PyType type() { + return Py.SLICE_TYPE; + } + + /// `slice.indices(length)` as `{start, stop, step}`, clamped as CPython does. + public int[] indices(int length) { + int st = step == Py.NONE ? 1 : (int) Py.index(step); //NOPMD CompareObjectsWithEquals - identity is meant + if (st == 0) { + throw Py.valueError("slice step cannot be zero"); + } + int lo; + int hi; + if (st > 0) { + lo = start == Py.NONE ? 0 : clamp(Py.index(start), length, 0, length); //NOPMD CompareObjectsWithEquals - identity is meant + hi = stop == Py.NONE ? length : clamp(Py.index(stop), length, 0, length); //NOPMD CompareObjectsWithEquals - identity is meant + } else { + lo = start == Py.NONE ? length - 1 : clamp(Py.index(start), length, -1, length - 1); //NOPMD CompareObjectsWithEquals - identity is meant + hi = stop == Py.NONE ? -1 : clamp(Py.index(stop), length, -1, length - 1); //NOPMD CompareObjectsWithEquals - identity is meant + } + return new int[] {lo, hi, st}; + } + + /// The step of a slice, checked to be nonzero. + static int step(Object step) { + int st = (int) Py.index(step); + if (st == 0) { + throw Py.valueError("slice step cannot be zero"); + } + return st; + } + + /// The first index a slice selects, as in [#indices(int)]. + static int lower(Object start, Object stop, int st, int length) { + if (st > 0) { + return start == Py.NONE ? 0 : clamp(Py.index(start), length, 0, length); //NOPMD CompareObjectsWithEquals - identity is meant + } + return start == Py.NONE ? length - 1 : clamp(Py.index(start), length, -1, length - 1); //NOPMD CompareObjectsWithEquals - identity is meant + } + + /// The bound a slice stops at, as in [#indices(int)]. + static int upper(Object stop, int st, int length) { + if (st > 0) { + return stop == Py.NONE ? length : clamp(Py.index(stop), length, 0, length); //NOPMD CompareObjectsWithEquals - identity is meant + } + return stop == Py.NONE ? -1 : clamp(Py.index(stop), length, -1, length - 1); //NOPMD CompareObjectsWithEquals - identity is meant + } + + /// The number of indices from `lo` towards `hi` by `st`. + static int count(int lo, int hi, int st) { + if (st > 0) { + return hi > lo ? (hi - lo + st - 1) / st : 0; + } + return lo > hi ? (lo - hi - st - 1) / -st : 0; + } + + private static int clamp(long v, int length, int min, int max) { + if (v < 0) { + v += length; + } + if (v < min) { + return min; + } + if (v > max) { + return max; + } + return (int) v; + } + + /// The number of elements `indices` selects. + public static int length(int[] r) { + if (r[2] > 0) { + return r[1] > r[0] ? (r[1] - r[0] + r[2] - 1) / r[2] : 0; + } + return r[0] > r[1] ? (r[0] - r[1] - r[2] - 1) / -r[2] : 0; + } + + @Override + public Object getAttrOrNull(String name) { + if ("start".equals(name)) { + return start; + } + if ("stop".equals(name)) { + return stop; + } + if ("step".equals(name)) { + return step; + } + return super.getAttrOrNull(name); + } + + @Override + public String repr() { + return "slice(" + Py.toRepr(start) + ", " + Py.toRepr(stop) + ", " + Py.toRepr(step) + ")"; + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyStaticMethod.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyStaticMethod.java new file mode 100644 index 00000000000..197c3afeda5 --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyStaticMethod.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +/// `staticmethod(func)`: never binds. +public final class PyStaticMethod extends PyObject { + final Object func; + + public PyStaticMethod(Object func) { + this.func = func; + } + + @Override + public PyType type() { + return Py.STATICMETHOD_TYPE; + } + + @Override + public Object bind(Object self) { + return func; + } + + @Override + public Object call(Object[] args, String[] kwNames) { + return Py.obj(func).call(args, kwNames); + } + + @Override + public Object getAttrOrNull(String attr) { + if ("__func__".equals(attr) || "__wrapped__".equals(attr)) { + return func; + } + if ("__isabstractmethod__".equals(attr) || "__doc__".equals(attr) || "__name__".equals(attr)) { + Object v = Py.obj(func).getAttrOrNull(attr); + return v != null ? v : "__isabstractmethod__".equals(attr) ? Py.FALSE : Py.NONE; + } + return super.getAttrOrNull(attr); + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyStr.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyStr.java new file mode 100644 index 00000000000..c3edd344244 --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyStr.java @@ -0,0 +1,348 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +/// Python's `str`: an immutable sequence of code points. +/// +/// A Python string IS a `java.lang.String`, the way an int is a `Long`: the +/// transpiled code and the runtime pass `String`s around, and nothing wraps +/// one. `PyStr` is the view [Py#obj(Object)] hands out for the operations +/// that go through the object protocol (attribute lookup, the operators the +/// fast paths in [Py] do not take); it is transient and never a Python value +/// itself. +/// +/// Python indexes by code point and Java by UTF-16 unit; the two agree unless +/// the text holds a supplementary character, so the slow path that translates +/// indices runs only for such strings. +public final class PyStr extends PyObject { + public final String value; + + public PyStr(String value) { + this.value = value; + } + + /// The text of a str value, or null when `o` is not a str. + public static String sv(Object o) { + if (o instanceof String) { + return (String) o; + } + if (o instanceof PyStr) { + return ((PyStr) o).value; + } + return null; + } + + @Override + public PyType type() { + return Py.STR_TYPE; + } + + /// The last long string whose surrogates were counted. Indexing a long + /// string in a loop would otherwise rescan it on every access; one entry + /// covers that loop. The holder is immutable, so a reader on another + /// thread sees either the old pair or the new one, never a mix. + private static final class Count { + final String s; + final int n; + + Count(String s, int n) { + this.s = s; + this.n = n; + } + } + + private static Count lastCount = new Count("", 0); + + /// High surrogates in `s`: zero for every string without a supplementary + /// character. + static int surrogates(String s) { + int len = s.length(); + if (len > 32) { + Count c = lastCount; + if (c.s == s) { //NOPMD CompareObjectsWithEquals - identity is meant + return c.n; + } + } + int n = 0; + for (int i = 0; i < len; i++) { + char c = s.charAt(i); + if (c >= '\uD800' && c <= '\uDBFF') { + n++; + } + } + if (len > 32) { + lastCount = new Count(s, n); + } + return n; + } + + /// Length in code points; a lone surrogate counts as one, as in Python. + public static int codePoints(String s) { + return s.codePointCount(0, s.length()); + } + + public int codePoints() { + return codePoints(value); + } + + /// The UTF-16 offset of code point `cp`. + static int offset(String s, int cp) { + if (surrogates(s) == 0) { + return cp; + } + int off = 0; + for (int i = 0; i < cp; i++) { + off += Character.isHighSurrogate(s.charAt(off)) && off + 1 < s.length() ? 2 : 1; + } + return off; + } + + int offset(int cp) { + return offset(value, cp); + } + + /// The code point index of UTF-16 offset `off`. + static int cpIndex(String s, int off) { + if (surrogates(s) == 0) { + return off; + } + int cp = 0; + for (int i = 0; i < off; i++) { + if (!Character.isLowSurrogate(s.charAt(i)) || i == 0) { + cp++; + } + } + return cp; + } + + int cpIndex(int off) { + return cpIndex(value, off); + } + + /// The substring of code points `[from, to)`. + public static String substr(String s, int from, int to) { + if (surrogates(s) == 0) { + return s.substring(from, to); + } + return s.substring(offset(s, from), offset(s, to)); + } + + public String substr(int from, int to) { + return substr(value, from, to); + } + + @Override + public Object getAttrOrNull(String name) { + // Methods bind to the string itself, never to this transient view. + Object attr = type().lookup(name); + if (attr == null && "__class__".equals(name)) { + return Py.STR_TYPE; + } + return attr != null ? Py.bindAttr(attr, value) : null; + } + + @Override + public String str() { + return value; + } + + @Override + public Object toJava() { + return value; + } + + @Override + public String repr() { + return repr(value); + } + /// CPython's quoting rules: single quotes unless the text holds a single + /// quote and no double quote. + public static String repr(String s) { + char q = s.indexOf('\'') >= 0 && s.indexOf('"') < 0 ? '"' : '\''; + StringBuilder sb = new StringBuilder(s.length() + 2); + sb.append(q); + for (int i = 0; i < s.length(); i++) { + char c = s.charAt(i); + if (c == q || c == '\\') { + sb.append('\\').append(c); + } else if (c == '\n') { + sb.append("\\n"); + } else if (c == '\r') { + sb.append("\\r"); + } else if (c == '\t') { + sb.append("\\t"); + } else if (c < 0x20 || c == 0x7f) { + sb.append("\\x"); + hex(sb, c, 2); + } else { + sb.append(c); + } + } + sb.append(q); + return sb.toString(); + } + + static void hex(StringBuilder sb, int v, int digits) { + String h = Integer.toHexString(v); + for (int i = h.length(); i < digits; i++) { + sb.append('0'); + } + sb.append(h); + } + + @Override + public boolean isTrue() { + return value.length() != 0; + } + + @Override + public int len() { + return codePoints(); + } + + @Override + public long hash() { + return value.hashCode(); + } + + @Override + public Object eq(Object o) { + String b = sv(o); + return b != null ? Py.bool(value.equals(b)) : null; + } + + private int cmp(Object o) { + return compare(value, sv(o)); + } + + /// Code point order; UTF-16 order differs only around surrogates. + static int compare(String a, String b) { + int n = Math.min(a.length(), b.length()); + for (int i = 0; i < n; i++) { + int ca = a.codePointAt(i); + int cb = b.codePointAt(i); + if (ca != cb) { + return ca < cb ? -1 : 1; + } + } + return a.length() - b.length(); + } + + @Override + public Object lt(Object o) { + return sv(o) != null ? Py.bool(cmp(o) < 0) : null; + } + + @Override + public Object le(Object o) { + return sv(o) != null ? Py.bool(cmp(o) <= 0) : null; + } + + @Override + public Object gt(Object o) { + return sv(o) != null ? Py.bool(cmp(o) > 0) : null; + } + + @Override + public Object ge(Object o) { + return sv(o) != null ? Py.bool(cmp(o) >= 0) : null; + } + + @Override + public Object add(Object o) { + String b = sv(o); + return b != null ? value.concat(b) : null; + } + + @Override + public Object mul(Object o) { + if (!(Py.isInt(o))) { + return null; + } + long n = Py.asLong(o); + if (n <= 0 || value.length() == 0) { + return Py.EMPTY_STR; + } + StringBuilder sb = new StringBuilder((int) Math.min(Integer.MAX_VALUE, value.length() * n)); + for (long i = 0; i < n; i++) { + sb.append(value); + } + return sb.toString(); + } + + @Override + public Object rmul(Object o) { + return mul(o); + } + + @Override + public Object mod(Object o) { + return Formatting.percent(value, o); + } + + @Override + public boolean contains(Object x) { + String b = sv(x); + if (b == null) { + throw Py.typeError("'in ' requires string as left operand, not " + Py.typeOf(x).name); + } + return value.indexOf(b) >= 0; + } + + @Override + public Object getItem(Object key) { + int n = codePoints(); + if (key instanceof PySlice) { + PySlice s = (PySlice) key; + int[] r = s.indices(n); + if (r[2] == 1) { + return r[1] <= r[0] ? Py.EMPTY_STR : substr(r[0], r[1]); + } + StringBuilder sb = new StringBuilder(); + int count = PySlice.length(r); + for (int i = 0, j = r[0]; i < count; i++, j += r[2]) { + sb.append(substr(j, j + 1)); + } + return sb.toString(); + } + long i = Py.index(key); + if (i < 0) { + i += n; + } + if (i < 0 || i >= n) { + throw Py.indexError("string index out of range"); + } + return Py.str(substr((int) i, (int) i + 1)); + } + + @Override + public Object iter() { + return new PyIter(codePoints()) { + @Override + Object at(int i) { + return Py.str(substr(i, i + 1)); + } + }; + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PySuper.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PySuper.java new file mode 100644 index 00000000000..e69f814d926 --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PySuper.java @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +/// `super()`: attribute lookup that starts after `cls` in the MRO of `self`. +/// +/// The transpiler turns the zero-argument form into `super(Class, self)` +/// statically, which is what CPython's `__class__` cell does at run time. +public final class PySuper extends PyObject { + final PyType cls; + final Object self; + + public PySuper(PyType cls, Object self) { + this.cls = cls; + this.self = self; + } + + @Override + public PyType type() { + return Py.OBJECT_TYPE; + } + + @Override + public Object getAttrOrNull(String name) { + PyType start = self instanceof PyType && ((PyType) self).isSubtype(cls) ? (PyType) self : Py.typeOf(self); + PyType[] mro = start.mro; + int i = 0; + while (i < mro.length && mro[i] != cls) { //NOPMD CompareObjectsWithEquals - identity is meant + i++; + } + for (i++; i < mro.length; i++) { + Object v = mro[i].dict.get(name); + if (v != null) { + if (v instanceof PyClassMethod) { + return new PyMethod(start, ((PyClassMethod) v).func); + } + if (v instanceof PyStaticMethod) { + return ((PyStaticMethod) v).func; + } + if (v instanceof PyProperty) { + return ((PyProperty) v).get(self); + } + if (self instanceof PyType && "__new__".equals(name)) { + return v; + } + // A class whose own MRO is searched gets the plain function; + // a class searched through its metaclass is an instance. + return start == self ? v : Py.bindAttr(v, self); //NOPMD CompareObjectsWithEquals - identity is meant + } + } + return null; + } + + @Override + public String repr() { + return ", " + Py.toRepr(self) + ">"; + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyTuple.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyTuple.java new file mode 100644 index 00000000000..b4fe03489a1 --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyTuple.java @@ -0,0 +1,171 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +/// Python's `tuple`. +public final class PyTuple extends PyObject { + public static final PyTuple EMPTY = new PyTuple(new Object[0]); + + final Object[] items; + + /// Wraps `items` without copying; the caller must not keep mutating it. + public PyTuple(Object[] items) { + this.items = items; + } + + public static PyTuple of(Object... items) { + return items.length == 0 ? EMPTY : new PyTuple(items); + } + + @Override + public PyType type() { + return Py.TUPLE_TYPE; + } + + public Object get(int i) { + return items[i]; + } + + public int size() { + return items.length; + } + + @Override + public int len() { + return items.length; + } + + @Override + public boolean isTrue() { + return items.length != 0; + } + + @Override + public String repr() { + if (items.length == 1) { + return "(" + Py.toRepr(items[0]) + ",)"; + } + return Py.joinRepr("(", items, items.length, ")"); + } + + @Override + public long hash() { + return Hashing.ofTuple(items); + } + + @Override + public Object eq(Object o) { + if (!(o instanceof PyTuple)) { + return null; + } + Object[] b = ((PyTuple) o).items; + return Py.bool(Py.seqEquals(items, items.length, b, b.length)); + } + + @Override + public Object lt(Object o) { + return o instanceof PyTuple ? Py.bool(cmp(o) < 0) : null; + } + + @Override + public Object le(Object o) { + return o instanceof PyTuple ? Py.bool(cmp(o) <= 0) : null; + } + + @Override + public Object gt(Object o) { + return o instanceof PyTuple ? Py.bool(cmp(o) > 0) : null; + } + + @Override + public Object ge(Object o) { + return o instanceof PyTuple ? Py.bool(cmp(o) >= 0) : null; + } + + private int cmp(Object o) { + Object[] b = ((PyTuple) o).items; + return Py.seqCompare(items, items.length, b, b.length); + } + + @Override + public Object add(Object o) { + if (!(o instanceof PyTuple)) { + return null; + } + Object[] b = ((PyTuple) o).items; + Object[] r = new Object[items.length + b.length]; + System.arraycopy(items, 0, r, 0, items.length); + System.arraycopy(b, 0, r, items.length, b.length); + return new PyTuple(r); + } + + @Override + public Object mul(Object o) { + if (!(Py.isInt(o))) { + return null; + } + long n = Math.max(0, Py.asLong(o)); + Object[] r = new Object[(int) (items.length * n)]; + for (int i = 0; i < n; i++) { + System.arraycopy(items, 0, r, i * items.length, items.length); + } + return new PyTuple(r); + } + + @Override + public Object rmul(Object o) { + return mul(o); + } + + @Override + public Object getItem(Object key) { + if (key instanceof PySlice) { + int[] r = ((PySlice) key).indices(items.length); + int n = PySlice.length(r); + Object[] out = new Object[n]; + for (int i = 0, j = r[0]; i < n; i++, j += r[2]) { + out[i] = items[j]; + } + return new PyTuple(out); + } + long i = Py.index(key); + if (i < 0) { + i += items.length; + } + if (i < 0 || i >= items.length) { + throw Py.indexError("tuple index out of range"); + } + return items[(int) i]; + } + + @Override + public Object iter() { + return new PyIter(items.length) { + @Override + Object at(int i) { + return items[i]; + } + }; + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyType.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyType.java new file mode 100644 index 00000000000..12afe5913b0 --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyType.java @@ -0,0 +1,977 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/// A Python type: a builtin (`int`, `list`, `ValueError`) or a class the +/// application defines. +/// +/// Attribute lookup walks the C3 method resolution order, which is computed +/// once when the type is created. Calling a type constructs an instance: a +/// builtin type through its [Factory], a class through `__new__`/`__init__`. +public final class PyType extends PyObject { + + /// How a builtin type builds its instances from call arguments. + public interface Factory { + Object create(PyType type, Object[] args, String[] kwNames); + } + + public String name; + public final PyType[] bases; + final PyType[] mro; + final Map dict = new LinkedHashMap(); + + /// Bumped by every change to any class namespace. Inline caches + /// (`AttrSite`) and the lookup cache below compare against it, so nothing + /// cached outlives the class state it was read from. Changing a class + /// after it is built is rare, so one counter for all of them costs + /// nothing in practice and needs no list of subclasses. + static int version; + + private static final Object MISSING = new PyObject() { + @Override + public PyType type() { + return Py.OBJECT_TYPE; + } + }; + private Map lookupCache; + private int lookupVersion = -1; + Factory factory; + /// The builtin type whose storage instances of this class extend, or `null`. + final PyType solidBase; + /// The class's metaclass: `type`, or a subclass of it (null while `type` + /// itself is being built, which is its own metaclass). + PyType meta; + + /// The direct subclasses, for `type.__subclasses__()`; null until one exists. + private List subclasses; + + PyType(String name, PyType[] bases, Factory factory) { + this.name = name; + this.bases = bases; + this.factory = factory; + this.mro = computeMro(this, bases); + for (int i = 0; i < bases.length; i++) { + if (bases[i].subclasses == null) { + bases[i].subclasses = new ArrayList(); + } + bases[i].subclasses.add(this); + } + PyType solid = null; + for (int i = 1; i < mro.length && solid == null; i++) { + if (mro[i].factory != null && mro[i] != Py.OBJECT_TYPE) { //NOPMD CompareObjectsWithEquals - identity is meant + solid = mro[i]; + } + } + this.solidBase = solid; + this.meta = Py.TYPE_TYPE; + } + + /// Builds the Java object behind an instance of a class the transpiler + /// compiled, whose attributes are fields of a generated subclass of + /// `PyInstance` (see `PyInstance.fieldIndex`). + public interface Layout { + PyInstance create(PyType type); + } + + private Layout layout; + private String[] compiledMethods; + private String[] layoutFields; + /// Set once a method the generated code calls directly may no longer be + /// what the class resolves that name to: a class in its MRO was given, + /// or lost, an attribute of that name after it was built. The generated + /// dispatch then goes the generic way for good. Kept per class, so + /// patching one class leaves the others on the fast path. + public boolean patched; + private static final List LAYOUTS = new ArrayList(); + + /// Gives this class a compiled layout. `methods` are the names whose + /// functions generated code calls without a lookup; `fields` the + /// attributes that live in fields. Only this exact class uses the layout: + /// a subclass built at run time gets ordinary instances, so a compiled + /// method can never run for a class that overrides it unseen. + public PyType layout(Layout l, String[] methods, String[] fields) { + layout = l; + compiledMethods = methods; + layoutFields = fields; + LAYOUTS.add(this); + return this; + } + + /// The compiled class this one's layout extends, or null. + private PyType layoutParent() { + return bases.length == 1 && bases[0].layout != null ? bases[0] : null; + } + + /// Whether the class has a compiled layout. + public boolean hasLayout() { + return layout != null; + } + + /// A new, uninitialised instance. + public PyInstance newInstance() { + PyInstance o = layout != null ? layout.create(this) : new PyInstance(this); + if (solidBase != null && BuiltinSubclass.mutable(solidBase)) { + // Filled by the builtin's __init__ (or the subclass's, through super()). + o.base = solidBase.factory.create(solidBase, Py.NO_ARGS, Py.NO_KW); + } + return o; + } + + /// Invalidates direct calls to `attr` and refuses what a compiled field + /// cannot honour, once the class namespace of `this` changes. + private void changed(String attr, Object value) { + for (int i = 0; i < LAYOUTS.size(); i++) { //NOPMD ForLoopCanBeForeach - indexed: a for-each allocates an iterator + PyType t = LAYOUTS.get(i); + if (!t.isSubtype(this)) { + continue; + } + if ("__setattr__".equals(attr) || "__getattribute__".equals(attr) || "__delattr__".equals(attr) + || value instanceof PyProperty && contains(t.layoutFields, attr)) { + // Generated code stores and reads the fields of these + // instances directly. The transpiler leaves a field to the + // generic path when the module itself assigns that name on + // the class (Layouts.plan); this is the case it cannot see. + throw Py.typeError("cannot add '" + attr + "' to class '" + t.name + + "' after it is built: its attributes are compiled"); + } + // A compiled method t inherits counts as much as its own. + for (PyType u = t; u != null && !t.patched; u = u.layoutParent()) { + if (contains(u.compiledMethods, attr)) { + t.patched = true; + } + } + } + } + + private static boolean contains(String[] a, String s) { + if (a != null) { + for (String e : a) { + if (e.equals(s)) { + return true; + } + } + } + return false; + } + + /// Creates a class the application defines, `class name(bases): ...`. + /// The generated code fills the namespace afterwards with [#define]. + public static PyType newClass(String name, Object[] bases) { + Object[] original = bases; + bases = mroEntries(bases); + PyType[] b; + if (bases.length == 0) { + b = new PyType[] {Py.OBJECT_TYPE}; + } else { + b = new PyType[bases.length]; + for (int i = 0; i < bases.length; i++) { + if (!(bases[i] instanceof PyType)) { + throw Py.typeError("bases must be types"); + } + b[i] = (PyType) bases[i]; + PyType solid = b[i].factory != null && b[i] != Py.OBJECT_TYPE ? b[i] : b[i].solidBase; //NOPMD CompareObjectsWithEquals - identity is meant + if (solid != null && solid != Py.TYPE_TYPE && !BuiltinSubclass.subclassable(solid)) { //NOPMD CompareObjectsWithEquals - identity is meant + throw Py.typeError("type '" + solid.name + "' is not an acceptable base type"); + } + } + } + PyType t = new PyType(name, b, null); + if (original != bases) { //NOPMD CompareObjectsWithEquals - identity is meant + t.dict.put("__orig_bases__", PyTuple.of(original)); + } + for (PyType e : b) { + PyType s = e.factory != null && e != Py.OBJECT_TYPE ? e : e.solidBase; //NOPMD CompareObjectsWithEquals - identity is meant + if (s != null && s != t.solidBase && !t.solidBase.isSubtype(s)) { //NOPMD CompareObjectsWithEquals - identity is meant + throw Py.typeError("multiple bases have instance lay-out conflict"); + } + } + return t; + } + + /// `bases` with each non-class base replaced by what its + /// `__mro_entries__` answers (`typing.Generic[T]`, `NamedTuple`), or + /// `bases` itself when every entry is a class. + static Object[] mroEntries(Object[] bases) { + List out = null; + for (int i = 0; i < bases.length; i++) { + Object base = bases[i]; + Object hook = base instanceof PyType ? null : Py.obj(base).getAttrOrNull("__mro_entries__"); + if (hook == null) { + if (out != null) { + out.add(base); + } + continue; + } + if (out == null) { + out = new ArrayList(); + for (int j = 0; j < i; j++) { + out.add(bases[j]); + } + } + Object entries = Py.obj(hook).call(new Object[] {PyTuple.of(bases)}, Py.NO_KW); + if (!(entries instanceof PyTuple)) { + throw Py.typeError("__mro_entries__ must return a tuple"); + } + Object[] items = ((PyTuple) entries).items; + for (Object item : items) { + if (!out.contains(item)) { + out.add(item); + } + } + } + return out == null ? bases : out.toArray(); + } + + /// Stores an attribute in the class namespace. + /// The class namespace's own binding of `attr`, without the MRO: what a + /// name read inside a class body sees. + private Set slotNames; + private boolean slotsKnown; + + /// The only attributes an instance may hold, when every class in the + /// MRO declares `__slots__` and none of them names `__dict__`; null when + /// instances have a `__dict__`. + Set slotNames() { + if (!slotsKnown) { + slotNames = computeSlots(); + slotsKnown = true; + } + return slotNames; + } + + private Set computeSlots() { + Set r = new HashSet(); + for (PyType t : mro) { + if (t == Py.OBJECT_TYPE) { //NOPMD CompareObjectsWithEquals - identity is meant + continue; + } + Object s = t.dict.get("__slots__"); + if (s == null) { + return null; + } + Object[] names = s instanceof String ? new Object[] {s} : Py.toArray(s); + for (Object n : names) { + String name = Py.toStr(n); + if ("__dict__".equals(name)) { + return null; + } + r.add(name); + } + } + return r; + } + + public Object own(String attr) { + return dict.get(attr); + } + + public PyType define(String attr, Object value) { + version++; + dict.put(attr, value); + return this; + } + + @Override + public PyType type() { + return meta != null ? meta : Py.TYPE_TYPE; + } + + /// Whether the metaclass is a class the application defines. + private boolean customMeta() { + return meta != null && meta != Py.TYPE_TYPE; //NOPMD CompareObjectsWithEquals - identity is meant + } + + /// The metaclass's own dunder `name`, or null when the metaclass is + /// `type` or does not override it. + private Object metaDunder(String name) { + if (!customMeta()) { + return null; + } + Object m = meta.lookup(name); + return m == null || m instanceof PyBuiltin ? null : m; + } + + /// `isinstance(o, this)`, through the metaclass's `__instancecheck__` + /// when it defines one (what `abc.ABCMeta` registration needs). + boolean instanceCheck(Object o) { + if (Py.typeOf(o).isSubtype(this)) { + return true; + } + Object m = metaDunder("__instancecheck__"); + return m != null && Py.isTrue(Py.obj(m).call(new Object[] {this, o}, Py.NO_KW)); + } + + /// `issubclass(c, this)`, through the metaclass's `__subclasscheck__`. + boolean subclassCheck(PyType c) { + Object m = metaDunder("__subclasscheck__"); + if (m != null) { + return Py.isTrue(Py.obj(m).call(new Object[] {this, c}, Py.NO_KW)); + } + return c.isSubtype(this); + } + + // ---- class creation ---------------------------------------------------- + + /// Class namespaces handed to a metaclass whose class object already + /// exists: the generated code builds a class before its metaclass runs, + /// so `type.__new__` adopts that object instead of making another. + /// Pairs of namespace and class, innermost last. + private static final List PROVISIONAL = new ArrayList(); + + /// The most derived of `explicit` and the metaclasses of `bases`. + static PyType winner(PyType explicit, PyType[] bases) { + PyType w = explicit; + for (PyType base : bases) { + PyType m = base.type(); + if (w.isSubtype(m)) { + continue; + } + if (m.isSubtype(w)) { + w = m; + continue; + } + throw Py.typeError("metaclass conflict: the metaclass of a derived class must be a (non-strict) " + + "subclass of the metaclasses of all its bases"); + } + return w; + } + + /// Completes `class k(..., **kw)` once its body has filled the namespace: + /// runs the metaclass, `__set_name__` and `__init_subclass__`. Returns + /// what the class name is bound to, which a metaclass may replace. + /// `kwNames` has null for a `**mapping` entry. + public static Object finish(PyType k, String[] kwNames, Object[] kwValues) { + List names = new ArrayList(); + List values = new ArrayList(); + Object explicit = null; + for (int i = 0; i < kwNames.length; i++) { + if (kwNames[i] == null) { + PyDict d = Py.toDict(kwValues[i]); + Object[] keys = d.keyArray(); + for (Object key : keys) { + addKw(names, values, Py.toStr(key), d.getItem(key)); + } + } else { + addKw(names, values, kwNames[i], kwValues[i]); + } + } + int mi = names.indexOf("metaclass"); + if (mi >= 0) { + explicit = values.remove(mi); + names.remove(mi); + } + String[] kn = names.toArray(new String[names.size()]); + Object[] kv = values.toArray(new Object[values.size()]); + k.implicitClassMethods(); + if (explicit != null && !(explicit instanceof PyType)) { + // A plain callable as the metaclass is simply called. + return callMeta(Py.obj(explicit), k, kn, kv); + } + PyType w = winner(explicit == null ? Py.TYPE_TYPE : (PyType) explicit, k.bases); + if (w == Py.TYPE_TYPE) { //NOPMD CompareObjectsWithEquals - identity is meant + k.created(kn, kv); + return k; + } + return callMeta(w, k, kn, kv); + } + + private static void addKw(List names, List values, String n, Object v) { + if (names.contains(n)) { + throw Py.typeError("__build_class__() got multiple values for keyword argument '" + n + "'"); + } + names.add(n); + values.add(v); + } + + private static Object callMeta(PyObject meta, PyType k, String[] kn, Object[] kv) { + PyDict ns = k.namespace(); + Object[] bs = new Object[k.bases.length]; + System.arraycopy(k.bases, 0, bs, 0, bs.length); + Object[] args = new Object[3 + kv.length]; + args[0] = k.name; + args[1] = new PyTuple(bs); + args[2] = ns; + System.arraycopy(kv, 0, args, 3, kv.length); + PROVISIONAL.add(ns); + PROVISIONAL.add(k); + try { + return meta.call(args, kn); + } finally { + PROVISIONAL.remove(PROVISIONAL.size() - 1); + PROVISIONAL.remove(PROVISIONAL.size() - 1); + } + } + + /// The class namespace as a dict, as a metaclass receives it. + private PyDict namespace() { + PyDict ns = new PyDict(); + for (Map.Entry e : dict.entrySet()) { + ns.setItem(Py.str(e.getKey()), e.getValue()); + } + return ns; + } + + /// `type.__new__(mcs, name, bases, ns, **kw)`. + private static boolean sameBases(PyType k, Object[] bases) { + Object[] want = mroEntries(bases); + if (want.length == 0) { + return k.bases.length == 1 && k.bases[0] == Py.OBJECT_TYPE; //NOPMD CompareObjectsWithEquals - identity is meant + } + if (want.length != k.bases.length) { + return false; + } + for (int i = 0; i < want.length; i++) { + if (want[i] != k.bases[i]) { //NOPMD CompareObjectsWithEquals - identity is meant + return false; + } + } + return true; + } + + static Object typeNew(Object[] args, String[] kw) { + int npos = args.length - kw.length; + if (npos < 1 || !(args[0] instanceof PyType)) { + throw Py.notATypeObject("type", args); + } + PyType mcs = (PyType) args[0]; + if (!mcs.isSubtype(Py.TYPE_TYPE)) { + throw Py.typeError("type.__new__(" + mcs.name + "): " + mcs.name + " is not a subtype of type"); + } + if (npos == 2 && kw.length == 0 && mcs == Py.TYPE_TYPE) { //NOPMD CompareObjectsWithEquals - identity is meant + return Py.typeOf(args[1]); + } + if (npos != 4) { + throw Py.typeError("type.__new__() takes exactly 3 arguments (" + (npos - 1) + " given)"); + } + PyDict ns = Py.toDict(args[3]); + String[] kn = kw; + Object[] kv = new Object[kw.length]; + System.arraycopy(args, npos, kv, 0, kw.length); + PyType k = null; + for (int i = PROVISIONAL.size() - 2; i >= 0 && k == null; i -= 2) { + if (PROVISIONAL.get(i) == ns) { //NOPMD CompareObjectsWithEquals - identity is meant + k = (PyType) PROVISIONAL.get(i + 1); + } + } + if (k != null && !sameBases(k, Py.toArray(args[2]))) { + // The metaclass asked for other bases (TypedDict makes a dict): + // that is a different class, not the one the code built. + k = null; + } + Object[] keys = ns.keyArray(); + if (k != null) { + // The class the generated code built: adopt the namespace the + // metaclass may have changed. + List keep = new ArrayList(); + for (Object key : keys) { + keep.add(Py.toStr(key)); + } + Object[] old = k.dict.keySet().toArray(); + for (Object x : old) { + if (!keep.contains(x)) { + k.dict.remove(x); + } + } + } else { + k = newClass(Py.toStr(args[1]), Py.toArray(args[2])); + } + for (Object key : keys) { + k.dict.put(Py.toStr(key), ns.getItem(key)); + } + version++; + k.meta = winner(mcs, k.bases); + k.implicitClassMethods(); + k.created(kn, kv); + return k; + } + + /// `__init_subclass__` and `__class_getitem__` are class methods even + /// when written as plain functions. + private void implicitClassMethods() { + String[] names = {"__init_subclass__", "__class_getitem__"}; + for (String e : names) { + Object f = dict.get(e); + if (f instanceof PyFunction) { + dict.put(e, new PyClassMethod(f)); + } + } + } + + /// `__set_name__` on the namespace, then the parent's `__init_subclass__`. + private void created(String[] kn, Object[] kv) { + Object[] entries = dict.entrySet().toArray(); + for (Object entry : entries) { + @SuppressWarnings("unchecked") + Map.Entry e = (Map.Entry) entry; + Object v = e.getValue(); + if (v instanceof PyInstance) { + Object sn = ((PyInstance) v).cls.lookup("__set_name__"); + if (sn != null) { + Py.obj(sn).call(new Object[] {v, this, e.getKey()}, Py.NO_KW); + } + } + } + for (int i = 1; i < mro.length; i++) { + Object is = mro[i].dict.get("__init_subclass__"); + if (is != null) { + Object f = is instanceof PyClassMethod ? ((PyClassMethod) is).func : is; + Py.obj(f).call(Py.prepend(this, kv), kn); + return; + } + } + if (kn.length > 0) { + throw Py.typeError(name + ".__init_subclass__() takes no keyword arguments"); + } + } + + // ---- what a metaclass can override ------------------------------------- + + @Override + public Object getItem(Object key) { + Object m = metaDunder("__getitem__"); + if (m != null) { + return Py.obj(m).call(new Object[] {this, key}, Py.NO_KW); + } + Object cg = lookup("__class_getitem__"); + if (cg != null) { + Object f = cg instanceof PyClassMethod ? ((PyClassMethod) cg).func : cg; + return Py.obj(f).call(new Object[] {this, key}, Py.NO_KW); + } + if (factory != null) { + // `list[int]`: a generic alias, which is the type itself here. + return this; + } + throw Py.typeError("type '" + name + "' is not subscriptable"); + } + + @Override + public Object iter() { + Object m = metaDunder("__iter__"); + if (m != null) { + return Py.obj(m).call(new Object[] {this}, Py.NO_KW); + } + return super.iter(); + } + + @Override + public int len() { + Object m = metaDunder("__len__"); + if (m != null) { + return (int) Py.asLong(Py.obj(m).call(new Object[] {this}, Py.NO_KW)); + } + return super.len(); + } + + @Override + public boolean contains(Object x) { + Object m = metaDunder("__contains__"); + if (m != null) { + return Py.isTrue(Py.obj(m).call(new Object[] {this, x}, Py.NO_KW)); + } + return super.contains(x); + } + + @Override + public boolean isTrue() { + Object m = metaDunder("__bool__"); + if (m != null) { + return Py.isTrue(Py.obj(m).call(new Object[] {this}, Py.NO_KW)); + } + m = metaDunder("__len__"); + return m == null || len() != 0; + } + + @Override + public String str() { + Object m = metaDunder("__str__"); + if (m != null) { + return Py.strValue(Py.obj(m).call(new Object[] {this}, Py.NO_KW), "__str__"); + } + return repr(); + } + + /// Looks `attr` up along the MRO, without binding. + public Object lookup(String attr) { + if (lookupVersion != version || lookupCache == null) { + lookupCache = new java.util.HashMap(); + lookupVersion = version; + } + Object c = lookupCache.get(attr); + if (c != null) { + return c == MISSING ? null : c; + } + Object found = null; + for (int i = 0; i < mro.length && found == null; i++) { + found = mro[i].dict.get(attr); + } + lookupCache.put(attr, found == null ? MISSING : found); + return found; + } + + /// Whether the class overrides `__getattribute__`, which takes every + /// attribute read out of the inline caches' hands. + boolean hasCustomGetattribute() { + Object g = lookup("__getattribute__"); + return g != null && !(g instanceof PyBuiltin); + } + + /// `issubclass(self, other)`. + public boolean isSubtype(PyType other) { + for (PyType e : mro) { + if (e == other) { //NOPMD CompareObjectsWithEquals - identity is meant + return true; + } + } + return false; + } + + @Override + public Object getAttrOrNull(String attr) { + if ("__name__".equals(attr)) { + return name; + } + if ("__qualname__".equals(attr)) { + Object q = dict.get(attr); + return q != null ? q : name; + } + if ("__mro__".equals(attr)) { + Object[] items = new Object[mro.length]; + System.arraycopy(mro, 0, items, 0, mro.length); + return new PyTuple(items); + } + if ("__dict__".equals(attr)) { + return namespace(); + } + if ("__subclasses__".equals(attr)) { + return PyBuiltin.positional("__subclasses__", a -> { + PyList out = new PyList(subclasses == null ? 0 : subclasses.size()); + if (subclasses != null) { + for (int i = 0; i < subclasses.size(); i++) { //NOPMD ForLoopCanBeForeach - indexed: a for-each allocates an iterator + out.append(subclasses.get(i)); + } + } + return out; + }); + } + if ("__bases__".equals(attr)) { + Object[] items = new Object[bases.length]; + System.arraycopy(bases, 0, items, 0, bases.length); + return new PyTuple(items); + } + Object v = lookup(attr); + if (v == null) { + if (customMeta()) { + Object mv = metaAttr(attr); + if (mv != null) { + return mv; + } + } + if ("__class__".equals(attr)) { + return type(); + } + // Built-in types carry no namespace entries for these. + if ("__module__".equals(attr)) { + return Py.str("builtins"); + } + if ("__doc__".equals(attr)) { + return Py.NONE; + } + return null; + } + // Reading a function through the class gives the plain function; + // classmethod and staticmethod know what they become. + if (v instanceof PyClassMethod) { + return new PyMethod(this, ((PyClassMethod) v).func); + } + if (v instanceof PyStaticMethod) { + return ((PyStaticMethod) v).func; + } + if (v instanceof PyInstance) { + Object get = ((PyInstance) v).cls.lookup("__get__"); + if (get != null) { + return Py.obj(get).call(new Object[] {v, Py.NONE, this}, Py.NO_KW); + } + } + return v; + } + + /// An attribute of the metaclass, read through the class: a method + /// bound to the class, a property computed on it. + private Object metaAttr(String attr) { + Object m = meta.lookup(attr); + if (m == null) { + Object ga = metaDunder("__getattr__"); + return ga == null ? null : Py.obj(ga).call(new Object[] {this, attr}, Py.NO_KW); + } + if (m instanceof PyProperty) { + return ((PyProperty) m).get(this); + } + if (m instanceof PyClassMethod) { + return new PyMethod(meta, ((PyClassMethod) m).func); + } + if (m instanceof PyStaticMethod) { + return ((PyStaticMethod) m).func; + } + return Py.bindAttr(m, this); + } + + @Override + public void setAttr(String attr, Object value) { + Object sa = metaDunder("__setattr__"); + if (sa != null) { + Py.obj(sa).call(new Object[] {this, attr, value}, Py.NO_KW); + return; + } + setAttrDirect(attr, value); + } + + /// `type.__setattr__(cls, attr, value)`. + public void setAttrDirect(String attr, Object value) { + if ("__name__".equals(attr)) { + if (!(value instanceof String) && !(value instanceof PyStr)) { + throw Py.typeError("can only assign string to " + name + ".__name__, not '" + + Py.typeOf(value).name + "'"); + } + name = Py.toStr(value); + version++; + return; + } + changed(attr, value); + version++; + dict.put(attr, value); + } + + @Override + public void delAttr(String attr) { + Object da = metaDunder("__delattr__"); + if (da != null) { + Py.obj(da).call(new Object[] {this, attr}, Py.NO_KW); + return; + } + delAttrDirect(attr); + } + + /// `type.__delattr__(cls, attr)`. + public void delAttrDirect(String attr) { + changed(attr, null); + version++; + if (dict.remove(attr) == null) { + throw Py.attributeError("type object '" + name + "' has no attribute '" + attr + "'"); + } + } + + @Override + public Object call(Object[] args, String[] kwNames) { + if (this == Py.TYPE_TYPE && args.length == 1 && kwNames.length == 0) { //NOPMD CompareObjectsWithEquals - identity is meant + return Py.typeOf(args[0]); + } + Object mc = metaDunder("__call__"); + if (mc != null) { + return Py.obj(mc).call(Py.prepend(this, args), kwNames); + } + return construct(args, kwNames); + } + + /// `type.__call__(cls, *args)`: `__new__`, then `__init__`. + Object construct(Object[] args, String[] kwNames) { + if (factory != null) { + // A builtin type builds and initialises its own instances. + return factory.create(this, args, kwNames); + } + Object newFn = lookup("__new__"); + Object obj; + if (newFn != null && newFn != Builtins.OBJECT_NEW) { //NOPMD CompareObjectsWithEquals - identity is meant + Object fn = newFn instanceof PyStaticMethod ? ((PyStaticMethod) newFn).func : newFn; + obj = Py.obj(fn).call(Py.prepend(this, args), kwNames); + if (!Py.typeOf(obj).isSubtype(this)) { + return obj; + } + } else { + obj = newInstance(); + } + Object init = lookup("__init__"); + if (init != null && init != Builtins.OBJECT_INIT) { //NOPMD CompareObjectsWithEquals - identity is meant + Object r = Py.obj(init).call(Py.prepend(obj, args), kwNames); + if (r != Py.NONE) { //NOPMD CompareObjectsWithEquals - identity is meant + throw Py.typeError("__init__() should return None, not '" + Py.typeOf(r).name + "'"); + } + } else if (args.length > 0 && newFn == Builtins.OBJECT_NEW) { //NOPMD CompareObjectsWithEquals - identity is meant + throw Py.typeError(name + "() takes no arguments"); + } + return obj; + } + + /// The `__init__` of a plain class (no `__new__`, no builtin base), the + /// case a positional constructor call takes without an argument array. + private Object plainInit() { + if (factory != null || this == Py.TYPE_TYPE || customMeta()) { //NOPMD CompareObjectsWithEquals - identity is meant + return null; + } + Object newFn = lookup("__new__"); + if (newFn != null && newFn != Builtins.OBJECT_NEW) { //NOPMD CompareObjectsWithEquals - identity is meant + return null; + } + Object init = lookup("__init__"); + return init instanceof PyFunction ? init : null; + } + + private static void initReturned(Object r) { + if (r != Py.NONE) { //NOPMD CompareObjectsWithEquals - identity is meant + throw Py.typeError("__init__() should return None, not '" + Py.typeOf(r).name + "'"); + } + } + + @Override + public Object call0() { + Object init = plainInit(); + if (init == null) { + return super.call0(); + } + PyInstance o = newInstance(); + initReturned(((PyFunction) init).call1(o)); + return o; + } + + @Override + public Object call1(Object a) { + Object init = plainInit(); + if (init == null) { + return super.call1(a); + } + PyInstance o = newInstance(); + initReturned(((PyFunction) init).call2(o, a)); + return o; + } + + @Override + public Object call2(Object a, Object b) { + Object init = plainInit(); + if (init == null) { + return super.call2(a, b); + } + PyInstance o = newInstance(); + initReturned(((PyFunction) init).call3(o, a, b)); + return o; + } + + @Override + public Object call3(Object a, Object b, Object c) { + Object init = plainInit(); + if (init == null) { + return super.call3(a, b, c); + } + PyInstance o = newInstance(); + initReturned(((PyFunction) init).call4(o, a, b, c)); + return o; + } + + @Override + public String repr() { + Object m = metaDunder("__repr__"); + if (m != null) { + return Py.strValue(Py.obj(m).call(new Object[] {this}, Py.NO_KW), "__repr__"); + } + Object q = dict.get("__qualname__"); + String qual = q instanceof String ? (String) q : name; + Object mod = dict.get("__module__"); + if (mod instanceof String && !"builtins".equals(mod)) { + return ""; + } + return ""; + } + + @Override + public long hash() { + return System.identityHashCode(this); + } + + @Override + public Object or(Object o) { + // `int | None` in annotations evaluates to a union; keep it inert. + return new PyTuple(new Object[] {this, o}); + } + + private static PyType[] computeMro(PyType self, PyType[] bases) { + List> seqs = new ArrayList>(); + for (PyType e : bases) { + List l = new ArrayList(); + for (int j = 0; j < e.mro.length; j++) { + l.add(e.mro[j]); + } + seqs.add(l); + } + List direct = new ArrayList(); + for (PyType base : bases) { + direct.add(base); + } + seqs.add(direct); + List out = new ArrayList(); + out.add(self); + while (true) { + boolean empty = true; + for (int i = 0; i < seqs.size(); i++) { //NOPMD ForLoopCanBeForeach - indexed: a for-each allocates an iterator + if (!seqs.get(i).isEmpty()) { + empty = false; + break; + } + } + if (empty) { + return out.toArray(new PyType[out.size()]); + } + PyType candidate = null; + for (int i = 0; i < seqs.size() && candidate == null; i++) { + List s = seqs.get(i); + if (s.isEmpty()) { + continue; + } + PyType head = s.get(0); + boolean inTail = false; + for (int j = 0; j < seqs.size() && !inTail; j++) { + List t = seqs.get(j); + inTail = t.indexOf(head) > 0; + } + if (!inTail) { + candidate = head; + } + } + if (candidate == null) { + throw Py.typeError("Cannot create a consistent method resolution order (MRO)"); + } + out.add(candidate); + for (int i = 0; i < seqs.size(); i++) { //NOPMD ForLoopCanBeForeach - indexed: a for-each allocates an iterator + List s = seqs.get(i); + if (!s.isEmpty() && s.get(0) == candidate) { //NOPMD CompareObjectsWithEquals - identity is meant + s.remove(0); + } + } + } + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyWeakRef.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyWeakRef.java new file mode 100644 index 00000000000..5a852016267 --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyWeakRef.java @@ -0,0 +1,119 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +import java.lang.ref.WeakReference; + +/// `weakref.ref`: a reference that does not keep its referent alive. +/// +/// The referent is held by a Java `WeakReference`, so it goes when the +/// garbage collector next runs rather than at the moment its last strong +/// reference is dropped, as it does under CPython's reference counting. +/// The callback is never called: no port can say when a referent died. +public final class PyWeakRef extends PyObject { + private final WeakReference ref; + private boolean hashed; + private long cachedHash; + final Object callback; + + PyWeakRef(Object referent, Object callback) { + if (!referenceable(referent)) { + throw Py.typeError("cannot create weak reference to '" + Py.typeOf(referent).name + "' object"); + } + this.ref = new WeakReference(referent); + this.callback = callback; + } + + /// The values CPython can weakly reference: instances of classes, functions, + /// classes, sets, methods, generators and modules. Ints, strs, tuples, + /// lists and dicts cannot be. + static boolean referenceable(Object o) { + return o instanceof PyInstance || o instanceof PyFunction || o instanceof PyType || o instanceof PySet + || o instanceof PyMethod || o instanceof PyGenerator || o instanceof PyModule; + } + + @Override + public PyType type() { + return Py.WEAKREF_TYPE; + } + + /// The referent, or `None` once it has been collected. + Object get() { + Object o = ref.get(); + return o == null ? Py.NONE : o; + } + + @Override + public Object call(Object[] args, String[] kw) { + if (args.length != 0 || kw.length != 0) { + throw Py.typeError("weakref() takes no arguments"); + } + return get(); + } + + @Override + public long hash() { + if (!hashed) { + Object o = ref.get(); + if (o == null) { + throw Py.typeError("weak object has gone away"); + } + cachedHash = Py.hashOf(o); + hashed = true; + } + return cachedHash; + } + + @Override + public Object eq(Object o) { + if (!(o instanceof PyWeakRef)) { + return Py.NOT_IMPLEMENTED; + } + Object a = ref.get(); + Object b = ((PyWeakRef) o).ref.get(); + if (a == null || b == null) { + return Py.bool(this == o); //NOPMD CompareObjectsWithEquals - identity is meant + } + return Py.bool(Py.isTrue(Py.eq(a, b))); + } + + @Override + public String repr() { + String id = Integer.toHexString(System.identityHashCode(this)); + Object o = ref.get(); + if (o == null) { + return ""; + } + return ""; + } + + @Override + public Object getAttrOrNull(String name) { + if ("__callback__".equals(name)) { + return callback == null ? Py.NONE : callback; + } + return super.getAttrOrNull(name); + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Shape.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Shape.java new file mode 100644 index 00000000000..482e1c91926 --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Shape.java @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +/// The layout of an instance's attributes: which name lives in which slot. +/// +/// Instances of a class that assign the same attributes in the same order +/// share one shape, so an attribute is an array element at an index every +/// one of them agrees on. A shape is immutable; adding an attribute moves the +/// instance to the shape that extends its own by that name, and the +/// transitions are remembered so the next instance reaches the very same +/// object. That identity is what an `AttrSite` compares. +final class Shape { + static final Shape EMPTY = new Shape(new String[0]); + + final String[] names; + private Shape[] next = new Shape[0]; + + private Shape(String[] names) { + this.names = names; + } + + int size() { + return names.length; + } + + /// The slot holding `name`, or -1. + int indexOf(String name) { + String[] n = names; + for (int i = 0; i < n.length; i++) { + if (n[i] == name) { //NOPMD CompareObjectsWithEquals - identity is meant + return i; + } + } + for (int i = 0; i < n.length; i++) { + if (n[i].equals(name)) { + return i; + } + } + return -1; + } + + /// This shape extended by `name`. + Shape with(String name) { + Shape[] t = next; + for (Shape e : t) { + String last = e.names[names.length]; + if (last == name || last.equals(name)) { //NOPMD CompareObjectsWithEquals - identity is meant + return e; + } + } + String[] n = new String[names.length + 1]; + System.arraycopy(names, 0, n, 0, names.length); + n[names.length] = name; + Shape s = new Shape(n); + Shape[] grown = new Shape[t.length + 1]; + System.arraycopy(t, 0, grown, 0, t.length); + grown[t.length] = s; + next = grown; + return s; + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java new file mode 100644 index 00000000000..00c088bb343 --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java @@ -0,0 +1,485 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +/// Standard library modules implemented in Java because CPython implements +/// them in C: `math`, `sys`, `time`. Each is built the first time it is +/// imported. +public final class StdModules { + + private StdModules() { + } + + /// Builds and registers module `name` if this class provides it. + static boolean provide(String name) { + if ("math".equals(name)) { + PyModule.registerModule(math()); + return true; + } + if ("sys".equals(name)) { + PyModule.registerModule(sys()); + return true; + } + if ("_time".equals(name)) { + PyModule.registerModule(time()); + return true; + } + if ("atexit".equals(name)) { + PyModule.registerModule(atexit()); + return true; + } + if ("_weakref".equals(name)) { + PyModule.registerModule(weakref()); + return true; + } + if ("_cn1fs".equals(name)) { + PyModule.registerModule(fileSystem()); + return true; + } + if ("builtins".equals(name)) { + PyModule m = new PyModule("builtins", new String[0]); + for (java.util.Map.Entry e : Builtins.NAMES.entrySet()) { + m.setAttr(e.getKey(), e.getValue()); + } + PyModule.registerModule(m); + return true; + } + return false; + } + + /// The `atexit` handlers, each its function followed by its arguments + /// (the trailing ones named by the matching entry of `EXIT_KW`). + private static final java.util.List EXIT_CALLS = new java.util.ArrayList(); + private static final java.util.List EXIT_KW = new java.util.ArrayList(); + + /// Runs the `atexit` handlers, last registered first, once each. The + /// generated `runMain()` calls this when the main module finishes, + /// whether or not it raised; a handler that raises is reported on + /// stderr and the rest still run, as in CPython. + public static void runExitHandlers() { + while (!EXIT_CALLS.isEmpty()) { + int last = EXIT_CALLS.size() - 1; + Object[] c = EXIT_CALLS.remove(last); + String[] kw = EXIT_KW.remove(last); + Object[] args = new Object[c.length - 1]; + System.arraycopy(c, 1, args, 0, args.length); + try { + Py.obj(c[0]).call(args, kw); + } catch (PyException e) { + System.err.println("Exception ignored in atexit callback " + Py.obj(c[0]).repr() + ":"); + System.err.println(e.getMessage()); + } + } + } + + private static PyModule atexit() { + PyModule m = new PyModule("atexit", new String[0]); + m.setAttr("register", new PyBuiltin("register", (args, kwNames) -> { + if (args.length - kwNames.length < 1) { + throw Py.typeError("register() takes at least 1 argument (0 given)"); + } + if (!Builtins.isCallable(args[0])) { + throw Py.typeError("the first argument must be callable"); + } + Object[] call = new Object[args.length]; + System.arraycopy(args, 0, call, 0, args.length); + EXIT_CALLS.add(call); + EXIT_KW.add(kwNames); + return args[0]; + })); + m.setAttr("unregister", PyBuiltin.positional("unregister", a -> { + for (int i = EXIT_CALLS.size() - 1; i >= 0; i--) { + if (Py.isTrue(Py.eq(EXIT_CALLS.get(i)[0], a[0]))) { + EXIT_CALLS.remove(i); + EXIT_KW.remove(i); + } + } + return Py.NONE; + })); + m.setAttr("_run_exitfuncs", PyBuiltin.positional("_run_exitfuncs", a -> { + runExitHandlers(); + return Py.NONE; + })); + m.setAttr("_clear", PyBuiltin.positional("_clear", a -> { + EXIT_CALLS.clear(); + EXIT_KW.clear(); + return Py.NONE; + })); + m.setAttr("_ncallbacks", PyBuiltin.positional("_ncallbacks", a -> PyInt.of(EXIT_CALLS.size()))); + return m; + } + + /// `_weakref`: the reference type the `weakref` module builds on. + private static PyModule weakref() { + PyModule m = new PyModule("_weakref", new String[0]); + PyType t = Py.WEAKREF_TYPE; + if (t.factory == null) { + t.define("__module__", "weakref"); + t.factory = (type, args, kw) -> { + Args a = new Args("ref", args, kw).check(2); + Object cb = a.get(1, "callback", Py.NONE); + return new PyWeakRef(a.get(0, "object", null), cb == Py.NONE ? null : cb); //NOPMD CompareObjectsWithEquals - identity is meant + }; + } + m.setAttr("ref", t); + m.setAttr("ReferenceType", t); + m.setAttr("referenceable", PyBuiltin.positional("referenceable", + a -> Py.bool(PyWeakRef.referenceable(a[0])))); + return m; + } + + /// Seconds east of UTC of the device's zone at instant `t` (seconds + /// since the epoch), daylight saving included. + private static int utcOffset(double t) { + java.util.TimeZone z = java.util.TimeZone.getDefault(); + java.util.Calendar c = java.util.Calendar.getInstance(z); + c.setTime(new java.util.Date((long) Math.floor(t * 1000))); + int ms = ((c.get(java.util.Calendar.HOUR_OF_DAY) * 60 + c.get(java.util.Calendar.MINUTE)) * 60 + + c.get(java.util.Calendar.SECOND)) * 1000 + c.get(java.util.Calendar.MILLISECOND); + return z.getOffset(1, c.get(java.util.Calendar.YEAR), c.get(java.util.Calendar.MONTH), + c.get(java.util.Calendar.DAY_OF_MONTH), c.get(java.util.Calendar.DAY_OF_WEEK), ms) / 1000; + } + + private interface FsOp { + Object run(PyFileSystem fs, Object[] a) throws java.io.IOException; + } + + private static void fs(PyModule m, String name, final FsOp op) { + m.setAttr(name, PyBuiltin.positional(name, a -> { + try { + return op.run(PyFileSystem.get(), a); + } catch (java.io.IOException e) { + throw Py.error(Py.OS_ERROR, String.valueOf(e.getMessage())); //NOPMD PreserveStackTrace - a Python exception has no Java cause + } + })); + } + + /// `_cn1fs`: the primitives `io` and `os` are written over. Paths are + /// absolute; the Python side resolves them against the working + /// directory and turns a missing file into the right exception. + private static PyModule fileSystem() { + PyModule m = new PyModule("_cn1fs", new String[0]); + fs(m, "home", (fs, a) -> Py.str(fs.home())); + fs(m, "read", (fs, a) -> { + java.io.InputStream in = fs.openInput(Py.toStr(a[0])); + try { + java.io.ByteArrayOutputStream out = new java.io.ByteArrayOutputStream(); + byte[] buf = new byte[8192]; + for (int n = in.read(buf); n > 0; n = in.read(buf)) { + out.write(buf, 0, n); + } + return new PyBytes(out.toByteArray()); + } finally { + in.close(); + } + }); + fs(m, "write", (fs, a) -> { + java.io.OutputStream out = fs.openOutput(Py.toStr(a[0]), Py.isTrue(a[2])); + try { + out.write(PyBytes.like(a[1])); + } finally { + out.close(); + } + return Py.NONE; + }); + fs(m, "exists", (fs, a) -> Py.bool(fs.exists(Py.toStr(a[0])))); + fs(m, "isdir", (fs, a) -> Py.bool(fs.isDirectory(Py.toStr(a[0])))); + fs(m, "listdir", (fs, a) -> { + String[] names = fs.list(Py.toStr(a[0])); + Object[] r = new Object[names.length]; + for (int i = 0; i < names.length; i++) { + r[i] = Py.str(names[i]); + } + return new PyList(r); + }); + fs(m, "mkdir", (fs, a) -> { + fs.mkdir(Py.toStr(a[0])); + return Py.NONE; + }); + fs(m, "remove", (fs, a) -> { + fs.delete(Py.toStr(a[0])); + return Py.NONE; + }); + fs(m, "rename", (fs, a) -> { + fs.rename(Py.toStr(a[0]), Py.toStr(a[1])); + return Py.NONE; + }); + fs(m, "size", (fs, a) -> PyInt.of(fs.size(Py.toStr(a[0])))); + fs(m, "mtime", (fs, a) -> Py.ofFloat(fs.lastModified(Py.toStr(a[0])) / 1000.0)); + return m; + } + + private interface D1 { + double f(double x); + } + + private static void fn1(PyModule m, String name, final D1 f) { + m.setAttr(name, PyBuiltin.positional(name, a -> Py.ofFloat(f.f(Py.asDouble(a[0]))))); + } + + private static PyModule math() { + PyModule m = new PyModule("math", new String[0]); + m.setAttr("pi", Py.ofFloat(Math.PI)); + m.setAttr("e", Py.ofFloat(Math.E)); + MathFunctions.install(m); + m.setAttr("tau", Py.ofFloat(2 * Math.PI)); + m.setAttr("inf", Py.ofFloat(Double.POSITIVE_INFINITY)); + m.setAttr("nan", Py.ofFloat(Double.NaN)); + m.setAttr("sqrt", PyBuiltin.positional("sqrt", a -> { + double x = Py.asDouble(a[0]); + if (x < 0) { + throw Py.valueError("math domain error"); + } + return Py.ofFloat(Math.sqrt(x)); + })); + fn1(m, "sin", Math::sin); + fn1(m, "cos", Math::cos); + fn1(m, "tan", Math::tan); + fn1(m, "asin", com.codename1.util.MathUtil::asin); + fn1(m, "acos", com.codename1.util.MathUtil::acos); + fn1(m, "atan", com.codename1.util.MathUtil::atan); + fn1(m, "exp", com.codename1.util.MathUtil::exp); + fn1(m, "fabs", Math::abs); + fn1(m, "degrees", Math::toDegrees); + fn1(m, "radians", Math::toRadians); + m.setAttr("atan2", PyBuiltin.positional("atan2", + a -> Py.ofFloat(com.codename1.util.MathUtil.atan2(Py.asDouble(a[0]), Py.asDouble(a[1]))))); + m.setAttr("pow", PyBuiltin.positional("pow", + a -> Py.ofFloat(com.codename1.util.MathUtil.pow(Py.asDouble(a[0]), Py.asDouble(a[1]))))); + m.setAttr("hypot", PyBuiltin.positional("hypot", a -> { + double s = 0; + for (int i = 0; i < a.length; i++) { + double v = Py.asDouble(a[i]); + s += v * v; + } + return Py.ofFloat(Math.sqrt(s)); + })); + m.setAttr("log", PyBuiltin.positional("log", a -> { + double x = Py.asDouble(a[0]); + if (x <= 0) { + throw Py.valueError("math domain error"); + } + double r = com.codename1.util.MathUtil.log(x); + if (a.length > 1) { + r /= com.codename1.util.MathUtil.log(Py.asDouble(a[1])); + } + return Py.ofFloat(r); + })); + m.setAttr("log10", PyBuiltin.positional("log10", + a -> Py.ofFloat(com.codename1.util.MathUtil.log10(Py.asDouble(a[0]))))); + m.setAttr("log2", PyBuiltin.positional("log2", + a -> Py.ofFloat(com.codename1.util.MathUtil.log(Py.asDouble(a[0])) / com.codename1.util.MathUtil.log(2)))); + m.setAttr("floor", PyBuiltin.positional("floor", a -> a[0] instanceof PyInt ? a[0] + : Builtins.toInt(Py.ofFloat(Math.floor(Py.asDouble(a[0])))))); + m.setAttr("ceil", PyBuiltin.positional("ceil", a -> a[0] instanceof PyInt ? a[0] + : Builtins.toInt(Py.ofFloat(Math.ceil(Py.asDouble(a[0])))))); + m.setAttr("trunc", PyBuiltin.positional("trunc", a -> Builtins.toInt(a[0]))); + m.setAttr("isnan", PyBuiltin.positional("isnan", a -> Py.bool(Double.isNaN(Py.asDouble(a[0]))))); + m.setAttr("isinf", PyBuiltin.positional("isinf", a -> Py.bool(Double.isInfinite(Py.asDouble(a[0]))))); + m.setAttr("isfinite", PyBuiltin.positional("isfinite", a -> { + double d = Py.asDouble(a[0]); + return Py.bool(!Double.isNaN(d) && !Double.isInfinite(d)); + })); + m.setAttr("copysign", PyBuiltin.positional("copysign", a -> { + double x = Math.abs(Py.asDouble(a[0])); + double s = Py.asDouble(a[1]); + return Py.ofFloat(s < 0 || s == 0 && 1 / s < 0 ? -x : x); + })); + m.setAttr("fmod", PyBuiltin.positional("fmod", a -> Py.ofFloat(Py.asDouble(a[0]) % Py.asDouble(a[1])))); + m.setAttr("gcd", PyBuiltin.positional("gcd", a -> { + com.codename1.util.BigInteger g = com.codename1.util.BigInteger.ZERO; + for (int i = 0; i < a.length; i++) { + g = g.gcd(Py.intView(Builtins.toInt(a[i])).toBig()); + } + return PyInt.of(g); + })); + m.setAttr("factorial", PyBuiltin.positional("factorial", a -> { + long n = Py.asLong(a[0]); + if (n < 0) { + throw Py.valueError("factorial() not defined for negative values"); + } + Object r = PyInt.of(1); + for (long i = 2; i <= n; i++) { + r = Py.mul(r, PyInt.of(i)); + } + return r; + })); + m.setAttr("isqrt", PyBuiltin.positional("isqrt", a -> { + com.codename1.util.BigInteger n = Py.intView(Builtins.toInt(a[0])).toBig(); + if (n.signum() < 0) { + throw Py.valueError("isqrt() argument must be nonnegative"); + } + if (n.signum() == 0) { + return PyInt.of(0); + } + com.codename1.util.BigInteger x = com.codename1.util.BigInteger.ONE.shiftLeft((n.bitLength() + 1) / 2); + while (true) { + com.codename1.util.BigInteger y = x.add(n.divide(x)).shiftRight(1); + if (y.compareTo(x) >= 0) { + return PyInt.of(x); + } + x = y; + } + })); + m.setAttr("fsum", PyBuiltin.positional("fsum", a -> Py.obj(Builtins.SUM).call(new Object[] {a[0], Py.ofFloat(0)}, Py.NO_KW))); + m.setAttr("prod", PyBuiltin.positional("prod", a -> { + Object r = PyInt.of(1); + Object it = Py.iter(a[0]); + for (Object v = Py.next(it); v != null; v = Py.next(it)) { + r = Py.mul(r, v); + } + return r; + })); + m.setAttr("comb", PyBuiltin.positional("comb", a -> { + long n = Py.asLong(a[0]); + long k = Py.asLong(a[1]); + if (k < 0 || k > n) { + return PyInt.of(0); + } + k = Math.min(k, n - k); + Object r = PyInt.of(1); + for (long i = 1; i <= k; i++) { + r = Py.floordiv(Py.mul(r, PyInt.of(n - k + i)), PyInt.of(i)); + } + return r; + })); + m.setAttr("isclose", new PyBuiltin("isclose", (args, kw) -> { + Args a = new Args("isclose", args, kw).check(2, "rel_tol", "abs_tol"); + double x = Py.asDouble(args[0]); + double y = Py.asDouble(args[1]); + double rel = Py.asDouble(a.get(-1, "rel_tol", Py.ofFloat(1e-9))); + double abs = Py.asDouble(a.get(-1, "abs_tol", Py.ofFloat(0))); + if (x == y) { + return Py.TRUE; + } + double diff = Math.abs(x - y); + return Py.bool(diff <= Math.abs(rel * y) || diff <= Math.abs(rel * x) || diff <= abs); + })); + return m; + } + + private static PyModule sys() { + PyModule m = new PyModule("sys", new String[0]); + m.setAttr("maxsize", PyInt.of(Long.MAX_VALUE)); + m.setAttr("platform", Py.str("codenameone")); + m.setAttr("byteorder", Py.str("little")); + m.setAttr("version", Py.str("3.13.0 (Codename One)")); + m.setAttr("version_info", PyTuple.of(PyInt.of(3), PyInt.of(13), PyInt.of(0), Py.str("final"), PyInt.of(0))); + Object[] argv = new Object[Py.argv.length]; + for (int i = 0; i < argv.length; i++) { + argv[i] = Py.argv[i]; + } + m.setAttr("argv", new PyList(argv)); + m.setAttr("path", new PyList()); + m.setAttr("modules", PyModule.SYS_MODULES); + m.setAttr("float_info", PyTuple.of(Py.ofFloat(Double.MAX_VALUE), PyInt.of(1024), PyInt.of(308), + Py.ofFloat(Double.MIN_NORMAL), PyInt.of(-1021), PyInt.of(-307), PyInt.of(15), PyInt.of(53), + Py.ofFloat(Math.ulp(1.0)), PyInt.of(2), PyInt.of(1))); + m.setAttr("getrecursionlimit", PyBuiltin.positional("getrecursionlimit", a -> PyInt.of(1000))); + m.setAttr("setrecursionlimit", PyBuiltin.positional("setrecursionlimit", a -> Py.NONE)); + m.setAttr("exception", PyBuiltin.positional("exception", a -> Py.handling())); + m.setAttr("exc_info", PyBuiltin.positional("exc_info", a -> { + Object e = Py.handling(); + if (e == Py.NONE) { //NOPMD CompareObjectsWithEquals - identity is meant + return PyTuple.of(Py.NONE, Py.NONE, Py.NONE); + } + return PyTuple.of(Py.typeOf(e), e, Py.NONE); + })); + m.setAttr("intern", PyBuiltin.positional("intern", a -> a[0])); + m.setAttr("exit", PyBuiltin.positional("exit", a -> { + throw new PyException(Py.SYSTEM_EXIT.call(a, Py.NO_KW)); + })); + Object stdout = new PyInstance(Py.OBJECT_TYPE) { + @Override + public Object getAttrOrNull(String name) { + if ("write".equals(name)) { + return PyBuiltin.positional("write", a -> { + Builtins.writeOut(Py.toStr(a[0])); + return PyInt.of(Py.len(a[0])); + }); + } + if ("flush".equals(name)) { + return PyBuiltin.positional("flush", a -> Py.NONE); + } + return super.getAttrOrNull(name); + } + }; + Object stderr = new PyInstance(Py.OBJECT_TYPE) { + @Override + public Object getAttrOrNull(String name) { + if ("write".equals(name)) { + return PyBuiltin.positional("write", a -> { + System.err.print(Py.toStr(a[0])); + return PyInt.of(Py.len(a[0])); + }); + } + if ("flush".equals(name)) { + return PyBuiltin.positional("flush", a -> Py.NONE); + } + return super.getAttrOrNull(name); + } + }; + m.setAttr("stdout", stdout); + m.setAttr("stderr", stderr); + m.setAttr("__stdout__", stdout); + m.setAttr("__stderr__", stderr); + sysModule = m; + originalStdout = stdout; + return m; + } + + /// The sys module once something imported it, and the stdout it started with. + private static PyModule sysModule; + private static Object originalStdout; + + /// `sys.stdout` when the program replaced it, else null (print writes + /// to the platform output directly). + static Object redirectedStdout() { + if (sysModule == null) { + return null; + } + Object so = sysModule.getAttrOrNull("stdout"); + return so == null || so == originalStdout || so == Py.NONE ? null : so; //NOPMD CompareObjectsWithEquals - identity is meant + } + + private static PyModule time() { + PyModule m = new PyModule("_time", new String[0]); + m.setAttr("utcoffset", PyBuiltin.positional("utcoffset", a -> PyInt.of(utcOffset(Py.asDouble(a[0]))))); + m.setAttr("tzname", PyBuiltin.positional("tzname", a -> Py.str(java.util.TimeZone.getDefault().getID()))); + m.setAttr("time", PyBuiltin.positional("time", a -> Py.ofFloat(System.currentTimeMillis() / 1000.0))); + m.setAttr("time_ns", PyBuiltin.positional("time_ns", a -> PyInt.of(System.currentTimeMillis() * 1000000L))); + m.setAttr("perf_counter", PyBuiltin.positional("perf_counter", a -> Py.ofFloat(System.nanoTime() / 1e9))); + m.setAttr("perf_counter_ns", PyBuiltin.positional("perf_counter_ns", a -> PyInt.of(System.nanoTime()))); + m.setAttr("monotonic", PyBuiltin.positional("monotonic", a -> Py.ofFloat(System.nanoTime() / 1e9))); + m.setAttr("sleep", PyBuiltin.positional("sleep", a -> { + try { + Thread.sleep((long) (Py.asDouble(a[0]) * 1000)); + } catch (InterruptedException e) { + throw Py.error(Py.KEYBOARD_INTERRUPT, ""); //NOPMD PreserveStackTrace - a Python exception has no Java cause + } + return Py.NONE; + })); + return m; + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/StrFormat.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/StrFormat.java new file mode 100644 index 00000000000..7d081afff32 --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/StrFormat.java @@ -0,0 +1,158 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +/// `str.format` and `str.format_map`: replacement fields with indices, +/// names, `.attr` and `[key]` lookups, `!r`/`!s`/`!a` conversions and nested +/// format specs. +final class StrFormat { + + private StrFormat() { + } + + static String format(String fmt, Object[] pos, Object named) { + StringBuilder sb = new StringBuilder(); + int auto = 0; + boolean manual = false; + int n = fmt.length(); + for (int i = 0; i < n; i++) { + char c = fmt.charAt(i); + if (c == '}') { + if (i + 1 < n && fmt.charAt(i + 1) == '}') { + sb.append('}'); + i++; + continue; + } + throw Py.valueError("Single '}' encountered in format string"); + } + if (c != '{') { + sb.append(c); + continue; + } + if (i + 1 < n && fmt.charAt(i + 1) == '{') { + sb.append('{'); + i++; + continue; + } + int depth = 1; + int j = i + 1; + while (j < n && depth > 0) { + char d = fmt.charAt(j); + if (d == '{') { + depth++; + } else if (d == '}') { + depth--; + } + j++; + } + if (depth != 0) { + throw Py.valueError("expected '}' before end of string"); + } + String field = fmt.substring(i + 1, j - 1); + i = j - 1; + String spec = ""; + char conv = 0; + int colon = indexOutsideBrackets(field, ':'); + if (colon >= 0) { + spec = field.substring(colon + 1); + field = field.substring(0, colon); + } + int bang = field.indexOf('!'); + if (bang >= 0) { + conv = field.charAt(bang + 1); + field = field.substring(0, bang); + } + int end = 0; + while (end < field.length() && field.charAt(end) != '.' && field.charAt(end) != '[') { + end++; + } + String head = field.substring(0, end); + Object v; + if (head.length() == 0) { + if (manual) { + throw Py.valueError("cannot switch from manual field specification to automatic field numbering"); + } + v = posArg(pos, auto++); + } else if (Character.isDigit(head.charAt(0))) { + manual = true; + v = posArg(pos, Integer.parseInt(head)); + } else { + v = Py.obj(named).getItem(Py.str(head)); + } + while (end < field.length()) { + if (field.charAt(end) == '.') { + int k = end + 1; + while (k < field.length() && field.charAt(k) != '.' && field.charAt(k) != '[') { + k++; + } + v = Py.obj(v).getAttr(field.substring(end + 1, k)); + end = k; + } else { + int k = field.indexOf(']', end); + String key = field.substring(end + 1, k); + boolean digits = key.length() > 0; + for (int q = 0; q < key.length() && digits; q++) { + digits = Character.isDigit(key.charAt(q)); + } + v = Py.obj(v).getItem(digits ? PyInt.of(Long.parseLong(key)) : Py.str(key)); + end = k + 1; + } + } + if (spec.indexOf('{') >= 0) { + spec = format(spec, pos, named); + } + if (conv == 'r') { + v = Py.str(Py.toRepr(v)); + } else if (conv == 's') { + v = Py.str(Py.toStr(v)); + } else if (conv == 'a') { + v = Py.obj(Builtins.ASCII).call(new Object[] {v}, Py.NO_KW); + } + sb.append(Formatting.format(v, spec)); + } + return sb.toString(); + } + + private static Object posArg(Object[] pos, int i) { + if (i >= pos.length) { + throw Py.indexError("Replacement index " + i + " out of range for positional args tuple"); + } + return pos[i]; + } + + private static int indexOutsideBrackets(String s, char c) { + int depth = 0; + for (int i = 0; i < s.length(); i++) { + char d = s.charAt(i); + if (d == '[') { + depth++; + } else if (d == ']') { + depth--; + } else if (d == c && depth == 0) { + return i; + } + } + return -1; + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/TypeMethods.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/TypeMethods.java new file mode 100644 index 00000000000..d303afc1ee3 --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/TypeMethods.java @@ -0,0 +1,1160 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +import com.codename1.util.BigInteger; + +/// The methods of the builtin types: `str.split`, `list.append`, +/// `dict.items` and the rest. Each receives the instance as `args[0]`. +final class TypeMethods { + + private TypeMethods() { + } + + private static void def(PyType t, String name, PyBuiltin.Positional p) { + t.define(name, PyBuiltin.method(t, name, p)); + } + + private static void defKw(PyType t, String name, PyBuiltin.Impl p) { + t.define(name, PyBuiltin.methodKw(t, name, p)); + } + + static void install() { + installStr(); + installList(); + installTuple(); + installDict(); + installSet(); + installNumbers(); + } + + // ---- helpers --------------------------------------------------------- + + private static String s(Object o) { + String v = PyStr.sv(o); + if (v == null) { + throw Py.typeError("descriptor requires a 'str' object but received a '" + Py.typeOf(o).name + "'"); + } + return v; + } + + private static String strArg(Object o, String fn) { + String v = PyStr.sv(o); + if (v == null) { + throw Py.typeError(fn + " arg must be None or str, not " + Py.typeOf(o).name); + } + return v; + } + + /// Locale-independent ASCII lower case. + static String asciiLower(String s) { + StringBuilder sb = null; + for (int i = 0; i < s.length(); i++) { + char c = s.charAt(i); + if (c >= 'A' && c <= 'Z') { + if (sb == null) { + sb = new StringBuilder(s); + } + sb.setCharAt(i, (char) (c + 32)); + } + } + return sb == null ? s : sb.toString(); + } + + /// Unicode case mapping per character, which (unlike + /// `String.toLowerCase()`) does not depend on the device locale. + static String mapCase(String s, int mode) { + if (mode <= 1) { + // ASCII: one array, and the string itself when nothing changes. + int n = s.length(); + char[] out = null; + int i = 0; + for (; i < n; i++) { + char c = s.charAt(i); + if (c >= 128) { + break; + } + char r = mode == 1 ? (c >= 'a' && c <= 'z' ? (char) (c - 32) : c) + : (c >= 'A' && c <= 'Z' ? (char) (c + 32) : c); + if (r != c && out == null) { + out = new char[n]; + s.getChars(0, i, out, 0); + } + if (out != null) { + out[i] = r; + } + } + if (i == n) { + return out == null ? s : new String(out); + } + } + StringBuilder sb = new StringBuilder(s.length()); + boolean prevCased = false; + for (int i = 0; i < s.length(); i++) { + char c = s.charAt(i); + char r; + switch (mode) { + case 0: + r = Character.toLowerCase(c); + break; + case 1: + r = Character.toUpperCase(c); + break; + case 2: + r = Character.isUpperCase(c) ? Character.toLowerCase(c) : Character.toUpperCase(c); + break; + default: + r = prevCased ? Character.toLowerCase(c) : Character.toUpperCase(c); + prevCased = Character.isLetter(c); + break; + } + if (mode == 1 && c == '\u00df') { + sb.append("SS"); + continue; + } + sb.append(r); + } + return sb.toString(); + } + + /// The words of `str.split()` one at a time, for a loop that only walks + /// them: a str cannot change, so splitting lazily yields the same words + /// as splitting first, without the list. + static final class Words extends PyObject { + private final String v; + private int pos; + + Words(String v) { + this.v = v; + } + + @Override + public PyType type() { + return Py.ITERATOR_TYPE; + } + + @Override + public Object iter() { + return this; + } + + @Override + public Object next() { + int n = v.length(); + int a = pos; + // The range test first: isSpace is a call, and words are mostly + // printable ASCII. + char c; + while (a < n && ((c = v.charAt(a)) == ' ' || (c < ' ' || c >= '\u0085') && isSpace(c))) { + a++; + } + if (a >= n) { + pos = n; + return null; + } + int b = a + 1; + while (b < n && ((c = v.charAt(b)) > ' ' && c < '\u0085' || c != ' ' && !isSpace(c))) { + b++; + } + pos = b; + return v.substring(a, b); + } + } + + /// The pieces of `str.split(sep)` one at a time (see Words). + static final class Pieces extends PyObject { + private final String v; + private final String sep; + private int pos; + + Pieces(String v, String sep) { + this.v = v; + this.sep = sep; + } + + @Override + public PyType type() { + return Py.ITERATOR_TYPE; + } + + @Override + public Object iter() { + return this; + } + + @Override + public Object next() { + int a = pos; + if (a < 0) { + return null; + } + int b = v.indexOf(sep, a); + if (b < 0) { + pos = -1; + return v.substring(a); + } + pos = b + sep.length(); + return v.substring(a, b); + } + } + + private static boolean isSpace(char c) { + if (c > ' ' && c < '\u0085') { + return false; + } + return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\u000b' || c == '\u000c' + || c == '\u001c' || c == '\u001d' || c == '\u001e' || c == '\u001f' || c == '\u0085' + || Character.isWhitespace(c) || c == '\u00a0'; + } + + private static String strip(String v, Object chars, boolean left, boolean right) { + int a = 0; + int b = v.length(); + String cs = chars == null || chars == Py.NONE ? null : strArg(chars, "strip"); //NOPMD CompareObjectsWithEquals - identity is meant + while (left && a < b && (cs == null ? isSpace(v.charAt(a)) : cs.indexOf(v.charAt(a)) >= 0)) { + a++; + } + while (right && b > a && (cs == null ? isSpace(v.charAt(b - 1)) : cs.indexOf(v.charAt(b - 1)) >= 0)) { + b--; + } + return v.substring(a, b); + } + + static PyList split(String v, Object sepO, long max) { + if (sepO == null || sepO == Py.NONE) { //NOPMD CompareObjectsWithEquals - identity is meant + // Counted first, so the list is one array of the right size. + int n = v.length(); + int count = 0; + for (int i = 0; i < n;) { + while (i < n && isSpace(v.charAt(i))) { + i++; + } + if (i >= n) { + break; + } + count++; + if (max >= 0 && count > max) { + break; + } + while (i < n && !isSpace(v.charAt(i))) { + i++; + } + } + Object[] parts = new Object[count]; + int k = 0; + int i = 0; + while (k < count) { + while (isSpace(v.charAt(i))) { + i++; + } + if (max >= 0 && k == max) { + parts[k++] = strip(v.substring(i), null, false, true); + break; + } + int j = i; + while (j < n && !isSpace(v.charAt(j))) { + j++; + } + parts[k++] = v.substring(i, j); + i = j; + } + return new PyList(parts); + } + PyList out = new PyList(); + String sep = strArg(sepO, "split"); + if (sep.length() == 0) { + throw Py.valueError("empty separator"); + } + int i = 0; + while (max < 0 || out.size() < max) { + int j = v.indexOf(sep, i); + if (j < 0) { + break; + } + out.append(Py.str(v.substring(i, j))); + i = j + sep.length(); + } + out.append(Py.str(v.substring(i))); + return out; + } + + private static PyList rsplit(String v, Object sepO, long max) { + if (max < 0) { + return split(v, sepO, max); + } + PyList rev = new PyList(); + if (sepO == null || sepO == Py.NONE) { //NOPMD CompareObjectsWithEquals - identity is meant + int j = v.length(); + while (true) { + while (j > 0 && isSpace(v.charAt(j - 1))) { + j--; + } + if (j <= 0) { + break; + } + if (rev.size() >= max) { + rev.append(Py.str(strip(v.substring(0, j), null, true, false))); + break; + } + int i = j; + while (i > 0 && !isSpace(v.charAt(i - 1))) { + i--; + } + rev.append(Py.str(v.substring(i, j))); + j = i; + } + } else { + String sep = strArg(sepO, "rsplit"); + int j = v.length(); + while (rev.size() < max) { + int i = v.lastIndexOf(sep, j - sep.length()); + if (i < 0) { + break; + } + rev.append(Py.str(v.substring(i + sep.length(), j))); + j = i; + } + rev.append(Py.str(v.substring(0, j))); + } + rev.reverse(); + return rev; + } + + /// Resolves the optional `start, end` arguments of find/count/startswith + /// to UTF-16 offsets. + private static int[] range(String self, Object[] a, int from) { + int n = PyStr.codePoints(self); + int[] r = new PySlice(a.length > from ? a[from] : Py.NONE, a.length > from + 1 ? a[from + 1] : Py.NONE, + Py.NONE).indices(n); + return new int[] {PyStr.offset(self, r[0]), PyStr.offset(self, Math.max(r[0], r[1]))}; + } + + private static int find(String self, Object[] a, boolean last) { + String sub = strArg(a[1], "find"); + int[] r = range(self, a, 2); + String v = self; + if (r[1] - r[0] < sub.length()) { + return -1; + } + int i = last ? v.lastIndexOf(sub, r[1] - sub.length()) : v.indexOf(sub, r[0]); + if (i < r[0] || i + sub.length() > r[1]) { + return -1; + } + return PyStr.cpIndex(self, i); + } + + private static boolean affix(String self, Object[] a, boolean start) { + int[] r = range(self, a, 2); + String v = self.substring(r[0], r[1]); + Object[] cands = a[1] instanceof PyTuple ? ((PyTuple) a[1]).items : new Object[] {a[1]}; + for (Object cand : cands) { + String c = strArg(cand, start ? "startswith" : "endswith"); + if (start ? v.startsWith(c) : v.endsWith(c)) { + return true; + } + } + return false; + } + + private static Object pad(Object[] a, int mode) { + String v = s(a[0]); + int width = (int) Py.asLong(a[1]); + String fill = a.length > 2 ? strArg(a[2], "fill") : " "; + int n = PyStr.codePoints((String) a[0]); + if (width <= n) { + return a[0]; + } + int total = width - n; + int left = mode == 0 ? 0 : mode == 1 ? total : total / 2 + (total & width & 1); + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < left; i++) { + sb.append(fill); + } + sb.append(v); + for (int i = left; i < total; i++) { + sb.append(fill); + } + return Py.str(sb.toString()); + } + + private static boolean all(String v, int kind) { + if (v.length() == 0) { + return false; + } + boolean cased = false; + for (int i = 0; i < v.length(); i++) { + char c = v.charAt(i); + boolean ok; + switch (kind) { + case 0: + ok = Character.isDigit(c); + break; + case 1: + ok = Character.isLetter(c); + break; + case 2: + ok = Character.isLetterOrDigit(c); + break; + case 3: + ok = isSpace(c); + break; + case 4: + ok = !Character.isLowerCase(c); + cased |= Character.isUpperCase(c); + break; + default: + ok = !Character.isUpperCase(c); + cased |= Character.isLowerCase(c); + break; + } + if (!ok) { + return false; + } + } + return kind < 4 || cased; + } + + /// `sep.join(items)`; a list is read in place, without an iterator. + static Object join(String sep, Object items) { + if (items instanceof PyList) { + PyList l = (PyList) items; + int n = l.size; + if (n == 1 && l.get(0) instanceof String) { + return l.get(0); + } + // String.join, which ParparVM builds in one allocation that the + // parts' bytes are copied straight into. + String[] parts = new String[n]; + for (int i = 0; i < n; i++) { + Object v = l.get(i); + if (!(v instanceof String)) { + throw Py.typeError("sequence item " + i + ": expected str instance, " + Py.typeOf(v).name + " found"); + } + parts[i] = (String) v; + } + return String.join(sep, parts); + } + Object it = Py.iter(items); + StringBuilder sb = new StringBuilder(); + boolean first = true; + int i = 0; + for (Object v = Py.next(it); v != null; v = Py.next(it), i++) { + if (!(v instanceof String)) { + throw Py.typeError("sequence item " + i + ": expected str instance, " + Py.typeOf(v).name + " found"); + } + if (!first) { + sb.append(sep); + } + first = false; + sb.append(((String) v)); + } + return Py.str(sb.toString()); + } + + private static void installStr() { + PyType t = Py.STR_TYPE; + def(t, "join", a -> join(s(a[0]), a[1])); + defKw(t, "split", (args, kw) -> { + Args a = new Args("split", args, kw).check(3, "sep", "maxsplit"); + return split(s(args[0]), a.get(1, "sep", Py.NONE), Py.asLong(a.get(2, "maxsplit", PyInt.of(-1)))); + }); + defKw(t, "rsplit", (args, kw) -> { + Args a = new Args("rsplit", args, kw).check(3, "sep", "maxsplit"); + return rsplit(s(args[0]), a.get(1, "sep", Py.NONE), Py.asLong(a.get(2, "maxsplit", PyInt.of(-1)))); + }); + defKw(t, "splitlines", (args, kw) -> { + Args a = new Args("splitlines", args, kw).check(2, "keepends"); + boolean keep = Py.isTrue(a.get(1, "keepends", Py.FALSE)); + String v = s(args[0]); + PyList out = new PyList(); + int i = 0; + int start = 0; + while (i < v.length()) { + char c = v.charAt(i); + if (c == '\n' || c == '\r' || c == '\u000b' || c == '\u000c' || c == '\u001c' || c == '\u001d' + || c == '\u001e' || c == '\u0085' || c == '\u2028' || c == '\u2029') { + int end = i; + i++; + if (c == '\r' && i < v.length() && v.charAt(i) == '\n') { + i++; + } + out.append(Py.str(v.substring(start, keep ? i : end))); + start = i; + } else { + i++; + } + } + if (start < v.length()) { + out.append(Py.str(v.substring(start))); + } + return out; + }); + def(t, "strip", a -> Py.str(strip(s(a[0]), a.length > 1 ? a[1] : null, true, true))); + def(t, "lstrip", a -> Py.str(strip(s(a[0]), a.length > 1 ? a[1] : null, true, false))); + def(t, "rstrip", a -> Py.str(strip(s(a[0]), a.length > 1 ? a[1] : null, false, true))); + def(t, "replace", a -> { + String v = s(a[0]); + String old = strArg(a[1], "replace"); + String nw = strArg(a[2], "replace"); + long count = a.length > 3 ? Py.asLong(a[3]) : -1; + if (count < 0) { + if (old.length() == 0) { + StringBuilder sb = new StringBuilder(nw); + for (int i = 0; i < v.length(); i++) { + sb.append(v.charAt(i)).append(nw); + } + return Py.str(sb.toString()); + } + return Py.str(v.replace(old, nw)); + } + StringBuilder sb = new StringBuilder(); + int i = 0; + for (long k = 0; k < count; k++) { + int j = v.indexOf(old, i); + if (j < 0) { + break; + } + sb.append(v, i, j).append(nw); + i = j + old.length(); + if (old.length() == 0) { + if (i < v.length()) { + sb.append(v.charAt(i)); + } + i++; + } + } + if (i <= v.length()) { + sb.append(v.substring(Math.min(i, v.length()))); + } + return Py.str(sb.toString()); + }); + def(t, "startswith", a -> Py.bool(affix(s(a[0]), a, true))); + def(t, "endswith", a -> Py.bool(affix(s(a[0]), a, false))); + def(t, "find", a -> PyInt.of(find(s(a[0]), a, false))); + def(t, "rfind", a -> PyInt.of(find(s(a[0]), a, true))); + def(t, "index", a -> { + int i = find(s(a[0]), a, false); + if (i < 0) { + throw Py.valueError("substring not found"); + } + return PyInt.of(i); + }); + def(t, "rindex", a -> { + int i = find(s(a[0]), a, true); + if (i < 0) { + throw Py.valueError("substring not found"); + } + return PyInt.of(i); + }); + def(t, "count", a -> { + String self = s(a[0]); + String sub = strArg(a[1], "count"); + int[] r = range(self, a, 2); + String v = self.substring(r[0], r[1]); + if (sub.length() == 0) { + return PyInt.of(PyStr.codePoints(v) + 1); + } + int n = 0; + for (int i = v.indexOf(sub); i >= 0; i = v.indexOf(sub, i + sub.length())) { + n++; + } + return PyInt.of(n); + }); + def(t, "upper", a -> Py.str(mapCase(s(a[0]), 1))); + def(t, "lower", a -> Py.str(mapCase(s(a[0]), 0))); + def(t, "casefold", a -> Py.str(mapCase(s(a[0]), 0))); + def(t, "swapcase", a -> Py.str(mapCase(s(a[0]), 2))); + def(t, "title", a -> Py.str(mapCase(s(a[0]), 3))); + def(t, "capitalize", a -> { + String v = s(a[0]); + if (v.length() == 0) { + return a[0]; + } + return Py.str(Character.toUpperCase(v.charAt(0)) + mapCase(v.substring(1), 0)); + }); + def(t, "isdigit", a -> Py.bool(all(s(a[0]), 0))); + def(t, "isdecimal", a -> Py.bool(all(s(a[0]), 0))); + def(t, "isnumeric", a -> Py.bool(all(s(a[0]), 0))); + def(t, "isalpha", a -> Py.bool(all(s(a[0]), 1))); + def(t, "isalnum", a -> Py.bool(all(s(a[0]), 2))); + def(t, "isspace", a -> Py.bool(all(s(a[0]), 3))); + def(t, "isupper", a -> Py.bool(all(s(a[0]), 4))); + def(t, "islower", a -> Py.bool(all(s(a[0]), 5))); + def(t, "isascii", a -> { + String v = s(a[0]); + for (int i = 0; i < v.length(); i++) { + if (v.charAt(i) > 127) { + return Py.FALSE; + } + } + return Py.TRUE; + }); + def(t, "isidentifier", a -> { + String v = s(a[0]); + if (v.length() == 0 || !(Character.isLetter(v.charAt(0)) || v.charAt(0) == '_')) { + return Py.FALSE; + } + for (int i = 1; i < v.length(); i++) { + if (!(Character.isLetterOrDigit(v.charAt(i)) || v.charAt(i) == '_')) { + return Py.FALSE; + } + } + return Py.TRUE; + }); + def(t, "ljust", a -> pad(a, 0)); + def(t, "rjust", a -> pad(a, 1)); + def(t, "center", a -> pad(a, 2)); + def(t, "zfill", a -> { + String v = s(a[0]); + int width = (int) Py.asLong(a[1]); + if (v.length() >= width) { + return a[0]; + } + String sign = v.length() > 0 && (v.charAt(0) == '-' || v.charAt(0) == '+') ? v.substring(0, 1) : ""; + StringBuilder sb = new StringBuilder(sign); + for (int i = v.length(); i < width; i++) { + sb.append('0'); + } + return Py.str(sb.append(v.substring(sign.length())).toString()); + }); + def(t, "partition", a -> { + String v = s(a[0]); + String sep = strArg(a[1], "partition"); + int i = v.indexOf(sep); + if (i < 0) { + return PyTuple.of(a[0], Py.EMPTY_STR, Py.EMPTY_STR); + } + return PyTuple.of(Py.str(v.substring(0, i)), a[1], Py.str(v.substring(i + sep.length()))); + }); + def(t, "rpartition", a -> { + String v = s(a[0]); + String sep = strArg(a[1], "rpartition"); + int i = v.lastIndexOf(sep); + if (i < 0) { + return PyTuple.of(Py.EMPTY_STR, Py.EMPTY_STR, a[0]); + } + return PyTuple.of(Py.str(v.substring(0, i)), a[1], Py.str(v.substring(i + sep.length()))); + }); + def(t, "removeprefix", a -> { + String v = s(a[0]); + String p = strArg(a[1], "removeprefix"); + return v.startsWith(p) ? Py.str(v.substring(p.length())) : a[0]; + }); + def(t, "removesuffix", a -> { + String v = s(a[0]); + String p = strArg(a[1], "removesuffix"); + return p.length() > 0 && v.endsWith(p) ? Py.str(v.substring(0, v.length() - p.length())) : a[0]; + }); + def(t, "expandtabs", a -> { + String v = s(a[0]); + int size = a.length > 1 ? (int) Py.asLong(a[1]) : 8; + StringBuilder sb = new StringBuilder(); + int col = 0; + for (int i = 0; i < v.length(); i++) { + char c = v.charAt(i); + if (c == '\t') { + int n = size <= 0 ? 0 : size - col % size; + for (int k = 0; k < n; k++) { + sb.append(' '); + } + col += n; + } else { + sb.append(c); + col = c == '\n' || c == '\r' ? 0 : col + 1; + } + } + return Py.str(sb.toString()); + }); + defKw(t, "format", (args, kw) -> { + Object[] pos = new Object[args.length - kw.length - 1]; + System.arraycopy(args, 1, pos, 0, pos.length); + PyDict named = new PyDict(); + for (int i = 0; i < kw.length; i++) { + named.setItem(Py.str(kw[i]), args[args.length - kw.length + i]); + } + return Py.str(StrFormat.format(s(args[0]), pos, named)); + }); + def(t, "format_map", a -> Py.str(StrFormat.format(s(a[0]), Py.NO_ARGS, a[1]))); + defKw(t, "encode", (a, kw) -> { + Object[] p = Args.bind("encode", a, kw, new String[] {"self", "encoding", "errors"}, + new Object[] {null, Py.str("utf-8"), Py.str("strict")}); + return new PyBytes(Codecs.encode(s(p[0]), Py.toStr(p[1]), Py.toStr(p[2]))); + }); + def(t, "__len__", a -> PyInt.of(Py.len(a[0]))); + def(t, "__contains__", a -> Py.bool(Py.obj(a[0]).contains(a[1]))); + def(t, "__getitem__", a -> Py.obj(a[0]).getItem(a[1])); + t.define("maketrans", new PyStaticMethod(PyBuiltin.positional("maketrans", a -> { + PyDict d = new PyDict(); + if (a.length == 1) { + PyDict src = (PyDict) a[0]; + Object[] keys = src.keyArray(); + for (Object key : keys) { + Object k = key instanceof String ? PyInt.of(((String) key).codePointAt(0)) : key; + d.setItem(k, src.getItem(key)); + } + return d; + } + String from = s(a[0]); + String to = s(a[1]); + for (int i = 0; i < from.length(); i++) { + d.setItem(PyInt.of(from.charAt(i)), PyInt.of(to.charAt(i))); + } + if (a.length > 2) { + String del = s(a[2]); + for (int i = 0; i < del.length(); i++) { + d.setItem(PyInt.of(del.charAt(i)), Py.NONE); + } + } + return d; + }))); + def(t, "translate", a -> { + String v = s(a[0]); + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < v.length(); i++) { + int cp = v.codePointAt(i); + if (cp > 0xffff) { + i++; + } + Object r; + try { + r = Py.obj(a[1]).getItem(PyInt.of(cp)); + } catch (PyException e) { + if (!e.matches(Py.LOOKUP_ERROR)) { + throw e; + } + sb.appendCodePoint(cp); + continue; + } + if (r == Py.NONE) { //NOPMD CompareObjectsWithEquals - identity is meant + continue; + } + if (Py.isInt(r)) { + sb.appendCodePoint((int) Py.asLong(r)); + } else { + sb.append(Py.toStr(r)); + } + } + return Py.str(sb.toString()); + }); + } + + // ---- list ------------------------------------------------------------ + + private static PyList l(Object o) { + if (!(o instanceof PyList)) { + throw Py.typeError("descriptor requires a 'list' object but received a '" + Py.typeOf(o).name + "'"); + } + return (PyList) o; + } + + private static void installList() { + PyType t = Py.LIST_TYPE; + def(t, "append", a -> { + l(a[0]).append(a[1]); + return Py.NONE; + }); + def(t, "extend", a -> { + l(a[0]).extend(a[1]); + return Py.NONE; + }); + def(t, "insert", a -> { + l(a[0]).insert(Py.index(a[1]), a[2]); + return Py.NONE; + }); + def(t, "pop", a -> l(a[0]).pop(a.length > 1 ? Py.index(a[1]) : -1)); + def(t, "remove", a -> { + l(a[0]).remove(a[1]); + return Py.NONE; + }); + def(t, "index", a -> PyInt.of(l(a[0]).index(a[1]))); + def(t, "count", a -> { + PyList self = l(a[0]); + int n = 0; + for (int i = 0; i < self.size; i++) { + if (Py.eqBool(self.get(i), a[1])) { + n++; + } + } + return PyInt.of(n); + }); + defKw(t, "sort", (args, kw) -> { + Args a = new Args("sort", args, kw).check(1, "key", "reverse"); + Object rev = a.kw("reverse"); + l(args[0]).sort(a.kw("key"), rev != null && Py.isTrue(rev)); + return Py.NONE; + }); + def(t, "reverse", a -> { + l(a[0]).reverse(); + return Py.NONE; + }); + def(t, "clear", a -> { + l(a[0]).clear(); + return Py.NONE; + }); + def(t, "copy", a -> new PyList(l(a[0]).toArray())); + def(t, "__len__", a -> PyInt.of(Py.len(a[0]))); + def(t, "__getitem__", a -> Py.obj(a[0]).getItem(a[1])); + def(t, "__setitem__", a -> { + Py.obj(a[0]).setItem(a[1], a[2]); + return Py.NONE; + }); + def(t, "__contains__", a -> Py.bool(Py.obj(a[0]).contains(a[1]))); + def(t, "__iter__", a -> Py.iter(a[0])); + } + + private static void installTuple() { + PyType t = Py.TUPLE_TYPE; + def(t, "count", a -> { + Object[] items = ((PyTuple) a[0]).items; + int n = 0; + for (Object item : items) { + if (Py.eqBool(item, a[1])) { + n++; + } + } + return PyInt.of(n); + }); + def(t, "index", a -> { + Object[] items = ((PyTuple) a[0]).items; + for (int i = 0; i < items.length; i++) { + if (Py.eqBool(items[i], a[1])) { + return PyInt.of(i); + } + } + throw Py.valueError("tuple.index(x): x not in tuple"); + }); + def(t, "__len__", a -> PyInt.of(Py.len(a[0]))); + def(t, "__getitem__", a -> Py.obj(a[0]).getItem(a[1])); + } + + // ---- dict ------------------------------------------------------------ + + private static PyDict d(Object o) { + if (!(o instanceof PyDict)) { + throw Py.typeError("descriptor requires a 'dict' object but received a '" + Py.typeOf(o).name + "'"); + } + return (PyDict) o; + } + + private static void installDict() { + PyType t = Py.DICT_TYPE; + def(t, "get", a -> { + Object v = d(a[0]).getOrNull(a[1]); + return v != null ? v : a.length > 2 ? a[2] : Py.NONE; + }); + def(t, "keys", a -> new PyDictView(d(a[0]), 0)); + def(t, "values", a -> new PyDictView(d(a[0]), 1)); + def(t, "items", a -> new PyDictView(d(a[0]), 2)); + def(t, "pop", a -> { + Object v = d(a[0]).remove(a[1]); + if (v == null) { + if (a.length > 2) { + return a[2]; + } + throw Py.keyError(a[1]); + } + return v; + }); + def(t, "popitem", a -> { + PyDict self = d(a[0]); + for (int i = self.used - 1; i >= 0; i--) { + if (self.keys[i] != null) { + Object k = self.keys[i]; + Object v = self.remove(k); + return PyTuple.of(k, v); + } + } + throw Py.error(Py.KEY_ERROR, "popitem(): dictionary is empty"); + }); + def(t, "setdefault", a -> { + PyDict self = d(a[0]); + Object v = self.getOrNull(a[1]); + if (v == null) { + v = a.length > 2 ? a[2] : Py.NONE; + self.setItem(a[1], v); + } + return v; + }); + defKw(t, "update", (args, kw) -> { + PyDict self = d(args[0]); + int npos = args.length - kw.length; + if (npos > 1) { + self.update(args[1]); + } + for (int i = 0; i < kw.length; i++) { + self.setItem(Py.str(kw[i]), args[npos + i]); + } + return Py.NONE; + }); + def(t, "clear", a -> { + d(a[0]).clear(); + return Py.NONE; + }); + def(t, "copy", a -> d(a[0]).copy()); + t.define("fromkeys", new PyClassMethod(PyBuiltin.positional("fromkeys", a -> { + PyDict r = new PyDict(); + Object v = a.length > 2 ? a[2] : Py.NONE; + Object it = Py.iter(a[1]); + for (Object k = Py.next(it); k != null; k = Py.next(it)) { + r.setItem(k, v); + } + return r; + }))); + def(t, "__getitem__", a -> Py.obj(a[0]).getItem(a[1])); + def(t, "__setitem__", a -> { + Py.obj(a[0]).setItem(a[1], a[2]); + return Py.NONE; + }); + def(t, "__contains__", a -> Py.bool(Py.obj(a[0]).contains(a[1]))); + def(t, "__len__", a -> PyInt.of(Py.len(a[0]))); + def(t, "__iter__", a -> Py.iter(a[0])); + } + + // ---- set ------------------------------------------------------------- + + private static PySet st(Object o) { + if (!(o instanceof PySet)) { + throw Py.typeError("descriptor requires a 'set' object but received a '" + Py.typeOf(o).name + "'"); + } + return (PySet) o; + } + + private static void installSet() { + PyType[] types = {Py.SET_TYPE, Py.FROZENSET_TYPE}; + for (PyType type : types) { + PyType t = type; + def(t, "union", a -> { + PySet r = st(a[0]); + for (int i = 1; i < a.length; i++) { + r = r.union(a[i]); + } + return r == a[0] ? r.copyAs(r.frozen) : r; + }); + def(t, "intersection", a -> { + PySet r = st(a[0]).copyAs(st(a[0]).frozen); + for (int i = 1; i < a.length; i++) { + r = r.intersection(a[i]); + } + return r; + }); + def(t, "difference", a -> { + PySet r = st(a[0]).copyAs(st(a[0]).frozen); + for (int i = 1; i < a.length; i++) { + r = r.difference(a[i]); + } + return r; + }); + def(t, "symmetric_difference", a -> st(a[0]).symmetricDifference(a[1])); + def(t, "issubset", a -> Py.bool(st(a[0]).isSubset(a[1]))); + def(t, "issuperset", a -> Py.bool(PySet.of(a[1], true).isSubset(a[0]))); + def(t, "isdisjoint", a -> Py.bool(st(a[0]).intersection(a[1]).len() == 0)); + def(t, "copy", a -> st(a[0]).copyAs(st(a[0]).frozen)); + def(t, "__contains__", a -> Py.bool(Py.obj(a[0]).contains(a[1]))); + def(t, "__len__", a -> PyInt.of(Py.len(a[0]))); + } + PyType t = Py.SET_TYPE; + def(t, "add", a -> { + st(a[0]).addItem(a[1]); + return Py.NONE; + }); + def(t, "discard", a -> { + st(a[0]).discard(a[1]); + return Py.NONE; + }); + def(t, "remove", a -> { + if (!st(a[0]).discard(a[1])) { + throw Py.keyError(a[1]); + } + return Py.NONE; + }); + def(t, "pop", a -> { + return st(a[0]).pop(); + }); + def(t, "clear", a -> { + st(a[0]).clear(); + return Py.NONE; + }); + def(t, "update", a -> { + PySet self = st(a[0]); + if (self.frozen) { + throw Py.attributeError("'frozenset' object has no attribute 'update'"); + } + for (int i = 1; i < a.length; i++) { + self.update(a[i]); + } + return Py.NONE; + }); + def(t, "intersection_update", a -> { + PySet self = st(a[0]); + self.replaceWith(self.intersection(a[1])); + return Py.NONE; + }); + def(t, "symmetric_difference_update", a -> { + PySet self = st(a[0]); + self.replaceWith(self.symmetricDifference(a[1])); + return Py.NONE; + }); + def(t, "difference_update", a -> { + PySet self = st(a[0]); + Object it = Py.iter(a[1]); + for (Object v = Py.next(it); v != null; v = Py.next(it)) { + self.discard(v); + } + return Py.NONE; + }); + } + + // ---- numbers --------------------------------------------------------- + + private static boolean little(Object order) { + String o = Py.toStr(order); + if ("little".equals(o)) { + return true; + } + if ("big".equals(o)) { + return false; + } + throw Py.valueError("byteorder must be either 'little' or 'big'"); + } + + private static byte[] toBytes(BigInteger v, long length, boolean little, boolean signed) { + if (length < 0) { + throw Py.valueError("length argument must be non-negative"); + } + if (!signed && v.signum() < 0) { + throw Py.error(Py.OVERFLOW_ERROR, "can't convert negative int to unsigned"); + } + int bits = signed ? v.bitLength() + 1 : v.bitLength(); + if (bits > length * 8 && !(v.signum() == 0 && length == 0)) { + throw Py.error(Py.OVERFLOW_ERROR, "int too big to convert"); + } + byte[] two = v.toByteArray(); + byte[] r = new byte[(int) length]; + byte fill = (byte) (v.signum() < 0 ? 0xFF : 0); + for (int i = 0; i < r.length; i++) { + int src = two.length - 1 - i; + byte bv = src >= 0 ? two[src] : fill; + r[little ? i : r.length - 1 - i] = bv; + } + return r; + } + + private static void installNumbers() { + def(Py.INT_TYPE, "bit_length", a -> { + if (a[0] instanceof Long) { + long v = ((Long) a[0]).longValue(); + return PyInt.of(v == Long.MIN_VALUE ? 64 : 64 - Long.numberOfLeadingZeros(Math.abs(v))); + } + return PyInt.of(Py.intView(a[0]).toBig().abs().bitLength()); + }); + defKw(Py.INT_TYPE, "to_bytes", (args, kw) -> { + Args a = new Args("to_bytes", args, kw).check(3, "length", "byteorder", "signed"); + return new PyBytes(toBytes(Py.intView(args[0]).toBig(), Py.asLong(a.get(1, "length", PyInt.of(1))), + little(a.get(2, "byteorder", Py.str("big"))), Py.isTrue(a.get(3, "signed", Py.FALSE)))); + }); + Py.INT_TYPE.define("from_bytes", new PyClassMethod(new PyBuiltin("from_bytes", (args, kw) -> { + Args a = new Args("from_bytes", args, kw).check(3, "bytes", "byteorder", "signed"); + Object src = a.get(1, "bytes", null); + byte[] b; + if (src instanceof PyBytes) { + b = ((PyBytes) src).value; + } else { + Object[] items = Py.toArray(src); + b = new byte[items.length]; + for (int i = 0; i < b.length; i++) { + b[i] = (byte) Py.asLong(items[i]); + } + } + if (little(a.get(2, "byteorder", Py.str("big")))) { + byte[] r = new byte[b.length]; + for (int i = 0; i < b.length; i++) { + r[i] = b[b.length - 1 - i]; + } + b = r; + } + BigInteger v = b.length == 0 ? BigInteger.ZERO + : Py.isTrue(a.get(3, "signed", Py.FALSE)) ? new BigInteger(b) : new BigInteger(1, b); + Object cls = args[0]; + return cls == Py.INT_TYPE ? PyInt.of(v) : Py.call(cls, PyInt.of(v)); //NOPMD CompareObjectsWithEquals - identity is meant + }))); + def(Py.INT_TYPE, "as_integer_ratio", a -> new PyTuple(new Object[] {a[0], PyInt.of(1)})); + def(Py.INT_TYPE, "is_integer", a -> Py.TRUE); + def(Py.INT_TYPE, "bit_count", a -> PyInt.of(Py.intView(a[0]).toBig().abs().bitCount())); + def(Py.INT_TYPE, "conjugate", a -> a[0]); + prop(Py.INT_TYPE, "real", a -> a[0]); + prop(Py.INT_TYPE, "imag", a -> PyInt.of(0)); + prop(Py.INT_TYPE, "numerator", a -> a[0]); + prop(Py.INT_TYPE, "denominator", a -> PyInt.of(1)); + prop(Py.FLOAT_TYPE, "real", a -> a[0]); + prop(Py.FLOAT_TYPE, "imag", a -> Py.ofFloat(0.0)); + def(Py.INT_TYPE, "__index__", a -> a[0]); + def(Py.INT_TYPE, "__int__", a -> a[0]); + def(Py.INT_TYPE, "__float__", a -> Py.ofFloat(Py.asDouble(a[0]))); + def(Py.FLOAT_TYPE, "is_integer", a -> { + double v = Py.asDouble(a[0]); + return Py.bool(!Double.isInfinite(v) && v == Math.floor(v)); + }); + def(Py.FLOAT_TYPE, "as_integer_ratio", a -> floatRatio(Py.asDouble(a[0]))); + def(Py.FLOAT_TYPE, "__float__", a -> a[0]); + def(Py.FLOAT_TYPE, "__int__", a -> Builtins.toInt(a[0])); + def(Py.FLOAT_TYPE, "conjugate", a -> a[0]); + } + + private static void prop(PyType t, String name, PyBuiltin.Positional body) { + t.define(name, new PyProperty(PyBuiltin.positional(name, body), null, null)); + } + + /// The exact fraction a double holds, in lowest terms. + private static Object floatRatio(double v) { + if (Double.isNaN(v)) { + throw Py.valueError("cannot convert NaN to integer ratio"); + } + if (Double.isInfinite(v)) { + throw Py.error(Py.OVERFLOW_ERROR, "cannot convert Infinity to integer ratio"); + } + long bits = Double.doubleToLongBits(v); + int exp = (int) ((bits >> 52) & 0x7ff); + long mant = bits & ((1L << 52) - 1); + if (exp == 0) { + exp = 1; + } else { + mant |= 1L << 52; + } + exp -= 1075; + if (mant == 0) { + return new PyTuple(new Object[] {PyInt.of(0), PyInt.of(1)}); + } + while ((mant & 1) == 0 && exp < 0) { + mant >>= 1; + exp++; + } + if (bits < 0) { + mant = -mant; + } + if (exp >= 0) { + return new PyTuple(new Object[] {PyInt.of(BigInteger.valueOf(mant).shiftLeft(exp)), PyInt.of(1)}); + } + return new PyTuple(new Object[] {PyInt.of(mant), PyInt.of(BigInteger.valueOf(1).shiftLeft(-exp))}); + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Typed.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Typed.java new file mode 100644 index 00000000000..0154c5f5d91 --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Typed.java @@ -0,0 +1,350 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +/// What the typed code the transpiler generates for a closed program calls. +/// +/// In such code a value whose type the whole-program inference proved is +/// held unboxed: a float as a `double`, a bool as a `boolean`, an instance +/// of a compiled class as that Java class, with None as Java null. These +/// helpers convert at the edges with the generic code, and implement the +/// operations whose Python meaning is not the Java one. A conversion that +/// finds a value of another type means the inference was wrong; it raises +/// rather than letting a cast that is not checked on every port go through. +public final class Typed { + /// A `while True` test javac cannot see as a constant: generated code + /// after such a loop must not be "unreachable" to it. + public static boolean yes = true; + + private Typed() { + } + + /// The receiver of an attribute access, which raises for None. + public static T nz(T o, String attr) { + if (o == null) { + throw Py.attributeError("'NoneType' object has no attribute '" + attr + "'"); + } + return o; + } + + /// The failing half of an inline None check on reaching `attr`. + public static T nzf(String attr) { + throw Py.attributeError("'NoneType' object has no attribute '" + attr + "'"); + } + + /// Subscripting None. + public static PyList nzl(PyList l) { + if (l == null) { + throw Py.typeError("'NoneType' object is not subscriptable"); + } + return l; + } + + public static Object box(Object o) { + return o == null ? Py.NONE : o; + } + + public static PyException mismatch(Object o) { + return Py.runtimeError("internal error: typed code received a value of type '" + Py.typeOf(o).name + "'"); + } + + public static boolean asBool(Object o) { + if (o == Py.TRUE) { //NOPMD CompareObjectsWithEquals - identity is meant + return true; + } + if (o == Py.FALSE) { //NOPMD CompareObjectsWithEquals - identity is meant + return false; + } + throw mismatch(o); + } + + public static double asFloat(Object o) { + if (o instanceof Double) { + return ((Double) o).doubleValue(); + } + throw mismatch(o); + } + + public static String asStr(Object o) { + if (o instanceof String) { + return (String) o; + } + if (o == Py.NONE) { //NOPMD CompareObjectsWithEquals - identity is meant + return null; + } + throw mismatch(o); + } + + public static PyList asList(Object o) { + if (o instanceof PyList) { + return (PyList) o; + } + if (o == Py.NONE) { //NOPMD CompareObjectsWithEquals - identity is meant + return null; + } + throw mismatch(o); + } + + /// `float(i)` of an int. + public static double toFloat(Object i) { + if (i instanceof Long) { + return ((Long) i).longValue(); + } + return Py.asDouble(i); + } + + /// A range bound, which must fit a Java long here. + public static long bound(Object i) { + if (i instanceof Long) { + return ((Long) i).longValue(); + } + return Py.asLong(i); + } + + // ---- ints ------------------------------------------------------------ + + public static boolean lt(Object a, Object b) { + if (a instanceof Long && b instanceof Long) { + return ((Long) a).longValue() < ((Long) b).longValue(); + } + return Py.isTrue(Py.lt(a, b)); + } + + public static boolean le(Object a, Object b) { + if (a instanceof Long && b instanceof Long) { + return ((Long) a).longValue() <= ((Long) b).longValue(); + } + return Py.isTrue(Py.le(a, b)); + } + + public static boolean gt(Object a, Object b) { + if (a instanceof Long && b instanceof Long) { + return ((Long) a).longValue() > ((Long) b).longValue(); + } + return Py.isTrue(Py.gt(a, b)); + } + + public static boolean ge(Object a, Object b) { + if (a instanceof Long && b instanceof Long) { + return ((Long) a).longValue() >= ((Long) b).longValue(); + } + return Py.isTrue(Py.ge(a, b)); + } + + public static boolean eq(Object a, Object b) { + if (a instanceof Long && b instanceof Long) { + return ((Long) a).longValue() == ((Long) b).longValue(); + } + return Py.eqBool(a, b); + } + + // ---- floats ---------------------------------------------------------- + + public static double div(double a, double b) { + if (b == 0) { + throw Py.zeroDivision("division by zero"); + } + return a / b; + } + + public static double floordiv(double a, double b) { + return Py.asDouble(Py.floordiv(Double.valueOf(a), Double.valueOf(b))); + } + + public static double mod(double a, double b) { + return Py.asDouble(Py.mod(Double.valueOf(a), Double.valueOf(b))); + } + + /// `a / b` of two ints. + public static double truediv(Object a, Object b) { + return Py.asDouble(Py.truediv(a, b)); + } + + // ---- lists ----------------------------------------------------------- + + public static Object get(PyList l, Object i) { + nzl(l); + if (i instanceof Long) { + long k = ((Long) i).longValue(); + if (k >= 0 && k < l.size) { + long[] n = l.ints; + if (n != null) { + return Long.valueOf(n[(int) k]); + } + double[] f = l.floats; + if (f != null) { + return Double.valueOf(f[(int) k]); + } + return l.items[(int) k]; + } + } + return l.getItem(i); + } + + public static double getF(PyList l, Object i) { + nzl(l); + double[] f = l.floats; + if (f != null && i instanceof Long) { + long k = ((Long) i).longValue(); + if (k >= 0 && k < l.size) { + return f[(int) k]; + } + } + return asFloat(l.getItem(i)); + } + + public static void set(PyList l, Object i, Object v) { + nzl(l); + if (i instanceof Long) { + long k = ((Long) i).longValue(); + if (k >= 0 && k < l.size) { + long[] n = l.ints; + if (n != null) { + if (v instanceof Long) { + n[(int) k] = ((Long) v).longValue(); + return; + } + } else if (l.floats == null) { + l.items[(int) k] = v; + return; + } + } + } + l.setItem(i, v); + } + + public static void setF(PyList l, Object i, double v) { + nzl(l); + double[] f = l.floats; + if (f != null && i instanceof Long) { + long k = ((Long) i).longValue(); + if (k >= 0 && k < l.size) { + f[(int) k] = v; + return; + } + } + l.setItem(i, Double.valueOf(v)); + } + + public static Object len(PyList l) { + return Long.valueOf(nzl(l).size); + } + + /// Iterating over None. + public static PyList iterable(PyList l) { + if (l == null) { + throw Py.typeError("'NoneType' object is not iterable"); + } + return l; + } + + /// Item `i` of a float list, `i` in range. + public static double atF(PyList l, int i) { + double[] f = l.floats; + return f != null ? f[i] : asFloat(l.get(i)); + } + + public static PyList list(Object[] items) { + return PyList.of(items); + } + + /// `list(range(stop))`. + public static PyList rangeList(Object stop) { + return rangeList(Long.valueOf(0), stop, Long.valueOf(1)); + } + + /// `list(range(start, stop))`. + public static PyList rangeList(Object start, Object stop) { + return rangeList(start, stop, Long.valueOf(1)); + } + + /// `list(range(start, stop, step))`. + public static PyList rangeList(Object start, Object stop, Object step) { + long a = bound(start); + long b = bound(stop); + long c = bound(step); + if (c == 0) { + throw Py.valueError("range() arg 3 must not be zero"); + } + PyList l = new PyList(); + if (c > 0) { + for (long i = a; i < b; i += c) { + l.append(Long.valueOf(i)); + if (b - i <= c) { + // The next step would pass the end (or overflow). + break; + } + } + } else { + for (long i = a; i > b; i += c) { + l.append(Long.valueOf(i)); + if (c == Long.MIN_VALUE || i - b <= -c) { + break; + } + } + } + return l; + } + + /// `list(l)`, a shallow copy. + public static PyList copy(PyList l) { + return iterable(l).getSlice(Py.NONE, Py.NONE, Py.NONE); + } + + public static Object append(PyList l, Object v) { + nzl(l).append(v); + return Py.NONE; + } + + public static boolean truthy(String s) { + return s != null && s.length() != 0; + } + + public static boolean truthy(PyList l) { + return l != null && l.size != 0; + } + + public static String concat(String a, String b) { + return a.concat(b); + } + + public static Object abs(Object i) { + if (i instanceof Long) { + long v = ((Long) i).longValue(); + if (v != Long.MIN_VALUE) { + return Long.valueOf(v < 0 ? -v : v); + } + } + return Py.call(Builtins.lookup("abs"), i); + } + + /// A None-typed value where a nullable one is expected. + public static T none(Object ignored) { + return null; + } + + /// An expression statement whose value is dropped. + public static void drop(Object ignored) { + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Utf8.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Utf8.java new file mode 100644 index 00000000000..b4e612310a2 --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Utf8.java @@ -0,0 +1,133 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +/// UTF-8 coding with Python's strict error behaviour. +final class Utf8 { + + private Utf8() { + } + + static byte[] encode(String s) { + java.io.ByteArrayOutputStream out = new java.io.ByteArrayOutputStream(s.length()); + for (int i = 0; i < s.length(); i++) { + int c = s.codePointAt(i); + if (c > 0xffff) { + i++; + } + if (c < 0x80) { + out.write(c); + } else if (c < 0x800) { + out.write(0xc0 | c >> 6); + out.write(0x80 | c & 0x3f); + } else if (c < 0x10000) { + out.write(0xe0 | c >> 12); + out.write(0x80 | c >> 6 & 0x3f); + out.write(0x80 | c & 0x3f); + } else { + out.write(0xf0 | c >> 18); + out.write(0x80 | c >> 12 & 0x3f); + out.write(0x80 | c >> 6 & 0x3f); + out.write(0x80 | c & 0x3f); + } + } + return out.toByteArray(); + } + + static String decode(byte[] b) { + return decode(b, "strict"); + } + + /// Decodes UTF-8 under a Python error handler: `strict` raises, + /// `replace` substitutes U+FFFD for each bad byte, `ignore` drops it. + static String decode(byte[] b, String errors) { + boolean strict = !"replace".equals(errors) && !"ignore".equals(errors); + boolean replace = "replace".equals(errors); + StringBuilder sb = new StringBuilder(b.length); + int i = 0; + while (i < b.length) { + int c = b[i] & 0xff; + int need; + int cp; + if (c < 0x80) { + sb.append((char) c); + i++; + continue; + } else if (c >= 0xc2 && c < 0xe0) { + need = 1; + cp = c & 0x1f; + } else if (c >= 0xe0 && c < 0xf0) { + need = 2; + cp = c & 0x0f; + } else if (c >= 0xf0 && c < 0xf5) { + need = 3; + cp = c & 0x07; + } else { + if (strict) { + throw decodeError(c, i); + } + if (replace) { + sb.append('\ufffd'); + } + i++; + continue; + } + boolean bad = false; + if (i + need >= b.length) { + if (strict) { + throw Py.error(Py.UNICODE_DECODE_ERROR, "'utf-8' codec can't decode bytes in position " + i + + "-" + (b.length - 1) + ": unexpected end of data"); + } + bad = true; + } else { + for (int k = 1; k <= need; k++) { + int cc = b[i + k] & 0xff; + if ((cc & 0xc0) != 0x80) { + if (strict) { + throw decodeError(c, i); + } + bad = true; + break; + } + cp = cp << 6 | cc & 0x3f; + } + } + if (bad) { + if (replace) { + sb.append('\ufffd'); + } + i++; + continue; + } + sb.appendCodePoint(cp); + i += need + 1; + } + return sb.toString(); + } + + private static PyException decodeError(int c, int pos) { + return Py.error(Py.UNICODE_DECODE_ERROR, "'utf-8' codec can't decode byte 0x" + Integer.toHexString(c) + + " in position " + pos + ": invalid start byte"); + } +} diff --git a/maven/python-transpiler/pom.xml b/maven/python-transpiler/pom.xml new file mode 100644 index 00000000000..38ae8f97f1c --- /dev/null +++ b/maven/python-transpiler/pom.xml @@ -0,0 +1,172 @@ + + + + + com.codenameone + codenameone + 8.0-SNAPSHOT + + 4.0.0 + + codenameone-python-transpiler + 8.0-SNAPSHOT + jar + codenameone-python-transpiler + + Build-time Python-to-Java transpiler. Parses Python with a hand-written + recursive descent parser (no Python installation and no parser + generator), and emits Java 17 source against + codenameone-python-runtime. The transpiler itself runs on Java 8, so + it can execute inside the Maven plugin. + + + + UTF-8 + 1.8 + 1.8 + + + + + + maven-compiler-plugin + + 1.8 + 1.8 + + + + com.github.spotbugs + spotbugs-maven-plugin + 4.7.3.4 + + Max + Low + false + true + ${project.build.directory} + ${project.basedir}/spotbugs-exclude.xml + + + + spotbugs + verify + + spotbugs + + + + + + + org.apache.maven.plugins + maven-pmd-plugin + 3.21.0 + + false + false + 1.8 + + ${project.basedir}/../core-unittests/pmd.xml + + + + + pmd + verify + + pmd + + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + 3.3.0 + + ${project.basedir}/../core-unittests/checkstyle.xml + ${project.build.sourceEncoding} + false + false + false + false + ${project.build.directory}/checkstyle-result.xml + xml + + + + validate + verify + + checkstyle + + + + checkstyle-zero-errors + verify + + check + + + true + error + 0 + true + + + + + + + + + + org.junit.jupiter + junit-jupiter + test + + + + + + + transpiler-tests + + [17,) + + + + com.codenameone + codenameone-python-runtime + ${project.version} + test + + + com.codenameone + codenameone-core + ${project.version} + test + + + + + transpiler-tests-need-jdk17 + + (,17) + + + true + + + + diff --git a/maven/python-transpiler/spotbugs-exclude.xml b/maven/python-transpiler/spotbugs-exclude.xml new file mode 100644 index 00000000000..fffa5af9893 --- /dev/null +++ b/maven/python-transpiler/spotbugs-exclude.xml @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + + + + + + diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/analyze/Desugar.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/analyze/Desugar.java new file mode 100644 index 00000000000..a2f7f2ab4ae --- /dev/null +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/analyze/Desugar.java @@ -0,0 +1,389 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.transpiler.analyze; + +import com.codename1.python.transpiler.ast.Node.Assign; +import com.codename1.python.transpiler.ast.Node.Call; +import com.codename1.python.transpiler.ast.Node.ClassDef; +import com.codename1.python.transpiler.ast.Node.Compare; +import com.codename1.python.transpiler.ast.Node.Constant; +import com.codename1.python.transpiler.ast.Node.ExceptHandler; +import com.codename1.python.transpiler.ast.Node.Expr; +import com.codename1.python.transpiler.ast.Node.ExprStmt; +import com.codename1.python.transpiler.ast.Node.For; +import com.codename1.python.transpiler.ast.Node.FunctionDef; +import com.codename1.python.transpiler.ast.Node.If; +import com.codename1.python.transpiler.ast.Node.Match; +import com.codename1.python.transpiler.ast.Node.MatchCase; +import com.codename1.python.transpiler.ast.Node.Name; +import com.codename1.python.transpiler.ast.Node.Pattern; +import com.codename1.python.transpiler.ast.Node.Raise; +import com.codename1.python.transpiler.ast.Node.Stmt; +import com.codename1.python.transpiler.ast.Node.Subscript; +import com.codename1.python.transpiler.ast.Node.Try; +import com.codename1.python.transpiler.ast.Node.TupleExpr; +import com.codename1.python.transpiler.ast.Node.UnaryOp; +import com.codename1.python.transpiler.ast.Node.While; +import com.codename1.python.transpiler.ast.Node.With; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/// Rewrites `match` statements and `try` statements with `except*` clauses +/// into ordinary statements, before scope analysis, so that every later pass +/// (and the generator lowering) sees nothing new. +/// +/// Each `match` case becomes a call of the runtime matcher with a +/// description of its pattern (see `PyMatch` in the runtime), which answers +/// the captured names or None: +/// +/// ``` +/// $mS = subject +/// $mD = False +/// $mB = __cn1_match__(("q", ("c", "x"), ("_",)), $mS) +/// if $mB is not None: +/// x = $mB["x"] +/// if guard: +/// $mD = True +/// body +/// if not $mD: +/// ...the next case... +/// ``` +/// +/// Value and class expressions in a pattern are evaluated in the +/// description, so they are looked up each time the case is tried, as in +/// CPython; the names a case binds are assigned only once it matched. +/// +/// A `try` with `except*` clauses keeps its body, `else` and `finally`, and +/// gets one `except BaseException` handler that splits the group clause by +/// clause through the runtime (see `ExceptionGroups` in the runtime): +/// +/// ``` +/// except BaseException as $xN: +/// $sN = __cn1_except_star__($xN) +/// $mN = __cn1_except_star_take__($sN, T1) +/// if $mN is not None: +/// e = $mN +/// try: +/// handler # a bare `raise` here re-raises $mN +/// except BaseException as $rN: +/// __cn1_except_star_raised__($sN, $rN) +/// ... +/// __cn1_except_star_finish__($sN) +/// ``` +public final class Desugar { + private int counter; + + private Desugar() { + } + + public static void rewrite(List body) { + new Desugar().block(body); + } + + private void block(List body) { + for (int i = 0; i < body.size(); i++) { + Stmt s = body.get(i); + children(s); + if (s instanceof Match) { + List r = lower((Match) s); + body.remove(i); + body.addAll(i, r); + i += r.size() - 1; + } else if (s instanceof Try && ((Try) s).star) { + body.set(i, exceptStar((Try) s)); + } + } + } + + private void children(Stmt s) { + if (s instanceof If) { + block(((If) s).body); + block(((If) s).orelse); + } else if (s instanceof While) { + block(((While) s).body); + block(((While) s).orelse); + } else if (s instanceof For) { + block(((For) s).body); + block(((For) s).orelse); + } else if (s instanceof With) { + block(((With) s).body); + } else if (s instanceof Try) { + Try t = (Try) s; + block(t.body); + for (ExceptHandler h : t.handlers) { + block(h.body); + } + block(t.orelse); + block(t.finalbody); + } else if (s instanceof FunctionDef) { + block(((FunctionDef) s).body); + } else if (s instanceof ClassDef) { + block(((ClassDef) s).body); + } else if (s instanceof Match) { + for (MatchCase c : ((Match) s).cases) { + block(c.body); + } + } + } + + private List lower(Match m) { + int n = ++counter; + String subject = "$mS" + n; + String done = "$mD" + n; + String bound = "$mB" + n; + List out = new ArrayList(); + out.add(assign(name(subject, m), m.subject, m)); + boolean flag = m.cases.size() > 1; + if (flag) { + out.add(assign(name(done, m), new Constant(Constant.Kind.FALSE, null).at(m.line, m.col), m)); + } + List into = out; + for (int i = 0; i < m.cases.size(); i++) { + MatchCase c = m.cases.get(i); + List cs = new ArrayList(); + Call call = new Call(name("__cn1_match__", c)).at(c.line, c.col); + call.args.add(describe(c.pattern)); + call.args.add(name(subject, c)); + cs.add(assign(name(bound, c), call, c)); + Compare test = new Compare(name(bound, c)).at(c.line, c.col); + test.ops.add("is not"); + test.comparators.add(new Constant(Constant.Kind.NONE, null).at(c.line, c.col)); + List matched = new ArrayList(); + List names = new ArrayList(); + captures(c.pattern, names); + for (String x : names) { + Expr v = new Subscript(name(bound, c), new Constant(Constant.Kind.STR, x).at(c.line, c.col)) + .at(c.line, c.col); + matched.add(assign(name(x, c), v, c)); + } + List run = new ArrayList(); + if (flag) { + run.add(assign(name(done, c), new Constant(Constant.Kind.TRUE, null).at(c.line, c.col), c)); + } + run.addAll(c.body); + if (c.guard != null) { + matched.add(new If(c.guard, run, new ArrayList()).at(c.line, c.col)); + } else { + matched.addAll(run); + } + cs.add(new If(test, matched, new ArrayList()).at(c.line, c.col)); + if (i == 0) { + into.addAll(cs); + } else { + Expr notDone = new UnaryOp("not", name(done, c)).at(c.line, c.col); + into.add(new If(notDone, cs, new ArrayList()).at(c.line, c.col)); + } + } + return out; + } + + private Stmt exceptStar(Try t) { + int n = ++counter; + String x = "$x" + n; + String st = "$s" + n; + String m = "$m" + n; + List h = new ArrayList(); + Call begin = new Call(name("__cn1_except_star__", t)).at(t.line, t.col); + begin.args.add(name(x, t)); + h.add(assign(name(st, t), begin, t)); + for (ExceptHandler c : t.handlers) { + Call take = new Call(name("__cn1_except_star_take__", c)).at(c.line, c.col); + take.args.add(name(st, c)); + take.args.add(c.type); + h.add(assign(name(m, c), take, c)); + Compare test = new Compare(name(m, c)).at(c.line, c.col); + test.ops.add("is not"); + test.comparators.add(new Constant(Constant.Kind.NONE, null).at(c.line, c.col)); + List run = new ArrayList(); + if (c.name != null) { + run.add(assign(name(c.name, c), name(m, c), c)); + } + reraise(c.body, m); + String r = "$r" + n; + Call rec = new Call(name("__cn1_except_star_raised__", c)).at(c.line, c.col); + rec.args.add(name(st, c)); + rec.args.add(name(r, c)); + List recBody = new ArrayList(); + recBody.add(new ExprStmt(rec).at(c.line, c.col)); + List hs = new ArrayList(); + hs.add(new ExceptHandler(name("BaseException", c), r, recBody).at(c.line, c.col)); + run.add(new Try(c.body, hs, new ArrayList(), new ArrayList(), false).at(c.line, c.col)); + h.add(new If(test, run, new ArrayList()).at(c.line, c.col)); + } + Call fin = new Call(name("__cn1_except_star_finish__", t)).at(t.line, t.col); + fin.args.add(name(st, t)); + h.add(new ExprStmt(fin).at(t.line, t.col)); + List one = new ArrayList(); + one.add(new ExceptHandler(name("BaseException", t), x, h).at(t.line, t.col)); + return new Try(t.body, one, t.orelse, t.finalbody, false).at(t.line, t.col); + } + + /// Points each bare `raise` of an `except*` clause at the part of the + /// group the clause took, rather than the whole exception being handled. + private static void reraise(List body, String m) { + for (int i = 0; i < body.size(); i++) { + Stmt s = body.get(i); + if (s instanceof Raise && ((Raise) s).exc == null) { + body.set(i, new Raise(name(m, s), null).at(s.line, s.col)); + } else if (s instanceof If) { + reraise(((If) s).body, m); + reraise(((If) s).orelse, m); + } else if (s instanceof While) { + reraise(((While) s).body, m); + reraise(((While) s).orelse, m); + } else if (s instanceof For) { + reraise(((For) s).body, m); + reraise(((For) s).orelse, m); + } else if (s instanceof With) { + reraise(((With) s).body, m); + } else if (s instanceof Try) { + // Its own handlers re-raise what they caught. + reraise(((Try) s).body, m); + reraise(((Try) s).orelse, m); + reraise(((Try) s).finalbody, m); + } + } + } + + /// The names `p` binds, in the order they first appear. + private static void captures(Pattern p, List names) { + if (p.name != null && p.kind != Pattern.Kind.VALUE && p.kind != Pattern.Kind.SINGLETON + && p.kind != Pattern.Kind.CLASS && !names.contains(p.name)) { + if (p.kind == Pattern.Kind.AS) { + for (Pattern q : p.patterns) { + captures(q, names); + } + names.add(p.name); + return; + } + names.add(p.name); + } + if (p.kind == Pattern.Kind.OR) { + if (!p.patterns.isEmpty()) { + captures(p.patterns.get(0), names); + } + return; + } + for (Pattern q : p.patterns) { + captures(q, names); + } + for (Pattern q : p.kwdPatterns) { + captures(q, names); + } + } + + private Expr describe(Pattern p) { + List t = new ArrayList(); + switch (p.kind) { + case WILDCARD: + t.add(str("_", p)); + break; + case CAPTURE: + t.add(str("c", p)); + t.add(str(p.name, p)); + break; + case VALUE: + t.add(str("v", p)); + t.add(p.value); + break; + case SINGLETON: + t.add(str("s", p)); + t.add(p.value); + break; + case STAR: + t.add(str("*", p)); + t.add(p.name == null ? none(p) : str(p.name, p)); + break; + case SEQUENCE: + t.add(str("q", p)); + for (Pattern q : p.patterns) { + t.add(describe(q)); + } + break; + case MAPPING: + t.add(str("m", p)); + t.add(tuple(new ArrayList(p.keys), p)); + List vs = new ArrayList(); + for (Pattern q : p.patterns) { + vs.add(describe(q)); + } + t.add(tuple(vs, p)); + t.add(p.name == null ? none(p) : str(p.name, p)); + break; + case CLASS: + t.add(str("k", p)); + t.add(p.value); + List pos = new ArrayList(); + for (Pattern q : p.patterns) { + pos.add(describe(q)); + } + t.add(tuple(pos, p)); + List kn = new ArrayList(); + List kp = new ArrayList(); + for (int i = 0; i < p.kwdAttrs.size(); i++) { + kn.add(str(p.kwdAttrs.get(i), p)); + kp.add(describe(p.kwdPatterns.get(i))); + } + t.add(tuple(kn, p)); + t.add(tuple(kp, p)); + break; + case OR: + t.add(str("|", p)); + for (Pattern q : p.patterns) { + t.add(describe(q)); + } + break; + case AS: + t.add(str("as", p)); + t.add(p.patterns.isEmpty() ? tuple(Collections.singletonList(str("_", p)), p) + : describe(p.patterns.get(0))); + t.add(str(p.name, p)); + break; + default: + throw new IllegalStateException(String.valueOf(p.kind)); + } + return tuple(t, p); + } + + private static Expr tuple(List items, Pattern at) { + return new TupleExpr(items).at(at.line, at.col); + } + + private static Expr str(String s, Pattern at) { + return new Constant(Constant.Kind.STR, s).at(at.line, at.col); + } + + private static Expr none(Pattern at) { + return new Constant(Constant.Kind.NONE, null).at(at.line, at.col); + } + + private static Name name(String id, com.codename1.python.transpiler.ast.Node at) { + return new Name(id).at(at.line, at.col); + } + + private static Stmt assign(Name target, Expr value, com.codename1.python.transpiler.ast.Node at) { + return new Assign(Collections.singletonList(target), value).at(at.line, at.col); + } +} diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/analyze/ScopeBuilder.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/analyze/ScopeBuilder.java new file mode 100644 index 00000000000..32c9ceb4d51 --- /dev/null +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/analyze/ScopeBuilder.java @@ -0,0 +1,469 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.transpiler.analyze; + +import com.codename1.python.transpiler.api.PythonSyntaxError; +import com.codename1.python.transpiler.ast.Node; +import com.codename1.python.transpiler.ast.Node.*; +import com.codename1.python.transpiler.ast.Scope; + +import java.util.ArrayList; +import java.util.List; + +/// Builds the [Scope] tree of a module with CPython's binding rules, then +/// decides which locals are cells (captured by an inner scope). +/// +/// Every function, lambda, class and comprehension node gets its scope +/// attached; the module scope is returned. +public final class ScopeBuilder { + private final String file; + private final List all = new ArrayList(); + private Scope cur; + + private ScopeBuilder(String file) { + this.file = file; + } + + public static Scope build(Node.Module m, String file) { + ScopeBuilder b = new ScopeBuilder(file); + Scope s = new Scope(Scope.Kind.MODULE, null, ""); + b.all.add(s); + b.cur = s; + Desugar.rewrite(m.body); + b.stmts(m.body); + b.resolveCells(); + return s; + } + + private PythonSyntaxError err(Node n, String msg) { + return new PythonSyntaxError(file, n.line, n.col, msg); + } + + private void bind(String n) { + cur.assigned.add(n); + } + + private Scope push(Scope.Kind k, String name) { + Scope s = new Scope(k, cur, name); + if (cur.kind == Scope.Kind.CLASS) { + s.enclosingClass = cur; + } else { + s.enclosingClass = cur.enclosingClass; + } + all.add(s); + cur = s; + return s; + } + + private void pop() { + cur = cur.parent; + } + + // ---- statements ------------------------------------------------------ + + private void stmts(List body) { + for (Stmt s : body) { + stmt(s); + } + } + + private void stmt(Stmt s) { + if (s instanceof FunctionDef) { + FunctionDef f = (FunctionDef) s; + exprs(f.decorators); + argDefaults(f.args); + bind(f.name); + f.scope = push(Scope.Kind.FUNCTION, f.name); + params(f.args); + stmts(f.body); + pop(); + } else if (s instanceof ClassDef) { + ClassDef c = (ClassDef) s; + exprs(c.decorators); + exprs(c.bases); + for (Keyword k : c.keywords) { + expr(k.value); + } + bind(c.name); + c.scope = push(Scope.Kind.CLASS, c.name); + stmts(c.body); + pop(); + } else if (s instanceof ExprStmt) { + expr(((ExprStmt) s).value); + } else if (s instanceof Assign) { + Assign a = (Assign) s; + expr(a.value); + for (Expr t : a.targets) { + target(t); + } + } else if (s instanceof AugAssign) { + AugAssign a = (AugAssign) s; + expr(a.value); + expr(a.target); + target(a.target); + } else if (s instanceof AnnAssign) { + AnnAssign a = (AnnAssign) s; + if (a.value != null) { + expr(a.value); + target(a.target); + } else if (a.target instanceof Name && cur.kind != Scope.Kind.MODULE) { + // An annotation alone still makes the name local to a function. + bind(((Name) a.target).id); + } else if (!(a.target instanceof Name)) { + expr(a.target); + } + } else if (s instanceof Return) { + expr(((Return) s).value); + } else if (s instanceof Delete) { + for (Expr t : ((Delete) s).targets) { + if (t instanceof Name) { + cur.used.add(((Name) t).id); + cur.deleted.add(((Name) t).id); + } + target(t); + } + } else if (s instanceof Raise) { + expr(((Raise) s).exc); + expr(((Raise) s).cause); + } else if (s instanceof Global) { + Global g = (Global) s; + for (String n : g.names) { + if (cur.kind == Scope.Kind.MODULE) { + if (g.nonlocal) { + throw err(s, "nonlocal declaration not allowed at module level"); + } + continue; + } + if (g.nonlocal) { + cur.nonlocals.add(n); + } else { + cur.globals.add(n); + cur.module().assigned.add(n); + } + } + } else if (s instanceof Assert) { + expr(((Assert) s).test); + expr(((Assert) s).msg); + } else if (s instanceof Import) { + for (Alias a : ((Import) s).names) { + if (a.asname != null) { + bind(a.asname); + } else { + int dot = a.name.indexOf('.'); + bind(dot < 0 ? a.name : a.name.substring(0, dot)); + } + } + } else if (s instanceof ImportFrom) { + for (Alias a : ((ImportFrom) s).names) { + if ("*".equals(a.name)) { + if (cur.kind != Scope.Kind.MODULE) { + throw err(s, "import * only allowed at module level"); + } + cur.hasStarImport = true; + } else { + bind(a.asname != null ? a.asname : a.name); + } + } + } else if (s instanceof If) { + If i = (If) s; + expr(i.test); + stmts(i.body); + stmts(i.orelse); + } else if (s instanceof While) { + While w = (While) s; + expr(w.test); + stmts(w.body); + stmts(w.orelse); + } else if (s instanceof For) { + For f = (For) s; + expr(f.iter); + target(f.target); + stmts(f.body); + stmts(f.orelse); + } else if (s instanceof With) { + With w = (With) s; + for (WithItem it : w.items) { + expr(it.context); + if (it.vars != null) { + target(it.vars); + } + } + stmts(w.body); + } else if (s instanceof Try) { + Try t = (Try) s; + stmts(t.body); + for (ExceptHandler h : t.handlers) { + expr(h.type); + if (h.name != null) { + bind(h.name); + cur.deleted.add(h.name); + } + stmts(h.body); + } + stmts(t.orelse); + stmts(t.finalbody); + } else if (s instanceof Match) { + Match m = (Match) s; + expr(m.subject); + for (MatchCase c : m.cases) { + pattern(c.pattern); + expr(c.guard); + stmts(c.body); + } + } else if (s instanceof TypeAlias) { + bind(((TypeAlias) s).name); + expr(((TypeAlias) s).value); + } + } + + private void pattern(Pattern p) { + if (p == null) { + return; + } + if (p.name != null && p.kind != Pattern.Kind.VALUE) { + bind(p.name); + } + if (p.value != null) { + expr(p.value); + } + exprs(p.keys); + for (Pattern q : p.patterns) { + pattern(q); + } + for (Pattern q : p.kwdPatterns) { + pattern(q); + } + } + + private void argDefaults(Arguments a) { + exprs(a.defaults); + for (Expr e : a.kwDefaults) { + expr(e); + } + } + + private void params(Arguments a) { + for (String n : a.names()) { + cur.params.add(n); + } + } + + private void target(Expr t) { + if (t instanceof Name) { + bind(((Name) t).id); + } else if (t instanceof TupleExpr) { + for (Expr e : ((TupleExpr) t).elts) { + target(e); + } + } else if (t instanceof ListExpr) { + for (Expr e : ((ListExpr) t).elts) { + target(e); + } + } else if (t instanceof Starred) { + target(((Starred) t).value); + } else { + // Attribute or subscript: its parts are reads. + if (t instanceof Attribute && ((Attribute) t).value instanceof Name) { + root().attrStores.add(((Name) ((Attribute) t).value).id + "." + ((Attribute) t).attr); + } + expr(t); + } + } + + private Scope root() { + Scope s = cur; + while (s.parent != null) { + s = s.parent; + } + return s; + } + + // ---- expressions ----------------------------------------------------- + + private void exprs(List es) { + for (Expr e : es) { + expr(e); + } + } + + private void expr(Expr e) { + if (e == null) { + return; + } + if (e instanceof Name) { + cur.used.add(((Name) e).id); + } else if (e instanceof Constant) { + return; + } else if (e instanceof JoinedStr) { + exprs(((JoinedStr) e).parts); + } else if (e instanceof FormattedValue) { + expr(((FormattedValue) e).value); + expr(((FormattedValue) e).spec); + } else if (e instanceof Attribute) { + expr(((Attribute) e).value); + } else if (e instanceof Subscript) { + expr(((Subscript) e).value); + expr(((Subscript) e).index); + } else if (e instanceof Slice) { + Slice s = (Slice) e; + expr(s.lower); + expr(s.upper); + expr(s.step); + } else if (e instanceof Starred) { + expr(((Starred) e).value); + } else if (e instanceof BinOp) { + expr(((BinOp) e).left); + expr(((BinOp) e).right); + } else if (e instanceof UnaryOp) { + expr(((UnaryOp) e).operand); + } else if (e instanceof BoolOp) { + exprs(((BoolOp) e).values); + } else if (e instanceof Compare) { + expr(((Compare) e).left); + exprs(((Compare) e).comparators); + } else if (e instanceof IfExp) { + IfExp i = (IfExp) e; + expr(i.test); + expr(i.body); + expr(i.orelse); + } else if (e instanceof NamedExpr) { + NamedExpr n = (NamedExpr) e; + expr(n.value); + walrus(n); + } else if (e instanceof Call) { + Call c = (Call) e; + if (c.func instanceof Name && ("setattr".equals(((Name) c.func).id) || "delattr".equals(((Name) c.func).id)) + && !c.args.isEmpty() && c.args.get(0) instanceof Name) { + Expr a = c.args.size() > 1 ? c.args.get(1) : null; + String attr = a instanceof Constant && ((Constant) a).kind == Constant.Kind.STR + ? ((Constant) a).text : null; + root().attrStores.add(((Name) c.args.get(0)).id + "." + (attr != null ? attr : "*")); + } + expr(c.func); + exprs(c.args); + for (Keyword k : c.keywords) { + expr(k.value); + } + if (c.func instanceof Name && "super".equals(((Name) c.func).id) && c.args.isEmpty()) { + // Zero-argument super() reads the first parameter, which + // must therefore be visible as a cell from nested scopes. + Scope f = cur; + while (f != null && f.kind == Scope.Kind.COMPREHENSION) { + f = f.parent; + } + if (f != null && f != cur && f.isFunctionLike() && !f.params.isEmpty()) { //NOPMD CompareObjectsWithEquals - identity is meant + cur.used.add(f.params.iterator().next()); + } + } + } else if (e instanceof ListExpr) { + exprs(((ListExpr) e).elts); + } else if (e instanceof TupleExpr) { + exprs(((TupleExpr) e).elts); + } else if (e instanceof SetExpr) { + exprs(((SetExpr) e).elts); + } else if (e instanceof DictExpr) { + DictExpr d = (DictExpr) e; + exprs(d.keys); + exprs(d.values); + } else if (e instanceof Comp) { + Comp c = (Comp) e; + // The first iterable is evaluated in the enclosing scope. + expr(c.generators.get(0).iter); + c.scope = push(Scope.Kind.COMPREHENSION, ""); + for (int i = 0; i < c.generators.size(); i++) { + Comprehension g = c.generators.get(i); + if (i > 0) { + expr(g.iter); + } + target(g.target); + exprs(g.ifs); + } + expr(c.elt); + expr(c.value); + pop(); + } else if (e instanceof Lambda) { + Lambda l = (Lambda) e; + argDefaults(l.args); + l.scope = push(Scope.Kind.LAMBDA, ""); + params(l.args); + expr(l.body); + pop(); + } else if (e instanceof Await) { + cur.hasYield = true; + expr(((Await) e).value); + } else if (e instanceof Yield) { + cur.hasYield = true; + cur.yields = true; + expr(((Yield) e).value); + } + } + + /// PEP 572: a walrus inside a comprehension binds in the nearest + /// enclosing scope that is not a comprehension. + private void walrus(NamedExpr n) { + String id = n.target.id; + if (cur.kind != Scope.Kind.COMPREHENSION) { + bind(id); + return; + } + Scope t = cur; + while (t.kind == Scope.Kind.COMPREHENSION) { + t = t.parent; + } + if (t.kind == Scope.Kind.CLASS) { + throw err(n, "assignment expression within a comprehension cannot be used in a class body"); + } + for (Scope s = cur; s != t; s = s.parent) { //NOPMD CompareObjectsWithEquals - identity is meant + if (t.kind == Scope.Kind.MODULE) { + s.globals.add(id); + } else { + s.nonlocals.add(id); + } + } + if (t.kind == Scope.Kind.MODULE || t.globals.contains(id)) { + t.module().assigned.add(id); + } else if (!t.nonlocals.contains(id)) { + t.assigned.add(id); + } + } + + // ---- cells ----------------------------------------------------------- + + private void resolveCells() { + for (Scope s : all) { + List names = new ArrayList(s.used); + names.addAll(s.nonlocals); + for (String n : names) { + if (s.resolve(n) != Scope.Binding.FREE) { + continue; + } + Scope owner = s.owner(n); + if (owner == null) { + throw new PythonSyntaxError(file, 0, 0, "no binding for nonlocal '" + n + "' found"); + } + owner.cells.add(n); + } + } + } +} diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/Main.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/Main.java new file mode 100644 index 00000000000..678c9642c4c --- /dev/null +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/Main.java @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.transpiler.api; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/// Command line: `Main [main module]`. +/// +/// Transpiles every `.py` under the source root and writes the Java sources +/// under the output root. Exit status is 0 on success, 1 when a source does +/// not transpile (the message names its file and line), 2 for a bad command +/// line. +public final class Main { + private Main() { + } + + public static void main(String[] args) throws IOException { + if (args.length < 2 || args.length > 3) { + System.err.println("usage: Main [main module]"); + System.exit(2); + return; + } + Path root = Paths.get(args[0]); + Map sources = new TreeMap(); + List files; + try (Stream s = Files.walk(root)) { + files = s.filter(p -> p.toString().endsWith(".py")).collect(Collectors.toCollection(ArrayList::new)); + } + for (Path p : files) { + sources.put(root.relativize(p).toString().replace(File.separatorChar, '/'), + new String(Files.readAllBytes(p), StandardCharsets.UTF_8)); + } + Map out; + try { + out = PythonTranspiler.transpile(sources, args.length == 3 ? args[2] : null); + } catch (PythonSyntaxError e) { + System.err.println(e.getMessage()); + System.exit(1); + return; + } + for (Map.Entry e : out.entrySet()) { + Path f = Paths.get(args[1], e.getKey()).toAbsolutePath(); + Path dir = f.getParent(); + if (dir != null) { + Files.createDirectories(dir); + } + Files.write(f, e.getValue().getBytes(StandardCharsets.US_ASCII)); + } + } +} diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonSyntaxError.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonSyntaxError.java new file mode 100644 index 00000000000..e8b54aa1c28 --- /dev/null +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonSyntaxError.java @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.transpiler.api; + +/// A Python construct the transpiler rejects, located in its source file. +/// +/// Raised both for invalid syntax and for valid Python this transpiler does +/// not support (`eval`, runtime `import`); the build fails naming the +/// position rather than emitting code that would misbehave on the device. +public final class PythonSyntaxError extends RuntimeException { + private static final long serialVersionUID = 1L; + + public final String file; + public final int line; + public final int col; + public final String reason; + + public PythonSyntaxError(String file, int line, int col, String reason) { + super(file + ":" + line + ":" + col + ": " + reason); + this.file = file; + this.line = line; + this.col = col; + this.reason = reason; + } +} diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java new file mode 100644 index 00000000000..3611f753efb --- /dev/null +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java @@ -0,0 +1,328 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.transpiler.api; + +import com.codename1.python.transpiler.analyze.ScopeBuilder; +import com.codename1.python.transpiler.ast.Node; +import com.codename1.python.transpiler.ast.Scope; +import com.codename1.python.transpiler.codegen.Closed; +import com.codename1.python.transpiler.codegen.Emitter; +import com.codename1.python.transpiler.parser.Parser; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Deque; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/// Transpiles a set of Python modules to Java sources. +/// +/// Input is keyed by the path of each `.py` file relative to its source root +/// (`pkg/mod.py`, `pkg/__init__.py`), which is what fixes its module name. +/// Output is keyed by the path of each generated `.java` file relative to the +/// generated-sources root: one class per module, and a `PythonModules` +/// registry whose `registerAll()` makes every module importable and whose +/// `runMain()` runs the entry module as `__main__`. +public final class PythonTranspiler { + /// The package every generated class is placed in. + public static final String PACKAGE = "com.codename1.generated.python"; + + /// Where the standard library modules written in Python live, as + /// resources of this jar. A program gets the ones it imports, and only + /// those; a module of its own with the same name wins. + private static final String STDLIB = "/com/codename1/python/stdlib/"; + + private PythonTranspiler() { + } + + /// @param sources relative `.py` path to source text + /// @param mainModule the module run as `__main__`, or null for a library + public static Map transpile(Map sources, String mainModule) { + Map out = new LinkedHashMap(); + Map classes = new LinkedHashMap(); + Map trees = new LinkedHashMap(); + List> bodies = new ArrayList>(); + for (Map.Entry e : sources.entrySet()) { + String path = e.getKey().replace('\\', '/'); + if (path.endsWith(".py")) { + Node.Module tree = Parser.parse(e.getValue(), path); + trees.put(path, tree); + bodies.add(tree.body); + } + } + boolean closed = Closed.check(bodies); + Set stdlib = addStdlib(sources, trees); + for (Map.Entry e : sources.entrySet()) { + String path = e.getKey().replace('\\', '/'); + if (!path.endsWith(".py")) { + continue; + } + String stem = path.substring(0, path.length() - 3); + boolean isPackage = "__init__".equals(stem) || stem.endsWith("/__init__"); + if (isPackage) { + stem = "__init__".equals(stem) ? "" : stem.substring(0, stem.length() - "/__init__".length()); + } + if (stem.isEmpty()) { + continue; + } + String module = stem.replace('/', '.'); + String cls = className(module); + Node.Module tree = trees.get(path); + Scope scope = ScopeBuilder.build(tree, path); + // The standard library is written to be correct without the + // whole-program types, and is left out of the closed-world check. + String java = new Emitter(path, module, isPackage, PACKAGE, cls, scope) + .closed(closed && !stdlib.contains(path)).emit(tree); + out.put(PACKAGE.replace('.', '/') + "/" + cls + ".java", java); + classes.put(module, cls); + } + if (mainModule != null && !classes.containsKey(mainModule)) { + throw new IllegalArgumentException("main module '" + mainModule + "' is not among the sources"); + } + out.put(PACKAGE.replace('.', '/') + "/PythonModules.java", registry(classes, mainModule)); + return out; + } + + /// Adds to `sources` (and `trees`) every standard library module the + /// program imports, directly or through another such module, and + /// returns their paths. + private static Set addStdlib(Map sources, Map trees) { + Set modules = new HashSet(); + for (String path : sources.keySet()) { + modules.add(moduleOf(path.replace('\\', '/'))); + } + Set added = new HashSet(); + Deque work = new ArrayDeque(trees.values()); + Map found = new LinkedHashMap(); + Set opens = new HashSet(); + for (Map.Entry e : sources.entrySet()) { + Node.Module t = trees.get(e.getKey().replace('\\', '/')); + if (t != null && mentionsOpen(e.getValue())) { + opens.add(t); + } + } + while (!work.isEmpty()) { + Node.Module tree = work.pop(); + List imported = new ArrayList(); + imports(tree.body, imported); + if (opens.contains(tree)) { + // The builtin open() is io.open, written in Python. + imported.add("io"); + } + for (String name : imported) { + String prefix = null; + for (String part : name.split("\\.")) { + prefix = prefix == null ? part : prefix + "." + part; + if (modules.contains(prefix)) { + continue; + } + String base = prefix.replace('.', '/'); + String path = base + ".py"; + String text = resource(path); + if (text == null) { + path = base + "/__init__.py"; + text = resource(path); + } + if (text == null) { + // A module the runtime provides in Java, or none. + break; + } + modules.add(prefix); + added.add(path); + Node.Module t = Parser.parse(text, path); + if (mentionsOpen(text)) { + opens.add(t); + } + trees.put(path, t); + found.put(path, text); + work.push(t); + } + } + } + sources.putAll(found); + return added; + } + + /// Whether `text` may call the builtin `open` (a cheap textual test: + /// bundling `io` for a false positive costs only its size). + private static boolean mentionsOpen(String text) { + if (text == null) { + return false; + } + int i = text.indexOf("open"); + while (i >= 0) { + char before = i == 0 ? ' ' : text.charAt(i - 1); + int after = i + 4; + char next = after < text.length() ? text.charAt(after) : ' '; + if (before != '.' && !Character.isJavaIdentifierPart(before) && !Character.isJavaIdentifierPart(next)) { + return true; + } + i = text.indexOf("open", i + 4); + } + return false; + } + + private static String moduleOf(String path) { + String stem = path.endsWith(".py") ? path.substring(0, path.length() - 3) : path; + if (stem.endsWith("/__init__")) { + stem = stem.substring(0, stem.length() - "/__init__".length()); + } + return stem.replace('/', '.'); + } + + /// Every absolute module name `body` imports, at any depth. + private static void imports(List body, List out) { + for (Node.Stmt s : body) { + if (s instanceof Node.Import) { + for (Node.Alias a : ((Node.Import) s).names) { + out.add(a.name); + } + } else if (s instanceof Node.ImportFrom) { + Node.ImportFrom f = (Node.ImportFrom) s; + if (f.level == 0 && f.module != null) { + out.add(f.module); + for (Node.Alias a : f.names) { + // `from pkg import sub` may name a submodule. + if (!"*".equals(a.name)) { + out.add(f.module + "." + a.name); + } + } + } + } else if (s instanceof Node.FunctionDef) { + imports(((Node.FunctionDef) s).body, out); + } else if (s instanceof Node.ClassDef) { + imports(((Node.ClassDef) s).body, out); + } else { + for (List b : bodiesOf(s)) { + imports(b, out); + } + } + } + } + + private static List> bodiesOf(Node.Stmt s) { + List> r = new ArrayList>(); + if (s instanceof Node.If) { + r.add(((Node.If) s).body); + r.add(((Node.If) s).orelse); + } else if (s instanceof Node.While) { + r.add(((Node.While) s).body); + r.add(((Node.While) s).orelse); + } else if (s instanceof Node.For) { + r.add(((Node.For) s).body); + r.add(((Node.For) s).orelse); + } else if (s instanceof Node.With) { + r.add(((Node.With) s).body); + } else if (s instanceof Node.Try) { + Node.Try t = (Node.Try) s; + r.add(t.body); + for (Node.ExceptHandler h : t.handlers) { + r.add(h.body); + } + r.add(t.orelse); + r.add(t.finalbody); + } else if (s instanceof Node.Match) { + for (Node.MatchCase c : ((Node.Match) s).cases) { + r.add(c.body); + } + } + return r; + } + + private static String resource(String path) { + InputStream in = PythonTranspiler.class.getResourceAsStream(STDLIB + path); + if (in == null) { + return null; + } + try { + try { + ByteArrayOutputStream b = new ByteArrayOutputStream(); + byte[] buf = new byte[8192]; + for (int n = in.read(buf); n > 0; n = in.read(buf)) { + b.write(buf, 0, n); + } + return new String(b.toByteArray(), StandardCharsets.UTF_8); + } finally { + in.close(); + } + } catch (IOException e) { + throw new IllegalStateException("cannot read the standard library module " + path, e); + } + } + + /// The Java class of a module: `py_` and the module name, with every + /// character a Java identifier cannot hold (including the dots) written + /// as `$` and its hex code. Python names never contain `$`, so two + /// modules never share a class. + public static String className(String module) { + StringBuilder sb = new StringBuilder("py_"); + for (int i = 0; i < module.length(); i++) { + char c = module.charAt(i); + if (c < 128 && (Character.isLetterOrDigit(c) || c == '_')) { + sb.append(c); + } else { + sb.append('$').append(Integer.toHexString(c)); + } + } + return sb.toString(); + } + + private static String registry(Map classes, String mainModule) { + StringBuilder sb = new StringBuilder(); + sb.append("// Generated by the Codename One Python transpiler.\n"); + sb.append("package ").append(PACKAGE).append(";\n\n"); + sb.append("/// Every transpiled Python module of this build.\n"); + sb.append("public final class PythonModules {\n"); + sb.append(" private static boolean registered;\n\n"); + sb.append(" private PythonModules() {\n }\n\n"); + sb.append(" /// Makes every module importable; idempotent.\n"); + sb.append(" public static void registerAll() {\n"); + sb.append(" if (registered) {\n return;\n }\n"); + sb.append(" registered = true;\n"); + for (String cls : classes.values()) { + sb.append(" ").append(cls).append(".register();\n"); + } + sb.append(" }\n\n"); + sb.append(" /// Runs the entry module as `__main__`.\n"); + sb.append(" public static void runMain() {\n"); + sb.append(" registerAll();\n"); + if (mainModule != null) { + sb.append(" try {\n"); + sb.append(" ").append(classes.get(mainModule)).append(".exec(\"__main__\");\n"); + sb.append(" } finally {\n"); + sb.append(" com.codename1.python.runtime.StdModules.runExitHandlers();\n"); + sb.append(" }\n"); + } + sb.append(" }\n"); + sb.append("}\n"); + return sb.toString(); + } +} diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/ast/Node.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/ast/Node.java new file mode 100644 index 00000000000..f00ea1af595 --- /dev/null +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/ast/Node.java @@ -0,0 +1,685 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.transpiler.ast; + +import java.util.ArrayList; +import java.util.List; + +/// The Python abstract syntax tree, close to CPython's `ast` module. +/// +/// Nodes are plain mutable holders; the parser builds them, the scope pass +/// annotates them, and the emitter walks them. +public abstract class Node { + public int line; + public int col; + + public T at(int l, int c) { + this.line = l; + this.col = c; + @SuppressWarnings("unchecked") + T self = (T) this; + return self; + } + + // ===================================================================== + // Expressions + // ===================================================================== + + public abstract static class Expr extends Node { + } + + public static final class Name extends Expr { + public final String id; + + public Name(String id) { + this.id = id; + } + } + + /// `None`, `True`, `False`, `...`, numbers, str and bytes. + public static final class Constant extends Expr { + public enum Kind { NONE, TRUE, FALSE, ELLIPSIS, INT, FLOAT, STR, BYTES } + + public final Kind kind; + /// Source text for numbers, decoded text for strings. + public final String text; + + public Constant(Kind kind, String text) { + this.kind = kind; + this.text = text; + } + } + + /// An f-string: literal pieces and replacement fields, in order. + public static final class JoinedStr extends Expr { + public final List parts = new ArrayList(); + } + + /// One `{value!conv:spec}` field of an f-string. `spec` is itself a + /// JoinedStr (it may hold nested fields) or null. + public static final class FormattedValue extends Expr { + public final Expr value; + public final char conversion; + public final JoinedStr spec; + + public FormattedValue(Expr value, char conversion, JoinedStr spec) { + this.value = value; + this.conversion = conversion; + this.spec = spec; + } + } + + public static final class Attribute extends Expr { + public final Expr value; + public final String attr; + + public Attribute(Expr value, String attr) { + this.value = value; + this.attr = attr; + } + } + + public static final class Subscript extends Expr { + public final Expr value; + public final Expr index; + + public Subscript(Expr value, Expr index) { + this.value = value; + this.index = index; + } + } + + public static final class Slice extends Expr { + public final Expr lower; + public final Expr upper; + public final Expr step; + + public Slice(Expr lower, Expr upper, Expr step) { + this.lower = lower; + this.upper = upper; + this.step = step; + } + } + + public static final class Starred extends Expr { + public final Expr value; + + public Starred(Expr value) { + this.value = value; + } + } + + public static final class BinOp extends Expr { + public final Expr left; + public final String op; + public final Expr right; + + public BinOp(Expr left, String op, Expr right) { + this.left = left; + this.op = op; + this.right = right; + } + } + + public static final class UnaryOp extends Expr { + public final String op; + public final Expr operand; + + public UnaryOp(String op, Expr operand) { + this.op = op; + this.operand = operand; + } + } + + /// `a and b and c` / `a or b`. + public static final class BoolOp extends Expr { + public final boolean isAnd; + public final List values; + + public BoolOp(boolean isAnd, List values) { + this.isAnd = isAnd; + this.values = values; + } + } + + /// `a < b <= c`: `ops.size() == comparators.size()`. + public static final class Compare extends Expr { + public final Expr left; + public final List ops = new ArrayList(); + public final List comparators = new ArrayList(); + + public Compare(Expr left) { + this.left = left; + } + } + + public static final class IfExp extends Expr { + public final Expr test; + public final Expr body; + public final Expr orelse; + + public IfExp(Expr test, Expr body, Expr orelse) { + this.test = test; + this.body = body; + this.orelse = orelse; + } + } + + /// `target := value`. + public static final class NamedExpr extends Expr { + public final Name target; + public final Expr value; + + public NamedExpr(Name target, Expr value) { + this.target = target; + this.value = value; + } + } + + public static final class Keyword extends Node { + /// null for `**mapping`. + public final String arg; + public final Expr value; + + public Keyword(String arg, Expr value) { + this.arg = arg; + this.value = value; + } + } + + public static final class Call extends Expr { + public final Expr func; + /// May contain Starred for `*seq`. + public final List args = new ArrayList(); + public final List keywords = new ArrayList(); + + public Call(Expr func) { + this.func = func; + } + } + + public static final class ListExpr extends Expr { + public final List elts; + + public ListExpr(List elts) { + this.elts = elts; + } + } + + public static final class TupleExpr extends Expr { + public final List elts; + + public TupleExpr(List elts) { + this.elts = elts; + } + } + + public static final class SetExpr extends Expr { + public final List elts; + + public SetExpr(List elts) { + this.elts = elts; + } + } + + /// `{k: v, **m}`: a null key marks `**value`. + public static final class DictExpr extends Expr { + public final List keys = new ArrayList(); + public final List values = new ArrayList(); + } + + public static final class Comprehension extends Node { + public final Expr target; + public final Expr iter; + public final List ifs = new ArrayList(); + public final boolean isAsync; + + public Comprehension(Expr target, Expr iter, boolean isAsync) { + this.target = target; + this.iter = iter; + this.isAsync = isAsync; + } + } + + /// List, set and dict comprehensions and generator expressions. + public static final class Comp extends Expr { + public enum Kind { LIST, SET, DICT, GENERATOR } + + public final Kind kind; + public final Expr elt; + /// The value for dict comprehensions. + public final Expr value; + public final List generators = new ArrayList(); + public Scope scope; + + public Comp(Kind kind, Expr elt, Expr value) { + this.kind = kind; + this.elt = elt; + this.value = value; + } + } + + public static final class Lambda extends Expr { + public final Arguments args; + public final Expr body; + public Scope scope; + + public Lambda(Arguments args, Expr body) { + this.args = args; + this.body = body; + } + } + + public static final class Await extends Expr { + public final Expr value; + + public Await(Expr value) { + this.value = value; + } + } + + public static final class Yield extends Expr { + public final Expr value; + public final boolean from; + + public Yield(Expr value, boolean from) { + this.value = value; + this.from = from; + } + } + + // ===================================================================== + // Function signatures + // ===================================================================== + + public static final class Arg extends Node { + public final String name; + public final Expr annotation; + + public Arg(String name, Expr annotation) { + this.name = name; + this.annotation = annotation; + } + } + + public static final class Arguments extends Node { + /// Positional-only followed by positional-or-keyword. + public final List positional = new ArrayList(); + public int positionalOnly; + /// Defaults for the last `defaults.size()` positional parameters. + public final List defaults = new ArrayList(); + public Arg vararg; + public final List kwonly = new ArrayList(); + /// Parallel to kwonly; null where required. + public final List kwDefaults = new ArrayList(); + public Arg kwarg; + + /// Every parameter name in binding order. + public List names() { + List r = new ArrayList(); + for (Arg a : positional) { + r.add(a.name); + } + if (vararg != null) { + r.add(vararg.name); + } + for (Arg a : kwonly) { + r.add(a.name); + } + if (kwarg != null) { + r.add(kwarg.name); + } + return r; + } + } + + // ===================================================================== + // Statements + // ===================================================================== + + public abstract static class Stmt extends Node { + } + + public static final class ExprStmt extends Stmt { + public final Expr value; + + public ExprStmt(Expr value) { + this.value = value; + } + } + + /// `a = b = value`. + public static final class Assign extends Stmt { + public final List targets; + public final Expr value; + + public Assign(List targets, Expr value) { + this.targets = targets; + this.value = value; + } + } + + public static final class AugAssign extends Stmt { + public final Expr target; + public final String op; + public final Expr value; + + public AugAssign(Expr target, String op, Expr value) { + this.target = target; + this.op = op; + this.value = value; + } + } + + public static final class AnnAssign extends Stmt { + public final Expr target; + public final Expr annotation; + public final Expr value; + + public AnnAssign(Expr target, Expr annotation, Expr value) { + this.target = target; + this.annotation = annotation; + this.value = value; + } + } + + public static final class Return extends Stmt { + public final Expr value; + + public Return(Expr value) { + this.value = value; + } + } + + public static final class Delete extends Stmt { + public final List targets; + + public Delete(List targets) { + this.targets = targets; + } + } + + public static final class Pass extends Stmt { + } + + public static final class Break extends Stmt { + } + + public static final class Continue extends Stmt { + } + + public static final class Raise extends Stmt { + public final Expr exc; + public final Expr cause; + + public Raise(Expr exc, Expr cause) { + this.exc = exc; + this.cause = cause; + } + } + + public static final class Global extends Stmt { + public final List names; + public final boolean nonlocal; + + public Global(List names, boolean nonlocal) { + this.names = names; + this.nonlocal = nonlocal; + } + } + + public static final class Assert extends Stmt { + public final Expr test; + public final Expr msg; + + public Assert(Expr test, Expr msg) { + this.test = test; + this.msg = msg; + } + } + + public static final class Alias extends Node { + public final String name; + public final String asname; + + public Alias(String name, String asname) { + this.name = name; + this.asname = asname; + } + } + + public static final class Import extends Stmt { + public final List names; + + public Import(List names) { + this.names = names; + } + } + + public static final class ImportFrom extends Stmt { + public final String module; + public final int level; + /// A single "*" alias for `from m import *`. + public final List names; + + public ImportFrom(String module, int level, List names) { + this.module = module; + this.level = level; + this.names = names; + } + } + + public static final class If extends Stmt { + public final Expr test; + public final List body; + public final List orelse; + + public If(Expr test, List body, List orelse) { + this.test = test; + this.body = body; + this.orelse = orelse; + } + } + + public static final class While extends Stmt { + public final Expr test; + public final List body; + public final List orelse; + + public While(Expr test, List body, List orelse) { + this.test = test; + this.body = body; + this.orelse = orelse; + } + } + + public static final class For extends Stmt { + public final Expr target; + public final Expr iter; + public final List body; + public final List orelse; + public final boolean isAsync; + + public For(Expr target, Expr iter, List body, List orelse, boolean isAsync) { + this.target = target; + this.iter = iter; + this.body = body; + this.orelse = orelse; + this.isAsync = isAsync; + } + } + + public static final class WithItem extends Node { + public final Expr context; + public final Expr vars; + + public WithItem(Expr context, Expr vars) { + this.context = context; + this.vars = vars; + } + } + + public static final class With extends Stmt { + public final List items; + public final List body; + public final boolean isAsync; + + public With(List items, List body, boolean isAsync) { + this.items = items; + this.body = body; + this.isAsync = isAsync; + } + } + + public static final class ExceptHandler extends Node { + public final Expr type; + public final String name; + public final List body; + + public ExceptHandler(Expr type, String name, List body) { + this.type = type; + this.name = name; + this.body = body; + } + } + + public static final class Try extends Stmt { + public final List body; + public final List handlers; + public final List orelse; + public final List finalbody; + public final boolean star; + + public Try(List body, List handlers, List orelse, List finalbody, + boolean star) { + this.body = body; + this.handlers = handlers; + this.orelse = orelse; + this.finalbody = finalbody; + this.star = star; + } + } + + public static final class FunctionDef extends Stmt { + public final String name; + public final Arguments args; + public final List body; + public final List decorators; + public final Expr returns; + public final boolean isAsync; + public Scope scope; + + public FunctionDef(String name, Arguments args, List body, List decorators, Expr returns, + boolean isAsync) { + this.name = name; + this.args = args; + this.body = body; + this.decorators = decorators; + this.returns = returns; + this.isAsync = isAsync; + } + } + + public static final class ClassDef extends Stmt { + public final String name; + public final List bases; + public final List keywords; + public final List body; + public final List decorators; + public Scope scope; + + public ClassDef(String name, List bases, List keywords, List body, + List decorators) { + this.name = name; + this.bases = bases; + this.keywords = keywords; + this.body = body; + this.decorators = decorators; + } + } + + /// `match subject:` with its cases. + public static final class Match extends Stmt { + public final Expr subject; + public final List cases; + + public Match(Expr subject, List cases) { + this.subject = subject; + this.cases = cases; + } + } + + public static final class MatchCase extends Node { + public final Pattern pattern; + public final Expr guard; + public final List body; + + public MatchCase(Pattern pattern, Expr guard, List body) { + this.pattern = pattern; + this.guard = guard; + this.body = body; + } + } + + /// A `match` pattern. + public static final class Pattern extends Node { + public enum Kind { VALUE, SINGLETON, CAPTURE, WILDCARD, SEQUENCE, STAR, MAPPING, CLASS, OR, AS } + + public final Kind kind; + /// VALUE/SINGLETON: the value; CLASS: the class expression. + public Expr value; + /// CAPTURE/STAR/AS/MAPPING-rest: the bound name (null for `_`). + public String name; + /// SEQUENCE/OR/CLASS-positional/MAPPING-values/AS-inner. + public final List patterns = new ArrayList(); + /// MAPPING keys. + public final List keys = new ArrayList(); + /// CLASS keyword attribute names, parallel to `kwdPatterns`. + public final List kwdAttrs = new ArrayList(); + public final List kwdPatterns = new ArrayList(); + + public Pattern(Kind kind) { + this.kind = kind; + } + } + + /// `type Name[T] = value`. + public static final class TypeAlias extends Stmt { + public final String name; + public final Expr value; + + public TypeAlias(String name, Expr value) { + this.name = name; + this.value = value; + } + } + + public static final class Module extends Node { + public final List body; + + public Module(List body) { + this.body = body; + } + } +} diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/ast/Scope.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/ast/Scope.java new file mode 100644 index 00000000000..7d4755510cc --- /dev/null +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/ast/Scope.java @@ -0,0 +1,160 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.transpiler.ast; + +import java.util.LinkedHashSet; +import java.util.Set; + +/// A Python scope and where each of its names lives, computed by the scope +/// pass with CPython's rules (LEGB, `global`, `nonlocal`, class bodies that +/// do not enclose their methods, comprehensions as scopes of their own). +public final class Scope { + + public enum Kind { MODULE, FUNCTION, CLASS, LAMBDA, COMPREHENSION } + + /// Where a name resolves from inside a scope. + public enum Binding { + /// A plain Java local of this scope. + LOCAL, + /// A local of this scope that an inner scope captures: a cell. + CELL, + /// A cell owned by an enclosing function scope. + FREE, + /// A module global with a fixed slot. + GLOBAL, + /// Not assigned anywhere visible: a builtin, or a run-time global. + BUILTIN, + /// An attribute of the class namespace being built. + CLASS + } + + public final Kind kind; + public final Scope parent; + public final String name; + public final Set assigned = new LinkedHashSet(); + public final Set params = new LinkedHashSet(); + public final Set globals = new LinkedHashSet(); + public final Set nonlocals = new LinkedHashSet(); + public final Set used = new LinkedHashSet(); + /// Locals captured by an inner scope. + public final Set cells = new LinkedHashSet(); + /// Names a `del` or an `except ... as` clause can unbind. + public final Set deleted = new LinkedHashSet(); + /// On the module scope: every `Name.attr` the module stores into or + /// deletes, anywhere in it (`setattr(Name, ...)` too; `Name.*` when the + /// attribute name is not a literal). + public final Set attrStores = new LinkedHashSet(); + public boolean hasYield; + /// Whether the body has a `yield` itself, not just an `await`: an async + /// function with one is an async generator. + public boolean yields; + public boolean hasStarImport; + /// The class this scope's function is a method of (for zero-arg super()). + public Scope enclosingClass; + /// Set on CLASS scopes: the Java variable holding the class being built. + public String classVar; + + public Scope(Kind kind, Scope parent, String name) { + this.kind = kind; + this.parent = parent; + this.name = name; + } + + public boolean isFunctionLike() { + return kind == Kind.FUNCTION || kind == Kind.LAMBDA || kind == Kind.COMPREHENSION; + } + + /// Whether `n` is a local (plain or cell) of this scope. + public boolean isLocal(String n) { + if (globals.contains(n) || nonlocals.contains(n)) { + return false; + } + return assigned.contains(n) || params.contains(n); + } + + public Scope module() { + Scope s = this; + while (s.parent != null) { + s = s.parent; + } + return s; + } + + /// Resolves `n` as read or written from inside this scope. + public Binding resolve(String n) { + if (kind == Kind.MODULE) { + return assigned.contains(n) ? Binding.GLOBAL : Binding.BUILTIN; + } + if (globals.contains(n)) { + // The scope pass gives every `global` name a module slot. + return Binding.GLOBAL; + } + if (kind == Kind.CLASS) { + if (assigned.contains(n)) { + return Binding.CLASS; + } + return enclosing(n); + } + if (nonlocals.contains(n)) { + return Binding.FREE; + } + if (isLocal(n)) { + return cells.contains(n) ? Binding.CELL : Binding.LOCAL; + } + return enclosing(n); + } + + /// Resolution of a name this scope does not bind: the nearest enclosing + /// function scope that binds it (skipping class bodies), else global. + private Binding enclosing(String n) { + for (Scope s = parent; s != null; s = s.parent) { + if (s.kind == Kind.CLASS) { + continue; + } + if (s.kind == Kind.MODULE) { + return s.assigned.contains(n) ? Binding.GLOBAL : Binding.BUILTIN; + } + if (s.globals.contains(n)) { + return Binding.GLOBAL; + } + if (s.isLocal(n) || s.nonlocals.contains(n)) { + return Binding.FREE; + } + } + return Binding.BUILTIN; + } + + /// The function scope that owns the cell a FREE name refers to. + public Scope owner(String n) { + for (Scope s = parent; s != null; s = s.parent) { + if (s.kind == Kind.CLASS || s.kind == Kind.MODULE) { + continue; + } + if (s.isLocal(n)) { + return s; + } + } + return null; + } +} diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Closed.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Closed.java new file mode 100644 index 00000000000..4ea535601cd --- /dev/null +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Closed.java @@ -0,0 +1,160 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.transpiler.codegen; + +import com.codename1.python.transpiler.ast.Node.*; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/// The whole-program check that lets `Infer` type a program at all. +/// +/// The transpiler sees every module of the application, so it can tell +/// whether anything in it does what would make static types unsound: +/// change a class or an instance through a name the compiler cannot see, +/// reach a module's globals from outside, or run code it never compiled. +/// One such use anywhere turns typing off for the whole program -- the +/// generic code stays, and is correct for all of it. +public final class Closed { + private static final Set CALLS = new HashSet(Arrays.asList("setattr", "delattr", "vars", + "globals", "locals", "exec", "eval", "compile", "__import__")); + private static final Set ATTRS = new HashSet(Arrays.asList("__dict__", "__class__", + "__setattr__", "__getattribute__", "__delattr__", "__new__", "__bases__", "__mro__", "__globals__", + "__code__", "__defaults__", "__closure__", "__init_subclass__", "__set_name__", "__slots__")); + private static final Set MODULES = new HashSet(Arrays.asList("copy", "pickle", "importlib", + "inspect", "ctypes", "types", "gc", "builtins", "abc", "dataclasses", "weakref")); + + private boolean open; + + private Closed() { + } + + /// Whether the program made of `modules` is closed. + public static boolean check(List> modules) { + Closed c = new Closed(); + for (List m : modules) { + Set classes = new HashSet(); + for (Stmt s : m) { + if (s instanceof ClassDef) { + classes.add(((ClassDef) s).name); + } else if (s instanceof Import) { + // A module object: stores through it reach its globals. + for (Alias a : ((Import) s).names) { + classes.add(a.asname != null ? a.asname : root(a.name)); + } + } + } + c.scan(m, classes); + } + return !c.open; + } + + private void scan(Object o, Set classes) { + if (open) { + return; + } + if (o instanceof Name && CALLS.contains(((Name) o).id)) { + // `type` is fine with one argument; the three-argument form makes a class. + open = true; + return; + } + if (o instanceof Call && ((Call) o).func instanceof Name && "type".equals(((Name) ((Call) o).func).id) + && ((Call) o).args.size() != 1) { + open = true; + return; + } + if (o instanceof Attribute && ATTRS.contains(((Attribute) o).attr)) { + open = true; + return; + } + if (o instanceof Delete) { + for (Expr t : ((Delete) o).targets) { + if (t instanceof Attribute) { + open = true; + return; + } + } + } + if (o instanceof Import) { + for (Alias a : ((Import) o).names) { + open |= MODULES.contains(root(a.name)); + } + } else if (o instanceof ImportFrom) { + ImportFrom f = (ImportFrom) o; + open |= f.module != null && MODULES.contains(root(f.module)); + } + if (o instanceof Assign || o instanceof AugAssign || o instanceof AnnAssign) { + // A store onto anything but a plain local name or `x.attr` where + // x is not a module or class: `mod.x = ...` reaches another + // module's globals, `Cls.x = ...` changes a class. + List targets = o instanceof Assign ? ((Assign) o).targets + : Arrays.asList(o instanceof AugAssign ? ((AugAssign) o).target : ((AnnAssign) o).target); + for (Expr t : targets) { + attrTargets(t, classes); + } + } + if (o instanceof ClassDef) { + ClassDef c = (ClassDef) o; + for (Expr b : c.bases) { + // Subclassing a class of another module, or anything + // computed, is a class the module's own analysis cannot see. + if (!(b instanceof Name) || !classes.contains(((Name) b).id) && !builtinBase(((Name) b).id)) { + open = true; + } + } + // A class decorator gets the class to change as it pleases. + open |= !c.keywords.isEmpty() || !c.decorators.isEmpty(); + } + Infer.children(o, x -> scan(x, classes)); + } + + private void attrTargets(Expr t, Set classes) { + if (t instanceof Attribute) { + Expr v = ((Attribute) t).value; + // A store through any other expression is a store on an instance. + open |= v instanceof Name && classes.contains(((Name) v).id); + } else if (t instanceof TupleExpr) { + for (Expr e : ((TupleExpr) t).elts) { + attrTargets(e, classes); + } + } else if (t instanceof ListExpr) { + for (Expr e : ((ListExpr) t).elts) { + attrTargets(e, classes); + } + } else if (t instanceof Starred) { + attrTargets(((Starred) t).value, classes); + } + } + + private static boolean builtinBase(String n) { + return "object".equals(n) || n.endsWith("Error") || "Exception".equals(n); + } + + private static String root(String dotted) { + int d = dotted.indexOf('.'); + return d < 0 ? dotted : dotted.substring(0, d); + } +} diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/DefAssign.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/DefAssign.java new file mode 100644 index 00000000000..fd219b53e17 --- /dev/null +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/DefAssign.java @@ -0,0 +1,387 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.transpiler.codegen; + +import com.codename1.python.transpiler.ast.Node.*; + +import java.util.Collections; +import java.util.HashSet; +import java.util.IdentityHashMap; +import java.util.List; +import java.util.Set; + +/// Definite assignment for a function's locals: the loads of a name that +/// every path to them has bound, and nothing since unbound. Such a load +/// needs no unbound-local check. +/// +/// Only ever an under-approximation: a construct this does not model leaves +/// the loads in it unmarked (still checked) and binds nothing after it. +/// Walrus targets, comprehensions and lambdas are not followed. +final class DefAssign { + final Set safe = Collections.newSetFromMap(new IdentityHashMap()); + + /// Marks the safe loads of `body`, a function whose parameters `params` + /// are bound on entry. + void function(List body, Set params) { + block(body, new HashSet(params)); + } + + /// The names bound after `b`, or null when it cannot complete normally. + private Set block(List b, Set in) { + Set cur = in; + for (Stmt s : b) { + if (cur == null) { + return null; + } + cur = stmt(s, cur); + } + return cur; + } + + private static Set join(Set a, Set b) { + if (a == null) { + return b; + } + if (b == null) { + return a; + } + Set r = new HashSet(a); + r.retainAll(b); + return r; + } + + private Set stmt(Stmt s, Set in) { + if (s instanceof ExprStmt) { + expr(((ExprStmt) s).value, in); + return in; + } + if (s instanceof Assign) { + Assign a = (Assign) s; + expr(a.value, in); + Set out = new HashSet(in); + for (Expr t : a.targets) { + bind(t, in, out); + } + return out; + } + if (s instanceof AugAssign) { + AugAssign a = (AugAssign) s; + Set out = new HashSet(in); + if (a.target instanceof Name) { + expr(a.target, in); + } + expr(a.value, in); + bind(a.target, in, out); + return out; + } + if (s instanceof AnnAssign) { + AnnAssign a = (AnnAssign) s; + if (a.value == null) { + return in; + } + expr(a.value, in); + Set out = new HashSet(in); + bind(a.target, in, out); + return out; + } + if (s instanceof Return) { + expr(((Return) s).value, in); + return null; + } + if (s instanceof Raise) { + expr(((Raise) s).exc, in); + expr(((Raise) s).cause, in); + return null; + } + if (s instanceof Break || s instanceof Continue) { + return null; + } + if (s instanceof Pass) { + return in; + } + if (s instanceof Assert) { + expr(((Assert) s).test, in); + expr(((Assert) s).msg, in); + return in; + } + if (s instanceof Delete) { + Set out = new HashSet(in); + for (Expr t : ((Delete) s).targets) { + if (t instanceof Name || t instanceof TupleExpr || t instanceof ListExpr) { + Set gone = new HashSet(); + delNames(t, gone); + out.removeAll(gone); + } else { + parts(t, in); + } + } + return out; + } + if (s instanceof If) { + If i = (If) s; + expr(i.test, in); + return join(block(i.body, in), block(i.orelse, in)); + } + if (s instanceof While) { + While w = (While) s; + // A later iteration sees what an earlier one unbound. + Set loop = new HashSet(in); + loop.removeAll(unbinds(w.body)); + expr(w.test, loop); + block(w.body, loop); + block(w.orelse, loop); + // The body may run no time at all, and a break skips the else. + return loop; + } + if (s instanceof For) { + For f = (For) s; + expr(f.iter, in); + Set loop = new HashSet(in); + loop.removeAll(unbinds(f.body)); + Set body = new HashSet(loop); + bind(f.target, loop, body); + block(f.body, body); + block(f.orelse, loop); + return loop; + } + if (s instanceof With) { + With w = (With) s; + Set cur = new HashSet(in); + for (WithItem it : w.items) { + expr(it.context, cur); + if (it.vars != null) { + bind(it.vars, cur, cur); + } + } + // A context manager may swallow an exception from the body. + block(w.body, cur); + return cur; + } + if (s instanceof Try) { + Try t = (Try) s; + Set body = block(t.body, in); + Set res = body == null ? null : block(t.orelse, body); + for (ExceptHandler h : t.handlers) { + expr(h.type, in); + Set hin = new HashSet(in); + if (h.name != null) { + hin.add(h.name); + } + Set hout = block(h.body, hin); + if (hout != null && h.name != null) { + // The name is unbound when the handler ends. + hout = new HashSet(hout); + hout.remove(h.name); + } + res = join(res, hout); + } + if (!t.finalbody.isEmpty()) { + Set fin = block(t.finalbody, in); + if (fin == null) { + return null; + } + if (res != null) { + res = new HashSet(res); + res.addAll(fin); + } + } + return res; + } + if (s instanceof FunctionDef) { + Set out = new HashSet(in); + out.add(((FunctionDef) s).name); + return out; + } + if (s instanceof ClassDef) { + Set out = new HashSet(in); + out.add(((ClassDef) s).name); + return out; + } + if (s instanceof Import) { + Set out = new HashSet(in); + for (Alias a : ((Import) s).names) { + out.add(a.asname != null ? a.asname : a.name.split("\\.")[0]); + } + return out; + } + if (s instanceof ImportFrom) { + Set out = new HashSet(in); + for (Alias a : ((ImportFrom) s).names) { + if (!"*".equals(a.name)) { + out.add(a.asname != null ? a.asname : a.name); + } + } + return out; + } + // Global, Match, TypeAlias: nothing marked, nothing bound. + return in; + } + + /// The names `del` or an `except ... as` clause in `b` can unbind, + /// outside nested functions and classes. + private static Set unbinds(List b) { + Set out = new HashSet(); + unbinds(b, out); + return out; + } + + private static void unbinds(List b, Set out) { + for (Stmt s : b) { + if (s instanceof Delete) { + for (Expr t : ((Delete) s).targets) { + delNames(t, out); + } + } else if (s instanceof If) { + unbinds(((If) s).body, out); + unbinds(((If) s).orelse, out); + } else if (s instanceof While) { + unbinds(((While) s).body, out); + unbinds(((While) s).orelse, out); + } else if (s instanceof For) { + unbinds(((For) s).body, out); + unbinds(((For) s).orelse, out); + } else if (s instanceof With) { + unbinds(((With) s).body, out); + } else if (s instanceof Try) { + Try t = (Try) s; + unbinds(t.body, out); + for (ExceptHandler h : t.handlers) { + if (h.name != null) { + out.add(h.name); + } + unbinds(h.body, out); + } + unbinds(t.orelse, out); + unbinds(t.finalbody, out); + } else if (s instanceof Match) { + for (MatchCase c : ((Match) s).cases) { + unbinds(c.body, out); + } + } + } + } + + private static void delNames(Expr t, Set out) { + if (t instanceof Name) { + out.add(((Name) t).id); + } else if (t instanceof TupleExpr) { + for (Expr e : ((TupleExpr) t).elts) { + delNames(e, out); + } + } else if (t instanceof ListExpr) { + for (Expr e : ((ListExpr) t).elts) { + delNames(e, out); + } + } + } + + /// Binds the names of target `t`; the parts of attribute and subscript + /// targets are loads. + private void bind(Expr t, Set in, Set out) { + if (t instanceof Name) { + out.add(((Name) t).id); + } else if (t instanceof TupleExpr) { + for (Expr e : ((TupleExpr) t).elts) { + bind(e, in, out); + } + } else if (t instanceof ListExpr) { + for (Expr e : ((ListExpr) t).elts) { + bind(e, in, out); + } + } else if (t instanceof Starred) { + bind(((Starred) t).value, in, out); + } else { + parts(t, in); + } + } + + private void parts(Expr t, Set in) { + if (t instanceof Attribute) { + expr(((Attribute) t).value, in); + } else if (t instanceof Subscript) { + expr(((Subscript) t).value, in); + expr(((Subscript) t).index, in); + } + } + + private void exprs(List es, Set in) { + for (Expr e : es) { + expr(e, in); + } + } + + private void expr(Expr e, Set in) { + if (e == null) { + return; + } + if (e instanceof Name) { + if (in.contains(((Name) e).id)) { + safe.add((Name) e); + } + } else if (e instanceof Attribute || e instanceof Subscript) { + parts(e, in); + } else if (e instanceof Slice) { + expr(((Slice) e).lower, in); + expr(((Slice) e).upper, in); + expr(((Slice) e).step, in); + } else if (e instanceof Starred) { + expr(((Starred) e).value, in); + } else if (e instanceof BinOp) { + expr(((BinOp) e).left, in); + expr(((BinOp) e).right, in); + } else if (e instanceof UnaryOp) { + expr(((UnaryOp) e).operand, in); + } else if (e instanceof BoolOp) { + exprs(((BoolOp) e).values, in); + } else if (e instanceof Compare) { + expr(((Compare) e).left, in); + exprs(((Compare) e).comparators, in); + } else if (e instanceof IfExp) { + expr(((IfExp) e).test, in); + expr(((IfExp) e).body, in); + expr(((IfExp) e).orelse, in); + } else if (e instanceof NamedExpr) { + expr(((NamedExpr) e).value, in); + } else if (e instanceof Call) { + Call c = (Call) e; + expr(c.func, in); + exprs(c.args, in); + for (Keyword k : c.keywords) { + expr(k.value, in); + } + } else if (e instanceof ListExpr) { + exprs(((ListExpr) e).elts, in); + } else if (e instanceof TupleExpr) { + exprs(((TupleExpr) e).elts, in); + } else if (e instanceof SetExpr) { + exprs(((SetExpr) e).elts, in); + } else if (e instanceof DictExpr) { + for (Expr k : ((DictExpr) e).keys) { + expr(k, in); + } + exprs(((DictExpr) e).values, in); + } + // Comprehensions, lambdas, f-strings, await, yield: left checked. + } +} diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Emitter.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Emitter.java new file mode 100644 index 00000000000..e88ac401b00 --- /dev/null +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Emitter.java @@ -0,0 +1,4849 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.transpiler.codegen; + +import com.codename1.python.transpiler.api.PythonSyntaxError; +import com.codename1.python.transpiler.ast.Node; +import com.codename1.python.transpiler.ast.Node.*; +import com.codename1.python.transpiler.ast.Scope; + +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/// Writes one Python module as one Java 17 class against +/// `com.codename1.python.runtime`. +/// +/// The shape of the output, so the rules below make sense: +/// +/// - Module globals live in a slot array (`G`); a read of slot `i` is `g(i)`, +/// which raises NameError while it is unbound. +/// - Every function body is a `PyFunction.Body` lambda. Its locals are Java +/// locals; a local an inner scope captures is a one-element array (a +/// cell), so the lambdas only ever capture effectively final values. +/// - Every Java name is unique within the class (`v12_x`, `c12_x`, `t12`), +/// because a Java lambda may not redeclare a local of its enclosing method. +/// - `return`, `raise`, `break` and `continue` are written `if (true) ...`: +/// Python allows dead code after them and Java rejects it as unreachable. +/// - Comprehensions are immediately invoked lambdas: they are scopes of +/// their own in Python 3. +public final class Emitter { + private static final Map BIN = new HashMap(); + private static final Map CMP = new HashMap(); + + static { + BIN.put("+", "add"); + BIN.put("-", "sub"); + BIN.put("*", "mul"); + BIN.put("/", "truediv"); + BIN.put("//", "floordiv"); + BIN.put("%", "mod"); + BIN.put("**", "pow"); + BIN.put("<<", "lshift"); + BIN.put(">>", "rshift"); + BIN.put("&", "and"); + BIN.put("|", "or"); + BIN.put("^", "xor"); + CMP.put("<", "lt"); + CMP.put("<=", "le"); + CMP.put(">", "gt"); + CMP.put(">=", "ge"); + CMP.put("==", "eq"); + CMP.put("!=", "ne"); + CMP.put("in", "in"); + CMP.put("not in", "notIn"); + CMP.put("is", "is"); + CMP.put("is not", "isNot"); + } + + /// One Java lambda (or the module body): owns a temp array. + private static final class Frame { + final int id; + final Scope scope; + int temps; + int ltemps; + + Frame(int id, Scope scope) { + this.id = id; + this.scope = scope; + } + } + + private static final class Loop { + final String label; + final String breakLabel; + + Loop(String label, String breakLabel) { + this.label = label; + this.breakLabel = breakLabel; + } + } + + private final String file; + private final String moduleName; + private final boolean isPackage; + private final String className; + private final String packageName; + + private final Map ids = new HashMap(); + private final Map constants = new LinkedHashMap(); + private final Map builtins = new LinkedHashMap(); + private final List slotNames = new ArrayList(); + private final Map slotIndex = new HashMap(); + private final StringBuilder fields = new StringBuilder(); //NOPMD AvoidStringBufferField - the generator's output + private int uid; + private final Scope moduleScope; + + private Frame frame; + private Scope sc; + private final List loops = new ArrayList(); + /// The lowering of the generator or coroutine body being emitted, if any. + private GenLowering gen; + /// Whether the next function() is an `async def`. + private boolean nextAsync; + final List handlers = new ArrayList(); + + public Emitter(String file, String moduleName, boolean isPackage, String packageName, String className, + Scope moduleScope) { + this.file = file; + this.moduleScope = moduleScope; + this.moduleName = moduleName; + this.isPackage = isPackage; + this.packageName = packageName; + this.className = className; + } + + PythonSyntaxError unsupported(Node n, String what) { + return new PythonSyntaxError(file, n.line, n.col, what + " is not supported yet"); + } + + /// Builtins that run, or look up, code the build never compiled. A + /// translated program has no compiler and no module loader at run time, + /// so each one is a build error at the line that names it -- never a + /// NameError the first time a device reaches it. + private static final Set DYNAMIC_CODE = new HashSet(Arrays.asList("eval", "exec", "compile", + "__import__")); + + /// `sys` members that reach the interpreter's frames, which a + /// translated program does not have. + private static final Set FRAME_HOOKS = new HashSet(Arrays.asList("settrace", "setprofile", + "_getframe", "gettrace", "getprofile")); + + private PythonSyntaxError dynamicCode(Node n, String what) { + return new PythonSyntaxError(file, n.line, n.col, what + + " is not supported: a translated program cannot run or inspect code that was not compiled with it"); + } + + private int id(Scope s) { + Integer i = ids.get(s); + if (i == null) { + i = Integer.valueOf(++uid); + ids.put(s, i); + } + return i.intValue(); + } + + String fresh(String prefix) { + return prefix + (++uid); + } + + // ---- module ---------------------------------------------------------- + + public String emit(Node.Module m) { + Scope ms = moduleScope; + slot("__name__"); + slot("__doc__"); + for (String n : ms.assigned) { + slot(n); + } + frame = new Frame(id(ms), ms); + sc = ms; + StringBuilder body = new StringBuilder(); + List stmts = m.body; + if (!stmts.isEmpty() && isDocstring(stmts.get(0))) { + body.append(" G[1] = ").append(constant((Constant) ((ExprStmt) stmts.get(0)).value)).append(";\n"); + } + findDirectLong(stmts); + layouts = Layouts.plan(stmts, "", ms.attrStores); + Infer infer = closed ? Infer.run(stmts, layouts, ms) : null; + if (infer != null && System.getProperty("cn1.python.dumpTypes") != null) { + System.err.println(infer.dump()); + } + if (infer != null) { + typedEmit = new TypedEmit(this, infer, layouts); + typedEmit.plan(); + } + block(stmts, body, 2); + + StringBuilder out = new StringBuilder(); + out.append("// Generated from ").append(javaComment(file)).append(" by the Codename One Python transpiler.\n"); + out.append("package ").append(packageName).append(";\n\n"); + out.append("import com.codename1.python.runtime.*;\n\n"); + out.append("@SuppressWarnings(\"all\")\n"); + out.append("public final class ").append(className).append(" {\n"); + out.append(" private static PyModule M;\n"); + out.append(" private static Object[] G;\n"); + for (Map.Entry e : builtins.entrySet()) { + out.append(" private static final Object ").append(e.getValue()).append(" = Builtins.lookup(") + .append(javaString(e.getKey())).append(");\n"); + } + out.append(fields); + out.append(layoutCode()); + if (typedEmit != null) { + out.append(typedEmit.members()); + } + out.append("\n private ").append(className).append("() {\n }\n\n"); + out.append(" /// Makes `import ").append(javaComment(moduleName)).append("` find this module.\n"); + out.append(" public static void register() {\n"); + out.append(" PyModule.register(").append(javaString(moduleName)).append(", m -> exec(") + .append(javaString(moduleName)).append("));\n }\n\n"); + out.append(" /// Runs the module body as module `runName` (`__main__` for the entry module).\n"); + out.append(" public static PyModule exec(String runName) {\n"); + out.append(" M = PyModule.begin(runName, new String[] {"); + for (int i = 0; i < slotNames.size(); i++) { + out.append(i == 0 ? "" : ", ").append(javaString(slotNames.get(i))); + } + out.append("});\n"); + out.append(" G = M.slots;\n"); + out.append(" G[0] = Py.str(runName);\n"); + if ((frame.temps > 0 || frame.ltemps > 0)) { + out.append(" ").append(tempDecls()).append("\n"); + } + out.append(body); + out.append(" return M;\n }\n\n"); + out.append(HELPERS); + out.append("}\n"); + return out.toString(); + } + + private static final String HELPERS = "" + + " private static Object g(int i) {\n" + + " Object v = G[i];\n" + + " return v != null ? v : M.unbound(i);\n" + + " }\n\n" + + " private static Object l(Object v, String n) {\n" + + " if (v == null) {\n" + + " throw Py.unboundLocal(n);\n" + + " }\n" + + " return v;\n" + + " }\n\n" + + " private static Object ul(String n) {\n" + + " throw Py.unboundLocal(n);\n" + + " }\n\n" + + " private static Object b(Object v, String n) {\n" + + " return v != null ? v : dyn(n);\n" + + " }\n\n" + + " private static Object dyn(String n) {\n" + + " Object v = M.getAttrOrNull(n);\n" + + " if (v == null) {\n" + + " v = Builtins.lookup(n);\n" + + " }\n" + + " if (v == null) {\n" + + " throw Py.nameError(n);\n" + + " }\n" + + " return v;\n" + + " }\n\n" + + " private static Object cr(PyType k, String n, int slot) {\n" + + " Object v = k.own(n);\n" + + " if (v != null) {\n" + + " return v;\n" + + " }\n" + + " if (slot >= 0 && G[slot] != null) {\n" + + " return G[slot];\n" + + " }\n" + + " return dyn(n);\n" + + " }\n\n" + + " private static void noop(Object v) {\n" + + " }\n"; + + boolean moduleScopeHasStarImport() { + return moduleScope.hasStarImport; + } + + int slot(String n) { + Integer i = slotIndex.get(n); + if (i == null) { + i = Integer.valueOf(slotNames.size()); + slotNames.add(n); + slotIndex.put(n, i); + } + return i.intValue(); + } + + private static boolean isDocstring(Stmt s) { + return s instanceof ExprStmt && ((ExprStmt) s).value instanceof Constant + && ((Constant) ((ExprStmt) s).value).kind == Constant.Kind.STR; + } + + // ---- names ----------------------------------------------------------- + + static String mangle(String n) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < n.length(); i++) { + char c = n.charAt(i); + if (c < 128 && (Character.isLetterOrDigit(c) || c == '_')) { + sb.append(c); + } else { + sb.append('$').append(Integer.toHexString(c)); + } + } + return sb.toString(); + } + + private String localVar(Scope s, String n) { + return "v" + id(s) + "_" + mangle(n); + } + + private String cellVar(Scope s, String n) { + return "c" + id(s) + "_" + mangle(n); + } + + private final DefAssign defAssign = new DefAssign(); + + private String load(Name nm) { + String n = nm.id; + if (DYNAMIC_CODE.contains(n) && sc.resolve(n) == Scope.Binding.BUILTIN) { + throw dynamicCode(nm, n + "()"); + } + if (inRegionFast && lazyIndex != null && lazyIndex[0].equals(n)) { + return "(" + lazyIndex[1] + " == null ? Long.valueOf(" + lazyIndex[2] + ") : " + localVar(sc, n) + ")"; + } + if (hoist != null && inRegionFast && !hoistAssigned.contains(n)) { + // The versioned loop reads it once, ahead of the loop: nothing in + // a region can rebind a name it does not assign (its calls run + // pure twins only). + Scope.Binding b = sc.resolve(n); + String raw = b == Scope.Binding.LOCAL ? localVar(sc, n) : b == Scope.Binding.GLOBAL ? "G[" + slot(n) + "]" : null; + if (raw != null) { + String h = hoist.get(n); + if (h == null) { + h = "h" + regionCount + "_" + hoist.size(); + hoist.put(n, h); + hoistRaw.put(h, raw); + } + return h; + } + } + switch (sc.resolve(n)) { + case LOCAL: + // A parameter is bound on entry; only `del` (or an except + // clause's implicit one) can unbind it. + if (sc.params.contains(n) && !sc.deleted.contains(n)) { + return localVar(sc, n); + } + // Inline: a call per read of a local is most of what a + // small method costs once translated. + if (defAssign.safe.contains(nm)) { + // Bound on every path here (DefAssign). + return localVar(sc, n); + } + return "(" + localVar(sc, n) + " != null ? " + localVar(sc, n) + " : ul(" + javaString(n) + "))"; + case CELL: + return "l(" + cellVar(sc, n) + "[0], " + javaString(n) + ")"; + case FREE: { + Scope o = sc.owner(n); + return "l(" + cellVar(o, n) + "[0], " + javaString(n) + ")"; + } + case GLOBAL: + return "g(" + slot(n) + ")"; + case CLASS: { + Integer s = slotIndex.get(n); + if (s == null && moduleScope.assigned.contains(n)) { + s = Integer.valueOf(slot(n)); + } + return "cr(" + sc.classVar + ", " + javaString(n) + ", " + (s == null ? -1 : s.intValue()) + ")"; + } + case BUILTIN: + default: + if (moduleScope.hasStarImport) { + return "dyn(" + javaString(n) + ")"; + } + String f = builtins.get(n); + if (f == null) { + f = "B" + builtins.size() + "_" + mangle(n); + builtins.put(n, f); + } + return "b(" + f + ", " + javaString(n) + ")"; + } + } + + void storeName(String n, String value, StringBuilder out, int ind) { + indent(out, ind); + switch (sc.resolve(n)) { + case LOCAL: { + String v = localVar(sc, n); + out.append(v).append(" = ").append(value).append(";\n"); + String h = shadowVar.get(v); + if (h != null) { + String k = shadowCls.get(v).javaName; + indent(out, ind); + out.append(h).append(" = ").append(v).append(" instanceof ").append(k).append(" ? (").append(k) + .append(") ").append(v).append(" : null;\n"); + } + return; + } + case CELL: + out.append(cellVar(sc, n)).append("[0] = ").append(value).append(";\n"); + return; + case FREE: + out.append(cellVar(sc.owner(n), n)).append("[0] = ").append(value).append(";\n"); + return; + case CLASS: + out.append(sc.classVar).append(".define(").append(javaString(n)).append(", ").append(value) + .append(");\n"); + return; + case GLOBAL: + case BUILTIN: + default: + out.append("G[").append(slot(n)).append("] = ").append(value).append(";\n"); + } + } + + /// The Java lvalue a walrus assigns, as an expression. + String nameLvalue(String n) { + switch (sc.resolve(n)) { + case LOCAL: + return localVar(sc, n); + case CELL: + return cellVar(sc, n) + "[0]"; + case FREE: + return cellVar(sc.owner(n), n) + "[0]"; + default: + return "G[" + slot(n) + "]"; + } + } + + // ---- statements ------------------------------------------------------ + + private static void indent(StringBuilder out, int ind) { + for (int i = 0; i < ind; i++) { + out.append(" "); + } + } + + private void block(List body, StringBuilder out, int ind) { + for (int i = 0; i < body.size();) { + int end = floatRegion(body, i, out, ind); + if (end > i) { + i = end; + continue; + } + stmt(body.get(i), out, ind); + i++; + } + } + + void stmt(Stmt s, StringBuilder out, int ind) { + if (s instanceof ExprStmt) { + Expr v = ((ExprStmt) s).value; + if (v instanceof Constant) { + return; + } + if (v instanceof Yield || v instanceof Await) { + throw unsupported(v, "generators and async"); + } + String e = expr(v); + indent(out, ind); + String app = appendFast(v, e); + if (app != null) { + out.append(app); + } else if (v instanceof Call && !e.startsWith("(")) { + out.append(e).append(";\n"); + } else { + out.append("noop(").append(e).append(");\n"); + } + } else if (s instanceof Assign) { + Assign a = (Assign) s; + if (a.targets.size() == 1 && isSlice(a.targets.get(0)) && isSlice(a.value)) { + // `x[a:b] = y[c:d]` copies straight between the lists. The + // source comes first in the argument list, as Python + // evaluates the right-hand side before the target. + Subscript dst = (Subscript) a.targets.get(0); + Subscript src = (Subscript) a.value; + Slice ds = (Slice) dst.index; + Slice ss = (Slice) src.index; + indent(out, ind); + out.append("Py.setsliceFrom(").append(expr(src.value)).append(", ").append(exprOrNone(ss.lower)) + .append(", ").append(exprOrNone(ss.upper)).append(", ").append(exprOrNone(ss.step)) + .append(", ").append(expr(dst.value)).append(", ").append(exprOrNone(ds.lower)) + .append(", ").append(exprOrNone(ds.upper)).append(", ").append(exprOrNone(ds.step)) + .append(");\n"); + return; + } + String counted = dictCount(a); + if (counted != null) { + indent(out, ind); + out.append(counted).append(";\n"); + return; + } + if (a.targets.size() == 1 && a.targets.get(0) instanceof Name + && recycled.contains(((Name) a.targets.get(0)).id) && ListRecycling.isEmptyList(a.value)) { + // As below: the previous list is emptied and kept. + String n = ((Name) a.targets.get(0)).id; + storeName(n, "Py.recycleList(" + localVar(sc, n) + ")", out, ind); + return; + } + if (a.targets.size() == 1 && a.targets.get(0) instanceof Name + && recycled.contains(((Name) a.targets.get(0)).id) && ListRecycling.isSliceCopy(a.value)) { + // The local's previous list is unreachable from anywhere + // else (ListRecycling), so the copy reuses its array. + String n = ((Name) a.targets.get(0)).id; + Subscript src = (Subscript) a.value; + Slice ss = (Slice) src.index; + storeName(n, "Py.recycleSlice(" + localVar(sc, n) + ", " + expr(src.value) + ", " + + exprOrNone(ss.lower) + ", " + exprOrNone(ss.upper) + ", " + exprOrNone(ss.step) + ")", out, ind); + return; + } + String v = expr(a.value); + if (a.targets.size() == 1 && a.targets.get(0) instanceof Name) { + storeName(((Name) a.targets.get(0)).id, v, out, ind); + return; + } + String tmp = fresh("s"); + indent(out, ind); + out.append("final Object ").append(tmp).append(" = ").append(v).append(";\n"); + for (Expr t : a.targets) { + store(t, tmp, out, ind); + } + } else if (s instanceof AugAssign) { + augAssign((AugAssign) s, out, ind); + } else if (s instanceof AnnAssign) { + AnnAssign a = (AnnAssign) s; + if (sc.kind == Scope.Kind.CLASS && a.target instanceof Name) { + indent(out, ind); + out.append("Py.setitem(").append(annotations()).append(", Py.str(") + .append(javaString(((Name) a.target).id)).append("), ").append(annotationExpr(a.annotation)) + .append(");\n"); + } + if (a.value != null) { + String v = expr(a.value); + if (a.target instanceof Name) { + storeName(((Name) a.target).id, v, out, ind); + } else { + String tmp = fresh("s"); + indent(out, ind); + out.append("final Object ").append(tmp).append(" = ").append(v).append(";\n"); + store(a.target, tmp, out, ind); + } + } + } else if (s instanceof Return) { + if (frame.scope.kind == Scope.Kind.MODULE || sc.kind == Scope.Kind.CLASS) { + throw new PythonSyntaxError(file, s.line, s.col, "'return' outside function"); + } + Expr v = ((Return) s).value; + indent(out, ind); + if (gen != null) { + // A generator's step returns its value with pc at -1. + out.append("if (true) { pc = -1; return ").append(v == null ? "Py.NONE" : expr(v)).append("; }\n"); + } else { + out.append("if (true) return ").append(v == null ? "Py.NONE" : expr(v)).append(";\n"); + } + } else if (s instanceof Delete) { + for (Expr t : ((Delete) s).targets) { + delete(t, out, ind); + } + } else if (s instanceof Pass || s instanceof Global) { + return; + } else if (s instanceof Break) { + if (loops.isEmpty()) { + throw new PythonSyntaxError(file, s.line, s.col, "'break' outside loop"); + } + indent(out, ind); + out.append("if (true) break ").append(loops.get(loops.size() - 1).breakLabel).append(";\n"); + } else if (s instanceof Continue) { + if (loops.isEmpty()) { + throw new PythonSyntaxError(file, s.line, s.col, "'continue' not properly in loop"); + } + indent(out, ind); + out.append("if (true) continue ").append(loops.get(loops.size() - 1).label).append(";\n"); + } else if (s instanceof Raise) { + Raise r = (Raise) s; + indent(out, ind); + if (r.exc == null) { + if (handlers.isEmpty()) { + out.append("if (true) throw Py.runtimeError(\"No active exception to reraise\");\n"); + } else { + out.append("if (true) throw ").append(handlers.get(handlers.size() - 1)).append(";\n"); + } + } else if (r.cause != null) { + out.append("if (true) throw Py.raise(").append(expr(r.exc)).append(", ").append(expr(r.cause)) + .append(");\n"); + } else { + out.append("if (true) throw Py.raise(").append(expr(r.exc)).append(");\n"); + } + } else if (s instanceof Assert) { + Assert a = (Assert) s; + indent(out, ind); + out.append("if (!(").append(cond(a.test)).append(")) {\n"); + indent(out, ind + 1); + if (a.msg == null) { + out.append("throw Py.raise(Py.ASSERTION_ERROR);\n"); + } else { + out.append("throw Py.raise(Py.call(Py.ASSERTION_ERROR, ").append(expr(a.msg)).append("));\n"); + } + indent(out, ind); + out.append("}\n"); + } else if (s instanceof Import) { + for (Alias a : ((Import) s).names) { + String load = "Py.importModule(" + javaString(a.name) + ")"; + if (a.asname != null) { + storeName(a.asname, load, out, ind); + } else { + int dot = a.name.indexOf('.'); + if (dot < 0) { + storeName(a.name, load, out, ind); + } else { + indent(out, ind); + out.append(load).append(";\n"); + String top = a.name.substring(0, dot); + storeName(top, "Py.importModule(" + javaString(top) + ")", out, ind); + } + } + } + } else if (s instanceof ImportFrom) { + importFrom((ImportFrom) s, out, ind); + } else if (s instanceof If) { + ifStmt((If) s, out, ind); + } else if (s instanceof While) { + whileStmt((While) s, out, ind); + } else if (s instanceof For) { + forStmt((For) s, out, ind); + } else if (s instanceof Try) { + tryStmt((Try) s, out, ind); + } else if (s instanceof With) { + With w = (With) s; + if (w.isAsync) { + throw unsupported(s, "async with"); + } + withItems(w, 0, out, ind); + } else if (s instanceof FunctionDef) { + FunctionDef f = (FunctionDef) s; + nextAsync = f.isAsync; + staticBody = layouts.staticNames.get(f); + staticClass = layouts.methodClass.get(f); + String fn = withMetadata(f, function(f.name, f.args, f.scope, f.body, null)); + storeName(f.name, decorate(f.decorators, fn), out, ind); + } else if (s instanceof ClassDef) { + classDef((ClassDef) s, out, ind); + } else if (s instanceof Match) { + throw unsupported(s, "match"); + } else if (s instanceof TypeAlias) { + TypeAlias t = (TypeAlias) s; + storeName(t.name, expr(t.value), out, ind); + } else { + throw unsupported(s, s.getClass().getSimpleName()); + } + } + + private String annotations() { + return "Py.getattr(" + sc.classVar + ", \"__annotations__\")"; + } + + /// `fn` with the function's docstring and annotations attached, when it + /// has either. Annotations are evaluated on first read of + /// `__annotations__` (the PEP 649 order), so a forward reference costs + /// nothing until something asks. + private String withMetadata(FunctionDef f, String fn) { + String doc = !f.body.isEmpty() && isDocstring(f.body.get(0)) + ? constant((Constant) ((ExprStmt) f.body.get(0)).value) : "null"; + List all = new ArrayList(f.args.positional); + if (f.args.vararg != null) { + all.add(f.args.vararg); + } + all.addAll(f.args.kwonly); + if (f.args.kwarg != null) { + all.add(f.args.kwarg); + } + StringBuilder ann = new StringBuilder(); + for (Arg a : all) { + if (a.annotation != null) { + ann.append(ann.length() == 0 ? "" : ", ").append(javaString(a.name)).append(", ") + .append(annotationExpr(a.annotation)); + } + } + if (f.returns != null) { + ann.append(ann.length() == 0 ? "" : ", ").append("\"return\", ").append(annotationExpr(f.returns)); + } + if ("null".equals(doc) && ann.length() == 0) { + return fn; + } + String thunk = ann.length() == 0 ? "null" : "() -> new Object[] {" + ann + "}"; + return "PyFunction.meta(" + fn + ", " + doc + ", " + thunk + ")"; + } + + /// An annotation's value: the evaluated expression when it only names + /// module globals and builtins (falling back to the source-ish form if + /// evaluating it raises), otherwise [#annotationValue]. + private String annotationExpr(Expr a) { + if (!evaluable(a)) { + return annotationValue(a); + } + return "PyFunction.annotation(() -> " + expr(a) + ", " + annotationValue(a) + ")"; + } + + private boolean evaluable(Expr a) { + if (a instanceof Constant) { + return true; + } + if (a instanceof Name) { + Scope.Binding b = sc.resolve(((Name) a).id); + return b == Scope.Binding.GLOBAL || b == Scope.Binding.BUILTIN; + } + if (a instanceof Attribute) { + return evaluable(((Attribute) a).value); + } + if (a instanceof Subscript) { + return evaluable(((Subscript) a).value) && evaluable(((Subscript) a).index); + } + if (a instanceof BinOp) { + return "|".equals(((BinOp) a).op) && evaluable(((BinOp) a).left) && evaluable(((BinOp) a).right); + } + if (a instanceof TupleExpr) { + for (Expr e : ((TupleExpr) a).elts) { + if (!evaluable(e)) { + return false; + } + } + return true; + } + if (a instanceof ListExpr) { + for (Expr e : ((ListExpr) a).elts) { + if (!evaluable(e)) { + return false; + } + } + return true; + } + return false; + } + + /// Annotations are not evaluated (as under `from __future__ import + /// annotations`): a simple name becomes its string, anything else None. + private String annotationValue(Expr a) { + if (a instanceof Name) { + return "Py.str(" + javaString(((Name) a).id) + ")"; + } + if (a instanceof Constant && ((Constant) a).kind == Constant.Kind.STR) { + return constant((Constant) a); + } + return "Py.NONE"; + } + + private String decorate(List decorators, String value) { + String v = value; + for (int i = decorators.size() - 1; i >= 0; i--) { + v = "Py.call(" + expr(decorators.get(i)) + ", " + v + ")"; + } + // Java evaluates the decorator expressions before the PyFunction's + // defaults, which is Python's order too. + return v; + } + + void store(Expr t, String value, StringBuilder out, int ind) { + if (t instanceof Name) { + storeName(((Name) t).id, value, out, ind); + } else if (t instanceof Attribute) { + Attribute a = (Attribute) t; + indent(out, ind); + if (layouts.fieldOwners.containsKey(a.attr)) { + String recv = expr(a.value); + String o; + String rl = rawLocal(recv); + if (rl != null && (rl.equals(selfLocal) || shadowVar.containsKey(rl))) { + // The receiver parameter, never rebound: read as it is. + o = rl; + } else { + o = fresh("o"); + out.append("final Object ").append(o).append(" = ").append(recv).append(";\n"); + } + fieldSet(a.attr, o, value, site(a.attr), out, ind); + return; + } + out.append(site(a.attr)).append(".set(").append(expr(a.value)).append(", ").append(value) + .append(");\n"); + } else if (t instanceof Subscript) { + Subscript sub = (Subscript) t; + indent(out, ind); + if (sub.index instanceof Slice) { + Slice sl = (Slice) sub.index; + out.append("Py.setslice(").append(expr(sub.value)).append(", ").append(exprOrNone(sl.lower)) + .append(", ").append(exprOrNone(sl.upper)).append(", ").append(exprOrNone(sl.step)) + .append(", ").append(value).append(");\n"); + } else { + out.append("Py.setitem(").append(expr(sub.value)).append(", ").append(expr(sub.index)).append(", ") + .append(value).append(");\n"); + } + } else if (t instanceof TupleExpr || t instanceof ListExpr) { + List elts = t instanceof TupleExpr ? ((TupleExpr) t).elts : ((ListExpr) t).elts; + int star = -1; + for (int i = 0; i < elts.size(); i++) { + if (elts.get(i) instanceof Starred) { + if (star >= 0) { + throw new PythonSyntaxError(file, t.line, t.col, "multiple starred expressions in assignment"); + } + star = i; + } + } + String arr = fresh("u"); + indent(out, ind); + out.append("final Object[] ").append(arr).append(" = "); + if (star < 0) { + out.append(onlyNames(t) ? "Py.unpackNames(" : "Py.unpack(").append(value).append(", ") + .append(elts.size()).append(");\n"); + } else { + out.append("Py.unpackStar(").append(value).append(", ").append(star).append(", ") + .append(elts.size() - star - 1).append(");\n"); + } + for (int i = 0; i < elts.size(); i++) { + Expr e = elts.get(i); + if (e instanceof Starred) { + e = ((Starred) e).value; + } + store(e, arr + "[" + i + "]", out, ind); + } + } else if (t instanceof Starred) { + throw new PythonSyntaxError(file, t.line, t.col, "starred assignment target must be in a list or tuple"); + } else { + throw new PythonSyntaxError(file, t.line, t.col, "cannot assign to expression"); + } + } + + /// `range(a)`, `range(a, b)` or `range(a, b, c)` with plain arguments. + private static boolean rangeCall(Expr e) { + if (!(e instanceof Call)) { + return false; + } + Call c = (Call) e; + if (!(c.func instanceof Name) || !"range".equals(((Name) c.func).id) || !c.keywords.isEmpty() + || c.args.isEmpty() || c.args.size() > 3) { + return false; + } + for (Expr a : c.args) { + if (a instanceof Starred) { + return false; + } + } + return true; + } + + /// `x.split()` with no arguments, which a loop walks word by word + /// (Py.splitWords) instead of splitting into a list first. + private static boolean splitWords(Expr e) { + if (!(e instanceof Call)) { + return false; + } + Call c = (Call) e; + return c.args.isEmpty() && c.keywords.isEmpty() && c.func instanceof Attribute + && "split".equals(((Attribute) c.func).attr); + } + + /// `x.split(sep)`, walked piece by piece the same way (Py.splitOn). + private static boolean splitOn(Expr e) { + if (!(e instanceof Call)) { + return false; + } + Call c = (Call) e; + return c.args.size() == 1 && !(c.args.get(0) instanceof Starred) && c.keywords.isEmpty() + && c.func instanceof Attribute && "split".equals(((Attribute) c.func).attr); + } + + private static boolean isSlice(Expr e) { + return e instanceof Subscript && ((Subscript) e).index instanceof Slice; + } + + /// Whether an unpacking target is made of plain names only, so storing + /// into it cannot change the sequence being unpacked. + private static boolean onlyNames(Expr t) { + if (t instanceof Name) { + return true; + } + List elts = t instanceof TupleExpr ? ((TupleExpr) t).elts + : t instanceof ListExpr ? ((ListExpr) t).elts : null; + if (elts == null) { + return false; + } + for (Expr e : elts) { + if (!onlyNames(e)) { + return false; + } + } + return true; + } + + private void delete(Expr t, StringBuilder out, int ind) { + indent(out, ind); + if (t instanceof Name) { + String n = ((Name) t).id; + switch (sc.resolve(n)) { + case LOCAL: + case CELL: + case FREE: + out.append("noop(").append(load((Name) t)).append(");\n"); + indent(out, ind); + out.append(nameLvalue(n)).append(" = null;\n"); + return; + case CLASS: + out.append(sc.classVar).append(".delAttr(").append(javaString(n)).append(");\n"); + return; + default: + out.append("M.delAttr(").append(javaString(n)).append(");\n"); + return; + } + } else if (t instanceof Attribute) { + Attribute a = (Attribute) t; + out.append("Py.obj(").append(expr(a.value)).append(").delAttr(").append(javaString(a.attr)).append(");\n"); + } else if (t instanceof Subscript) { + Subscript s = (Subscript) t; + out.append("Py.obj(").append(expr(s.value)).append(").delItem(").append(expr(s.index)).append(");\n"); + } else if (t instanceof TupleExpr || t instanceof ListExpr) { + out.setLength(out.length() - ind * 4); + for (Expr e : t instanceof TupleExpr ? ((TupleExpr) t).elts : ((ListExpr) t).elts) { + delete(e, out, ind); + } + } else { + throw new PythonSyntaxError(file, t.line, t.col, "cannot delete expression"); + } + } + + /// `cur op= value` as an expression, fused when the right side is float + /// arithmetic. + private String augValue(String op, String cur, AugAssign a) { + if (numNode(new BinOp(AUG_CURRENT, a.op, a.value))) { + String f = fused(new BinOp(AUG_CURRENT, a.op, a.value), cur); + if (f != null) { + return f; + } + } + String in = inlineArith(a.op, cur, expr(a.value), true); + if (in != null) { + return in; + } + return "Py." + op + "(" + cur + ", " + expr(a.value) + ")"; + } + + /// Whether `e` always evaluates to a bool: `not`, an identity or + /// membership test, a bool constant, or `and`/`or` of those. A rich + /// comparison may return anything a `__lt__` returns. + private static boolean boolValued(Expr e) { + if (e instanceof UnaryOp) { + return "not".equals(((UnaryOp) e).op); + } + if (e instanceof Constant) { + return ((Constant) e).kind == Constant.Kind.TRUE || ((Constant) e).kind == Constant.Kind.FALSE; + } + if (e instanceof Compare) { + for (String op : ((Compare) e).ops) { + if (!"is".equals(op) && !"is not".equals(op) && !"in".equals(op) && !"not in".equals(op)) { + return false; + } + } + return true; + } + if (e instanceof BoolOp) { + for (Expr v : ((BoolOp) e).values) { + if (!boolValued(v)) { + return false; + } + } + return true; + } + return false; + } + + private void augAssign(AugAssign a, StringBuilder out, int ind) { + String bin = BIN.get(a.op); + String op = bin == null ? null : "i" + bin; + if (op == null) { + throw unsupported(a, "operator " + a.op + "="); + } + if (a.target instanceof Name) { + Name n = (Name) a.target; + storeName(n.id, augValue(op, load(n), a), out, ind); + } else if (a.target instanceof Attribute) { + Attribute at = (Attribute) a.target; + String recv = expr(at.value); + String o; + String rl = rawLocal(recv); + if (rl != null && (rl.equals(selfLocal) || shadowVar.containsKey(rl)) + && layouts.fieldOwners.containsKey(at.attr)) { + o = rl; + } else { + o = fresh("o"); + indent(out, ind); + out.append("final Object ").append(o).append(" = ").append(recv).append(";\n"); + } + indent(out, ind); + if (layouts.fieldOwners.containsKey(at.attr)) { + String nv = fresh("s"); + out.append("final Object ").append(nv).append(" = ") + .append(augValue(op, fieldGet(at.attr, o, site(at.attr)), a)).append(";\n"); + fieldSet(at.attr, o, nv, site(at.attr), out, ind); + } else { + out.append(site(at.attr)).append(".set(").append(o).append(", ") + .append(augValue(op, site(at.attr) + ".get(" + o + ")", a)).append(");\n"); + } + } else if (a.target instanceof Subscript) { + Subscript sb = (Subscript) a.target; + String o = fresh("o"); + String k = fresh("k"); + indent(out, ind); + out.append("final Object ").append(o).append(" = ").append(expr(sb.value)).append(";\n"); + indent(out, ind); + out.append("final Object ").append(k).append(" = ").append(expr(sb.index)).append(";\n"); + indent(out, ind); + out.append("Py.setitem(").append(o).append(", ").append(k).append(", ") + .append(augValue(op, "Py.getitem(" + o + ", " + k + ")", a)).append(");\n"); + } else { + throw new PythonSyntaxError(file, a.line, a.col, "illegal expression for augmented assignment"); + } + } + + private void importFrom(ImportFrom f, StringBuilder out, int ind) { + String mod = resolveRelative(f); + if ("__future__".equals(mod)) { + return; + } + String m = fresh("m"); + indent(out, ind); + out.append("final PyModule ").append(m).append(" = Py.importModule(").append(javaString(mod)).append(");\n"); + for (Alias a : f.names) { + if ("*".equals(a.name)) { + indent(out, ind); + out.append("Py.importStar(M, ").append(m).append(");\n"); + } else { + storeName(a.asname != null ? a.asname : a.name, + "Py.importFrom(" + m + ", " + javaString(a.name) + ")", out, ind); + } + } + } + + private String resolveRelative(ImportFrom f) { + if (f.level == 0) { + return f.module; + } + String base = moduleName; + int up = isPackage ? f.level - 1 : f.level; + for (int i = 0; i < up; i++) { + int dot = base.lastIndexOf('.'); + if (dot < 0) { + if (i == up - 1 && !base.isEmpty()) { + base = ""; + continue; + } + throw new PythonSyntaxError(file, f.line, f.col, "attempted relative import beyond top-level package"); + } + base = base.substring(0, dot); + } + if (f.module == null) { + return base; + } + return base.isEmpty() ? f.module : base + "." + f.module; + } + + private void ifStmt(If s, StringBuilder out, int ind) { + indent(out, ind); + out.append("if (").append(cond(s.test)).append(") {\n"); + block(s.body, out, ind + 1); + If cur = s; + while (cur.orelse.size() == 1 && cur.orelse.get(0) instanceof If) { + cur = (If) cur.orelse.get(0); + indent(out, ind); + out.append("} else if (").append(cond(cur.test)).append(") {\n"); + block(cur.body, out, ind + 1); + } + if (!cur.orelse.isEmpty()) { + indent(out, ind); + out.append("} else {\n"); + block(cur.orelse, out, ind + 1); + } + indent(out, ind); + out.append("}\n"); + } + + private void whileStmt(While w, StringBuilder out, int ind) { + String label = fresh("L"); + String outer = w.orelse.isEmpty() ? label : fresh("E"); + if (!w.orelse.isEmpty()) { + indent(out, ind); + out.append(outer).append(": {\n"); + ind++; + } + indent(out, ind); + out.append(label).append(": while (").append(cond(w.test)).append(") {\n"); + loops.add(new Loop(label, outer)); + block(w.body, out, ind + 1); + loops.remove(loops.size() - 1); + indent(out, ind); + out.append("}\n"); + if (!w.orelse.isEmpty()) { + block(w.orelse, out, ind); + indent(out, ind - 1); + out.append("}\n"); + } + } + + private void forStmt(For f, StringBuilder out, int ind) { + if (f.isAsync) { + throw unsupported(f, "async for"); + } + String label = fresh("L"); + String outer = f.orelse.isEmpty() ? label : fresh("E"); + if (!f.orelse.isEmpty()) { + indent(out, ind); + out.append(outer).append(": {\n"); + ind++; + } + Map lc = loopCarried(f); + if (lc != null) { + for (String[] c : lc.values()) { + indent(out, ind); + out.append("double ").append(c[0]).append(" = 0;\n"); + indent(out, ind); + out.append("boolean ").append(c[1]).append(" = false;\n"); + } + } + String it = fresh("i"); + String x = fresh("x"); + String[] lazy = null; + int vpos = -1; + String[] vsig = null; + if (rangeCall(f.iter)) { + // `for x in range(...)` counts in a long while `range` is still + // the builtin and its arguments are ints, so the loop allocates + // no range object and no iterator; anything else iterates. + List ra = ((Call) f.iter).args; + String rf = fresh("rf"); + indent(out, ind); + out.append("final Object ").append(rf).append(" = ").append(expr(((Call) f.iter).func)).append(";\n"); + String[] av = new String[3]; + for (int i = 0; i < ra.size(); i++) { + av[i] = fresh("ra"); + indent(out, ind); + out.append("final Object ").append(av[i]).append(" = ").append(expr(ra.get(i))).append(";\n"); + } + String start = ra.size() == 1 ? "Py.of(0L)" : av[0]; + String stop = ra.size() == 1 ? av[0] : av[1]; + String step = ra.size() == 3 ? av[2] : "Py.of(1L)"; + String ri = fresh("ri"); + String re = fresh("re"); + String rs = fresh("rs"); + indent(out, ind); + out.append("long ").append(ri).append(" = 0, ").append(re).append(" = 0, ").append(rs).append(" = 0;\n"); + if (lc != null && f.target instanceof Name && sc.resolve(((Name) f.target).id) == Scope.Binding.LOCAL) { + // The loop body is one float region (loopCarried), so the + // index stays a long: the region reads `ri` itself, and the + // local gets its Long only when the general path runs or the + // loop ends. A counted loop then allocates nothing. + lazy = new String[] {((Name) f.target).id, it, ri, rs, fresh("rst")}; + String[] saved = lazyIndex; + lazyIndex = lazy; + for (Stmt st : f.body) { + // A plain int result would come out a float. + if (st instanceof Assign && regionInt(((Assign) st).value)) { + lazy = null; + } + } + lazyIndex = saved; + } + indent(out, ind); + out.append("PyObject ").append(it).append(" = null;\n"); + indent(out, ind); + out.append("if (Py.isIntRange(").append(rf).append(", ").append(start).append(", ").append(stop) + .append(", ").append(step).append(")) {\n"); + indent(out, ind + 1); + out.append(ri).append(" = Py.asLong(").append(start).append(");\n"); + indent(out, ind + 1); + out.append(re).append(" = Py.asLong(").append(stop).append(");\n"); + indent(out, ind + 1); + out.append(rs).append(" = Py.asLong(").append(step).append(");\n"); + indent(out, ind); + out.append("} else {\n"); + indent(out, ind + 1); + out.append(it).append(" = Py.iter(Py.call(").append(rf); + for (int i = 0; i < ra.size(); i++) { + out.append(", ").append(av[i]); + } + out.append("));\n"); + indent(out, ind); + out.append("}\n"); + if (lazy != null) { + indent(out, ind); + out.append("final long ").append(lazy[4]).append(" = ").append(ri).append(";\n"); + vpos = out.length(); + vsig = new String[] {it, ri, re, rs, Integer.toString(ind)}; + } + indent(out, ind); + out.append(label).append(": for (Object ").append(x).append("; (").append(x).append(" = ").append(it) + .append(" != null ? ").append(it).append(".next() : (").append(rs).append(" > 0 ? ").append(ri) + .append(" < ").append(re).append(" : ").append(ri).append(" > ").append(re) + .append(") ? ").append(lazy != null ? "Py.NONE" : "Long.valueOf(" + ri + ")") + .append(" : null) != null; ").append(ri).append(" += ").append(rs).append(") {\n"); + } else { + // A list is walked by index (re-reading its length every step, as + // its own iterator does), so the loop allocates no iterator. + String sq = fresh("sq"); + String ls = fresh("ls"); + String li = fresh("li"); + indent(out, ind); + String src; + if (splitWords(f.iter)) { + src = "Py.splitWords(" + expr(((Attribute) ((Call) f.iter).func).value) + ")"; + } else if (splitOn(f.iter)) { + Call sc1 = (Call) f.iter; + src = "Py.splitOn(" + expr(((Attribute) sc1.func).value) + ", " + expr(sc1.args.get(0)) + ")"; + } else { + src = expr(f.iter); + } + out.append("final Object ").append(sq).append(" = ").append(src).append(";\n"); + indent(out, ind); + out.append("final PyList ").append(ls).append(" = ").append(sq).append(" instanceof PyList ? (PyList) ") + .append(sq).append(" : null;\n"); + indent(out, ind); + out.append("final PyObject ").append(it).append(" = ").append(ls).append(" != null ? null : Py.iter(") + .append(sq).append(");\n"); + indent(out, ind); + out.append("int ").append(li).append(" = 0;\n"); + indent(out, ind); + out.append(label).append(": for (Object ").append(x).append("; (").append(x).append(" = ").append(it) + .append(" != null ? ").append(it).append(".next() : ").append(li).append(" < ").append(ls) + .append(".size() ? ").append(ls).append(".get(").append(li).append("++) : null) != null;) {\n"); + } + if (lazy != null) { + indent(out, ind + 1); + out.append("if (").append(it).append(" != null) {\n"); + store(f.target, x, out, ind + 2); + indent(out, ind + 1); + out.append("}\n"); + } else { + store(f.target, x, out, ind + 1); + } + loops.add(new Loop(label, outer)); + Map savedCarried = carried; + String[] savedLazy = lazyIndex; + carried = lc; + lazyIndex = lazy; + String[] savedVersion = version; + StringBuilder savedVersionOut = versionOut; + StringBuilder vout = new StringBuilder(); + version = vsig; + versionOut = vout; + block(f.body, out, ind + 1); + version = savedVersion; + versionOut = savedVersionOut; + carried = savedCarried; + lazyIndex = savedLazy; + if (vpos >= 0) { + out.insert(vpos, vout); + } + loops.remove(loops.size() - 1); + indent(out, ind); + out.append("}\n"); + if (lazy != null) { + indent(out, ind); + out.append("if (").append(it).append(" == null && ").append(lazy[2]).append(" != ").append(lazy[4]) + .append(") {\n"); + indent(out, ind + 1); + out.append(localVar(sc, lazy[0])).append(" = Long.valueOf(").append(lazy[2]).append(" - ") + .append(lazy[3]).append(");\n"); + indent(out, ind); + out.append("}\n"); + } + if (lc != null) { + // Only a local something else reads needs its object back; the + // marks let resolveRegions drop the write-back of the others. + indent(out, ind); + out.append(REGION_START).append("\n"); + for (Map.Entry e : lc.entrySet()) { + indent(out, ind); + out.append(IF_SHARED).append(localVar(sc, e.getKey())).append("*/"); + StringBuilder w = new StringBuilder(); + syncCarried(e.getKey(), e.getValue(), w, 0); + out.append(w.toString().replace("\n", " ").trim()).append(COND_END).append("\n"); + } + indent(out, ind); + out.append(REGION_END).append("\n"); + } + if (!f.orelse.isEmpty()) { + block(f.orelse, out, ind); + indent(out, ind - 1); + out.append("}\n"); + } + } + + private void tryStmt(Try t, StringBuilder out, int ind) { + if (t.star) { + throw unsupported(t, "except*"); + } + boolean fin = !t.finalbody.isEmpty(); + if (fin) { + indent(out, ind); + out.append("try {\n"); + ind++; + } + if (t.handlers.isEmpty()) { + block(t.body, out, ind); + } else { + String ok = t.orelse.isEmpty() ? null : fresh("ok"); + if (ok != null) { + indent(out, ind); + out.append("boolean ").append(ok).append(" = false;\n"); + } + indent(out, ind); + out.append("try {\n"); + block(t.body, out, ind + 1); + if (ok != null) { + indent(out, ind + 1); + out.append(ok).append(" = true;\n"); + } + String th = fresh("th"); + String pe = fresh("pe"); + indent(out, ind); + out.append("} catch (Throwable ").append(th).append(") {\n"); + indent(out, ind + 1); + out.append("final PyException ").append(pe).append(" = Py.convert(").append(th).append(");\n"); + handlers.add(pe); + boolean catchAll = false; + for (int i = 0; i < t.handlers.size(); i++) { + ExceptHandler h = t.handlers.get(i); + indent(out, ind + 1); + if (h.type == null) { + if (i != t.handlers.size() - 1) { + throw new PythonSyntaxError(file, h.line, h.col, "default 'except:' must be last"); + } + out.append(i == 0 ? "{\n" : "} else {\n"); + catchAll = true; + } else { + out.append(i == 0 ? "if (" : "} else if (").append(pe).append(".matches(").append(expr(h.type)) + .append(")) {\n"); + } + if (h.name != null) { + storeName(h.name, pe + ".value", out, ind + 2); + } + // sys.exception() answers this exception while the body runs. + String hv = fresh("hv"); + indent(out, ind + 2); + out.append("final Object ").append(hv).append(" = Py.enterHandler(").append(pe).append(");\n"); + indent(out, ind + 2); + out.append("try {\n"); + block(h.body, out, ind + 3); + indent(out, ind + 2); + out.append("} finally {\n"); + indent(out, ind + 3); + out.append("Py.exitHandler(").append(hv).append(");\n"); + indent(out, ind + 2); + out.append("}\n"); + if (h.name != null && sc.kind != Scope.Kind.CLASS) { + indent(out, ind + 2); + out.append(nameLvalue(h.name)).append(" = null;\n"); + } + } + if (!catchAll) { + indent(out, ind + 1); + out.append("} else {\n"); + indent(out, ind + 2); + out.append("throw ").append(pe).append(";\n"); + } + indent(out, ind + 1); + out.append("}\n"); + handlers.remove(handlers.size() - 1); + indent(out, ind); + out.append("}\n"); + if (ok != null) { + indent(out, ind); + out.append("if (").append(ok).append(") {\n"); + block(t.orelse, out, ind + 1); + indent(out, ind); + out.append("}\n"); + } + } + if (fin) { + ind--; + indent(out, ind); + out.append("} finally {\n"); + block(t.finalbody, out, ind + 1); + indent(out, ind); + out.append("}\n"); + } + } + + private void withItems(With w, int i, StringBuilder out, int ind) { + if (i == w.items.size()) { + block(w.body, out, ind); + return; + } + WithItem item = w.items.get(i); + String mgr = fresh("w"); + String exit = fresh("ex"); + String ok = fresh("ok"); + String th = fresh("th"); + String pe = fresh("pe"); + indent(out, ind); + out.append("final Object ").append(mgr).append(" = ").append(expr(item.context)).append(";\n"); + indent(out, ind); + out.append("final Object ").append(exit).append(" = Py.getattr(").append(mgr) + .append(", \"__exit__\");\n"); + String entered = "Py.callMethod(" + mgr + ", \"__enter__\")"; + if (item.vars != null) { + String ev = fresh("en"); + indent(out, ind); + out.append("final Object ").append(ev).append(" = ").append(entered).append(";\n"); + store(item.vars, ev, out, ind); + } else { + indent(out, ind); + out.append(entered).append(";\n"); + } + indent(out, ind); + out.append("boolean ").append(ok).append(" = true;\n"); + indent(out, ind); + out.append("try {\n"); + withItems(w, i + 1, out, ind + 1); + indent(out, ind); + out.append("} catch (Throwable ").append(th).append(") {\n"); + indent(out, ind + 1); + out.append(ok).append(" = false;\n"); + indent(out, ind + 1); + out.append("final PyException ").append(pe).append(" = Py.convert(").append(th).append(");\n"); + indent(out, ind + 1); + out.append("if (!Py.isTrue(Py.call(").append(exit).append(", ").append("Py.typeOf(").append(pe).append(".value), ") + .append(pe).append(".value, Py.NONE))) {\n"); + indent(out, ind + 2); + out.append("throw ").append(pe).append(";\n"); + indent(out, ind + 1); + out.append("}\n"); + indent(out, ind); + out.append("} finally {\n"); + indent(out, ind + 1); + out.append("if (").append(ok).append(") {\n"); + indent(out, ind + 2); + out.append("Py.call(").append(exit).append(", Py.NONE, Py.NONE, Py.NONE);\n"); + indent(out, ind + 1); + out.append("}\n"); + indent(out, ind); + out.append("}\n"); + } + + private void classDef(ClassDef c, StringBuilder out, int ind) { + Layouts.Cls lc = layouts.classes.get(c); + String k = lc != null ? lc.typeVar : "k" + id(c.scope); + c.scope.classVar = k; + StringBuilder bases = new StringBuilder(); + StringBuilder stars = new StringBuilder(); + boolean starred = false; + for (Expr b : c.bases) { + boolean s = b instanceof Starred; + starred |= s; + bases.append(bases.length() == 0 ? "" : ", ").append(expr(s ? ((Starred) b).value : b)); + stars.append(stars.length() == 0 ? "" : ", ").append(s); + } + // The keywords (metaclass= among them) are evaluated before the + // body, as the bases are; the metaclass runs after it (see + // PyType.finish). + StringBuilder kwNames = new StringBuilder(); + StringBuilder kwValues = new StringBuilder(); + for (Keyword kw : c.keywords) { + String kv = fresh("kw"); + indent(out, ind); + out.append("final Object ").append(kv).append(" = ").append(expr(kw.value)).append(";\n"); + kwNames.append(kwNames.length() == 0 ? "" : ", ").append(kw.arg == null ? "null" : javaString(kw.arg)); + kwValues.append(kwValues.length() == 0 ? "" : ", ").append(kv); + } + List decos = new ArrayList(); + for (Expr d : c.decorators) { + String dv = fresh("d"); + indent(out, ind); + out.append("final Object ").append(dv).append(" = ").append(expr(d)).append(";\n"); + decos.add(dv); + } + indent(out, ind); + if (lc != null) { + fields.append(" private static PyType ").append(k).append(";\n"); + out.append(k); + } else { + out.append("final PyType ").append(k); + } + out.append(" = PyType.newClass(").append(javaString(c.name)); + if (starred) { + out.append(", Py.spread(new Object[] {").append(bases).append("}, new boolean[] {").append(stars) + .append("}));\n"); + } else { + out.append(", new Object[] {").append(bases).append("});\n"); + } + indent(out, ind); + out.append(k).append(".define(\"__module__\", G[0]);\n"); + indent(out, ind); + out.append(k).append(".define(\"__qualname__\", Py.str(").append(javaString(c.name)).append("));\n"); + boolean anns = false; + for (Stmt s : c.body) { + if (s instanceof AnnAssign && ((AnnAssign) s).target instanceof Name) { + anns = true; + } + } + if (anns) { + indent(out, ind); + out.append(k).append(".define(\"__annotations__\", new PyDict());\n"); + } + List body = c.body; + if (!body.isEmpty() && isDocstring(body.get(0))) { + indent(out, ind); + out.append(k).append(".define(\"__doc__\", ").append(constant((Constant) ((ExprStmt) body.get(0)).value)) + .append(");\n"); + } + Scope saved = sc; + List savedLoops = new ArrayList(loops); + loops.clear(); + sc = c.scope; + block(body, out, ind); + sc = saved; + loops.addAll(savedLoops); + if (lc != null) { + registerLayout(lc, out, ind); + } + String v = fresh("c"); + indent(out, ind); + out.append("final Object ").append(v).append(" = PyType.finish(").append(k); + if (c.keywords.isEmpty()) { + out.append(", Py.NO_KW, Py.NO_ARGS);\n"); + } else { + out.append(", new String[] {").append(kwNames).append("}, new Object[] {").append(kwValues).append("});\n"); + } + for (int i = decos.size() - 1; i >= 0; i--) { + v = "Py.call(" + decos.get(i) + ", " + v + ")"; + } + storeName(c.name, v, out, ind); + } + + // ---- functions ------------------------------------------------------- + + /// `__qualname__`: the enclosing classes and functions, as CPython + /// names them. Comprehensions are inlined (PEP 709) and add nothing. + private static String qualname(Scope fs) { + StringBuilder q = new StringBuilder(fs.name); + for (Scope s = fs.parent; s != null && s.kind != Scope.Kind.MODULE; s = s.parent) { + if (s.kind == Scope.Kind.CLASS) { + q.insert(0, s.name + "."); + } else if (s.kind != Scope.Kind.COMPREHENSION) { + q.insert(0, s.name + ".."); + } + } + return q.toString(); + } + + /// A `new PyFunction(...)` expression. `exprBody` is set for a lambda. + private String function(String name, Arguments args, Scope fs, List body, Expr exprBody) { + boolean async = nextAsync; + nextAsync = false; + boolean generator = body != null && (async || fs.hasYield); + GenLowering savedGen = gen; + gen = null; + if (body != null) { + defAssign.function(body, fs.params); + } + String asStatic = staticBody; + Layouts.Cls asClass = staticClass; + staticBody = null; + staticClass = null; + String savedSelfLocal = selfLocal; + Layouts.Cls savedSelfCls = selfCls; + String savedSelfTyped = selfTyped; + Map savedShadowVar = shadowVar; + Map savedShadowCls = shadowCls; + shadowVar = new HashMap(); + shadowCls = new HashMap(); + selfLocal = null; + selfCls = null; + selfTyped = null; + StringBuilder sig = new StringBuilder(); + List names = args.names(); + sig.append("new PyFunction(").append(javaString(name)).append(", new String[] {"); + for (int i = 0; i < names.size(); i++) { + sig.append(i == 0 ? "" : ", ").append(javaString(names.get(i))); + } + sig.append("}, ").append(args.positional.size()).append(", ").append(args.vararg != null).append(", ") + .append(args.kwonly.size()).append(", ").append(args.kwarg != null).append(", "); + if (args.defaults.isEmpty()) { + sig.append("Py.NO_ARGS, "); + } else { + sig.append("new Object[] {"); + for (int i = 0; i < args.defaults.size(); i++) { + sig.append(i == 0 ? "" : ", ").append(expr(args.defaults.get(i))); + } + sig.append("}, "); + } + boolean anyKwDefault = false; + for (Expr e : args.kwDefaults) { + anyKwDefault |= e != null; + } + if (!anyKwDefault) { + sig.append("null, "); + } else { + sig.append("new Object[] {"); + for (int i = 0; i < args.kwDefaults.size(); i++) { + Expr e = args.kwDefaults.get(i); + sig.append(i == 0 ? "" : ", ").append(e == null ? "null" : expr(e)); + } + sig.append("}, "); + } + + Frame savedFrame = frame; + Scope savedScope = sc; + List savedLoops = new ArrayList(loops); + List savedHandlers = new ArrayList(handlers); + loops.clear(); + handlers.clear(); + frame = new Frame(id(fs), fs); + sc = fs; + boolean savedHandlersFlag = fnHasHandlers; + Map savedCarried = carried; + carried = null; + // A generator's locals are fields that outlive a step, so none is + // carried in a Java local across a loop. + fnHasHandlers = body != null && (generator || hasHandlers(body)); + Set savedRecycled = recycled; + recycled = body != null ? ListRecycling.find(body, names, fs) : java.util.Collections.emptySet(); + List pnames = args.names(); + boolean directParams = args.vararg == null && args.kwarg == null && args.kwonly.isEmpty() && pnames.size() <= 4; + if (asStatic != null && asClass != null && directParams && !pnames.isEmpty() + && !fs.assigned.contains(pnames.get(0)) && !fs.cells.contains(pnames.get(0))) { + selfLocal = localVar(fs, pnames.get(0)); + selfCls = asClass; + selfTyped = "k" + frame.id; + } + if (body != null && !generator && !layouts.fieldOwners.isEmpty()) { + for (Map.Entry e : ShadowTypes.of(body, layouts).entrySet()) { + String n = e.getKey(); + if (fs.resolve(n) != Scope.Binding.LOCAL || fs.cells.contains(n)) { + continue; + } + String v = localVar(fs, n); + if (v.equals(selfLocal)) { + continue; + } + shadowVar.put(v, "h" + frame.id + "_" + mangle(n)); + shadowCls.put(v, e.getValue()); + } + } + StringBuilder b = new StringBuilder(); + int ind = 3; + if (exprBody != null) { + String e = expr(exprBody); + indent(b, ind); + b.append("return ").append(e).append(";\n"); + } else if (generator) { + gen = new GenLowering(this, file, fs); + gen.asyncGen = async && fs.yields; + b.append(resolveRegions(gen.step(body))); + } else { + block(body, b, ind); + indent(b, ind); + b.append("return Py.NONE;\n"); + String settled = resolveRegions(b.toString()); + b.setLength(0); + b.append(settled); + Infer.Fn tf = typedEmit != null ? typedEmit.fnFor(fs) : null; + if (tf != null) { + // Enter the typed version (TypedEmit) when the arguments fit. + List locals = new ArrayList(); + for (String n : names) { + locals.add(localVar(fs, n)); + } + StringBuilder g = new StringBuilder(); + for (String line : typedEmit.entry(tf, locals).split("\n")) { + indent(g, ind); + g.append(line).append('\n'); + } + b.insert(0, g); + } + } + String a = "a" + frame.id; + // Only positional parameters, at most four: the body takes them as + // Java parameters (PyFunction.Body0..Body4), so a call allocates no + // argument array. + boolean direct = args.vararg == null && args.kwarg == null && args.kwonly.isEmpty() && names.size() <= 4; + StringBuilder pro = new StringBuilder(); + Set declared = new LinkedHashSet(); + for (int i = 0; i < names.size(); i++) { + String n = names.get(i); + declared.add(n); + indent(pro, ind); + if (fs.cells.contains(n)) { + pro.append("final Object[] ").append(cellVar(fs, n)).append(" = new Object[] {").append(param(a, i, direct)) + .append("};\n"); + } else { + pro.append("Object ").append(localVar(fs, n)).append(" = ").append(param(a, i, direct)) + .append(";\n"); + } + } + for (String n : fs.assigned) { + if (declared.contains(n) || !fs.isLocal(n)) { + continue; + } + declared.add(n); + indent(pro, ind); + if (fs.cells.contains(n)) { + pro.append("final Object[] ").append(cellVar(fs, n)).append(" = new Object[1];\n"); + } else { + pro.append("Object ").append(localVar(fs, n)).append(" = null;\n"); + } + } + if (selfTyped != null) { + indent(pro, ind); + pro.append("final ").append(selfCls.javaName).append(' ').append(selfTyped).append(" = ").append(selfLocal) + .append(" instanceof ").append(selfCls.javaName).append(" ? (").append(selfCls.javaName).append(") ") + .append(selfLocal).append(" : null;\n"); + } + for (Map.Entry e : shadowVar.entrySet()) { + String k = shadowCls.get(e.getKey()).javaName; + indent(pro, ind); + // Every local is declared above (parameters with their value, + // the rest null), so testing it here is right for both. + pro.append(k).append(' ').append(e.getValue()).append(" = ").append(e.getKey()).append(" instanceof ") + .append(k).append(" ? (").append(k).append(") ").append(e.getKey()).append(" : null;\n"); + } + selfLocal = savedSelfLocal; + selfCls = savedSelfCls; + selfTyped = savedSelfTyped; + shadowVar = savedShadowVar; + shadowCls = savedShadowCls; + String temps = frame.temps > 0 || frame.ltemps > 0 ? tempDecls() : null; + if (generator) { + // The locals become fields of the generator object, and the + // body its step(); temporaries never live across a suspension. + StringBuilder g = new StringBuilder(); + indent(g, ind); + if (async && fs.yields) { + g.append("return new PyGenerator.Async(").append(javaString(name)).append(") {\n"); + } else { + g.append("return new PyGenerator(").append(javaString(name)).append(", ").append(async).append(") {\n"); + } + g.append(pro.toString().replaceAll("(?m)^ {12}", " ")); + g.append(gen.fields()); + if (gen.usesReturnField()) { + g.append(" Object ret$;\n"); + } + g.append("\n @Override\n protected Object step() {\n"); + if (temps != null) { + g.append(" ").append(temps).append("\n"); + } + g.append(b); + g.append(" }\n"); + indent(g, ind); + g.append("};\n"); + pro.setLength(0); + b.setLength(0); + b.append(g); + } else if (temps != null) { + indent(pro, ind); + pro.append(temps).append("\n"); + } + gen = savedGen; + frame = savedFrame; + sc = savedScope; + fnHasHandlers = savedHandlersFlag; + recycled = savedRecycled; + carried = savedCarried; + loops.clear(); + loops.addAll(savedLoops); + handlers.clear(); + handlers.addAll(savedHandlers); + + StringBuilder r = new StringBuilder(sig); + r.append('('); + StringBuilder plist = new StringBuilder(); + StringBuilder alist = new StringBuilder(); + if (direct) { + for (int i = 0; i < names.size(); i++) { + plist.append(i == 0 ? "" : ", ").append("Object ").append(param(a, i, true)); + alist.append(i == 0 ? "" : ", ").append(param(a, i, true)); + } + } else { + plist.append("Object[] ").append(a); + alist.append(a); + } + r.append(plist); + if (asStatic != null && direct) { + // A compiled method: the body is a static method the class + // layout's dispatch calls directly (see Layouts). + fields.append(" static Object ").append(asStatic).append("(").append(plist).append(") {\n") + .append(pro).append(b).append(" }\n"); + r.append(") -> ").append(asStatic).append("(").append(alist).append("))"); + } else { + r.append(") -> {\n").append(pro).append(b); + indent(r, ind - 1); + r.append("})"); + } + Expr ret = generator ? null : pureReturn(args, names, body, exprBody); + if (ret != null) { + String pure = pureVariant(names, ret); + if (pure != null) { + r.append(".pure(").append(pure).append(")"); + String pureL = pureLongVariant(names, ret); + if (pureL != null) { + boolean module = sc == moduleScope && directLong.containsKey(name); //NOPMD CompareObjectsWithEquals - identity is meant + String method = module ? "PL_" + mangle(name) : "PL" + fusedCount++; + StringBuilder lsig = new StringBuilder(); + StringBuilder lcall = new StringBuilder(); + for (int i = 0; i < names.size(); i++) { + lsig.append(i == 0 ? "" : ", ").append("long q").append(i); + lcall.append(i == 0 ? "" : ", ").append("q").append(i); + } + fields.append(" private static double ").append(method).append("(").append(lsig) + .append(") {\n").append(pureL).append(" }\n"); + r.append(".pure((PyFunction.PureL").append(names.size()).append(") (").append(lsig) + .append(") -> ").append(method).append("(").append(lcall).append("))"); + if (module) { + fields.append(" private static Object DEF_").append(mangle(name)).append(";\n"); + r.insert(0, "(DEF_" + mangle(name) + " = ").append(")"); + } + } + } + } + String q = qualname(fs); + if (!q.equals(name)) { + r.append(".qual(").append(javaString(q)).append(")"); + } + return r.toString(); + } + + /// Parameter `i` of a function body: its own Java parameter, or an + /// element of the argument array. + private static String param(String a, int i, boolean direct) { + return direct ? a + "_" + i : a + "[" + i + "]"; + } + + // ---- expressions ----------------------------------------------------- + + private String temp() { + return "t" + frame.id + "_" + (frame.temps++); + } + + /// The temporaries of the current frame, as plain locals: an array of + /// them would be one more allocation on every call. + private String tempDecls() { + StringBuilder sb = new StringBuilder(); + if (frame.temps > 0) { + sb.append("Object "); + for (int i = 0; i < frame.temps; i++) { + sb.append(i == 0 ? "" : ", ").append("t").append(frame.id).append("_").append(i).append(" = null"); + } + sb.append(";"); + } + if (frame.ltemps > 0) { + sb.append(sb.length() == 0 ? "" : " ").append("long "); + for (int i = 0; i < frame.ltemps; i++) { + sb.append(i == 0 ? "" : ", ").append("j").append(frame.id).append("_").append(i).append(" = 0"); + } + sb.append(";"); + } + return sb.toString(); + } + + /// The int constants among the hoisted fields: an operand known to be + /// one needs no test and no unboxing. + private final Map constLongs = new HashMap(); + + /// The Java long for operand `v` once `t` holds it (and has been tested + /// when not a constant). + /// The hoisted constant for int `l`, known to the inline int paths. + private String intConst(long l) { + String k = hoist("Object", "Py.of(" + l + "L)"); + constLongs.put(k, Long.valueOf(l)); + return k; + } + + private String longOf(String v, String t) { + Long c = constLongs.get(v); + return c != null ? c.longValue() + "L" : "((Long) " + t + ").longValue()"; + } + + /// `(t = v) instanceof Long`, or true for an int constant (still + /// assigned, for the slow path). + private String isLong(String v, String t) { + return constLongs.containsKey(v) ? "(" + t + " = " + v + ") != null" : "(" + t + " = " + v + ") instanceof Long"; + } + + private String ltemp() { + return "j" + frame.id + "_" + (frame.ltemps++); + } + + /// The truth of `v` without a call for a bool or None. + private String truth(String v) { + String t = temp(); + return "((" + t + " = " + v + ") == Py.TRUE || " + t + " != Py.FALSE && " + t + " != Py.NONE && (" + t + + " instanceof Long ? ((Long) " + t + ").longValue() != 0 : Py.isTrue(" + t + ")))"; + } + + /// A Java boolean for `l op r` with the int case inline: both operands + /// are evaluated, in order, before either is tested. + private String inlineCmp(String op, String l, String r) { + String jop; + String slow; + switch (op) { + case "<": + jop = "<"; + slow = "Py.ltBool("; + break; + case "==": + jop = "=="; + slow = "Py.eqBool("; + break; + case "!=": + jop = "!="; + slow = "!Py.eqBool("; + break; + case "<=": + jop = "<="; + slow = "Py.isTrue(Py.le("; + break; + case ">": + jop = ">"; + slow = "Py.isTrue(Py.gt("; + break; + case ">=": + jop = ">="; + slow = "Py.isTrue(Py.ge("; + break; + default: + return null; + } + String t = temp(); + String u = temp(); + String close = slow.startsWith("Py.isTrue") ? "))" : ")"; + return "((" + isLong(l, t) + " & " + isLong(r, u) + ") ? " + longOf(l, t) + " " + jop + " " + longOf(r, u) + " : " + + slow + t + ", " + u + close + ")"; + } + + /// `l + r` or `l - r` with the int case inline (overflow goes the + /// generic way, to a big int); null for any other operator. + private String inlineArith(String op, String l, String r, boolean inPlace) { + if (!"+".equals(op) && !"-".equals(op)) { + return null; + } + String t = temp(); + String u = temp(); + String x = ltemp(); + String y = ltemp(); + String z = ltemp(); + String overflow = "+".equals(op) + ? "((" + x + " ^ " + z + ") & (" + y + " ^ " + z + ")) >= 0" + : "((" + x + " ^ " + y + ") & (" + x + " ^ " + z + ")) >= 0"; + String slow = "Py." + (inPlace ? "i" : "") + BIN.get(op) + "(" + t + ", " + u + ")"; + return "((" + isLong(l, t) + " & " + isLong(r, u) + ") && ((" + z + " = (" + x + " = " + longOf(l, t) + ") " + op + + " (" + y + " = " + longOf(r, u) + ")) == " + z + " & " + overflow + ") ? (Object) Long.valueOf(" + z + + ") : " + slow + ")"; + } + + /// A Java boolean expression for a Python condition. + String cond(Expr e) { + if (e instanceof Constant && ((Constant) e).kind == Constant.Kind.TRUE) { + // Not the literal: javac would then reject the code after a + // `while True` it cannot see a break out of. + return "(Py.TRUE == Py.TRUE)"; + } + if (e instanceof UnaryOp && "not".equals(((UnaryOp) e).op)) { + return "!(" + cond(((UnaryOp) e).operand) + ")"; + } + if (e instanceof BoolOp) { + BoolOp b = (BoolOp) e; + StringBuilder sb = new StringBuilder("("); + for (int i = 0; i < b.values.size(); i++) { + if (i > 0) { + sb.append(b.isAnd ? " && " : " || "); + } + sb.append(cond(b.values.get(i))); + } + return sb.append(")").toString(); + } + if (e instanceof Compare) { + Compare c = (Compare) e; + StringBuilder sb = new StringBuilder("("); + String left = expr(c.left); + for (int i = 0; i < c.ops.size(); i++) { + String right = expr(c.comparators.get(i)); + if (i < c.ops.size() - 1) { + String t = temp(); + right = "(" + t + " = " + right + ")"; + sb.append(cmpBool(c.ops.get(i), left, right)).append(" && "); + left = t; + } else { + sb.append(cmpBool(c.ops.get(i), left, right)); + } + } + return sb.append(")").toString(); + } + if (e instanceof Constant) { + Constant k = (Constant) e; + if (k.kind == Constant.Kind.TRUE) { + return "Py.isTrue(Py.TRUE)"; + } + if (k.kind == Constant.Kind.FALSE || k.kind == Constant.Kind.NONE) { + return "Py.isTrue(Py.FALSE)"; + } + } + return truth(expr(e)); + } + + private String cmpBool(String op, String l, String r) { + String in = inlineCmp(op, l, r); + if (in != null) { + return in; + } + switch (op) { + case "==": + return "Py.eqBool(" + l + ", " + r + ")"; + case "!=": + return "!Py.eqBool(" + l + ", " + r + ")"; + case "<": + return "Py.ltBool(" + l + ", " + r + ")"; + case "is": + return "(" + l + " == " + r + ")"; + case "is not": + return "(" + l + " != " + r + ")"; + default: + return "Py.isTrue(Py." + CMP.get(op) + "(" + l + ", " + r + "))"; + } + } + + String expr(Expr e) { + if (e instanceof Name) { + return load((Name) e); + } + if (e instanceof Constant) { + return constant((Constant) e); + } + if (e instanceof BinOp) { + BinOp b = (BinOp) e; + String f = fused(e); + if (f != null) { + return f; + } + String op = BIN.get(b.op); + if (op == null) { + return "Py.callMethod(" + expr(b.left) + ", \"__matmul__\", " + expr(b.right) + ")"; + } + String in = inlineArith(b.op, expr(b.left), expr(b.right), false); + if (in != null) { + return in; + } + return "Py." + op + "(" + expr(b.left) + ", " + expr(b.right) + ")"; + } + if (e instanceof UnaryOp) { + UnaryOp u = (UnaryOp) e; + switch (u.op) { + case "not": + return "(" + cond(e) + " ? Py.TRUE : Py.FALSE)"; + case "-": { + if (u.operand instanceof Constant && ((Constant) u.operand).kind == Constant.Kind.INT) { + BigInteger v = parseInt(((Constant) u.operand).text); + if (v.bitLength() < 63) { + return intConst(-v.longValue()); + } + } + String t = temp(); + String j = ltemp(); + return "((" + t + " = " + expr(u.operand) + ") instanceof Long && (" + j + " = ((Long) " + t + + ").longValue()) != Long.MIN_VALUE ? (Object) Long.valueOf(-" + j + ") : Py.neg(" + t + "))"; + } + case "+": + return "Py.pos(" + expr(u.operand) + ")"; + default: + return "Py.invert(" + expr(u.operand) + ")"; + } + } + if (e instanceof BoolOp) { + BoolOp b = (BoolOp) e; + if (boolValued(e)) { + // Every operand is a bool, so the result is one too. + return "(" + cond(e) + " ? Py.TRUE : Py.FALSE)"; + } + String r = expr(b.values.get(b.values.size() - 1)); + for (int i = b.values.size() - 2; i >= 0; i--) { + String t = temp(); + String test = truth(t + " = " + expr(b.values.get(i))); + r = "(" + (b.isAnd ? "!" : "") + test + " ? " + t + " : " + r + ")"; + } + return r; + } + if (e instanceof Compare) { + Compare c = (Compare) e; + if (c.ops.size() == 1 && boolValued(e)) { + return "(" + cond(e) + " ? Py.TRUE : Py.FALSE)"; + } + if (c.ops.size() == 1) { + return "Py." + CMP.get(c.ops.get(0)) + "(" + expr(c.left) + ", " + expr(c.comparators.get(0)) + ")"; + } + // a < b < c: each link's value; stop at the first false one. + String left = expr(c.left); + List parts = new ArrayList(); + for (int i = 0; i < c.ops.size(); i++) { + String right = expr(c.comparators.get(i)); + if (i < c.ops.size() - 1) { + String t = temp(); + parts.add("Py." + CMP.get(c.ops.get(i)) + "(" + left + ", " + t + " = " + right + ")"); + left = t; + } else { + parts.add("Py." + CMP.get(c.ops.get(i)) + "(" + left + ", " + right + ")"); + } + } + String r = parts.get(parts.size() - 1); + for (int i = parts.size() - 2; i >= 0; i--) { + String t = temp(); + r = "(!Py.isTrue(" + t + " = " + parts.get(i) + ") ? " + t + " : " + r + ")"; + } + return r; + } + if (e instanceof IfExp) { + IfExp i = (IfExp) e; + return "(" + cond(i.test) + " ? " + expr(i.body) + " : " + expr(i.orelse) + ")"; + } + if (e instanceof NamedExpr) { + NamedExpr n = (NamedExpr) e; + String n2 = n.target.id; + if (sc.resolve(n2) == Scope.Binding.CLASS) { + throw unsupported(e, "an assignment expression in a class body"); + } + return "(" + nameLvalue(n2) + " = " + expr(n.value) + ")"; + } + if (e instanceof Attribute) { + Attribute a = (Attribute) e; + if (FRAME_HOOKS.contains(a.attr) && a.value instanceof Name && "sys".equals(((Name) a.value).id)) { + throw dynamicCode(a, "sys." + a.attr); + } + if (layouts.fieldOwners.containsKey(a.attr)) { + return fieldGet(a.attr, expr(a.value), site(a.attr)); + } + return site(a.attr) + ".get(" + expr(a.value) + ")"; + } + if (e instanceof Subscript) { + Subscript s = (Subscript) e; + if (s.index instanceof Slice) { + Slice sl = (Slice) s.index; + return "Py.getslice(" + expr(s.value) + ", " + exprOrNone(sl.lower) + ", " + exprOrNone(sl.upper) + + ", " + exprOrNone(sl.step) + ")"; + } + String ci = constIndex(s.index); + if (ci != null) { + return "Py.getitem(" + expr(s.value) + ", " + ci + ")"; + } + return "Py.getitem(" + expr(s.value) + ", " + expr(s.index) + ")"; + } + if (e instanceof Slice) { + Slice s = (Slice) e; + return "Py.slice(" + exprOrNone(s.lower) + ", " + exprOrNone(s.upper) + ", " + exprOrNone(s.step) + ")"; + } + if (e instanceof Call) { + return call((Call) e); + } + if (e instanceof ListExpr) { + return display("new PyList(", ((ListExpr) e).elts, ")", "Py.list("); + } + if (e instanceof TupleExpr) { + List elts = ((TupleExpr) e).elts; + if (elts.isEmpty()) { + return "PyTuple.EMPTY"; + } + return display("new PyTuple(", elts, ")", "Py.tuple("); + } + if (e instanceof SetExpr) { + List elts = ((SetExpr) e).elts; + if (elts.size() >= 3 && allConstant(elts)) { + return display("PySet.ofConstants(", elts, ")", "PySet.ofConstants("); + } + return display("Py.set(", ((SetExpr) e).elts, ")", "Py.set("); + } + if (e instanceof DictExpr) { + DictExpr d = (DictExpr) e; + boolean spread = false; + for (Expr k : d.keys) { + spread |= k == null; + } + StringBuilder sb = new StringBuilder(spread ? "Py.dictSpread(" : "Py.dict("); + for (int i = 0; i < d.keys.size(); i++) { + Expr k = d.keys.get(i); + sb.append(i == 0 ? "" : ", ").append(k == null ? "null" : expr(k)).append(", ") + .append(expr(d.values.get(i))); + } + return sb.append(")").toString(); + } + if (e instanceof JoinedStr) { + return joined((JoinedStr) e); + } + if (e instanceof FormattedValue) { + JoinedStr j = new JoinedStr(); + j.parts.add(e); + return joined(j); + } + if (e instanceof Lambda) { + Lambda l = (Lambda) e; + if (l.scope.hasYield) { + throw unsupported(e, "generators and async"); + } + return function("", l.args, l.scope, null, l.body); + } + if (e instanceof Comp) { + return comprehension((Comp) e); + } + if (e instanceof Starred) { + throw new PythonSyntaxError(file, e.line, e.col, "can't use starred expression here"); + } + if (e instanceof Yield || e instanceof Await) { + throw unsupported(e, "generators and async"); + } + throw unsupported(e, e.getClass().getSimpleName()); + } + + private String exprOrNone(Expr e) { + return e == null ? "Py.NONE" : expr(e); + } + + /// A list/tuple/set display; one with `*x` items goes through Py.spread. + private String display(String open, List elts, String close, String varargsOpen) { + boolean star = false; + for (Expr x : elts) { + star |= x instanceof Starred; + } + if (!star) { + StringBuilder sb = new StringBuilder(varargsOpen); + for (int i = 0; i < elts.size(); i++) { + sb.append(i == 0 ? "" : ", ").append(expr(elts.get(i))); + } + return sb.append(")").toString(); + } + StringBuilder parts = new StringBuilder("new Object[] {"); + StringBuilder flags = new StringBuilder("new boolean[] {"); + for (int i = 0; i < elts.size(); i++) { + Expr x = elts.get(i); + boolean s = x instanceof Starred; + parts.append(i == 0 ? "" : ", ").append(expr(s ? ((Starred) x).value : x)); + flags.append(i == 0 ? "" : ", ").append(s); + } + return open + "Py.spread(" + parts + "}, " + flags + "})" + close; + } + + private String joined(JoinedStr j) { + List parts = new ArrayList(); + for (Expr p : j.parts) { + if (p instanceof Constant) { + parts.add(constant((Constant) p)); + continue; + } + FormattedValue f = (FormattedValue) p; + String v = expr(f.value); + if (f.conversion == 'r') { + v = "Py.str(Py.toRepr(" + v + "))"; + } else if (f.conversion == 's') { + v = "Py.str(Py.strOf(" + v + "))"; + } else if (f.conversion == 'a') { + v = "Py.call(Builtins.ASCII, " + v + ")"; + } + String spec; + if (f.spec == null) { + spec = "\"\""; + } else if (f.spec.parts.size() == 1 && f.spec.parts.get(0) instanceof Constant) { + spec = javaString(((Constant) f.spec.parts.get(0)).text); + } else if (f.spec.parts.isEmpty()) { + spec = "\"\""; + } else { + spec = "Py.strOf(" + joined(f.spec) + ")"; + } + parts.add("Py.format(" + v + ", " + spec + ")"); + } + if (parts.size() == 1) { + return parts.get(0); + } + StringBuilder sb = new StringBuilder("Py.concat("); + for (int i = 0; i < parts.size(); i++) { + sb.append(i == 0 ? "" : ", ").append(parts.get(i)); + } + return sb.append(")").toString(); + } + + private String call(Call c) { + if (c.func instanceof Name && c.args.isEmpty() && c.keywords.isEmpty()) { + String fn = ((Name) c.func).id; + if (("globals".equals(fn) || "locals".equals(fn)) && sc.resolve(fn) == Scope.Binding.BUILTIN) { + if ("locals".equals(fn) && sc.kind != Scope.Kind.MODULE) { + // A function's locals are Java variables; there is no + // namespace to hand out. + throw unsupported(c, "locals() inside a function or class"); + } + // A snapshot: the module's globals are fields, not a dict. + return "M.globals()"; + } + } + if (c.func instanceof Name && "super".equals(((Name) c.func).id) && c.args.isEmpty() + && c.keywords.isEmpty() && sc.resolve("super") == Scope.Binding.BUILTIN) { + return zeroArgSuper(c); + } + if (c.func instanceof Name && "len".equals(((Name) c.func).id) && c.args.size() == 1 + && !(c.args.get(0) instanceof Starred) && c.keywords.isEmpty() + && sc.resolve("len") == Scope.Binding.BUILTIN) { + // Still the global's value that is called, so a module that + // rebinds `len` later gets its own function. + return "Py.len(" + expr(c.func) + ", " + expr(c.args.get(0)) + ")"; + } + boolean spread = false; + for (Expr a : c.args) { + spread |= a instanceof Starred; + } + for (Keyword k : c.keywords) { + spread |= k.arg == null; + } + if (spread) { + StringBuilder parts = new StringBuilder("new Object[] {"); + StringBuilder kinds = new StringBuilder("new int[] {"); + StringBuilder names = new StringBuilder("new String[] {"); + int i = 0; + for (Expr a : c.args) { + boolean s = a instanceof Starred; + parts.append(i == 0 ? "" : ", ").append(expr(s ? ((Starred) a).value : a)); + kinds.append(i == 0 ? "" : ", ").append(s ? 1 : 0); + names.append(i == 0 ? "" : ", ").append("null"); + i++; + } + for (Keyword k : c.keywords) { + parts.append(i == 0 ? "" : ", ").append(expr(k.value)); + kinds.append(i == 0 ? "" : ", ").append(k.arg == null ? 2 : 3); + names.append(i == 0 ? "" : ", ").append(k.arg == null ? "null" : javaString(k.arg)); + i++; + } + return "Py.callSpread(" + expr(c.func) + ", " + parts + "}, " + kinds + "}, " + names + "})"; + } + StringBuilder args = new StringBuilder(); + for (Expr a : c.args) { + args.append(", ").append(expr(a)); + } + if (!c.keywords.isEmpty()) { + StringBuilder kw = new StringBuilder("new String[] {"); + for (int i = 0; i < c.keywords.size(); i++) { + Keyword k = c.keywords.get(i); + args.append(", ").append(expr(k.value)); + kw.append(i == 0 ? "" : ", ").append(javaString(k.arg)); + } + String w = hoist("String[]", kw.append("}").toString()); + String arr = args.length() == 0 ? "Py.NO_ARGS" : "new Object[] {" + args.substring(2) + "}"; + return "Py.callKw(" + expr(c.func) + ", " + arr + ", " + w + ")"; + } + if (c.func instanceof Attribute) { + Attribute fa = (Attribute) c.func; + if (FRAME_HOOKS.contains(fa.attr) && fa.value instanceof Name && "sys".equals(((Name) fa.value).id)) { + throw dynamicCode(fa, "sys." + fa.attr); + } + Attribute a = (Attribute) c.func; + int n = c.args.size(); + String recv = expr(a.value); + if (n <= 3 && layouts.dispatch.contains(a.attr + "/" + n)) { + usedCall.add(a.attr + "/" + n); + String helper = "c_" + mangle(a.attr) + "_" + n + "("; + String direct = directCall(a.attr, n, recv, args.toString(), helper, site(a.attr)); + return direct != null ? direct : helper + recv + args + ", " + site(a.attr) + ")"; + } + if (n <= 3) { + return site(a.attr) + ".call" + n + "(" + recv + args + ")"; + } + return site(a.attr) + ".callN(" + recv + ", new Object[] {" + args.substring(2) + "})"; + } + return "Py.call(" + expr(c.func) + args + ")"; + } + + private String zeroArgSuper(Call c) { + Scope f = sc; + while (f != null && f.kind == Scope.Kind.COMPREHENSION) { + f = f.parent; + } + if (f == null || !f.isFunctionLike() || f.enclosingClass == null || f.params.isEmpty()) { + throw new PythonSyntaxError(file, c.line, c.col, "super(): no arguments"); + } + String first = f.params.iterator().next(); + Name n = new Name(first); + n.line = c.line; + n.col = c.col; + return "new PySuper(" + f.enclosingClass.classVar + ", " + load(n) + ")"; + } + + private String comprehension(Comp c) { + if (c.kind == Comp.Kind.GENERATOR || Linearize.awaited(c)) { + return generatorExpression(c); + } + Frame savedFrame = frame; + Scope savedScope = sc; + String first = expr(c.generators.get(0).iter); + Scope cs = c.scope; + frame = new Frame(id(cs), cs); + sc = cs; + String a = "a" + frame.id; + String r = fresh("r"); + StringBuilder b = new StringBuilder(); + int ind = 3; + int depth = ind; + for (int i = 0; i < c.generators.size(); i++) { + Comprehension g = c.generators.get(i); + if (g.isAsync) { + throw unsupported(g, "async comprehensions"); + } + String it = fresh("i"); + String x = fresh("x"); + indent(b, depth); + b.append("final PyObject ").append(it).append(" = Py.iter(").append(i == 0 ? a + "[0]" : expr(g.iter)) + .append(");\n"); + indent(b, depth); + b.append("for (Object ").append(x).append("; (").append(x).append(" = ").append(it) + .append(".next()) != null;) {\n"); + depth++; + store(g.target, x, b, depth); + for (Expr cond : g.ifs) { + indent(b, depth); + b.append("if (").append(cond(cond)).append(") {\n"); + depth++; + } + } + indent(b, depth); + switch (c.kind) { + case DICT: + b.append(r).append(".setItem(").append(expr(c.elt)).append(", ").append(expr(c.value)) + .append(");\n"); + break; + case SET: + b.append(r).append(".addItem(").append(expr(c.elt)).append(");\n"); + break; + default: + b.append(r).append(".append(").append(expr(c.elt)).append(");\n"); + break; + } + while (depth > ind) { + depth--; + indent(b, depth); + b.append("}\n"); + } + StringBuilder pro = new StringBuilder(); + for (String n : cs.assigned) { + if (!cs.isLocal(n)) { + continue; + } + indent(pro, ind); + if (cs.cells.contains(n)) { + pro.append("final Object[] ").append(cellVar(cs, n)).append(" = new Object[1];\n"); + } else { + pro.append("Object ").append(localVar(cs, n)).append(" = null;\n"); + } + } + if ((frame.temps > 0 || frame.ltemps > 0)) { + indent(pro, ind); + pro.append(tempDecls()).append("\n"); + } + String type; + switch (c.kind) { + case DICT: + type = "PyDict"; + break; + case SET: + type = "PySet"; + break; + default: + type = "PyList"; + break; + } + indent(pro, ind); + pro.append("final ").append(type).append(" ").append(r).append(" = new ").append(type).append("();\n"); + frame = savedFrame; + sc = savedScope; + StringBuilder out = new StringBuilder(); + out.append("((PyFunction.Body) (Object[] ").append(a).append(") -> {\n").append(pro).append(b); + indent(out, ind); + // Generator expressions are evaluated eagerly for now: an iterator + // over the materialised list. + out.append("return ").append(c.kind == Comp.Kind.GENERATOR ? "Py.iter(" + r + ")" : r).append(";\n"); + indent(out, ind - 1); + out.append("}).run(new Object[] {").append(first).append("})"); + return out.toString(); + } + + /// A generator expression: a generator whose body is the loops of the + /// comprehension around `yield elt`, lazy as in Python. The first + /// iterable is evaluated here, in the enclosing scope, and handed in as + /// the hidden local `.0`. + private String generatorExpression(Comp c) { + Frame savedFrame = frame; + Scope savedScope = sc; + GenLowering savedGen = gen; + boolean async = Linearize.isAsync(c); + boolean genexp = c.kind == Comp.Kind.GENERATOR; + String first = expr(c.generators.get(0).iter); + Scope cs = c.scope; + frame = new Frame(id(cs), cs); + sc = cs; + cs.assigned.add(".0"); + List body = new ArrayList(); + Expr acc = new Name(".r").at(c.line, c.col); + if (genexp) { + body.add(new ExprStmt(new Yield(c.elt, false).at(c.elt.line, c.elt.col)).at(c.line, c.col)); + } else if (c.kind == Comp.Kind.DICT) { + // An async comprehension: a coroutine that fills `.r` and returns it. + cs.assigned.add(".r"); + body.add(new Assign(java.util.Collections.singletonList( + new Subscript(acc, c.elt).at(c.line, c.col)), c.value).at(c.line, c.col)); + } else { + cs.assigned.add(".r"); + Call add = new Call(new Attribute(acc, c.kind == Comp.Kind.SET ? "add" : "append").at(c.line, c.col)); + add.args.add(c.elt); + body.add(new ExprStmt(add.at(c.elt.line, c.elt.col)).at(c.line, c.col)); + } + for (int i = c.generators.size() - 1; i >= 0; i--) { + Comprehension g = c.generators.get(i); + for (int j = g.ifs.size() - 1; j >= 0; j--) { + Expr cond = g.ifs.get(j); + body = new ArrayList(java.util.Collections.singletonList( + new If(cond, body, new ArrayList()).at(cond.line, cond.col))); + } + Expr iter = i == 0 ? new Name(".0").at(g.line, g.col) : g.iter; + body = new ArrayList(java.util.Collections.singletonList( + new For(g.target, iter, body, new ArrayList(), g.isAsync).at(g.line, g.col))); + } + if (!genexp) { + body.add(new Return(acc).at(c.line, c.col)); + } + gen = new GenLowering(this, file, cs); + gen.asyncGen = async && genexp; + String step = resolveRegions(gen.step(body)); + String a = "a" + frame.id; + StringBuilder r = new StringBuilder(); + r.append("((PyFunction.Body) (Object[] ").append(a).append(") -> {\n"); + indent(r, 3); + if (!genexp) { + r.append("return new PyGenerator(\"<").append(c.kind.name().toLowerCase(java.util.Locale.ROOT)) + .append("comp>\", true) {\n"); + } else if (async) { + r.append("return new PyGenerator.Async(\"\") {\n"); + } else { + r.append("return new PyGenerator(\"\", false) {\n"); + } + String empty = c.kind == Comp.Kind.DICT ? "new PyDict()" : c.kind == Comp.Kind.SET ? "new PySet()" : "new PyList()"; + for (String n : cs.assigned) { + if (!cs.isLocal(n)) { + continue; + } + String init = ".0".equals(n) ? a + "[0]" : ".r".equals(n) ? empty : null; + if (cs.cells.contains(n)) { + r.append(" final Object[] ").append(cellVar(cs, n)).append(" = new Object[] {") + .append(init == null ? "null" : init).append("};\n"); + } else { + r.append(" Object ").append(localVar(cs, n)).append(" = ").append(init == null ? "null" : init) + .append(";\n"); + } + } + if (gen.usesReturnField()) { + r.append(" Object ret$;\n"); + } + r.append(gen.fields()); + r.append("\n @Override\n protected Object step() {\n"); + if (frame.temps > 0 || frame.ltemps > 0) { + r.append(" ").append(tempDecls()).append("\n"); + } + r.append(step).append(" }\n"); + indent(r, 3); + r.append("};\n"); + indent(r, 2); + String firstIter = c.generators.get(0).isAsync ? first : "Py.iter(" + first + ")"; + r.append("}).run(new Object[] {").append(firstIter).append("})"); + gen = savedGen; + frame = savedFrame; + sc = savedScope; + return r.toString(); + } + + // ---- constants ------------------------------------------------------- + + private int sites; + + // ---- compiled class layouts (see Layouts) ----------------------------- + + private Layouts layouts = new Layouts(); + /// Whole-program types, when the program is closed (see `Closed`). + private TypedEmit typedEmit; + private boolean closed; + + /// Whether the program this module belongs to passed `Closed.check`. + public Emitter closed(boolean c) { + closed = c; + return this; + } + /// The static method name the next `function` call puts its body in. + private String staticBody; + /// The class of the compiled method `staticBody` names. + private Layouts.Cls staticClass; + /// In a compiled method whose receiver parameter is never rebound: that + /// parameter's Java local, its class, and the local holding it cast to + /// that class, or null when it is not an instance of it. The receiver's + /// type is tested once on entry instead of at every field access. + private String selfLocal; + private Layouts.Cls selfCls; + private String selfTyped; + /// Java local -> the typed copy kept of it, for the locals of the current + /// function read for compiled fields (see ShadowTypes). The copy is set + /// at every store to the local and each access checks `copy == local`, + /// so a copy left stale only sends the access down the generic path. + private Map shadowVar = new HashMap(); + private Map shadowCls = new HashMap(); + private final Set usedCall = new LinkedHashSet(); + + /// Gives a compiled class its layout once its body has run, if its base + /// is still the compiled class the layout extends. + private void registerLayout(Layouts.Cls lc, StringBuilder out, int ind) { + indent(out, ind); + out.append("if ("); + if (lc.parent != null) { + out.append(lc.typeVar).append(".bases.length == 1 && ").append(lc.typeVar).append(".bases[0] == ") + .append(lc.parent.typeVar).append(" && ").append(lc.parent.typeVar).append(".hasLayout()"); + } else { + out.append("true"); + } + out.append(") {\n"); + indent(out, ind + 1); + out.append(lc.typeVar).append(".layout(t -> new ").append(lc.javaName).append("(t), new String[] {"); + int i = 0; + for (String m : lc.methods.keySet()) { + out.append(i++ == 0 ? "" : ", ").append(javaString(m)); + } + out.append("}, new String[] {"); + i = 0; + for (String f : lc.allFields) { + out.append(i++ == 0 ? "" : ", ").append(javaString(f)); + } + out.append("});\n"); + indent(out, ind); + out.append("}\n"); + } + + private String layoutCode() { + if (layouts.classes.isEmpty()) { + return ""; + } + StringBuilder o = new StringBuilder(); + List disp = new ArrayList(); + for (String d : layouts.dispatch) { + int at = d.lastIndexOf('/'); + disp.add(new String[] {d.substring(0, at), d.substring(at + 1)}); + } + o.append("\n static class LB extends PyInstance {\n"); + o.append(" LB(PyType t) {\n super(t);\n }\n"); + for (String[] d : disp) { + o.append(" Object m_").append(mangle(d[0])).append("(").append(dparams(Integer.parseInt(d[1]))) + .append(") {\n return null;\n }\n"); + } + if (typedEmit != null) { + o.append(typedEmit.layoutMembers(null)); + } + o.append(" }\n"); + for (Layouts.Cls k : layouts.classes.values()) { + String sup = k.parent != null ? k.parent.javaName : "LB"; + int base = k.allFields.size() - k.fields.size(); + o.append("\n static class ").append(k.javaName).append(" extends ").append(sup).append(" {\n"); + for (String f : k.fields) { + String td = typedEmit != null ? typedEmit.fieldDecl(k, f) : null; + o.append(" ").append(td != null ? td : "Object f_" + mangle(f) + ";").append("\n"); + } + o.append(" private static final String[] FN = {"); + for (int i = 0; i < k.allFields.size(); i++) { + o.append(i == 0 ? "" : ", ").append(javaString(k.allFields.get(i))); + } + o.append("};\n"); + o.append(" ").append(k.javaName).append("(PyType t) {\n super(t);\n }\n"); + if (!k.fields.isEmpty()) { + o.append(" @Override\n protected int fieldIndex(String n) {\n switch (n) {\n"); + for (int i = 0; i < k.fields.size(); i++) { + o.append(" case ").append(javaString(k.fields.get(i))).append(":\n return ") + .append(base + i).append(";\n"); + } + o.append(" default:\n return super.fieldIndex(n);\n }\n }\n"); + o.append(" @Override\n protected Object getField(int i) {\n switch (i) {\n"); + for (int i = 0; i < k.fields.size(); i++) { + String f = k.fields.get(i); + boolean t = typedEmit != null && typedEmit.typedField(k, f) != null; + o.append(" case ").append(base + i).append(":\n return ") + .append(t ? typedEmit.fieldGetCase(k, f) : "f_" + mangle(f)).append(";\n"); + } + o.append(" default:\n return super.getField(i);\n }\n }\n"); + o.append(" @Override\n protected void setField(int i, Object v) {\n switch (i) {\n"); + for (int i = 0; i < k.fields.size(); i++) { + String f = k.fields.get(i); + boolean t = typedEmit != null && typedEmit.typedField(k, f) != null; + o.append(" case ").append(base + i).append(":\n ") + .append(t ? typedEmit.fieldSetCase(k, f, "v") : "f_" + mangle(f) + " = v;") + .append("\n return;\n"); + } + o.append(" default:\n super.setField(i, v);\n }\n }\n"); + o.append(" @Override\n protected String[] fieldNames() {\n return FN;\n }\n"); + } + for (String[] d : disp) { + Layouts.Cls r = Layouts.resolver(k, d[0]); + Layouts.Cls rp = k.parent == null ? null : Layouts.resolver(k.parent, d[0]); + if (r == rp) { //NOPMD CompareObjectsWithEquals - identity is meant + continue; + } + int ar = Integer.parseInt(d[1]); + o.append(" @Override\n Object m_").append(mangle(d[0])).append("(").append(dparams(ar)) + .append(") {\n"); + FunctionDef f = r == null ? null : r.methods.get(d[0]); + if (f != null && f.args.names().size() - 1 == ar) { + o.append(" if (cls.patched || extra) {\n return null;\n }\n"); + o.append(" return ").append(layouts.staticNames.get(f)).append("(this"); + for (int i = 0; i < ar; i++) { + o.append(", a").append(i); + } + o.append(");\n"); + } else { + o.append(" return null;\n"); + } + o.append(" }\n"); + } + if (typedEmit != null) { + o.append(typedEmit.layoutMembers(k)); + } + o.append(" }\n"); + } + for (String d : usedCall) { + int at = d.lastIndexOf('/'); + String n = d.substring(0, at); + int ar = Integer.parseInt(d.substring(at + 1)); + StringBuilder args = new StringBuilder(); + for (int i = 0; i < ar; i++) { + args.append(", a").append(i); + } + o.append("\n private static Object c_").append(mangle(n)).append("_").append(ar).append("(Object o") + .append(ar == 0 ? "" : ", " + dparams(ar)).append(", AttrSite s) {\n"); + o.append(" if (o instanceof LB) {\n Object r = ((LB) o).m_").append(mangle(n)).append("(") + .append(args.length() == 0 ? "" : args.substring(2)).append(");\n"); + o.append(" if (r != null) {\n return r;\n }\n }\n"); + o.append(" return s.call").append(ar).append("(o").append(args).append(");\n }\n"); + } + return o.toString(); + } + + /// `recv.name` for a compiled field: the field itself when the receiver + /// is of a class that declares it and the field is set, else the site. + private String fieldGet(String name, String recv, String site) { + String raw = rawLocal(recv); + Layouts.Cls own = raw != null && raw.equals(selfLocal) ? selfOwner(name) : null; + if (own != null) { + String f = selfTyped + ".f_" + mangle(name); + return "(" + selfTyped + " != null && " + f + " != null ? " + f + " : " + site + ".get(" + recv + "))"; + } + String h = raw != null ? shadowVar.get(raw) : null; + Layouts.Cls hk = h != null ? ownerFor(name, shadowCls.get(raw)) : null; + if (raw != null) { + // A local needs no temps (each a frame slot under ParparVM): the + // field is read twice, a load, and an unbound local fails every + // instanceof and raises from the generic path. + StringBuilder sb = new StringBuilder("("); + if (hk != null) { + String f = h + ".f_" + mangle(name); + sb.append(h).append(" == ").append(raw).append(" && ").append(f).append(" != null ? ").append(f) + .append(" : "); + } + for (Layouts.Cls k : layouts.fieldOwners.get(name)) { + String f = "((" + k.javaName + ") " + raw + ").f_" + mangle(name); + sb.append(raw).append(" instanceof ").append(k.javaName).append(" && ").append(f) + .append(" != null ? ").append(f).append(" : "); + } + return sb.append(site).append(".get(").append(recv).append("))").toString(); + } + String t = temp(); + String u = temp(); + StringBuilder sb = new StringBuilder("("); + String first = "(" + t + " = " + recv + ")"; + for (Layouts.Cls k : layouts.fieldOwners.get(name)) { + sb.append(first).append(" instanceof ").append(k.javaName).append(" && (").append(u).append(" = ((") + .append(k.javaName).append(") ").append(t).append(").f_").append(mangle(name)).append(") != null ? ") + .append(u).append(" : "); + first = t; + } + return sb.append(site).append(".get(").append(t).append("))").toString(); + } + + private static final java.util.regex.Pattern RAW_LOCAL = + java.util.regex.Pattern.compile("\\(?([a-z][A-Za-z0-9_]*) != null \\? \\1 : ul\\(\"[^\"]*\"\\)\\)|[a-z][A-Za-z0-9_]*"); + + /// The Java local a load expression reads, when it is one: a bare + /// variable, or the unbound-checked load of one. + private static String rawLocal(String e) { + java.util.regex.Matcher m = RAW_LOCAL.matcher(e); + if (!m.matches()) { + return null; + } + return m.group(1) != null ? m.group(1) : e; + } + + /// The class declaring field `name` that the compiled method's receiver + /// class is or extends, or null when there is none (or no such method). + private Layouts.Cls selfOwner(String name) { + return ownerFor(name, selfCls); + } + + /// The class declaring field `name` that `cls` is or extends, or null. + private Layouts.Cls ownerFor(String name, Layouts.Cls cls) { + if (cls == null) { + return null; + } + List owners = layouts.fieldOwners.get(name); + for (Layouts.Cls c = cls; c != null; c = c.parent) { + if (owners.contains(c)) { + return c; + } + } + return null; + } + + /// `o.name = v` for a compiled field; `o` and `v` are already evaluated. + private void fieldSet(String name, String o, String v, String site, StringBuilder out, int ind) { + indent(out, ind); + if (o.equals(selfLocal) && selfOwner(name) != null) { + String f = selfTyped + ".f_" + mangle(name); + out.append("if (").append(selfTyped).append(" != null && (").append(f).append(" != null || !") + .append(selfTyped).append(".extra)) {\n"); + indent(out, ind + 1); + out.append(f).append(" = ").append(v).append(";\n"); + indent(out, ind); + out.append("} else {\n"); + indent(out, ind + 1); + out.append(site).append(".set(").append(o).append(", ").append(v).append(");\n"); + indent(out, ind); + out.append("}\n"); + return; + } + String h = shadowVar.get(o); + if (h != null && ownerFor(name, shadowCls.get(o)) != null) { + String f = h + ".f_" + mangle(name); + out.append("if (").append(h).append(" == ").append(o).append(" && (").append(f).append(" != null || !") + .append(h).append(".extra)) {\n"); + indent(out, ind + 1); + out.append(f).append(" = ").append(v).append(";\n"); + indent(out, ind); + out.append("} else "); + } + for (Layouts.Cls k : layouts.fieldOwners.get(name)) { + out.append("if (").append(o).append(" instanceof ").append(k.javaName).append(" && (((") + .append(k.javaName).append(") ").append(o).append(").f_").append(mangle(name)) + .append(" != null || !((").append(k.javaName).append(") ").append(o).append(").extra)) {\n"); + indent(out, ind + 1); + out.append("((").append(k.javaName).append(") ").append(o).append(").f_").append(mangle(name)).append(" = ") + .append(v).append(";\n"); + indent(out, ind); + out.append("} else "); + } + out.append("{\n"); + indent(out, ind + 1); + out.append(site).append(".set(").append(o).append(", ").append(v).append(");\n"); + indent(out, ind); + out.append("}\n"); + } + + /// `d[k] = d.get(k, c) + e` as one probe of `d`, when every operand is + /// a plain name or constant; `Py.dictCount` falls back to the two calls + /// for anything but an exact dict. Null when the statement is not that. + private String dictCount(Assign a) { + if (a.targets.size() != 1 || !(a.targets.get(0) instanceof Subscript) || !(a.value instanceof BinOp)) { + return null; + } + Subscript t = (Subscript) a.targets.get(0); + BinOp b = (BinOp) a.value; + if (!"+".equals(b.op) || !(t.value instanceof Name) || !(t.index instanceof Name) + || !(b.left instanceof Call)) { + return null; + } + Call c = (Call) b.left; + if (!(c.func instanceof Attribute) || !"get".equals(((Attribute) c.func).attr) + || !(((Attribute) c.func).value instanceof Name) || c.args.size() != 2 || !c.keywords.isEmpty() + || !(c.args.get(0) instanceof Name)) { + return null; + } + String d = ((Name) t.value).id; + String k = ((Name) t.index).id; + if (!d.equals(((Name) ((Attribute) c.func).value).id) || !k.equals(((Name) c.args.get(0)).id)) { + return null; + } + String[] parts = {expr(t.value), expr(t.index), expr(c.args.get(1)), expr(b.right)}; + for (String p : parts) { + if (rawLocal(p) == null && !SIMPLE_ARG.matcher(p).matches()) { + return null; + } + } + return "Py.dictCount(" + parts[0] + ", " + parts[1] + ", " + parts[2] + ", " + parts[3] + ")"; + } + + private static final java.util.regex.Pattern SITE_CALL1 = + java.util.regex.Pattern.compile("(S[0-9]+)\\.call1\\(([A-Za-z][A-Za-z0-9_]*), (.*)\\)", java.util.regex.Pattern.DOTALL); + + /// `x.append(v)` as a statement, `e` being its generated call: the list's + /// own append straight away when `x` holds a list (a final class whose + /// method no program can replace), the site otherwise. Null when the + /// statement is not that. + private String appendFast(Expr v, String e) { + if (!(v instanceof Call)) { + return null; + } + Call c = (Call) v; + if (!(c.func instanceof Attribute) || !"append".equals(((Attribute) c.func).attr) || c.args.size() != 1 + || c.args.get(0) instanceof Starred || !c.keywords.isEmpty()) { + return null; + } + java.util.regex.Matcher m = SITE_CALL1.matcher(e); + if (!m.matches()) { + return null; + } + String recv = m.group(2); + String arg = m.group(3); + return "if (" + recv + " instanceof PyList) { ((PyList) " + recv + ").append(" + arg + "); } else { " + e + + "; }\n"; + } + + private static final java.util.regex.Pattern SIMPLE_ARG = + java.util.regex.Pattern.compile("[A-Za-z][A-Za-z0-9_]*|Py\\.(NONE|TRUE|FALSE)"); + + /// `recv.name(args)` calling the compiled body straight from the call + /// site, for each compiled class that resolves `name` to one: a static + /// call after an `instanceof`, instead of the `c_` helper's virtual + /// dispatch. Null when that is not certain for every compiled class or + /// an argument is not a plain value (each branch repeats them). + private String directCall(String name, int n, String recv, String args, String helper, String site) { + for (String a : args.isEmpty() ? new String[0] : args.substring(2).split(", ")) { + if (rawLocal(a) == null && !SIMPLE_ARG.matcher(a).matches()) { + return null; + } + } + List all = new ArrayList(layouts.classes.values()); + StringBuilder sb = new StringBuilder("("); + String raw = rawLocal(recv); + String t = raw; + String first; + if (raw == null) { + t = temp(); + first = "(" + t + " = " + recv + ")"; + } else { + first = raw; + } + int branches = 0; + // Subclasses were planned after their bases: test them first. + for (int i = all.size() - 1; i >= 0; i--) { + Layouts.Cls k = all.get(i); + Layouts.Cls r = Layouts.resolver(k, name); + if (r == null || k.parent != null && Layouts.resolver(k.parent, name) == r) { //NOPMD CompareObjectsWithEquals - identity is meant + // Answered by a base's branch, or not a class attribute here. + continue; + } + FunctionDef f = r.methods.get(name); + if (f == null || f.args.names().size() - 1 != n || branches >= 3) { + return null; + } + branches++; + String c = "((" + k.javaName + ") " + t + ")"; + sb.append(first).append(" instanceof ").append(k.javaName).append(" && !").append(c).append(".cls.patched && !") + .append(c).append(".extra ? ").append(layouts.staticNames.get(f)).append("(").append(t).append(args) + .append(") : "); + first = t; + } + if (branches == 0) { + return null; + } + return sb.append(helper).append(raw != null ? recv : t).append(args).append(", ").append(site).append("))") + .toString(); + } + + private static String dparams(int n) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < n; i++) { + sb.append(i == 0 ? "" : ", ").append("Object a").append(i); + } + return sb.toString(); + } + + /// A fresh inline cache for one attribute access. Never shared between two + /// accesses, unlike `hoist`: each site caches what ITS receivers look like. + private String site(String attr) { + String f = "S" + (++sites); + fields.append(" private static final AttrSite ").append(f).append(" = new AttrSite(") + .append(javaString(attr)).append(");\n"); + return f; + } + + private String hoist(String type, String init) { + String key = type + " " + init; + String f = constants.get(key); + if (f == null) { + f = "K" + constants.size(); + constants.put(key, f); + fields.append(" private static final ").append(type).append(' ').append(f).append(" = ").append(init) + .append(";\n"); + } + return f; + } + + String constant(Constant c) { + switch (c.kind) { + case NONE: + return "Py.NONE"; + case TRUE: + return "Py.TRUE"; + case FALSE: + return "Py.FALSE"; + case ELLIPSIS: + return "Py.ELLIPSIS"; + case STR: + if (c.text.isEmpty()) { + return "Py.EMPTY_STR"; + } + // A str is a java.lang.String; the literal is interned already. + return javaString(c.text); + case BYTES: { + StringBuilder sb = new StringBuilder("new PyBytes(new byte[] {"); + for (int i = 0; i < c.text.length(); i++) { + sb.append(i == 0 ? "" : ", ").append((int) (byte) c.text.charAt(i)); + } + return hoist("PyBytes", sb.append("})").toString()); + } + case FLOAT: { + String text = c.text.replace("_", ""); + if (text.endsWith("j") || text.endsWith("J")) { + double im = Double.parseDouble(text.substring(0, text.length() - 1)); + String imLit = Double.isInfinite(im) ? "Double.POSITIVE_INFINITY" : Double.toString(im); + return hoist("Object", "new PyComplex(0.0, " + imLit + ")"); + } + double d = Double.parseDouble(text); + String lit = Double.isInfinite(d) ? "Double.POSITIVE_INFINITY" : Double.toString(d); + return hoist("Object", "Py.of(" + lit + ")"); + } + case INT: + default: { + BigInteger v = parseInt(c.text); + if (v.bitLength() < 64) { + return intConst(v.longValue()); + } + return hoist("Object", "PyInt.parse(\"" + v.toString() + "\", 10)"); + } + } + } + + private BigInteger parseInt(String text) { + String t = text.replace("_", ""); + String l = t.length() > 1 ? t.substring(0, 2) : ""; + if ("0x".equals(l) || "0X".equals(l)) { + return new BigInteger(t.substring(2), 16); + } + if ("0o".equals(l) || "0O".equals(l)) { + return new BigInteger(t.substring(2), 8); + } + if ("0b".equals(l) || "0B".equals(l)) { + return new BigInteger(t.substring(2), 2); + } + return new BigInteger(t); + } + + // ---- Java literals --------------------------------------------------- + + /// A Java string literal for any text, in ASCII. Line breaks, quotes and + /// backslashes use their short escapes: a backslash-u escape for them + /// would be decoded by javac before the literal is even lexed. + public static String javaString(String s) { + StringBuilder sb = new StringBuilder("\""); + for (int i = 0; i < s.length(); i++) { + char c = s.charAt(i); + switch (c) { + case '\n': + sb.append("\\n"); + break; + case '\r': + sb.append("\\r"); + break; + case '\t': + sb.append("\\t"); + break; + case '"': + sb.append("\\\""); + break; + case '\\': + sb.append("\\\\"); + break; + default: + if (c < 0x20 || c > 0x7e) { + String h = Integer.toHexString(c); + sb.append("\\u"); + for (int k = h.length(); k < 4; k++) { + sb.append('0'); + } + sb.append(h); + } else { + sb.append(c); + } + } + } + return sb.append('"').toString(); + } + + private static String javaComment(String s) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < s.length(); i++) { + char c = s.charAt(i); + sb.append(c >= 0x20 && c < 0x7f && c != '\\' ? c : '?'); + } + return sb.toString(); + } + + /// Whether CPython's optimizer folds every element to a constant, which + /// turns a set display of three or more into a frozenset constant. + private static boolean allConstant(List elts) { + for (Expr x : elts) { + if (!isConstant(x)) { + return false; + } + } + return true; + } + + // ---- float expression fusion ------------------------------------------ + // + // An arithmetic tree such as `(dx * dx + dy * dy + dz * dz) ** 0.5` is + // emitted as one static helper taking the tree's leaves. When every leaf + // is a float the helper computes the whole tree in Java `double` and + // boxes only the result; otherwise (or when a step would raise: a zero + // divisor, a pow that overflows or leaves the reals) it evaluates the + // tree through the generic operators exactly as it would have been + // written. The leaves are still evaluated once each, left to right, at + // the call site. + + private int fusedCount; + + /// Stands for the current value of an augmented assignment's target in + /// a fused tree; the root operator of such a tree is the in-place one. + private static final Name AUG_CURRENT = new Name("$current"); + + private static boolean fuseNode(Expr e) { + if (e instanceof BinOp) { + String op = ((BinOp) e).op; + return "+".equals(op) || "-".equals(op) || "*".equals(op) || "/".equals(op) || "**".equals(op); + } + return e instanceof UnaryOp && "-".equals(((UnaryOp) e).op); + } + + private static boolean intConst(Expr e) { + if (!(e instanceof Constant) || ((Constant) e).kind != Constant.Kind.INT) { + return false; + } + String t = ((Constant) e).text; + if (t.length() > 15 || (t.length() > 1 && t.charAt(0) == '0')) { + return false; + } + for (int i = 0; i < t.length(); i++) { + if (t.charAt(i) < '0' || t.charAt(i) > '9') { + return false; + } + } + return true; + } + + private static boolean floatConst(Expr e) { + return e instanceof Constant && ((Constant) e).kind == Constant.Kind.FLOAT + && !((Constant) e).text.matches(".*[jJ]"); + } + + /// Whether the tree, given float leaves, is float at every node; -1 when + /// it cannot be fused at all. + private static int fuseCheck(Expr e) { + if (fuseNode(e)) { + if (e instanceof UnaryOp) { + return fuseCheck(((UnaryOp) e).operand); + } + int l = fuseCheck(((BinOp) e).left); + int r = fuseCheck(((BinOp) e).right); + if (l < 0 || r < 0 || l == 0 && r == 0) { + return -1; + } + return 1; + } + if (intConst(e)) { + return 0; + } + if (e instanceof Constant && !floatConst(e)) { + return -1; + } + return 1; + } + + private String fused(Expr root) { + return fused(root, null); + } + + private String fused(Expr root, String current) { + if (numOps(root) < 2) { + return null; + } + // Which nodes are int and which float, when every leaf has the type + // the tree implies for it: an operand of a bit, shift, floor or + // modulo operator is an int, and so is every other occurrence of the + // same name; any other leaf is taken for a float. + Set intCtx = java.util.Collections.newSetFromMap(new java.util.IdentityHashMap()); + Set intNames = new HashSet(); + markInts(root, false, intCtx, intNames); + Map types = new java.util.IdentityHashMap(); + if (numType(root, intCtx, intNames, types) == 0) { + return null; + } + List leaves = new ArrayList(); + numLeaves(root, leaves); + if (leaves.isEmpty()) { + return null; + } + String name = "F" + fusedCount++; + StringBuilder m = new StringBuilder(); + m.append(" private static Object ").append(name).append("("); + for (int i = 0; i < leaves.size(); i++) { + m.append(i == 0 ? "" : ", ").append("Object p").append(i); + } + m.append(") {\n"); + numPath(root, leaves, types, "fast", m); + // A tree that only took its leaves for floats also gets an all-int + // path when it has nothing an int cannot do: `i * n + j`. + boolean anyFloat = false; + for (Expr l : leaves) { + anyFloat |= types.get(l).charValue() == 'f'; + } + if (anyFloat) { + Map ints = new java.util.IdentityHashMap(); + Set all = java.util.Collections.newSetFromMap(new java.util.IdentityHashMap()); + all.addAll(leaves); + if (numType(root, all, new HashSet(), ints) == 'i') { + numPath(root, leaves, ints, "ints", m); + } + } + int[] counters = new int[2]; + String boxed = fuseBoxed(root, counters); + if (current != null) { + boxed = "Py.i" + boxed.substring(3); + } + m.append(" return ").append(boxed).append(";\n }\n"); + fields.append(m); + StringBuilder call = new StringBuilder(name).append("("); + for (int i = 0; i < leaves.size(); i++) { + Expr leaf = leaves.get(i); + call.append(i == 0 ? "" : ", ").append(leaf == AUG_CURRENT ? current : expr(leaf)); + } + return call.append(")").toString(); + } + + /// The raw-double twin of a function whose body is `return ` (PyFunction.PureD1..4), or null. + private String pureVariant(List params, Expr ret) { + if (ret == null || params.isEmpty() || params.size() > 4 || numOps(ret) < 1) { + return null; + } + Set intCtx = java.util.Collections.newSetFromMap(new java.util.IdentityHashMap()); + Set intNames = new HashSet(); + markInts(ret, false, intCtx, intNames); + Map types = new java.util.IdentityHashMap(); + if (numType(ret, intCtx, intNames, types) != 'f') { + return null; + } + List leaves = new ArrayList(); + numLeaves(ret, leaves); + StringBuilder m = new StringBuilder(); + m.append("(PyFunction.PureD").append(params.size()).append(") ("); + for (int i = 0; i < params.size(); i++) { + m.append(i == 0 ? "" : ", ").append("Object q").append(i); + } + m.append(") -> {\n"); + for (int i = 0; i < leaves.size(); i++) { + Expr l = leaves.get(i); + int k = l instanceof Name ? params.indexOf(((Name) l).id) : -1; + if (k < 0) { + return null; + } + m.append(" final Object p").append(i).append(" = q").append(k).append(";\n"); + } + numPath(ret, leaves, types, "fast", m, true); + return m.append(" return Double.NaN;\n }").toString(); + } + + /// A module-level function with a long twin, called directly: the static + /// method `PL_` holds the twin and `DEF_` the function object + /// its `def` made, so a call site that finds that very object in the + /// global calls the method, which the C compiler can inline. Filled + /// before the module is emitted, as callers may come first. + private final Map directLong = new HashMap(); + + private void findDirectLong(List body) { + Map defs = new HashMap(); + for (Stmt s : body) { + if (s instanceof FunctionDef) { + String n = ((FunctionDef) s).name; + defs.put(n, Integer.valueOf(defs.containsKey(n) ? 2 : 1)); + } + } + for (Stmt s : body) { + if (!(s instanceof FunctionDef) || defs.get(((FunctionDef) s).name).intValue() != 1) { + continue; + } + FunctionDef f = (FunctionDef) s; + List names = f.args.names(); + Expr ret = pureReturn(f.args, names, f.body, null); + if (ret != null && !f.isAsync && !f.scope.hasYield && pureVariant(names, ret) != null + && pureLongVariant(names, ret) != null) { + directLong.put(f.name, Integer.valueOf(names.size())); + } + } + } + + /// The single returned expression a pure twin is made from, or null. + private Expr pureReturn(Arguments args, List names, List body, Expr exprBody) { + boolean direct = args.vararg == null && args.kwarg == null && args.kwonly.isEmpty() && names.size() <= 4; + if (!direct || !args.defaults.isEmpty()) { + return null; + } + Expr ret = exprBody; + List rest = body; + if (rest != null && rest.size() == 2 && isDocstring(rest.get(0))) { + rest = rest.subList(1, 2); + } + if (ret == null && rest != null && rest.size() == 1 && rest.get(0) instanceof Return) { + ret = ((Return) rest.get(0)).value; + } + return ret; + } + + /// The same twin taking its parameters as raw longs + /// (PyFunction.PureL1..2), for a body whose every leaf is an int + /// parameter: callers holding unboxed ints then box nothing. + private String pureLongVariant(List params, Expr ret) { + if (params.size() > 2) { + return null; + } + Set intCtx = java.util.Collections.newSetFromMap(new java.util.IdentityHashMap()); + Set intNames = new HashSet(); + markInts(ret, false, intCtx, intNames); + Map types = new java.util.IdentityHashMap(); + if (numType(ret, intCtx, intNames, types) != 'f') { + return null; + } + List leaves = new ArrayList(); + numLeaves(ret, leaves); + StringBuilder m = new StringBuilder(); + m.append(" fast: {\n"); + for (int i = 0; i < leaves.size(); i++) { + Expr l = leaves.get(i); + int k = l instanceof Name ? params.indexOf(((Name) l).id) : -1; + if (k < 0 || types.get(l).charValue() != 'i') { + return null; + } + m.append(" long d").append(i).append(" = q").append(k).append(";\n"); + } + String r = numPrim(ret, types, "fast", m, new int[2]); + m.append(" return ").append(r).append(";\n }\n"); + return m.append(" return Double.NaN;\n").toString(); + } + + private static boolean intOnlyOp(String op) { + return "//".equals(op) || "%".equals(op) || "<<".equals(op) || ">>".equals(op) || "&".equals(op) + || "|".equals(op) || "^".equals(op); + } + + /// A node of a fused tree: arithmetic, bit and shift operators, negation. + private static boolean numNode(Expr e) { + if (e instanceof BinOp) { + String op = ((BinOp) e).op; + return fuseNode(e) || intOnlyOp(op); + } + return e instanceof UnaryOp && ("-".equals(((UnaryOp) e).op) || "~".equals(((UnaryOp) e).op)); + } + + private static int numOps(Expr e) { + if (!numNode(e)) { + return 0; + } + if (e instanceof UnaryOp) { + return 1 + numOps(((UnaryOp) e).operand); + } + return 1 + numOps(((BinOp) e).left) + numOps(((BinOp) e).right); + } + + private static void numLeaves(Expr e, List out) { + if (numNode(e)) { + if (e instanceof UnaryOp) { + numLeaves(((UnaryOp) e).operand, out); + } else { + numLeaves(((BinOp) e).left, out); + numLeaves(((BinOp) e).right, out); + } + } else if (!(e instanceof Constant)) { + out.add(e); + } + } + + private static void markInts(Expr e, boolean ctx, Set intCtx, Set intNames) { + if (!numNode(e)) { + if (ctx) { + intCtx.add(e); + if (e instanceof Name) { + intNames.add(((Name) e).id); + } + } + return; + } + if (e instanceof UnaryOp) { + UnaryOp u = (UnaryOp) e; + markInts(u.operand, ctx || "~".equals(u.op), intCtx, intNames); + return; + } + BinOp b = (BinOp) e; + boolean c = intOnlyOp(b.op) || ctx && ("+".equals(b.op) || "-".equals(b.op) || "*".equals(b.op)); + markInts(b.left, c, intCtx, intNames); + markInts(b.right, c, intCtx, intNames); + } + + /// The value of an int literal that fits a Java long, or null. + private static Long intLiteral(Expr e) { + if (!(e instanceof Constant) || ((Constant) e).kind != Constant.Kind.INT) { + return null; + } + String t = ((Constant) e).text.replace("_", "").toLowerCase(java.util.Locale.ROOT); + int radix = 10; + if (t.startsWith("0x")) { + radix = 16; + } else if (t.startsWith("0o")) { + radix = 8; + } else if (t.startsWith("0b")) { + radix = 2; + } + if (radix != 10) { + t = t.substring(2); + } + try { + BigInteger v = new BigInteger(t, radix); + return v.bitLength() < 63 ? Long.valueOf(v.longValue()) : null; + } catch (NumberFormatException ex) { + return null; + } + } + + /// 'i' or 'f' for `e` (recorded in `types`), or 0 when the tree cannot + /// be fused with those leaf types. + private static char numType(Expr e, Set intCtx, Set intNames, Map types) { + char t; + if (!numNode(e)) { + if (e instanceof Constant) { + t = intLiteral(e) != null ? 'i' : floatConst(e) ? 'f' : 0; + } else { + t = intCtx.contains(e) || e instanceof Name && intNames.contains(((Name) e).id) ? 'i' : 'f'; + } + } else if (e instanceof UnaryOp) { + UnaryOp u = (UnaryOp) e; + t = numType(u.operand, intCtx, intNames, types); + if ("~".equals(u.op) && t != 'i') { + t = 0; + } + } else { + BinOp b = (BinOp) e; + char l = numType(b.left, intCtx, intNames, types); + char r = numType(b.right, intCtx, intNames, types); + if (l == 0 || r == 0) { + t = 0; + } else if (intOnlyOp(b.op)) { + t = l == 'i' && r == 'i' ? 'i' : 0; + } else if ("/".equals(b.op)) { + t = 'f'; + } else if ("**".equals(b.op)) { + t = l == 'i' && r == 'i' ? 0 : 'f'; + } else { + t = l == 'i' && r == 'i' ? 'i' : 'f'; + } + } + if (t != 0) { + types.put(e, Character.valueOf(t)); + } + return t; + } + + /// One typed fast path: taken when every leaf has the type `types` + /// gives it, left by `break label` when a step would leave the range + /// Java computes exactly (an int overflow, a zero divisor, an int too + /// large to convert exactly). + private void numPath(Expr root, List leaves, Map types, String label, StringBuilder m) { + numPath(root, leaves, types, label, m, false); + } + + /// `raw`: the path returns the double itself (root type float). + private void numPath(Expr root, List leaves, Map types, String label, StringBuilder m, + boolean raw) { + StringBuilder guard = new StringBuilder(); + StringBuilder unbox = new StringBuilder(); + // A name read twice is one value (a fused tree calls nothing in + // between), so it is tested and unboxed once. + Map seen = new HashMap(); + for (int i = 0; i < leaves.size(); i++) { + boolean isInt = types.get(leaves.get(i)).charValue() == 'i'; + String box = isInt ? "Long" : "Double"; + if (leaves.get(i) instanceof Name) { + String key = ((Name) leaves.get(i)).id + (isInt ? ":i" : ":f"); + Integer first = seen.get(key); + if (first != null) { + unbox.append(" ").append(isInt ? "long" : "double").append(" d").append(i) + .append(" = d").append(first).append(";\n"); + continue; + } + seen.put(key, Integer.valueOf(i)); + } + guard.append(guard.length() == 0 ? "" : " && ").append("p").append(i).append(" instanceof ").append(box); + unbox.append(" ").append(isInt ? "long" : "double").append(" d").append(i).append(" = ((") + .append(box).append(") p").append(i).append(").").append(isInt ? "longValue" : "doubleValue") + .append("();\n"); + } + m.append(" ").append(label).append(": if (").append(guard).append(") {\n").append(unbox); + int[] counters = new int[2]; + String r = numPrim(root, types, label, m, counters); + boolean isInt = types.get(root).charValue() == 'i'; + if (raw) { + m.append(" return ").append(r).append(";\n }\n"); + } else { + m.append(" return ").append(isInt ? "Long" : "Double").append(".valueOf(").append(r) + .append(");\n }\n"); + } + } + + private static String asDouble(String v, char t) { + return t == 'i' ? "((double) " + v + ")" : v; + } + + private static void bail(StringBuilder m, String cond, String label) { + m.append(" if (").append(cond).append(") {\n break ").append(label) + .append(";\n }\n"); + } + + /// counters[0]: next leaf; counters[1]: next temp. + private String numPrim(Expr e, Map types, String label, StringBuilder m, int[] counters) { + char ty = types.get(e).charValue(); + if (!numNode(e)) { + if (e instanceof Constant) { + Long v = intLiteral(e); + if (v != null) { + return ty == 'i' ? "(" + v + "L)" : "(" + v + ".0)"; + } + double d = Double.parseDouble(((Constant) e).text.replace("_", "")); + return Double.isInfinite(d) ? "Double.POSITIVE_INFINITY" : "(" + Double.toString(d) + ")"; + } + return "d" + counters[0]++; + } + String t = "t" + counters[1]++; + String jt = ty == 'i' ? "long" : "double"; + if (e instanceof UnaryOp) { + UnaryOp u = (UnaryOp) e; + String a = numPrim(u.operand, types, label, m, counters); + if ("~".equals(u.op)) { + m.append(" long ").append(t).append(" = ~").append(a).append(";\n"); + return t; + } + if (ty == 'i') { + bail(m, a + " == Long.MIN_VALUE", label); + } + m.append(" ").append(jt).append(' ').append(t).append(" = -").append(a).append(";\n"); + return t; + } + BinOp b = (BinOp) e; + char lt = types.get(b.left).charValue(); + char rt = types.get(b.right).charValue(); + String l = numPrim(b.left, types, label, m, counters); + String r = numPrim(b.right, types, label, m, counters); + if (ty == 'f') { + if ("/".equals(b.op) && lt == 'i' && rt == 'i') { + // Exact as doubles, so one rounding, as Python's int division. + bail(m, l + " > 9007199254740992L || " + l + " < -9007199254740992L || " + r + + " > 9007199254740992L || " + r + " < -9007199254740992L", label); + } + l = asDouble(l, lt); + r = asDouble(r, rt); + if ("**".equals(b.op)) { + m.append(" double ").append(t).append(" = Math.pow(").append(l).append(", ").append(r) + .append(");\n"); + bail(m, "Double.isNaN(" + t + ") || Double.isInfinite(" + t + ")", label); + return t; + } + if ("/".equals(b.op)) { + bail(m, r + " == 0.0", label); + } + m.append(" double ").append(t).append(" = ").append(l).append(' ').append(b.op).append(' ') + .append(r).append(";\n"); + return t; + } + String op = b.op; + if ("+".equals(op)) { + m.append(" long ").append(t).append(" = ").append(l).append(" + ").append(r).append(";\n"); + bail(m, "((" + l + " ^ " + t + ") & (" + r + " ^ " + t + ")) < 0", label); + } else if ("-".equals(op)) { + m.append(" long ").append(t).append(" = ").append(l).append(" - ").append(r).append(";\n"); + bail(m, "((" + l + " ^ " + r + ") & (" + l + " ^ " + t + ")) < 0", label); + } else if ("*".equals(op)) { + m.append(" long ").append(t).append(" = ").append(l).append(" * ").append(r).append(";\n"); + // Two operands that fit in 32 bits cannot overflow 64, which + // settles nearly every product without the division. + bail(m, "(((" + l + " + 0x80000000L) | (" + r + " + 0x80000000L)) >>> 32) != 0 && " + l + + " != 0 && (" + t + " / " + l + " != " + r + " || " + l + " == -1 && " + r + + " == Long.MIN_VALUE)", label); + } else if ("&".equals(op) || "|".equals(op) || "^".equals(op)) { + m.append(" long ").append(t).append(" = ").append(l).append(' ').append(op).append(' ') + .append(r).append(";\n"); + } else if (">>".equals(op)) { + bail(m, r + " < 0", label); + m.append(" long ").append(t).append(" = ").append(r).append(" > 63 ? (").append(l) + .append(" < 0 ? -1L : 0L) : ").append(l).append(" >> ").append(r).append(";\n"); + } else if ("<<".equals(op)) { + bail(m, r + " < 0 || " + r + " > 62 || (" + l + " << " + r + " >> " + r + ") != " + l, label); + m.append(" long ").append(t).append(" = ").append(l).append(" << ").append(r).append(";\n"); + } else if ("//".equals(op)) { + bail(m, r + " == 0 || " + l + " == Long.MIN_VALUE && " + r + " == -1", label); + m.append(" long ").append(t).append(" = ").append(l).append(" / ").append(r).append(";\n"); + m.append(" if (").append(t).append(" * ").append(r).append(" != ").append(l).append(" && (") + .append(l).append(" < 0) != (").append(r).append(" < 0)) {\n ").append(t) + .append("--;\n }\n"); + } else { + bail(m, r + " == 0", label); + m.append(" long ").append(t).append(" = ").append(l).append(" % ").append(r).append(";\n"); + m.append(" if (").append(t).append(" != 0 && (").append(t).append(" < 0) != (").append(r) + .append(" < 0)) {\n ").append(t).append(" += ").append(r) + .append(";\n }\n"); + } + return t; + } + + private String fuseBoxed(Expr e, int[] counters) { + if (!numNode(e)) { + if (e instanceof Constant) { + return constant((Constant) e); + } + return "p" + counters[0]++; + } + if (e instanceof UnaryOp) { + String a = fuseBoxed(((UnaryOp) e).operand, counters); + return "~".equals(((UnaryOp) e).op) ? "Py.invert(" + a + ")" : "Py.neg(" + a + ")"; + } + BinOp b = (BinOp) e; + String l = fuseBoxed(b.left, counters); + String r = fuseBoxed(b.right, counters); + return "Py." + BIN.get(b.op) + "(" + l + ", " + r + ")"; + } + + // ---- float regions ----------------------------------------------------- + // + // A run of straight-line float statements (`dx = a - b`, `mag = ...`, + // `v[0] -= dx * m`) is computed in Java doubles as one region. A local + // the region assigns before anything reads it, and that nothing outside + // the function's regions reads at all, never exists as an object: that + // is decided after the whole function is emitted, from its Java text + // (resolveRegions). Every other local the region assigns is boxed as + // it is assigned, and every store into a list is boxed as it is stored. + // + // Any operand that is not a float -- an int, a user object, a missing + // list item -- makes the region stop before the statement that read it + // ("bail"); the boxed locals are then completed and the remaining + // statements run through the generic operators, exactly as written. A + // region that keeps bailing (int arithmetic that only looks like float + // arithmetic) turns its fast path off after a few tries, so it costs a + // counter test. Reads in the fast path never call user code: names, and + // list or tuple items by int index, are all it reads. + + private static final String REGION_START = "/*@R{*/"; + private static final String REGION_END = "/*@}R*/"; + private static final String IF_SHARED = "/*@S:"; + private static final String IF_PRIVATE = "/*@P:"; + private static final String COND_END = "/*@E*/"; + private static final int REGION_BAILS = 8; + + private int regionCount; + /// Locals some region reads before assigning: never private. + private final Set regionShared = new HashSet(); + + /// Locals of the function being emitted whose slice copies reuse their + /// previous list ([ListRecycling]). + private Set recycled = java.util.Collections.emptySet(); + /// While a loop body that is one region is emitted: each local it + /// assigns, mapped to {the double that carries it across iterations, + /// the flag saying that double is the current value}. + private Map carried; + /// The `range` loop whose index the current region reads unboxed: + /// {name, iterator var (null while counting), long index, step, start}. + private String[] lazyIndex; + /// Inside a region's fast path, where a lazy index's local is stale. + private boolean inRegionFast; + /// While a lazy `range` loop's body is emitted: {iterator var, long + /// index, end, step, indent}, asking the region that is the whole body + /// for a versioned copy of its fast path ([#versionedLoop]). + private String[] version; + /// Where that copy goes; the loop inserts it ahead of itself. + private StringBuilder versionOut; //NOPMD AvoidStringBufferField - the generator's output + /// While the versioned copy is emitted: each invariant name read, mapped + /// to the final that holds it; null otherwise. + private Map hoist; + /// The raw (non-throwing) read behind each of those finals. + private Map hoistRaw; + /// The names the versioned region assigns, which are never invariant. + private Set hoistAssigned; + /// The lists the versioned region stores items into. + private Set hoistStored; + /// Each hoisted float list read by index: {its double[], its size}. + private Map hoistFloats; + /// In a region's fast path, local -> {double[], count} it read once on + /// entry: lists the region subscripts with ints and never rebinds. + private Map regionArrays; + /// Conditions on hoisted values the versioned loop is entered under. + private List hoistTests; + /// Unboxed copies of hoisted values, declared once those tests passed. + private List hoistDecls; + /// Whether the function being emitted has a try, with or match: a loop + /// then keeps no local as a double, since a handler could read it. + private boolean fnHasHandlers; + + /// Writes a carried local back to its object, once, when code that + /// reads the object runs next. + private void syncCarried(String id, String[] c, StringBuilder r, int ind) { + indent(r, ind); + r.append("if (").append(c[1]).append(") {\n"); + indent(r, ind + 1); + r.append(localVar(sc, id)).append(" = Double.valueOf(").append(c[0]).append(");\n"); + indent(r, ind + 1); + r.append(c[1]).append(" = false;\n"); + indent(r, ind); + r.append("}\n"); + } + + /// The locals a loop body keeps as doubles across its iterations: the + /// body must be one float region (nothing in it can break out or run + /// a handler), or null. + private Map loopCarried(For f) { + if (sc == null || sc.kind != Scope.Kind.FUNCTION || fnHasHandlers || !f.orelse.isEmpty() + || f.body.isEmpty()) { + return null; + } + Map c = new LinkedHashMap(); + for (Stmt s : f.body) { + if (!regionStmt(s)) { + return null; + } + Expr t = s instanceof Assign ? ((Assign) s).targets.get(0) : ((AugAssign) s).target; + if (t instanceof Name && !c.containsKey(((Name) t).id)) { + String n = fresh("lc"); + c.put(((Name) t).id, new String[] {n + "_d", n + "_f"}); + } + } + if (f.target instanceof Name && c.containsKey(((Name) f.target).id)) { + return null; + } + return c; + } + + private static boolean hasHandlers(List body) { + for (Stmt s : body) { + if (s instanceof Try || s instanceof With || s instanceof Match) { + return true; + } + if (s instanceof If && (hasHandlers(((If) s).body) || hasHandlers(((If) s).orelse)) + || s instanceof While && (hasHandlers(((While) s).body) || hasHandlers(((While) s).orelse)) + || s instanceof For && (hasHandlers(((For) s).body) || hasHandlers(((For) s).orelse))) { + return true; + } + } + return false; + } + + private static boolean regionOp(String op) { + return "+".equals(op) || "-".equals(op) || "*".equals(op) || "/".equals(op); + } + + /// A list or tuple read the fast path can do without user code. + private boolean regionItem(Expr e) { + if (!(e instanceof Subscript)) { + return false; + } + Subscript sub = (Subscript) e; + return sub.value instanceof Name && !(sub.index instanceof Slice) + && (constIndex(sub.index) != null || sub.index instanceof Name); + } + + private boolean regionLeaf(Expr e) { + return e instanceof Name || regionItem(e) || intConst(e) || floatConst(e) || regionCall(e); + } + + /// `f(a, b)` with plain names or numbers for arguments: the fast path + /// asks `f` for its raw double (Py.callD), which runs no Python code, + /// and leaves the region when it gets none. + private static boolean regionCall(Expr e) { + if (!(e instanceof Call)) { + return false; + } + Call c = (Call) e; + if (!(c.func instanceof Name) || !c.keywords.isEmpty() || c.args.isEmpty() || c.args.size() > 4) { + return false; + } + for (Expr a : c.args) { + if (!(a instanceof Name) && !intConst(a) && !floatConst(a)) { + return false; + } + } + return true; + } + + /// Whether `e` is float arithmetic over region leaves. + private boolean regionExpr(Expr e) { + if (e instanceof BinOp) { + BinOp b = (BinOp) e; + return (regionOp(b.op) || "**".equals(b.op)) && regionExpr(b.left) && regionExpr(b.right); + } + if (e instanceof UnaryOp) { + return "-".equals(((UnaryOp) e).op) && regionExpr(((UnaryOp) e).operand); + } + return regionLeaf(e); + } + + private boolean localName(Expr e) { + return e instanceof Name && sc.resolve(((Name) e).id) == Scope.Binding.LOCAL; + } + + /// Whether a statement can be part of a float region, and has some + /// arithmetic in it (a plain copy is left alone). + private boolean regionStmt(Stmt s) { + if (s instanceof Assign) { + Assign a = (Assign) s; + if (a.targets.size() != 1 || !(fuseNode(a.value)) || !regionExpr(a.value) || fuseCheck(a.value) != 1) { + return false; + } + Expr t = a.targets.get(0); + return localName(t) || regionItem(t); + } + if (s instanceof AugAssign) { + AugAssign a = (AugAssign) s; + // `n += 1` is int arithmetic in practice; a region would only bail. + return regionOp(a.op) && !intConst(a.value) && (localName(a.target) || regionItem(a.target)) + && regionExpr(a.value); + } + return false; + } + + /// Emits the region starting at `start`, if there is one, and returns the + /// index after it; returns `start` when there is none. + private int floatRegion(List body, int start, StringBuilder out, int ind) { + if (sc == null || sc.kind != Scope.Kind.FUNCTION) { + return start; + } + int end = start; + while (end < body.size() && regionStmt(body.get(end))) { + end++; + } + // Worth a region: two statements (a local can then stay a double), or + // any store into a list (the list can keep the float unboxed). + boolean anyItem = false; + for (int i = start; i < end; i++) { + Stmt s = body.get(i); + Expr t = s instanceof Assign ? ((Assign) s).targets.get(0) : ((AugAssign) s).target; + anyItem |= t instanceof Subscript; + } + boolean wholeLoop = carried != null && start == 0 && end == body.size(); + if (end == start || end - start < 2 && !anyItem && !wholeLoop) { + return start; + } + Map keep = carried; + if (!wholeLoop) { + carried = null; + } + try { + return floatRegion(body, start, end, out, ind); + } finally { + carried = keep; + } + } + + private int floatRegion(List body, int start, int end, StringBuilder out, int ind) { + List run = body.subList(start, end); + int n = regionCount++; + String label = "fr" + n; + String done = "frs" + n; + String bails = "FR" + n; + fields.append(" private static int ").append(bails).append(";\n"); + + // Doubles for every local the region assigns; which of them are read + // before being assigned decides who may stay private. + Map firstStore = new LinkedHashMap(); + Set readFirst = new HashSet(); + for (int i = 0; i < run.size(); i++) { + Stmt s = run.get(i); + List reads = new ArrayList(); + Expr target; + if (s instanceof Assign) { + target = ((Assign) s).targets.get(0); + regionReads(((Assign) s).value, reads); + } else { + target = ((AugAssign) s).target; + if (target instanceof Name) { + reads.add(target); + } + regionReads(((AugAssign) s).value, reads); + } + if (target instanceof Subscript) { + regionReads(target, reads); + } + for (Expr r : reads) { + String id = ((Name) r).id; + if (sc.resolve(id) == Scope.Binding.LOCAL && !firstStore.containsKey(id)) { + readFirst.add(id); + } + } + if (target instanceof Name && !firstStore.containsKey(((Name) target).id)) { + firstStore.put(((Name) target).id, Integer.valueOf(i)); + } + } + for (String id : readFirst) { + regionShared.add(localVar(sc, id)); + } + if (carried != null) { + firstStore.keySet().removeAll(carried.keySet()); + } + String[] vsig = version; + version = null; + if (vsig != null && carried != null && firstStore.isEmpty()) { + versionedLoop(run, vsig, bails, n); + } + + StringBuilder r = new StringBuilder(); + Map arrays = regionArrays(run, n); + indent(r, ind); + r.append(REGION_START).append("\n"); + indent(r, ind); + r.append("int ").append(done).append(" = 0;\n"); + for (String id : firstStore.keySet()) { + indent(r, ind); + r.append("double ").append(localVar(sc, id)).append("_d = 0;\n"); + } + indent(r, ind); + r.append("if (").append(bails).append(" < ").append(REGION_BAILS).append(") {\n"); + for (Map.Entry e : arrays.entrySet()) { + // A list that is not all floats reads as empty: every index + // misses and leaves the fast path. + String v = localVar(sc, e.getKey()); + indent(r, ind + 1); + r.append("final double[] ").append(e.getValue()[0]).append(" = Py.floatArray(").append(v).append(");\n"); + indent(r, ind + 1); + r.append("final int ").append(e.getValue()[1]).append(" = Py.floatCount(").append(v).append(");\n"); + } + indent(r, ind + 1); + r.append(label).append(": {\n"); + Set inDoubles = new HashSet(); + for (int i = 0; i < run.size(); i++) { + inRegionFast = true; + regionArrays = arrays; + try { + regionFast(run.get(i), inDoubles, label, r, ind + 2); + } finally { + regionArrays = null; + } + inRegionFast = false; + indent(r, ind + 2); + r.append(done).append(" = ").append(i + 1).append(";\n"); + } + indent(r, ind + 1); + r.append("}\n"); + indent(r, ind + 1); + r.append("if (").append(done).append(" < ").append(run.size()).append(") {\n"); + indent(r, ind + 2); + r.append(bails).append("++;\n"); + indent(r, ind + 1); + r.append("}\n"); + indent(r, ind); + r.append("}\n"); + indent(r, ind); + r.append("if (").append(done).append(" < ").append(run.size()).append(") {\n"); + if (lazyIndex != null) { + indent(r, ind + 1); + r.append("if (").append(lazyIndex[1]).append(" == null) {\n"); + indent(r, ind + 2); + r.append(localVar(sc, lazyIndex[0])).append(" = Long.valueOf(").append(lazyIndex[2]).append(");\n"); + indent(r, ind + 1); + r.append("}\n"); + } + if (carried != null) { + for (Map.Entry e : carried.entrySet()) { + syncCarried(e.getKey(), e.getValue(), r, ind + 1); + } + } + for (Map.Entry e : firstStore.entrySet()) { + String v = localVar(sc, e.getKey()); + indent(r, ind + 1); + r.append(IF_PRIVATE).append(v).append("*/if (").append(done).append(" > ").append(e.getValue()) + .append(") {\n"); + indent(r, ind + 2); + r.append(v).append(" = Double.valueOf(").append(v).append("_d);\n"); + indent(r, ind + 1); + r.append("}").append(COND_END).append("\n"); + } + for (int i = 0; i < run.size(); i++) { + indent(r, ind + 1); + r.append("if (").append(done).append(" <= ").append(i).append(") {\n"); + stmt(run.get(i), r, ind + 2); + indent(r, ind + 1); + r.append("}\n"); + } + indent(r, ind); + r.append("}\n"); + indent(r, ind); + r.append(REGION_END).append("\n"); + out.append(r); + return end; + } + + /// A copy of a counted loop's only region, run ahead of the loop: the + /// fast path in a tight loop, with every name it does not assign read + /// once before the loop and none of the general loop's bookkeeping. An + /// iteration that cannot finish fast stops it before committing anything + /// (the fast path only commits at its end, and the calls in it are pure), + /// and the general loop then resumes at that same index. + private void versionedLoop(List run, String[] v, String bails, int n) { + int ind = Integer.parseInt(v[4]); + String vl = "VL" + n; + String fl = "fv" + n; + // A store into a list commits at once, so only the last statement, + // after which nothing can bail, may make one. + for (int i = 0; i < run.size(); i++) { + Stmt s = run.get(i); + Expr t = s instanceof Assign ? ((Assign) s).targets.get(0) : ((AugAssign) s).target; + // A store into something unnamed may be any list. + if (t instanceof Subscript && (i < run.size() - 1 || !(((Subscript) t).value instanceof Name))) { + return; + } + } + // The carried doubles are written to shadows that commit when the + // iteration finishes, so a bail part way leaves them as they were. + Map real = carried; + Map shadow = new LinkedHashMap(); + StringBuilder enter = new StringBuilder(); + StringBuilder commit = new StringBuilder(); + for (Map.Entry e : real.entrySet()) { + String[] c = e.getValue(); + String[] s = {"s" + c[0], "s" + c[1]}; + shadow.put(e.getKey(), s); + indent(enter, ind + 4); + enter.append("double ").append(s[0]).append(" = ").append(c[0]).append(";\n"); + indent(enter, ind + 4); + enter.append("boolean ").append(s[1]).append(" = ").append(c[1]).append(";\n"); + indent(commit, ind + 4); + commit.append(c[0]).append(" = ").append(s[0]).append(";\n"); + indent(commit, ind + 4); + commit.append(c[1]).append(" = ").append(s[1]).append(";\n"); + } + hoist = new LinkedHashMap(); + hoistRaw = new LinkedHashMap(); + hoistAssigned = new HashSet(); + hoistStored = new HashSet(); + hoistFloats = new LinkedHashMap(); + hoistTests = new ArrayList(); + hoistDecls = new ArrayList(); + for (Stmt s : run) { + Expr t = s instanceof Assign ? ((Assign) s).targets.get(0) : ((AugAssign) s).target; + if (t instanceof Name) { + hoistAssigned.add(((Name) t).id); + } else if (t instanceof Subscript && ((Subscript) t).value instanceof Name) { + hoistStored.add(((Name) ((Subscript) t).value).id); + } + } + StringBuilder body = new StringBuilder(); + Set inDoubles = new HashSet(); + carried = shadow; + try { + for (Stmt s : run) { + inRegionFast = true; + regionFast(s, inDoubles, fl, body, ind + 4); + inRegionFast = false; + } + } finally { + inRegionFast = false; + carried = real; + } + Map raw = hoistRaw; + hoist = null; + hoistRaw = null; + hoistAssigned = null; + Map floats = hoistFloats; + List tests = hoistTests; + List decls = hoistDecls; + hoistStored = null; + hoistFloats = null; + hoistTests = null; + hoistDecls = null; + StringBuilder r = versionOut; + indent(r, ind); + r.append("if (").append(v[0]).append(" == null && ").append(bails).append(" < ").append(REGION_BAILS) + .append(") {\n"); + StringBuilder test = new StringBuilder(); + for (Map.Entry e : raw.entrySet()) { + indent(r, ind + 1); + r.append("final Object ").append(e.getKey()).append(" = ").append(e.getValue()).append(";\n"); + test.append(" && ").append(e.getKey()).append(" != null"); + } + for (Map.Entry e : floats.entrySet()) { + String[] fa = e.getValue(); + indent(r, ind + 1); + r.append("final double[] ").append(fa[0]).append(" = Py.floatArray(").append(e.getKey()).append(");\n"); + indent(r, ind + 1); + r.append("final int ").append(fa[1]).append(" = Py.floatCount(").append(e.getKey()).append(");\n"); + } + for (String t : tests) { + test.append(" && ").append(t); + } + // An unbound name is left to the general loop, which raises for it + // where the program reads it. + indent(r, ind + 1); + r.append("if (true").append(test).append(") {\n"); + for (String d : decls) { + indent(r, ind + 2); + r.append(d).append("\n"); + } + indent(r, ind + 2); + r.append(vl).append(": while (").append(v[3]).append(" > 0 ? ").append(v[1]).append(" < ").append(v[2]) + .append(" : ").append(v[1]).append(" > ").append(v[2]).append(") {\n"); + indent(r, ind + 3); + r.append(fl).append(": {\n"); + r.append(enter); + r.append(body); + r.append(commit); + indent(r, ind + 4); + r.append(v[1]).append(" += ").append(v[3]).append(";\n"); + indent(r, ind + 4); + r.append("continue ").append(vl).append(";\n"); + indent(r, ind + 3); + r.append("}\n"); + indent(r, ind + 3); + r.append("break;\n"); + indent(r, ind + 2); + r.append("}\n"); + indent(r, ind + 1); + r.append("}\n"); + indent(r, ind); + r.append("}\n"); + } + + /// In the versioned loop, a call of the module's own long twin whose + /// function and non-index arguments are all hoisted: the checks the + /// general path makes per call (the global still being that function, + /// each argument an int) are made once on entry, and the call is the + /// twin alone. Null when the call is not of that shape. + private StringBuilder hoistedDirectCall(Call c) { + if (!(c.func instanceof Name)) { + return null; + } + Integer arity = directLong.get(((Name) c.func).id); + if (arity == null || arity.intValue() != c.args.size() || sc.resolve(((Name) c.func).id) != Scope.Binding.GLOBAL) { + return null; + } + String fe = expr(c.func); + if (!hoist.containsValue(fe)) { + return null; + } + List parts = new ArrayList(); + List tests = new ArrayList(); + List decls = new ArrayList(); + for (Expr a : c.args) { + if (lazyIndex != null && a instanceof Name && lazyIndex[0].equals(((Name) a).id)) { + parts.add(lazyIndex[2]); + } else if (intLiteral(a) != null) { + parts.add(intLiteral(a) + "L"); + } else { + String ae = expr(a); + if (!hoist.containsValue(ae)) { + return null; + } + tests.add(ae + " instanceof Long"); + decls.add("final long " + ae + "L = ((Long) " + ae + ").longValue();"); + parts.add(ae + "L"); + } + } + String m = mangle(((Name) c.func).id); + tests.add(0, fe + " == DEF_" + m); + for (String t : tests) { + if (!hoistTests.contains(t)) { + hoistTests.add(t); + } + } + for (String d : decls) { + if (!hoistDecls.contains(d)) { + hoistDecls.add(d); + } + } + StringBuilder call = new StringBuilder("PL_").append(m).append("("); + for (int i = 0; i < parts.size(); i++) { + call.append(i > 0 ? ", " : "").append(parts.get(i)); + } + // The caller closes the call. + return call; + } + + /// The lists a region may read through a `double[]` taken on entry: a + /// local it subscripts with an int and never assigns. Only when the region + /// calls nothing, since a call could grow the list and replace its array; + /// the region's own stores write the array in place. + private Map regionArrays(List run, int n) { + Map m = new LinkedHashMap(); + if (hoist != null) { + return m; + } + Set assigned = new HashSet(); + List subs = new ArrayList(); + for (Stmt s : run) { + Expr t = s instanceof Assign ? ((Assign) s).targets.get(0) : ((AugAssign) s).target; + Expr v = s instanceof Assign ? ((Assign) s).value : ((AugAssign) s).value; + if (t instanceof Name) { + assigned.add(((Name) t).id); + } else if (!regionSubs(t, subs)) { + return m; + } + if (!regionSubs(v, subs)) { + return m; + } + } + for (Subscript sub : subs) { + if (!(sub.value instanceof Name) || constIndex(sub.index) == null && !(sub.index instanceof Name)) { + continue; + } + String id = ((Name) sub.value).id; + if (assigned.contains(id) || sc.resolve(id) != Scope.Binding.LOCAL || m.containsKey(id)) { + continue; + } + int k = m.size(); + m.put(id, new String[] {"fra" + n + "_" + k, "frn" + n + "_" + k}); + } + return m; + } + + /// Collects the subscripts of a region expression; false when it calls. + private static boolean regionSubs(Expr e, List out) { + if (e instanceof Call) { + return false; + } + if (e instanceof BinOp) { + return regionSubs(((BinOp) e).left, out) && regionSubs(((BinOp) e).right, out); + } + if (e instanceof UnaryOp) { + return regionSubs(((UnaryOp) e).operand, out); + } + if (e instanceof Subscript) { + out.add((Subscript) e); + return regionSubs(((Subscript) e).value, out) && regionSubs(((Subscript) e).index, out); + } + return true; + } + + /// The hoisted array of a region subscript with an unboxed index, after + /// a bounds check that leaves the fast path; null when not hoisted. + private String regionArrayRef(Expr base, String k, String label, StringBuilder r, int ind) { + String[] a = regionArrays == null || k == null || !(base instanceof Name) ? null + : regionArrays.get(((Name) base).id); + if (a == null) { + return null; + } + indent(r, ind); + r.append("if (").append(k).append(" < 0 || ").append(k).append(" >= ").append(a[1]).append(") {\n"); + indent(r, ind + 1); + r.append("break ").append(label).append(";\n"); + indent(r, ind); + r.append("}\n"); + return a[0] + "[(int) " + k + "]"; + } + + /// The names a region expression reads, in order. + private void regionReads(Expr e, List out) { + if (e instanceof BinOp) { + regionReads(((BinOp) e).left, out); + regionReads(((BinOp) e).right, out); + } else if (e instanceof UnaryOp) { + regionReads(((UnaryOp) e).operand, out); + } else if (e instanceof Name) { + out.add(e); + } else if (e instanceof Subscript) { + regionReads(((Subscript) e).value, out); + regionReads(((Subscript) e).index, out); + } else if (e instanceof Call) { + regionReads(((Call) e).func, out); + for (Expr a : ((Call) e).args) { + regionReads(a, out); + } + } + } + + /// One statement of a region's fast path. + private void regionFast(Stmt s, Set inDoubles, String label, StringBuilder r, int ind) { + Expr target = s instanceof Assign ? ((Assign) s).targets.get(0) : ((AugAssign) s).target; + Expr value = s instanceof Assign ? ((Assign) s).value : ((AugAssign) s).value; + indent(r, ind); + r.append("{\n"); + int[] tmp = new int[1]; + String obj = null; + String idx = null; + String slot = null; + if (target instanceof Subscript) { + Subscript ts = (Subscript) target; + String tk = constIndex(ts.index); + if (tk == null && regionArrays != null && ts.value instanceof Name + && regionArrays.containsKey(((Name) ts.value).id)) { + tk = lazyLong(ts.index, label, r, ind + 1); + } + slot = regionArrayRef(ts.value, tk, label, r, ind + 1); + } + // A hoisted slot needs no object or index temps. + if (slot == null && target instanceof Subscript) { + Subscript sub = (Subscript) target; + obj = "fo" + tmp[0]++; + idx = "fi" + tmp[0]++; + indent(r, ind + 1); + r.append("final Object ").append(obj).append(" = ").append(expr(sub.value)).append(";\n"); + String ci = constIndex(sub.index); + String li = lazyLong(sub.index, label, r, ind + 1); + indent(r, ind + 1); + if (li != null || ci != null) { + r.append("final long ").append(idx).append(" = ").append(li != null ? li : ci).append(";\n"); + } else { + r.append("final Object ").append(idx).append(" = ").append(expr(sub.index)).append(";\n"); + } + } + String cur = null; + if (s instanceof AugAssign) { + if (slot != null) { + cur = slot; + } else if (target instanceof Subscript) { + cur = regionItemLoad(obj, idx, label, tmp, r, ind + 1); + } else { + cur = regionName((Name) target, inDoubles, label, tmp, r, ind + 1); + } + } + String v = regionValue(value, inDoubles, label, tmp, r, ind + 1); + if (cur != null) { + String op = ((AugAssign) s).op; + if ("/".equals(op)) { + indent(r, ind + 1); + r.append("if (").append(v).append(" == 0.0) {\n"); + indent(r, ind + 2); + r.append("break ").append(label).append(";\n"); + indent(r, ind + 1); + r.append("}\n"); + } + v = "(" + cur + " " + op + " " + v + ")"; + } + indent(r, ind + 1); + if (slot != null) { + r.append(slot).append(" = ").append(v).append(";\n"); + } else if (target instanceof Subscript) { + r.append("Py.setFloat(").append(obj).append(", ").append(idx).append(", ").append(v).append(");\n"); + } else if (carried != null && carried.containsKey(((Name) target).id)) { + String[] c = carried.get(((Name) target).id); + r.append(c[0]).append(" = ").append(v).append(";\n"); + indent(r, ind + 1); + r.append(c[1]).append(" = true;\n"); + inDoubles.add(((Name) target).id); + } else { + String lv = localVar(sc, ((Name) target).id); + r.append(lv).append("_d = ").append(v).append(";\n"); + indent(r, ind + 1); + r.append(IF_SHARED).append(lv).append("*/").append(lv).append(" = Double.valueOf(").append(lv) + .append("_d);").append(COND_END).append("\n"); + inDoubles.add(((Name) target).id); + } + indent(r, ind); + r.append("}\n"); + } + + /// Loads a boxed operand, leaving the region unless it is a float. + private String regionLoad(String source, String label, int[] tmp, StringBuilder r, int ind) { + String o = "fv" + tmp[0]++; + indent(r, ind); + r.append("final Object ").append(o).append(" = ").append(source).append(";\n"); + indent(r, ind); + r.append("if (!(").append(o).append(" instanceof Double)) {\n"); + indent(r, ind + 1); + r.append("break ").append(label).append(";\n"); + indent(r, ind); + r.append("}\n"); + String d = "fd" + tmp[0]++; + indent(r, ind); + r.append("final double ").append(d).append(" = ((Double) ").append(o).append(").doubleValue();\n"); + return d; + } + + /// Reads a list or tuple item unboxed, leaving the region unless it is a + /// float. + private String regionItemLoad(String o, String i, String label, int[] tmp, StringBuilder r, int ind) { + String d = "fd" + tmp[0]++; + indent(r, ind); + r.append("final double ").append(d).append(" = Py.floatItem(").append(o).append(", ").append(i).append(");\n"); + indent(r, ind); + r.append("if (").append(d).append(" != ").append(d).append(") {\n"); + indent(r, ind + 1); + r.append("break ").append(label).append(";\n"); + indent(r, ind); + r.append("}\n"); + return d; + } + + private String regionName(Name nm, Set inDoubles, String label, int[] tmp, StringBuilder r, int ind) { + String li = lazyLong(nm, label, r, ind); + if (li != null) { + return "((double) " + li + ")"; + } + String[] c = carried == null ? null : carried.get(nm.id); + if (c != null) { + if (inDoubles.contains(nm.id)) { + return c[0]; + } + // The loop holds it as a double once some iteration stored it. + String d = "fd" + tmp[0]++; + String o = "fv" + tmp[0]++; + indent(r, ind); + r.append("final double ").append(d).append(";\n"); + indent(r, ind); + r.append("if (").append(c[1]).append(") {\n"); + indent(r, ind + 1); + r.append(d).append(" = ").append(c[0]).append(";\n"); + indent(r, ind); + r.append("} else {\n"); + indent(r, ind + 1); + r.append("final Object ").append(o).append(" = ").append(expr(nm)).append(";\n"); + indent(r, ind + 1); + r.append("if (!(").append(o).append(" instanceof Double)) {\n"); + indent(r, ind + 2); + r.append("break ").append(label).append(";\n"); + indent(r, ind + 1); + r.append("}\n"); + indent(r, ind + 1); + r.append(d).append(" = ((Double) ").append(o).append(").doubleValue();\n"); + indent(r, ind); + r.append("}\n"); + return d; + } + if (inDoubles.contains(nm.id) && sc.resolve(nm.id) == Scope.Binding.LOCAL) { + return localVar(sc, nm.id) + "_d"; + } + return regionLoad(expr(nm), label, tmp, r, ind); + } + + /// The unboxed index of the lazy `range` loop when `e` names it, else + /// null. Counting by a `range` that is not the builtin leaves the region. + private String lazyLong(Expr e, String label, StringBuilder r, int ind) { + if (lazyIndex == null || !(e instanceof Name) || !lazyIndex[0].equals(((Name) e).id)) { + return null; + } + if (hoist != null) { + // The versioned loop runs only while counting. + return lazyIndex[2]; + } + indent(r, ind); + r.append("if (").append(lazyIndex[1]).append(" != null) {\n"); + indent(r, ind + 1); + r.append("break ").append(label).append(";\n"); + indent(r, ind); + r.append("}\n"); + return lazyIndex[2]; + } + + /// A call argument as the object the callee sees: a local the region + /// holds as a double is boxed for it. + private String regionBoxed(Expr a, Set inDoubles) { + if (a instanceof Name) { + String id = ((Name) a).id; + String[] c = carried == null ? null : carried.get(id); + if (c != null) { + return "(" + c[1] + " ? Double.valueOf(" + c[0] + ") : " + expr(a) + ")"; + } + if (inDoubles.contains(id) && sc.resolve(id) == Scope.Binding.LOCAL) { + return "Double.valueOf(" + localVar(sc, id) + "_d)"; + } + } + return expr(a); + } + + /// Whether `e` is int arithmetic over the lazy loop index and int + /// literals (`j`, `j + 1`, `-j * 2`): Python computes it exactly, as an + /// int, so the region does too, in a long, before any float sees it. + private boolean regionInt(Expr e) { + if (e instanceof Name) { + return lazyIndex != null && lazyIndex[0].equals(((Name) e).id); + } + if (e instanceof Constant) { + return intLiteral(e) != null; + } + if (e instanceof UnaryOp) { + return "-".equals(((UnaryOp) e).op) && regionInt(((UnaryOp) e).operand); + } + if (e instanceof BinOp) { + String op = ((BinOp) e).op; + return ("+".equals(op) || "-".equals(op) || "*".equals(op)) && regionInt(((BinOp) e).left) + && regionInt(((BinOp) e).right); + } + return false; + } + + /// The long value of a regionInt expression, leaving the region where + /// Python's int would outgrow 64 bits. + private String regionLong(Expr e, String label, int[] tmp, StringBuilder r, int ind) { + if (e instanceof Constant) { + return "(" + intLiteral(e) + "L)"; + } + if (e instanceof Name) { + return lazyLong(e, label, r, ind); + } + String t = "fl" + tmp[0]++; + if (e instanceof UnaryOp) { + String a = regionLong(((UnaryOp) e).operand, label, tmp, r, ind); + regionBail(a + " == Long.MIN_VALUE", label, r, ind); + indent(r, ind); + r.append("final long ").append(t).append(" = -").append(a).append(";\n"); + return t; + } + BinOp b = (BinOp) e; + String l = regionLong(b.left, label, tmp, r, ind); + String rt = regionLong(b.right, label, tmp, r, ind); + indent(r, ind); + r.append("final long ").append(t).append(" = ").append(l).append(' ').append(b.op).append(' ').append(rt) + .append(";\n"); + if ("+".equals(b.op)) { + regionBail("((" + l + " ^ " + t + ") & (" + rt + " ^ " + t + ")) < 0", label, r, ind); + } else if ("-".equals(b.op)) { + regionBail("((" + l + " ^ " + rt + ") & (" + l + " ^ " + t + ")) < 0", label, r, ind); + } else { + regionBail("(((" + l + " + 0x80000000L) | (" + rt + " + 0x80000000L)) >>> 32) != 0 && " + l + + " != 0 && (" + t + " / " + l + " != " + rt + " || " + l + " == -1 && " + rt + + " == Long.MIN_VALUE)", label, r, ind); + } + return t; + } + + private static void regionBail(String cond, String label, StringBuilder r, int ind) { + indent(r, ind); + r.append("if (").append(cond).append(") {\n"); + indent(r, ind + 1); + r.append("break ").append(label).append(";\n"); + indent(r, ind); + r.append("}\n"); + } + + /// The fast path's double for `e`, its leaves read left to right. + private String regionValue(Expr e, Set inDoubles, String label, int[] tmp, StringBuilder r, int ind) { + if (!(e instanceof Constant) && regionInt(e)) { + // One rounding, from the exact int, as float(int) does. + return "((double) " + regionLong(e, label, tmp, r, ind) + ")"; + } + if (e instanceof Constant) { + if (floatConst(e)) { + double d = Double.parseDouble(((Constant) e).text.replace("_", "")); + return Double.isInfinite(d) ? "Double.POSITIVE_INFINITY" : "(" + Double.toString(d) + ")"; + } + return "(" + ((Constant) e).text + ".0)"; + } + if (e instanceof Name) { + return regionName((Name) e, inDoubles, label, tmp, r, ind); + } + if (e instanceof Call) { + Call c = (Call) e; + String d = "fd" + tmp[0]++; + boolean longs = c.args.size() <= 2; + boolean anyLazy = false; + for (Expr a : c.args) { + boolean lz = lazyIndex != null && a instanceof Name && lazyIndex[0].equals(((Name) a).id); + anyLazy |= lz; + longs &= lz || intLiteral(a) != null || a instanceof Name && !(carried != null + && carried.containsKey(((Name) a).id)) && !inDoubles.contains(((Name) a).id); + } + StringBuilder call = longs && anyLazy && hoist != null ? hoistedDirectCall(c) : null; + // A call fully specialized in the versioned loop is emitted below. + if (call == null && longs && anyLazy) { + // An unboxed index among the arguments: the long twin when + // the others are ints too, so nothing is boxed; otherwise the + // object call, boxing the index. + String fn = "ff" + tmp[0]++; + indent(r, ind); + r.append("final Object ").append(fn).append(" = ").append(expr(c.func)).append(";\n"); + StringBuilder cond = new StringBuilder("true"); + StringBuilder typed = new StringBuilder("Py.callDL(").append(fn); + StringBuilder boxed = new StringBuilder("Py.callD(").append(fn); + for (Expr a : c.args) { + String li = lazyLong(a, label, r, ind); + if (li != null) { + typed.append(", ").append(li); + boxed.append(", Long.valueOf(").append(li).append(")"); + } else if (intLiteral(a) != null) { + typed.append(", ").append(intLiteral(a)).append("L"); + boxed.append(", ").append(expr(a)); + } else { + String ao = "fa" + tmp[0]++; + indent(r, ind); + r.append("final Object ").append(ao).append(" = ").append(expr(a)).append(";\n"); + cond.append(" && ").append(ao).append(" instanceof Long"); + typed.append(", ((Long) ").append(ao).append(").longValue()"); + boxed.append(", ").append(ao); + } + } + Integer arity = c.func instanceof Name ? directLong.get(((Name) c.func).id) : null; + if (arity != null && arity.intValue() == c.args.size() + && sc.resolve(((Name) c.func).id) == Scope.Binding.GLOBAL) { + // The module's own function, unless the global was rebound. + String m = mangle(((Name) c.func).id); + typed.replace(0, ("Py.callDL(" + fn + ", ").length(), "PL_" + m + "("); + cond.append(" && ").append(fn).append(" == DEF_").append(m); + String generic = "Py.callDL(" + fn + ", " + typed.substring(("PL_" + m + "(").length()); + call = new StringBuilder("(").append(cond).append(") ? ").append(typed).append(") : (") + .append(cond.substring(0, cond.lastIndexOf(" && "))).append(") ? ").append(generic) + .append(") : ").append(boxed); + } else { + call = new StringBuilder("(").append(cond).append(") ? ").append(typed).append(") : ") + .append(boxed); + } + } else { + call = new StringBuilder("Py.callD(").append(expr(c.func)); + for (Expr a : c.args) { + call.append(", ").append(regionBoxed(a, inDoubles)); + } + } + indent(r, ind); + r.append("final double ").append(d).append(" = ").append(call).append(");\n"); + indent(r, ind); + r.append("if (").append(d).append(" != ").append(d).append(") {\n"); + indent(r, ind + 1); + r.append("break ").append(label).append(";\n"); + indent(r, ind); + r.append("}\n"); + return d; + } + if (e instanceof Subscript) { + Subscript sub = (Subscript) e; + String ci = constIndex(sub.index); + String o = "fo" + tmp[0]++; + String i = "fi" + tmp[0]++; + String li = lazyLong(sub.index, label, r, ind); + String ref = regionArrayRef(sub.value, li != null ? li : ci, label, r, ind); + if (ref != null) { + String d = "fd" + tmp[0]++; + indent(r, ind); + r.append("final double ").append(d).append(" = ").append(ref).append(";\n"); + return d; + } + String base = expr(sub.value); + if ((li != null || ci != null) && hoist != null && hoist.containsValue(base) + && !hoistStored.contains(((Name) sub.value).id)) { + // The versioned loop reads a list nothing in it stores into + // straight from the double[] it took on entry: no call, and + // a region cannot grow or retype a list. + String[] fa = hoistFloats.get(base); + if (fa == null) { + fa = new String[] {base + "f", base + "n"}; + hoistFloats.put(base, fa); + } + String k = li != null ? li : ci; + String d = "fd" + tmp[0]++; + indent(r, ind); + r.append("if (").append(k).append(" < 0 || ").append(k).append(" >= ").append(fa[1]).append(") {\n"); + indent(r, ind + 1); + r.append("break ").append(label).append(";\n"); + indent(r, ind); + r.append("}\n"); + indent(r, ind); + r.append("final double ").append(d).append(" = ").append(fa[0]).append("[(int) ").append(k).append("];\n"); + return d; + } + indent(r, ind); + r.append("final Object ").append(o).append(" = ").append(base).append(";\n"); + if (li != null || ci != null) { + // An index already unboxed (or a constant) is passed as a long. + return regionItemLoad(o, li != null ? li : ci, label, tmp, r, ind); + } + indent(r, ind); + r.append("final Object ").append(i).append(" = ").append(expr(sub.index)).append(";\n"); + return regionItemLoad(o, i, label, tmp, r, ind); + } + String t = "ft" + tmp[0]++; + if (e instanceof UnaryOp) { + String a = regionValue(((UnaryOp) e).operand, inDoubles, label, tmp, r, ind); + indent(r, ind); + r.append("final double ").append(t).append(" = -").append(a).append(";\n"); + return t; + } + BinOp b = (BinOp) e; + if ("/".equals(b.op) && regionInt(b.left) && regionInt(b.right)) { + // int / int is correctly rounded in Python; dividing the doubles + // is too while both convert exactly. + String ll = regionLong(b.left, label, tmp, r, ind); + String rl = regionLong(b.right, label, tmp, r, ind); + regionBail(ll + " > (1L << 53) || " + ll + " < -(1L << 53) || " + rl + " > (1L << 53) || " + rl + + " < -(1L << 53) || " + rl + " == 0", label, r, ind); + indent(r, ind); + r.append("final double ").append(t).append(" = ((double) ").append(ll).append(") / ((double) ") + .append(rl).append(");\n"); + return t; + } + String lv = regionValue(b.left, inDoubles, label, tmp, r, ind); + String rv = regionValue(b.right, inDoubles, label, tmp, r, ind); + indent(r, ind); + if ("**".equals(b.op) && regionInt(b.left) && regionInt(b.right)) { + // int ** int is an int (or a float for a negative power): + // the general path decides. + r.append("if (").append(lv).append(" == ").append(lv).append(") {\n"); + indent(r, ind + 1); + r.append("break ").append(label).append(";\n"); + indent(r, ind); + r.append("}\n"); + indent(r, ind); + } + if ("**".equals(b.op)) { + r.append("final double ").append(t).append(" = Math.pow(").append(lv).append(", ").append(rv) + .append(");\n"); + indent(r, ind); + r.append("if (Double.isNaN(").append(t).append(") || Double.isInfinite(").append(t).append(")) {\n"); + indent(r, ind + 1); + r.append("break ").append(label).append(";\n"); + indent(r, ind); + r.append("}\n"); + return t; + } + if ("/".equals(b.op)) { + r.append("if (").append(rv).append(" == 0.0) {\n"); + indent(r, ind + 1); + r.append("break ").append(label).append(";\n"); + indent(r, ind); + r.append("}\n"); + indent(r, ind); + } + r.append("final double ").append(t).append(" = ").append(lv).append(' ').append(b.op).append(' ').append(rv) + .append(";\n"); + return t; + } + + /// Settles the regions of one emitted function body: a local stays a + /// double only if no region reads it before assigning it and no text + /// outside the regions names it at all. + private String resolveRegions(String body) { + if (body.indexOf(REGION_START) < 0) { + return body; + } + StringBuilder outside = new StringBuilder(); + int at = 0; + while (true) { + int st = body.indexOf(REGION_START, at); + if (st < 0) { + outside.append(body, at, body.length()); + break; + } + outside.append(body, at, st); + at = body.indexOf(REGION_END, st) + REGION_END.length(); + } + String rest = outside.toString(); + StringBuilder r = new StringBuilder(); + at = 0; + while (true) { + int a = nextCond(body, at); + if (a < 0) { + r.append(body, at, body.length()); + break; + } + r.append(body, at, a); + boolean ifPrivate = body.startsWith(IF_PRIVATE, a); + int nameEnd = body.indexOf("*/", a + IF_SHARED.length()); + String v = body.substring(a + IF_SHARED.length(), nameEnd); + int e = body.indexOf(COND_END, nameEnd); + boolean priv = !regionShared.contains(v) && !mentions(rest, v); + if (priv == ifPrivate) { + r.append(body, nameEnd + 2, e); + } + at = e + COND_END.length(); + } + return r.toString().replace(REGION_START + "\n", "").replace(REGION_END + "\n", "") + .replaceAll("(?m)^ +\n", ""); + } + + private static int nextCond(String s, int from) { + int a = s.indexOf(IF_SHARED, from); + int b = s.indexOf(IF_PRIVATE, from); + return a < 0 ? b : b < 0 ? a : Math.min(a, b); + } + + /// Whether `text` names the Java identifier `v`. + private static boolean mentions(String text, String v) { + int i = text.indexOf(v); + while (i >= 0) { + int e = i + v.length(); + boolean before = i > 0 && Character.isJavaIdentifierPart(text.charAt(i - 1)); + boolean after = e < text.length() && Character.isJavaIdentifierPart(text.charAt(e)); + if (!before && !after) { + return true; + } + i = text.indexOf(v, e); + } + return false; + } + + /// A subscript written as a small decimal int literal, as Java `long` + /// source, or null: such an index needs no int object at all. + private static String constIndex(Expr x) { + String sign = ""; + if (x instanceof UnaryOp && "-".equals(((UnaryOp) x).op)) { + sign = "-"; + x = ((UnaryOp) x).operand; + } + if (!(x instanceof Constant) || ((Constant) x).kind != Constant.Kind.INT) { + return null; + } + String t = ((Constant) x).text; + if (t.length() == 0 || t.length() > 9 || (t.length() > 1 && t.charAt(0) == '0')) { + return null; + } + for (int k = 0; k < t.length(); k++) { + if (t.charAt(k) < '0' || t.charAt(k) > '9') { + return null; + } + } + return sign + t + "L"; + } + + private static boolean isConstant(Expr x) { + if (x instanceof Constant) { + return true; + } + if (x instanceof UnaryOp) { + UnaryOp u = (UnaryOp) x; + return ("-".equals(u.op) || "+".equals(u.op)) && u.operand instanceof Constant + && (((Constant) u.operand).kind == Constant.Kind.INT + || ((Constant) u.operand).kind == Constant.Kind.FLOAT); + } + if (x instanceof BinOp) { + // Folded when both sides are; CPython also declines a few that + // would make huge results, which no set literal writes. + return isConstant(((BinOp) x).left) && isConstant(((BinOp) x).right); + } + if (x instanceof TupleExpr) { + return allConstant(((TupleExpr) x).elts); + } + return false; + } + +} diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/GenLowering.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/GenLowering.java new file mode 100644 index 00000000000..ce4c5c21fdf --- /dev/null +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/GenLowering.java @@ -0,0 +1,706 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.transpiler.codegen; + +import com.codename1.python.transpiler.api.PythonSyntaxError; +import com.codename1.python.transpiler.ast.Node.Assign; +import com.codename1.python.transpiler.ast.Node.Await; +import com.codename1.python.transpiler.ast.Node.Break; +import com.codename1.python.transpiler.ast.Node.ClassDef; +import com.codename1.python.transpiler.ast.Node.Continue; +import com.codename1.python.transpiler.ast.Node.ExceptHandler; +import com.codename1.python.transpiler.ast.Node.Expr; +import com.codename1.python.transpiler.ast.Node.ExprStmt; +import com.codename1.python.transpiler.ast.Node.For; +import com.codename1.python.transpiler.ast.Node.FunctionDef; +import com.codename1.python.transpiler.ast.Node.If; +import com.codename1.python.transpiler.ast.Node.Name; +import com.codename1.python.transpiler.ast.Node.Return; +import com.codename1.python.transpiler.ast.Node.Stmt; +import com.codename1.python.transpiler.ast.Node.Try; +import com.codename1.python.transpiler.ast.Node.While; +import com.codename1.python.transpiler.ast.Node.With; +import com.codename1.python.transpiler.ast.Node.WithItem; +import com.codename1.python.transpiler.ast.Node.Yield; +import com.codename1.python.transpiler.ast.Scope; +import java.util.ArrayList; +import java.util.List; + +/// Lowers the body of a generator or coroutine into the `step` method of a +/// `PyGenerator` subclass: a `switch (pc)` whose cases are the stretches of +/// the body between suspension points. +/// +/// A statement that cannot suspend (and does not jump out of a lowered loop +/// or `finally`) is emitted by the [Emitter] as it would be anywhere else; the +/// rest become states. Exceptions are routed by a table: every state records +/// the state that handles an exception raised in it (an `except` dispatch, a +/// `finally`, or none), and the `catch` around the `switch` jumps there with +/// the exception in `exc`. A `finally` is entered with a pending action in a +/// field -- fall through, re-raise, return, or jump to a state -- and performs +/// it when its body completes, which is how `break`, `continue` and `return` +/// run the `finally` blocks they leave. +final class GenLowering { + /// Pending actions of a `finally`; a jump is the target state, 0 or more. + private static final int NORMAL = -1; + private static final int RERAISE = -2; + private static final int RETURN = -3; + private static final int SUPPRESSED = -4; + + private static final int IND = 5; + + private final Emitter em; + private final String file; + private final Linearize lin; + private final StringBuilder fields = new StringBuilder(); //NOPMD AvoidStringBufferField - the generator's output + private final StringBuilder out = new StringBuilder(); //NOPMD AvoidStringBufferField - the generator's output + private final List handlerOf = new ArrayList(); + private final List loops = new ArrayList(); + private final List fins = new ArrayList(); + private int handler = -1; + private boolean retField; + + /// A lowered loop: where `break` and `continue` go, and how many + /// lowered `finally` blocks enclose it. + private static final class Loop { + final int breakTo; + final int continueTo; + final int depth; + + Loop(int breakTo, int continueTo, int depth) { + this.breakTo = breakTo; + this.continueTo = continueTo; + this.depth = depth; + } + } + + /// A lowered `finally`, and the jumps routed through it. + private static final class Fin { + final int entry; + final String pending; + final String saved; + final List jumps = new ArrayList(); + boolean returns; + + Fin(int entry, String pending, String saved) { + this.entry = entry; + this.pending = pending; + this.saved = saved; + } + } + + /// Whether this is an async generator, whose yields are wrapped. + boolean asyncGen; + + GenLowering(Emitter em, String file, Scope scope) { + this.em = em; + this.file = file; + this.lin = new Linearize(file, scope); + } + + /// The fields the lowered body needs besides the function's locals. + String fields() { + return fields.toString(); + } + + /// The body of `step()`: the dispatch loop over `body`'s states. + String step(List body) { + begin(newState()); + block(body); + line("if (true) {"); + line(" pc = -1;"); + line(" return Py.NONE;"); + line("}"); + StringBuilder r = new StringBuilder(); + r.append(" d$: while (true) {\n"); + r.append(" try {\n"); + r.append(" switch (pc) {\n"); + r.append(out); + r.append(" }\n"); + r.append(" default:\n"); + r.append(" throw new IllegalStateException();\n"); + r.append(" }\n"); + r.append(" } catch (Throwable t$) {\n"); + r.append(" final PyException x$ = Py.convert(t$);\n"); + r.append(" switch (pc) {\n"); + boolean any = false; + for (int h = 0; h < handlerOf.size(); h++) { + StringBuilder labels = new StringBuilder(); + for (int s = 0; s < handlerOf.size(); s++) { + if (handlerOf.get(s).intValue() == h) { + labels.append(" case ").append(s).append(":\n"); + } + } + if (labels.length() > 0) { + any = true; + r.append(labels).append(" pc = ").append(h).append(";\n") + .append(" break;\n"); + } + } + r.append(" default:\n"); + r.append(" throw x$;\n"); + r.append(" }\n"); + if (any) { + r.append(" exc = x$;\n"); + } + r.append(" }\n"); + r.append(" }\n"); + return r.toString(); + } + + // ---- states ------------------------------------------------------------ + + private int newState() { + handlerOf.add(Integer.valueOf(handler)); + return handlerOf.size() - 1; + } + + private int newState(int h) { + handlerOf.add(Integer.valueOf(h)); + return handlerOf.size() - 1; + } + + private void begin(int s) { + if (out.length() > 0) { + out.append(" }\n"); + } + out.append(" case ").append(s).append(": {\n"); + } + + private void line(String code) { + for (int i = 0; i < IND + 2; i++) { + out.append(" "); + } + out.append(code).append('\n'); + } + + private static String goTo(int s) { + return "if (true) { pc = " + s + "; continue d$; }"; + } + + private void jump(int s) { + line(goTo(s)); + } + + private String field(String type, String prefix) { + String f = em.fresh(prefix) + "$"; + fields.append(" ").append(type).append(' ').append(f).append(";\n"); + return f; + } + + // ---- statements -------------------------------------------------------- + + private void block(List body) { + for (Stmt s : body) { + stmt(s); + } + } + + /// Whether `s` has to become states: it suspends, or leaves a lowered + /// loop or `finally` in a way only states can follow. + private boolean lowered(Stmt s) { + if (Linearize.suspends(s)) { + return true; + } + if (!fins.isEmpty() && returns(s)) { + return true; + } + return !loops.isEmpty() && jumpsOut(s); + } + + private static boolean returns(Stmt s) { + if (s instanceof Return) { + return true; + } + if (s instanceof FunctionDef || s instanceof ClassDef) { + return false; + } + for (List b : Linearize.bodies(s)) { + for (Stmt x : b) { + if (returns(x)) { + return true; + } + } + } + return false; + } + + /// A `break` or `continue` that belongs to an enclosing loop. + private static boolean jumpsOut(Stmt s) { + if (s instanceof Break || s instanceof Continue) { + return true; + } + if (s instanceof For || s instanceof While || s instanceof FunctionDef || s instanceof ClassDef) { + // Their own loop's, or their own: a lowered loop's never reaches + // inside one emitted as Java. + return s instanceof For && jumpsOutOfElse(((For) s).orelse) + || s instanceof While && jumpsOutOfElse(((While) s).orelse); + } + for (List b : Linearize.bodies(s)) { + for (Stmt x : b) { + if (jumpsOut(x)) { + return true; + } + } + } + return false; + } + + private static boolean jumpsOutOfElse(List orelse) { + for (Stmt x : orelse) { + if (jumpsOut(x)) { + return true; + } + } + return false; + } + + private void stmt(Stmt s) { + if (!lowered(s)) { + em.stmt(s, out, IND + 2); + return; + } + List rep = lin.stmt(s); + if (rep != null) { + block(rep); + return; + } + if (s instanceof ExprStmt) { + suspend(((ExprStmt) s).value); + line("noop(" + result(((ExprStmt) s).value) + ");"); + } else if (s instanceof Assign) { + Assign a = (Assign) s; + suspend(a.value); + String r = result(a.value); + if (a.targets.size() == 1) { + em.store(a.targets.get(0), r, out, IND + 2); + } else { + String t = em.fresh("s"); + line("final Object " + t + " = " + r + ";"); + for (Expr x : a.targets) { + em.store(x, t, out, IND + 2); + } + } + } else if (s instanceof Return) { + Expr v = ((Return) s).value; + returnWith(v == null ? "Py.NONE" : em.expr(v)); + } else if (s instanceof Break) { + if (loops.isEmpty()) { + throw new PythonSyntaxError(file, s.line, s.col, "'break' outside loop"); + } + Loop l = loops.get(loops.size() - 1); + jumpTo(l.breakTo, l.depth); + } else if (s instanceof Continue) { + if (loops.isEmpty()) { + throw new PythonSyntaxError(file, s.line, s.col, "'continue' not properly in loop"); + } + Loop l = loops.get(loops.size() - 1); + jumpTo(l.continueTo, l.depth); + } else if (s instanceof If) { + ifStmt((If) s); + } else if (s instanceof While) { + whileStmt((While) s); + } else if (s instanceof For) { + forStmt((For) s); + } else if (s instanceof Try) { + tryStmt((Try) s); + } else if (s instanceof With) { + withStmt((With) s); + } else { + throw em.unsupported(s, "await or yield inside " + s.getClass().getSimpleName()); + } + } + + /// Emits the suspension at the top of `v`, if it has one, ending the + /// current state; the code after it runs on resumption. + private void suspend(Expr v) { + if (v instanceof Yield && !((Yield) v).from) { + Expr x = ((Yield) v).value; + int k = newState(); + String y = x == null ? "Py.NONE" : em.expr(x); + if (asyncGen) { + y = "new PyGenerator.Yielded(" + y + ")"; + } + line("if (true) { pc = " + k + "; return " + y + "; }"); + begin(k); + } else if (v instanceof Yield) { + delegate("Py.iter(" + em.expr(((Yield) v).value) + ")"); + } else if (v instanceof Await) { + delegate("PyGenerator.awaitable(" + em.expr(((Await) v).value) + ")"); + } + } + + /// The Java value of `v` once [#suspend] has run. + private String result(Expr v) { + if (v instanceof Yield && !((Yield) v).from) { + return "resumed()"; + } + if (v instanceof Yield || v instanceof Await) { + return "delegateResult"; + } + return em.expr(v); + } + + /// `yield from` an iterator: one state that re-enters until it is done. + private void delegate(String iterator) { + line("delegate = " + iterator + ";"); + int k = newState(); + jump(k); + begin(k); + line("final Object r$ = delegateStep();"); + line("if (r$ != null) {"); + line(" return r$;"); + line("}"); + } + + /// Awaits `java`, an awaitable, inside the current state sequence; the + /// result is then `delegateResult`. + private void await(String java) { + delegate("PyGenerator.awaitable(" + java + ")"); + } + + private void returnWith(String value) { + if (fins.isEmpty()) { + line("if (true) { pc = -1; return " + value + "; }"); + return; + } + retField = true; + line("ret$ = " + value + ";"); + returnThrough(); + } + + /// Returns `ret$`, running the lowered `finally` blocks on the way. + private void returnThrough() { + if (fins.isEmpty()) { + line("if (true) { pc = -1; return ret$; }"); + return; + } + Fin f = fins.get(fins.size() - 1); + f.returns = true; + line(f.pending + " = " + RETURN + ";"); + jump(f.entry); + } + + /// Jumps to state `target`, which `depth` lowered `finally` blocks + /// enclose, through the ones in between. + private void jumpTo(int target, int depth) { + if (fins.size() <= depth) { + jump(target); + return; + } + Fin f = fins.get(fins.size() - 1); + f.jumps.add(new int[] {target, depth}); + line(f.pending + " = " + target + ";"); + jump(f.entry); + } + + private void ifStmt(If s) { + int end = newState(); + int els = s.orelse.isEmpty() ? end : newState(); + line("if (!(" + em.cond(s.test) + ")) " + goTo(els)); + block(s.body); + jump(end); + if (!s.orelse.isEmpty()) { + begin(els); + block(s.orelse); + jump(end); + } + begin(end); + } + + private void whileStmt(While w) { + int top = newState(); + int end = newState(); + int els = w.orelse.isEmpty() ? end : newState(); + jump(top); + begin(top); + line("if (!(" + em.cond(w.test) + ")) " + goTo(els)); + loops.add(new Loop(end, top, fins.size())); + block(w.body); + loops.remove(loops.size() - 1); + jump(top); + if (!w.orelse.isEmpty()) { + begin(els); + block(w.orelse); + jump(end); + } + begin(end); + } + + private void forStmt(For f) { + String it = field("PyObject", "i"); + int top = newState(); + int end = newState(); + int els = f.orelse.isEmpty() ? end : newState(); + String x = em.fresh("x"); + if (f.isAsync) { + line(it + " = Py.obj(Py.callMethod(" + em.expr(f.iter) + ", \"__aiter__\"));"); + jump(top); + begin(top); + line("delegate = PyGenerator.awaitable(Py.callMethod(" + it + ", \"__anext__\"));"); + int k = newState(); + jump(k); + begin(k); + line("final Object r$;"); + line("try {"); + line(" r$ = delegateStep();"); + line("} catch (PyException e$) {"); + line(" if (e$.matches(Py.STOP_ASYNC_ITERATION)) " + goTo(els)); + line(" throw e$;"); + line("}"); + line("if (r$ != null) {"); + line(" return r$;"); + line("}"); + line("final Object " + x + " = delegateResult;"); + } else { + line(it + " = Py.iter(" + em.expr(f.iter) + ");"); + jump(top); + begin(top); + line("final Object " + x + " = " + it + ".next();"); + line("if (" + x + " == null) " + goTo(els)); + } + em.store(f.target, x, out, IND + 2); + loops.add(new Loop(end, top, fins.size())); + block(f.body); + loops.remove(loops.size() - 1); + jump(top); + if (!f.orelse.isEmpty()) { + begin(els); + block(f.orelse); + jump(end); + } + begin(end); + line(it + " = null;"); + } + + /// Opens a `finally`: its entry state and the field of its pending action. + private Fin openFin() { + return new Fin(newState(), field("int", "p"), field("PyException", "e")); + } + + /// The state that records a raise and enters `f`. + private int raiseInto() { + return newState(); + } + + /// The end of a `finally` body: performs the pending action. + private void closeFin(Fin f, int after) { + line("if (" + f.pending + " == " + RERAISE + ") {"); + line(" throw " + f.saved + ";"); + line("}"); + for (int[] j : f.jumps) { + line("if (" + f.pending + " == " + j[0] + ") {"); + jumpTo(j[0], j[1]); + line("}"); + } + if (f.returns) { + line("if (" + f.pending + " == " + RETURN + ") {"); + returnThrough(); + line("}"); + } + jump(after); + } + + private void tryStmt(Try t) { + if (t.star) { + throw em.unsupported(t, "except*"); + } + int outer = handler; + Fin fin = null; + int finRaise = -1; + if (!t.finalbody.isEmpty()) { + fin = openFin(); + finRaise = raiseInto(); + } + int inner = fin != null ? finRaise : outer; + int after = newState(); + int dispatch = t.handlers.isEmpty() ? -1 : newState(inner); + if (fin != null) { + fins.add(fin); + } + handler = dispatch >= 0 ? dispatch : inner; + int body = newState(); + jump(body); + begin(body); + block(t.body); + handler = inner; + if (!t.orelse.isEmpty()) { + int els = newState(); + jump(els); + begin(els); + block(t.orelse); + } + leave(fin, after); + if (dispatch >= 0) { + begin(dispatch); + String e = field("PyException", "e"); + line(e + " = exc;"); + List bodies = new ArrayList(); + boolean catchAll = false; + for (int i = 0; i < t.handlers.size(); i++) { + ExceptHandler h = t.handlers.get(i); + int hs = newState(); + bodies.add(Integer.valueOf(hs)); + if (h.type == null) { + if (i != t.handlers.size() - 1) { + throw new PythonSyntaxError(file, h.line, h.col, "default 'except:' must be last"); + } + jump(hs); + catchAll = true; + } else { + line("if (" + e + ".matches(" + em.expr(h.type) + ")) " + goTo(hs)); + } + } + if (!catchAll) { + line("if (true) throw " + e + ";"); + } + for (int i = 0; i < t.handlers.size(); i++) { + ExceptHandler h = t.handlers.get(i); + begin(bodies.get(i).intValue()); + if (h.name != null) { + em.storeName(h.name, e + ".value", out, IND + 2); + } + em.handlers.add(e); + block(h.body); + em.handlers.remove(em.handlers.size() - 1); + if (h.name != null) { + line(em.nameLvalue(h.name) + " = null;"); + } + line(e + " = null;"); + leave(fin, after); + } + } + handler = outer; + if (fin != null) { + fins.remove(fins.size() - 1); + begin(finRaise); + line(fin.pending + " = " + RERAISE + ";"); + line(fin.saved + " = exc;"); + jump(fin.entry); + begin(fin.entry); + block(t.finalbody); + closeFin(fin, after); + } + begin(after); + } + + /// Normal completion of a protected block: through `fin`, if there is one. + private void leave(Fin fin, int after) { + if (fin == null) { + jump(after); + } else { + line(fin.pending + " = " + NORMAL + ";"); + jump(fin.entry); + } + } + + private void withStmt(With w) { + WithItem item = w.items.get(0); + boolean async = w.isAsync; + String mgr = field("Object", "m"); + String exit = field("Object", "x"); + line(mgr + " = " + em.expr(item.context) + ";"); + line(exit + " = Py.getattr(" + mgr + ", \"" + (async ? "__aexit__" : "__exit__") + "\");"); + String entered; + if (async) { + await("Py.callMethod(" + mgr + ", \"__aenter__\")"); + entered = "delegateResult"; + } else { + entered = "Py.callMethod(" + mgr + ", \"__enter__\")"; + } + if (item.vars != null) { + String ev = em.fresh("en"); + line("final Object " + ev + " = " + entered + ";"); + em.store(item.vars, ev, out, IND + 2); + } else { + line("noop(" + entered + ");"); + } + int outer = handler; + Fin fin = openFin(); + int finRaise = raiseInto(); + int after = newState(); + int dispatch = newState(finRaise); + fins.add(fin); + handler = dispatch; + int body = newState(); + jump(body); + begin(body); + block(w.body); + handler = finRaise; + leave(fin, after); + begin(dispatch); + String e = field("PyException", "e"); + line(e + " = exc;"); + String call = "Py.call(" + exit + ", Py.typeOf(" + e + ".value), " + e + ".value, Py.NONE)"; + String suppressed; + if (async) { + await(call); + suppressed = "delegateResult"; + } else { + suppressed = call; + } + line("if (!Py.isTrue(" + suppressed + ")) {"); + line(" throw " + e + ";"); + line("}"); + line(fin.pending + " = " + SUPPRESSED + ";"); + jump(fin.entry); + handler = outer; + fins.remove(fins.size() - 1); + begin(finRaise); + line(fin.pending + " = " + RERAISE + ";"); + line(fin.saved + " = exc;"); + jump(fin.entry); + begin(fin.entry); + // The exit already ran for an exception the body raised; a raise + // out of the exit itself (RERAISE) must not run it again. + int skip = newState(); + line("if (" + fin.pending + " == " + SUPPRESSED + " || " + fin.pending + " == " + RERAISE + ") " + + goTo(skip)); + String clean = "Py.call(" + exit + ", Py.NONE, Py.NONE, Py.NONE)"; + if (async) { + await(clean); + } else { + line(clean + ";"); + } + jump(skip); + begin(skip); + line(mgr + " = null;"); + line(exit + " = null;"); + closeFin(fin, after); + begin(after); + } + + /// Whether `body` needs a lowered body at all. + static boolean needed(FunctionDef f) { + return f.isAsync || f.scope.hasYield; + } + + /// Whether a name the body binds is one of the hidden ones. + static boolean hidden(Name n) { + return n.id.startsWith("$y"); + } + + boolean usesReturnField() { + return retField; + } +} diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Infer.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Infer.java new file mode 100644 index 00000000000..9beab7ab71b --- /dev/null +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Infer.java @@ -0,0 +1,1644 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.transpiler.codegen; + +import com.codename1.python.transpiler.ast.Node; +import com.codename1.python.transpiler.ast.Node.*; +import com.codename1.python.transpiler.ast.Scope; + +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.IdentityHashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/// Whole-module type inference over a closed program. +/// +/// Runs only when `Closed` found nothing in the application that could +/// change a class, an instance's attributes or a module global behind the +/// compiler's back. Then every value that reaches a parameter, a field, a +/// list element or a global is one this pass sees, and the types it gives +/// are sound: typed code never needs to check them again, and a function +/// is entered typed after one check of its arguments at the door. +/// +/// It is a flow-insensitive fixpoint. Each round walks the whole module -- +/// nested functions, lambdas and class bodies included, where names are +/// simply `DYN` -- joining the type of every stored value into where it is +/// stored: a local of a compiled function, a parameter at each call site +/// that names its callee statically, a field, the elements of the lists +/// created at an allocation site, a global. A value that reaches somewhere +/// the pass cannot follow escapes: a function or class used as a value gets +/// `DYN` parameters, and a list that escapes gets `DYN` elements. +final class Infer { + /// A function the pass types: a compiled method, or a plain function at + /// the top of the module with a stable name. + static final class Fn { + final FunctionDef def; + /// The class, for a method. + final Layouts.Cls cls; + final List names; + final Ty[] params; + final Map locals = new LinkedHashMap(); + Ty ret = Ty.BOT; + boolean escapes; + + Fn(FunctionDef def, Layouts.Cls cls) { + this.def = def; + this.cls = cls; + this.names = def.args.names(); + this.params = new Ty[names.size()]; + for (int i = 0; i < params.length; i++) { + params[i] = Ty.BOT; + } + } + + Ty param(String n) { + int i = names.indexOf(n); + return i < 0 ? null : params[i]; + } + } + + final Layouts layouts; + final Scope moduleScope; + final Map fns = new LinkedHashMap(); + /// Stable global name -> the function or class it is bound to, once. + final Map globalFns = new HashMap(); + final Map globalClasses = new HashMap(); + final Map globals = new HashMap(); + /// Field name -> type, per declaring class. + final Map> fields = new LinkedHashMap>(); + /// Fields every instance has set once its `__init__` returned. + final Map> definite = new HashMap>(); + /// Classes something the pass cannot see may subclass or patch. + final Set open = new HashSet(); + final Map siteOf = new IdentityHashMap(); + final List siteElem = new ArrayList(); + final Set escapedSites = new HashSet(); + boolean allListsEscape; + + private boolean changed; + private final Map moduleBindings = new HashMap(); + + private Infer(Layouts layouts, Scope moduleScope) { + this.layouts = layouts; + this.moduleScope = moduleScope; + } + + /// Infers the module, or returns null when it gives up (no fixpoint). + static Infer run(List body, Layouts layouts, Scope moduleScope) { + Infer in = new Infer(layouts, moduleScope); + in.collect(body); + for (int round = 0; round < 60; round++) { + in.changed = false; + in.walkBlock(body, null, moduleScope); + in.unify(); + if (!in.changed) { + return in; + } + } + return null; + } + + // ---- setup ----------------------------------------------------------- + + private void collect(List body) { + countBindings(body); + for (Layouts.Cls k : layouts.classes.values()) { + fields.put(k, new LinkedHashMap()); + for (String f : k.fields) { + fields.get(k).put(f, Ty.BOT); + } + for (FunctionDef f : k.typable.values()) { + Fn fn = new Fn(f, k); + fn.params[0] = Ty.inst(k, false); + fns.put(f, fn); + } + } + for (Stmt s : body) { + if (s instanceof FunctionDef) { + FunctionDef f = (FunctionDef) s; + if (stable(f.name) && f.decorators.isEmpty() && !f.isAsync && !f.scope.hasYield + && f.args.vararg == null && f.args.kwarg == null && f.args.kwonly.isEmpty() + && f.scope.cells.isEmpty() && f.args.names().size() <= 8) { + Fn fn = new Fn(f, null); + fns.put(f, fn); + globalFns.put(f.name, fn); + } + } else if (s instanceof ClassDef) { + Layouts.Cls k = layouts.classes.get(s); + if (k != null && stable(((ClassDef) s).name)) { + globalClasses.put(((ClassDef) s).name, k); + } + } + } + markOpen(body); + for (Layouts.Cls k : layouts.classes.values()) { + definite.put(k, initDefinite(k, new HashSet())); + } + } + + private boolean stable(String n) { + Integer c = moduleBindings.get(n); + return c != null && c.intValue() == 1; + } + + /// Counts the bindings of each global at the top level, and every + /// `global` declaration in a function as one more (such a store is + /// invisible to a module-level count). + private void countBindings(List body) { + for (Stmt s : body) { + List names = new ArrayList(); + if (s instanceof FunctionDef) { + names.add(((FunctionDef) s).name); + } else if (s instanceof ClassDef) { + names.add(((ClassDef) s).name); + } else if (s instanceof Assign) { + for (Expr t : ((Assign) s).targets) { + targetNames(t, names); + } + } else if (s instanceof AugAssign) { + targetNames(((AugAssign) s).target, names); + targetNames(((AugAssign) s).target, names); + } else if (s instanceof AnnAssign) { + targetNames(((AnnAssign) s).target, names); + } else if (s instanceof Import) { + for (Alias a : ((Import) s).names) { + String n = a.asname != null ? a.asname : a.name; + int dot = n.indexOf('.'); + names.add(dot < 0 ? n : n.substring(0, dot)); + } + } else if (s instanceof ImportFrom) { + for (Alias a : ((ImportFrom) s).names) { + names.add(a.asname != null ? a.asname : a.name); + } + } else { + // A compound statement: everything it binds counts twice, + // since it may run any number of times. + for (String n : moduleScope.assigned) { + if (binds(s, n)) { + names.add(n); + names.add(n); + } + } + } + for (String n : names) { + Integer c = moduleBindings.get(n); + moduleBindings.put(n, Integer.valueOf(c == null ? 1 : c.intValue() + 1)); + } + } + globalDecls(body); + } + + private void globalDecls(Object o) { + if (o instanceof Global && !((Global) o).nonlocal) { + for (String n : ((Global) o).names) { + moduleBindings.put(n, Integer.valueOf(99)); + } + } + children(o, x -> globalDecls(x)); + } + + private static boolean binds(Stmt s, String n) { + List out = new ArrayList(); + boundIn(s, out); + return out.contains(n); + } + + /// The names a statement stores into, outside nested scopes. + private static void boundIn(Object o, List out) { + if (o instanceof Lambda || o instanceof Comp) { + return; + } + if (o instanceof FunctionDef) { + out.add(((FunctionDef) o).name); + return; + } + if (o instanceof ClassDef) { + out.add(((ClassDef) o).name); + return; + } + if (o instanceof Assign) { + for (Expr t : ((Assign) o).targets) { + targetNames(t, out); + } + } else if (o instanceof AugAssign) { + targetNames(((AugAssign) o).target, out); + } else if (o instanceof AnnAssign) { + targetNames(((AnnAssign) o).target, out); + } else if (o instanceof For) { + targetNames(((For) o).target, out); + } else if (o instanceof WithItem && ((WithItem) o).vars != null) { + targetNames(((WithItem) o).vars, out); + } else if (o instanceof NamedExpr) { + out.add(((NamedExpr) o).target.id); + } else if (o instanceof ExceptHandler && ((ExceptHandler) o).name != null) { + out.add(((ExceptHandler) o).name); + } else if (o instanceof Import) { + for (Alias a : ((Import) o).names) { + String n = a.asname != null ? a.asname : a.name; + int dot = n.indexOf('.'); + out.add(dot < 0 ? n : n.substring(0, dot)); + } + } else if (o instanceof ImportFrom) { + for (Alias a : ((ImportFrom) o).names) { + out.add(a.asname != null ? a.asname : a.name); + } + } else if (o instanceof Delete) { + for (Expr t : ((Delete) o).targets) { + targetNames(t, out); + } + } + children(o, x -> boundIn(x, out)); + } + + private static void targetNames(Expr t, List out) { + if (t instanceof Name) { + out.add(((Name) t).id); + } else if (t instanceof TupleExpr) { + for (Expr e : ((TupleExpr) t).elts) { + targetNames(e, out); + } + } else if (t instanceof ListExpr) { + for (Expr e : ((ListExpr) t).elts) { + targetNames(e, out); + } + } else if (t instanceof Starred) { + targetNames(((Starred) t).value, out); + } + } + + /// A class some class the layout planner rejected derives from: its + /// instances may run methods this pass does not see, so neither it nor + /// its ancestors are typed. + private void markOpen(Object o) { + if (o instanceof ClassDef) { + ClassDef c = (ClassDef) o; + if (!layouts.classes.containsKey(c)) { + for (Expr b : c.bases) { + if (b instanceof Name) { + Layouts.Cls k = byName(((Name) b).id); + for (; k != null; k = k.parent) { + open.add(k); + } + } + } + } + } + children(o, x -> markOpen(x)); + } + + private Layouts.Cls byName(String n) { + for (Layouts.Cls k : layouts.classes.values()) { + if (k.def.name.equals(n)) { + return k; + } + } + return null; + } + + // ---- definite assignment of fields ------------------------------------ + + private Set initDefinite(Layouts.Cls k, Set seen) { + Set r = new HashSet(); + Layouts.Cls owner = Layouts.resolver(k, "__init__"); + if (owner == null || !seen.add(owner)) { + return r; + } + FunctionDef f = owner.typable.get("__init__"); + if (f == null || f.args.names().isEmpty()) { + return r; + } + String self = f.args.names().get(0); + if (f.scope.assigned.contains(self)) { + return r; + } + for (Stmt s : f.body) { + if (s instanceof Pass || s instanceof ExprStmt && ((ExprStmt) s).value instanceof Constant) { + continue; + } + if (s instanceof Assign && ((Assign) s).targets.size() == 1) { + Assign a = (Assign) s; + Expr t = a.targets.get(0); + if (t instanceof Attribute && ((Attribute) t).value instanceof Name + && ((Name) ((Attribute) t).value).id.equals(self) && !mentions(a.value, self)) { + r.add(((Attribute) t).attr); + continue; + } + } + if (s instanceof ExprStmt && ((ExprStmt) s).value instanceof Call) { + Call c = (Call) ((ExprStmt) s).value; + if (c.func instanceof Attribute && "__init__".equals(((Attribute) c.func).attr) + && ((Attribute) c.func).value instanceof Name && c.keywords.isEmpty() && !c.args.isEmpty() + && c.args.get(0) instanceof Name && ((Name) c.args.get(0)).id.equals(self)) { + Layouts.Cls base = globalClasses.get(((Name) ((Attribute) c.func).value).id); + boolean clean = base != null && isAncestor(base, owner); + for (int i = 1; i < c.args.size() && clean; i++) { + clean = !mentions(c.args.get(i), self); + } + if (clean) { + r.addAll(initDefinite(base, seen)); + continue; + } + } + } + if (mentions(s, self)) { + break; + } + } + return r; + } + + private static boolean isAncestor(Layouts.Cls a, Layouts.Cls k) { + for (Layouts.Cls x = k.parent; x != null; x = x.parent) { + if (x == a) { //NOPMD CompareObjectsWithEquals - identity is meant + return true; + } + } + return false; + } + + static boolean mentions(Object o, String n) { + boolean[] r = new boolean[1]; + mentionsIn(o, n, r); + return r[0]; + } + + private static void mentionsIn(Object o, String n, boolean[] r) { + if (r[0]) { + return; + } + if (o instanceof Name && ((Name) o).id.equals(n)) { + r[0] = true; + return; + } + children(o, x -> mentionsIn(x, n, r)); + } + + /// Whether field `f` of class `k` (declared there or inherited) is set + /// on every instance of `k` and of all its subclasses. + boolean definitelySet(Layouts.Cls owner, String f) { + for (Layouts.Cls k : layouts.classes.values()) { + if (isSelfOrAncestor(owner, k) && !definite.get(k).contains(f)) { + return false; + } + } + return true; + } + + static boolean isSelfOrAncestor(Layouts.Cls a, Layouts.Cls k) { + for (Layouts.Cls x = k; x != null; x = x.parent) { + if (x == a) { //NOPMD CompareObjectsWithEquals - identity is meant + return true; + } + } + return false; + } + + // ---- joins ----------------------------------------------------------- + + private Ty joinInto(Ty cur, Ty v) { + Ty r = Ty.join(cur, v); + if (!r.equals(cur)) { + changed = true; + } + if (r.k != Ty.K.LIST) { + escape(v); + } + return r; + } + + /// A value went somewhere the pass does not follow. + void escape(Ty v) { + if (v != null && v.k == Ty.K.LIST) { + if (v.sites.length == 0) { + if (!allListsEscape) { + allListsEscape = true; + changed = true; + } + } else { + for (int s : v.sites) { + if (escapedSites.add(Integer.valueOf(s))) { + changed = true; + } + } + } + } + } + + private void escapeFn(Fn f) { + if (f != null && !f.escapes) { + f.escapes = true; + changed = true; + } + if (f != null) { + // Callers through a value are unknown, but the typed version + // is entered from the generic one only through a guard that + // checks every argument (TypedEmit.entry): a parameter keeps + // what its direct callers pass, and one nobody calls directly + // takes the type its uses suggest. A wrong guess costs the + // guard, never correctness. + for (int i = f.cls != null ? 1 : 0; i < f.params.length; i++) { + if (f.params[i].k == Ty.K.BOT) { + f.params[i] = joinInto(f.params[i], guess(f.def, f.names.get(i))); + } + } + } + } + + /// The type the uses of parameter `p` in `def` suggest: INT when it is + /// a `range` bound, a list repeat count or an operand next to an int + /// constant; FLOAT next to a float constant; DYN when the uses say + /// nothing or disagree. + static Ty guess(FunctionDef def, String p) { + Ty[] g = {null}; + boolean[] bad = {false}; + for (Stmt s : def.body) { + guessIn(s, p, g, bad); + } + return bad[0] || g[0] == null ? Ty.DYN : g[0]; + } + + private static void guessIn(Object o, String p, Ty[] g, boolean[] bad) { + if (o instanceof Call && ((Call) o).func instanceof Name && "range".equals(((Name) ((Call) o).func).id)) { + for (Expr a : ((Call) o).args) { + if (isName(a, p)) { + vote(Ty.INT, g, bad); + } + } + } else if (o instanceof BinOp) { + BinOp b = (BinOp) o; + if ("*".equals(b.op) && (b.left instanceof ListExpr && isName(b.right, p))) { + vote(Ty.INT, g, bad); + } else if (isName(b.left, p) || isName(b.right, p)) { + Ty c = constType(isName(b.left, p) ? b.right : b.left); + if (c != null) { + vote(c, g, bad); + } + } + } else if (o instanceof Compare && ((Compare) o).ops.size() == 1) { + Compare c = (Compare) o; + Expr r = c.comparators.get(0); + if (isName(c.left, p) || isName(r, p)) { + Ty t = constType(isName(c.left, p) ? r : c.left); + if (t != null) { + vote(t, g, bad); + } + } + } else if (o instanceof Assign) { + // Rebinding the parameter: its later uses say nothing of the + // argument. + for (Expr t : ((Assign) o).targets) { + if (isName(t, p)) { + bad[0] = true; + } + } + } + Infer.children(o, x -> guessIn(x, p, g, bad)); + } + + private static boolean isName(Expr e, String p) { + return e instanceof Name && ((Name) e).id.equals(p); + } + + private static Ty constType(Expr e) { + if (e instanceof Constant) { + Constant c = (Constant) e; + if (c.kind == Constant.Kind.INT) { + return Ty.INT; + } + if (c.kind == Constant.Kind.FLOAT && !c.text.matches(".*[jJ]")) { + return Ty.FLOAT; + } + } + return null; + } + + private static void vote(Ty t, Ty[] g, boolean[] bad) { + if (g[0] != null && !g[0].equals(t)) { + bad[0] = true; + } + g[0] = t; + } + + private void escapeClass(Layouts.Cls k) { + Layouts.Cls o = Layouts.resolver(k, "__init__"); + if (o != null && o.typable.get("__init__") != null) { + escapeFn(fns.get(o.typable.get("__init__"))); + } + } + + /// Every compiled method `name` may be reached with arguments the pass + /// cannot type. + private void escapeMethods(String name) { + for (Fn f : fns.values()) { + if (f.cls != null && f.def.name.equals(name)) { + escapeFn(f); + } + } + } + + Ty elem(Ty list) { + if (list.k != Ty.K.LIST || allListsEscape || list.sites.length == 0) { + return Ty.DYN; + } + Ty r = Ty.BOT; + for (int s : list.sites) { + if (escapedSites.contains(Integer.valueOf(s))) { + return Ty.DYN; + } + r = Ty.join(r, siteElem.get(s)); + } + return r; + } + + private void storeElem(Ty list, Ty v) { + if (list.k != Ty.K.LIST || list.sites.length == 0) { + escape(v); + return; + } + for (int s : list.sites) { + siteElem.set(s, joinInto(siteElem.get(s), v)); + } + } + + private int site(Expr e) { + Integer s = siteOf.get(e); + if (s == null) { + s = Integer.valueOf(siteElem.size()); + siteElem.add(Ty.BOT); + siteOf.put(e, s); + } + return s.intValue(); + } + + private void unify() { + // Methods of one name and arity are called through one virtual + // signature (TypedEmit), so they share their parameter types. + Map> groups = new LinkedHashMap>(); + for (Fn f : fns.values()) { + if (f.cls != null && !f.def.name.equals("__init__")) { + String key = f.def.name + "/" + f.params.length; + List g = groups.get(key); + if (g == null) { + g = new ArrayList(); + groups.put(key, g); + } + g.add(f); + } + } + for (List g : groups.values()) { + if (g.size() < 2) { + continue; + } + int n = g.get(0).params.length; + for (int i = 1; i < n; i++) { + Ty j = Ty.BOT; + for (Fn f : g) { + j = Ty.join(j, f.params[i]); + } + for (Fn f : g) { + f.params[i] = joinInto(f.params[i], j); + } + } + Ty r = Ty.BOT; + for (Fn f : g) { + r = Ty.join(r, f.ret); + } + for (Fn f : g) { + f.ret = joinInto(f.ret, r); + } + } + } + + // ---- walking --------------------------------------------------------- + + /// `fn` is the compiled function being walked, or null where names are + /// not tracked (the module body tracks globals instead). + private void walkBlock(List body, Fn fn, Scope sc) { + for (Stmt s : body) { + walk(s, fn, sc); + } + } + + private void walk(Stmt s, Fn fn, Scope sc) { + if (s instanceof ExprStmt) { + type(((ExprStmt) s).value, fn, sc); + } else if (s instanceof Assign) { + Assign a = (Assign) s; + Ty v = type(a.value, fn, sc); + for (Expr t : a.targets) { + store(t, v, a.value, fn, sc); + } + } else if (s instanceof AnnAssign) { + AnnAssign a = (AnnAssign) s; + if (a.value != null) { + store(a.target, type(a.value, fn, sc), a.value, fn, sc); + } else if (!(a.target instanceof Name)) { + type(a.target, fn, sc); + } + } else if (s instanceof AugAssign) { + AugAssign a = (AugAssign) s; + Ty cur = type(a.target, fn, sc); + Ty v = type(a.value, fn, sc); + Ty r = binop(a.op, cur, v, a.value); + if (cur.k == Ty.K.LIST && "+".equals(a.op)) { + storeElem(cur, elem(v)); + r = cur; + } + store(a.target, r, null, fn, sc); + } else if (s instanceof Return) { + Return r = (Return) s; + Ty v = r.value == null ? Ty.NONE : type(r.value, fn, sc); + if (fn != null && sc == fn.def.scope) { //NOPMD CompareObjectsWithEquals - identity is meant + fn.ret = joinInto(fn.ret, v); + } else { + escape(v); + } + } else if (s instanceof If) { + If i = (If) s; + type(i.test, fn, sc); + walkBlock(i.body, fn, sc); + walkBlock(i.orelse, fn, sc); + } else if (s instanceof While) { + While w = (While) s; + type(w.test, fn, sc); + walkBlock(w.body, fn, sc); + walkBlock(w.orelse, fn, sc); + } else if (s instanceof For) { + For f = (For) s; + Ty it = iterElem(f.iter, fn, sc); + store(f.target, it, null, fn, sc); + walkBlock(f.body, fn, sc); + walkBlock(f.orelse, fn, sc); + } else if (s instanceof FunctionDef) { + FunctionDef d = (FunctionDef) s; + for (Expr e : d.decorators) { + type(e, fn, sc); + } + for (Expr e : d.args.defaults) { + type(e, fn, sc); + } + for (Expr e : d.args.kwDefaults) { + if (e != null) { + type(e, fn, sc); + } + } + Fn inner = fns.get(d); + if (inner != null) { + // Defaults fill parameters the call leaves out. + int first = inner.params.length - d.args.defaults.size(); + for (int i = 0; i < d.args.defaults.size(); i++) { + inner.params[first + i] = joinInto(inner.params[first + i], type(d.args.defaults.get(i), fn, sc)); + } + if (!d.decorators.isEmpty() || inner.cls == null && !globalFns.containsValue(inner)) { + escapeFn(inner); + } + for (int i = 0; i < inner.params.length; i++) { + inner.locals.put(inner.names.get(i), Ty.join(inner.locals.get(inner.names.get(i)), inner.params[i])); + } + walkBlock(d.body, inner, d.scope); + if (!endsAbruptly(d.body)) { + inner.ret = joinInto(inner.ret, Ty.NONE); + } + if (inner.def.name.startsWith("__") && !"__init__".equals(inner.def.name)) { + // The runtime calls these itself, with anything. + escapeFn(inner); + inner.ret = joinInto(inner.ret, Ty.DYN); + } + } else { + walkBlock(d.body, null, d.scope); + } + if (sc.kind == Scope.Kind.MODULE && fns.get(d) == null) { + storeName(d.name, Ty.DYN, fn, sc); + } + } else if (s instanceof ClassDef) { + ClassDef c = (ClassDef) s; + for (Expr b : c.bases) { + if (!(b instanceof Name && globalClasses.containsKey(((Name) b).id))) { + type(b, fn, sc); + } + } + for (Expr d : c.decorators) { + type(d, fn, sc); + } + walkBlock(c.body, null, c.scope); + } else if (s instanceof Try) { + Try t = (Try) s; + walkBlock(t.body, fn, sc); + for (ExceptHandler h : t.handlers) { + if (h.type != null) { + type(h.type, fn, sc); + } + if (h.name != null) { + storeName(h.name, Ty.DYN, fn, sc); + } + walkBlock(h.body, fn, sc); + } + walkBlock(t.orelse, fn, sc); + walkBlock(t.finalbody, fn, sc); + } else if (s instanceof With) { + With w = (With) s; + for (WithItem i : w.items) { + type(i.context, fn, sc); + if (i.vars != null) { + store(i.vars, Ty.DYN, null, fn, sc); + } + } + walkBlock(w.body, fn, sc); + } else if (s instanceof Import || s instanceof ImportFrom) { + List names = new ArrayList(); + if (s instanceof Import) { + for (Alias a : ((Import) s).names) { + String n = a.asname != null ? a.asname : a.name; + int dot = n.indexOf('.'); + names.add(dot < 0 ? n : n.substring(0, dot)); + } + } else { + for (Alias a : ((ImportFrom) s).names) { + names.add(a.asname != null ? a.asname : a.name); + } + } + for (String n : names) { + storeName(n, Ty.DYN, fn, sc); + } + } else { + // Raise, Assert, Delete, Match, Global, Pass, Break, Continue: + // whatever they evaluate escapes. + anyChildren(s, fn, sc); + } + } + + private static boolean endsAbruptly(List body) { + if (body.isEmpty()) { + return false; + } + Stmt s = body.get(body.size() - 1); + if (s instanceof Return || s instanceof Raise) { + return true; + } + if (s instanceof If) { + If i = (If) s; + return endsAbruptly(i.body) && endsAbruptly(i.orelse); + } + if (s instanceof While) { + While w = (While) s; + return w.test instanceof Constant && ((Constant) w.test).kind == Constant.Kind.TRUE && !hasBreak(w.body); + } + return false; + } + + static boolean hasBreak(List body) { + for (Stmt s : body) { + if (s instanceof Break) { + return true; + } + if (s instanceof If && (hasBreak(((If) s).body) || hasBreak(((If) s).orelse))) { + return true; + } + if (s instanceof Try) { + Try t = (Try) s; + if (hasBreak(t.body) || hasBreak(t.orelse) || hasBreak(t.finalbody)) { + return true; + } + for (ExceptHandler h : t.handlers) { + if (hasBreak(h.body)) { + return true; + } + } + } + if (s instanceof With && hasBreak(((With) s).body)) { + return true; + } + } + return false; + } + + /// The type of the items `for x in it` binds. + Ty iterElem(Expr it, Fn fn, Scope sc) { + if (rangeCall(it, sc)) { + for (Expr a : ((Call) it).args) { + type(a, fn, sc); + } + return Ty.INT; + } + Ty t = type(it, fn, sc); + if (t.k == Ty.K.LIST) { + return elem(t); + } + escape(t); + return Ty.DYN; + } + + boolean rangeCall(Expr e, Scope sc) { + if (!(e instanceof Call)) { + return false; + } + Call c = (Call) e; + return c.func instanceof Name && "range".equals(((Name) c.func).id) && c.keywords.isEmpty() + && c.args.size() >= 1 && c.args.size() <= 3 && sc.resolve("range") == Scope.Binding.BUILTIN + && !moduleScope.hasStarImport && !anyStarred(c.args); + } + + private static boolean anyStarred(List l) { + for (Expr e : l) { + if (e instanceof Starred) { + return true; + } + } + return false; + } + + private void store(Expr t, Ty v, Expr value, Fn fn, Scope sc) { + if (t instanceof Name) { + storeName(((Name) t).id, v, fn, sc); + } else if (t instanceof Attribute) { + Attribute a = (Attribute) t; + Ty r = type(a.value, fn, sc); + storeField(r, a.attr, v); + } else if (t instanceof Subscript) { + Subscript s = (Subscript) t; + Ty r = type(s.value, fn, sc); + if (s.index instanceof Slice) { + anyChildren(s.index, fn, sc); + if (r.k == Ty.K.LIST) { + storeElem(r, elem(v)); + } else { + escape(v); + } + } else { + type(s.index, fn, sc); + if (r.k == Ty.K.LIST) { + storeElem(r, v); + } else if (r.k != Ty.K.BOT) { + escape(v); + } + } + } else if (t instanceof TupleExpr || t instanceof ListExpr) { + List elts = t instanceof TupleExpr ? ((TupleExpr) t).elts : ((ListExpr) t).elts; + List src = value instanceof TupleExpr ? ((TupleExpr) value).elts + : value instanceof ListExpr ? ((ListExpr) value).elts : null; + boolean starred = anyStarred(elts); + if (src != null && src.size() == elts.size() && !starred && !anyStarred(src)) { + // `a, b = x, y` pairs each target with its own value. + for (int i = 0; i < elts.size(); i++) { + store(elts.get(i), type(src.get(i), fn, sc), null, fn, sc); + } + return; + } + Ty each = Ty.DYN; + if (value != null && rangeCall(value, sc)) { + each = Ty.INT; + } else if (v.k == Ty.K.LIST) { + each = elem(v); + } else { + escape(v); + } + for (Expr e : elts) { + store(e instanceof Starred ? ((Starred) e).value : e, e instanceof Starred ? Ty.DYN : each, null, fn, + sc); + } + } else if (t instanceof Starred) { + store(((Starred) t).value, Ty.DYN, null, fn, sc); + } else { + anyChildren(t, fn, sc); + escape(v); + } + } + + private void storeName(String n, Ty v, Fn fn, Scope sc) { + switch (sc.resolve(n)) { + case LOCAL: + if (fn != null && sc == fn.def.scope) { //NOPMD CompareObjectsWithEquals - identity is meant + fn.locals.put(n, joinInto(fn.locals.get(n), v)); + } else { + escape(v); + } + return; + case GLOBAL: + globals.put(n, joinInto(globals.get(n), v)); + return; + default: + escape(v); + } + } + + void storeField(Ty recv, String attr, Ty v) { + if (recv.k == Ty.K.BOT || recv.k == Ty.K.NONE) { + return; + } + List owners = layouts.fieldOwners.get(attr); + if (owners == null) { + escape(v); + return; + } + boolean any = false; + for (Layouts.Cls o : owners) { + boolean reach = recv.k != Ty.K.INST || isSelfOrAncestor(o, recv.cls) || isSelfOrAncestor(recv.cls, o); + if (reach) { + Map m = fields.get(o); + m.put(attr, joinInto(m.get(attr), v)); + any = true; + } + } + if (!any) { + escape(v); + } + } + + /// The declaring class of field `attr` for an instance of `k`. + static Layouts.Cls fieldOwner(Layouts.Cls k, String attr) { + for (Layouts.Cls c = k; c != null; c = c.parent) { + if (c.fields.contains(attr)) { + return c; + } + } + return null; + } + + Ty fieldType(Layouts.Cls owner, String attr) { + Ty t = fields.get(owner).get(attr); + return t == null ? Ty.DYN : t; + } + + /// Compiled methods a call `recv.m(...)` on an instance of `k` may run: + /// the one `k` resolves, and every override in a subclass of `k`. Null + /// when the name may resolve to anything else (a field, a class + /// attribute that is not a compiled method). + List candidates(Layouts.Cls k, String m) { + if (layouts.fieldOwners.containsKey(m)) { + return null; + } + List r = new ArrayList(); + Layouts.Cls res = Layouts.resolver(k, m); + if (res != null) { + FunctionDef f = res.typable.get(m); + if (f == null) { + return null; + } + r.add(f); + } + for (Layouts.Cls c : layouts.classes.values()) { + if (c != k && isSelfOrAncestor(k, c) && c.bodyNames.contains(m)) { //NOPMD CompareObjectsWithEquals - identity is meant + FunctionDef f = c.typable.get(m); + if (f == null) { + return null; + } + if (!r.contains(f)) { + r.add(f); + } + } + } + return r.isEmpty() ? null : r; + } + + private void anyChildren(Object o, Fn fn, Scope sc) { + children(o, x -> { + if (x instanceof Expr) { + escape(type((Expr) x, fn, sc)); + } else if (x instanceof Stmt) { + walk((Stmt) x, fn, sc); + } else { + anyChildren(x, fn, sc); + } + }); + } + + // ---- expressions ----------------------------------------------------- + + Ty nameType(String n, Fn fn, Scope sc) { + switch (sc.resolve(n)) { + case LOCAL: + if (fn != null && sc == fn.def.scope) { //NOPMD CompareObjectsWithEquals - identity is meant + Ty t = fn.locals.get(n); + return t == null ? Ty.BOT : t; + } + return Ty.DYN; + case GLOBAL: { + Fn f = globalFns.get(n); + if (f != null) { + escapeFn(f); + return Ty.DYN; + } + Layouts.Cls k = globalClasses.get(n); + if (k != null) { + escapeClass(k); + return Ty.DYN; + } + Ty t = globals.get(n); + return t == null ? Ty.BOT : t; + } + default: + return Ty.DYN; + } + } + + Ty type(Expr e, Fn fn, Scope sc) { + if (e instanceof Constant) { + switch (((Constant) e).kind) { + case NONE: + return Ty.NONE; + case TRUE: + case FALSE: + return Ty.BOOL; + case INT: + return Ty.INT; + case FLOAT: + return ((Constant) e).text.endsWith("j") || ((Constant) e).text.endsWith("J") ? Ty.DYN + : Ty.FLOAT; + case STR: + return Ty.STR; + default: + return Ty.DYN; + } + } + if (e instanceof Name) { + return nameType(((Name) e).id, fn, sc); + } + if (e instanceof Attribute) { + Attribute a = (Attribute) e; + if (a.value instanceof Name && globalClasses.containsKey(((Name) a.value).id) + && sc.resolve(((Name) a.value).id) == Scope.Binding.GLOBAL) { + // `Cls.attr` as a value: a method taken off the class runs + // with arguments nobody tracks. + escapeMethods(a.attr); + return Ty.DYN; + } + Ty r = type(a.value, fn, sc); + return attrType(r, a.attr); + } + if (e instanceof Call) { + return call((Call) e, fn, sc); + } + if (e instanceof BinOp) { + BinOp b = (BinOp) e; + Ty l = type(b.left, fn, sc); + Ty r = type(b.right, fn, sc); + if ("*".equals(b.op) && b.left instanceof ListExpr && l.k == Ty.K.LIST) { + return Ty.list(l.sites, false); + } + if ("+".equals(b.op) && l.k == Ty.K.LIST && r.k == Ty.K.LIST) { + int s = site(b); + siteElem.set(s, joinInto(siteElem.get(s), Ty.join(elem(l), elem(r)))); + return Ty.list(new int[] {s}, false); + } + if (l.k == Ty.K.BOT || r.k == Ty.K.BOT) { + return Ty.BOT; + } + Ty t = binop(b.op, l, r, b.right); + if (t.k == Ty.K.DYN) { + escape(l); + escape(r); + } + return t; + } + if (e instanceof UnaryOp) { + UnaryOp u = (UnaryOp) e; + Ty t = type(u.operand, fn, sc); + if ("not".equals(u.op)) { + return Ty.BOOL; + } + if (t.k == Ty.K.BOT) { + return Ty.BOT; + } + if (t.k == Ty.K.FLOAT && !t.opt) { + return Ty.FLOAT; + } + if ((t.k == Ty.K.INT || t.k == Ty.K.BOOL) && !"+".equals(u.op)) { + return Ty.INT; + } + if (t.k == Ty.K.INT && "+".equals(u.op)) { + return Ty.INT; + } + escape(t); + return Ty.DYN; + } + if (e instanceof BoolOp) { + Ty r = Ty.BOT; + for (Expr v : ((BoolOp) e).values) { + r = Ty.join(r, type(v, fn, sc)); + } + return r; + } + if (e instanceof Compare) { + Compare c = (Compare) e; + Ty l = type(c.left, fn, sc); + boolean plain = true; + for (int i = 0; i < c.ops.size(); i++) { + Ty r = type(c.comparators.get(i), fn, sc); + String op = c.ops.get(i); + if (!("is".equals(op) || "is not".equals(op) || "in".equals(op) || "not in".equals(op) + || plainCompare(l) && plainCompare(r))) { + plain = false; + } + if ("in".equals(op) || "not in".equals(op)) { + escape(r); + escape(l); + } + l = r; + } + return plain ? Ty.BOOL : Ty.DYN; + } + if (e instanceof IfExp) { + IfExp i = (IfExp) e; + type(i.test, fn, sc); + return Ty.join(type(i.body, fn, sc), type(i.orelse, fn, sc)); + } + if (e instanceof Subscript) { + Subscript s = (Subscript) e; + Ty r = type(s.value, fn, sc); + if (s.index instanceof Slice) { + anyChildren(s.index, fn, sc); + if (r.k == Ty.K.LIST && !r.opt) { + return Ty.list(r.sites, false); + } + escape(r); + return Ty.DYN; + } + Ty i = type(s.index, fn, sc); + if (r.k == Ty.K.BOT || r.k == Ty.K.NONE || i.k == Ty.K.BOT) { + return Ty.BOT; + } + if (r.k == Ty.K.LIST && (i.k == Ty.K.INT || i.k == Ty.K.BOOL)) { + return elem(r); + } + escape(r); + escape(i); + return Ty.DYN; + } + if (e instanceof ListExpr) { + ListExpr l = (ListExpr) e; + int s = site(l); + for (Expr x : l.elts) { + if (x instanceof Starred) { + Ty t = type(((Starred) x).value, fn, sc); + siteElem.set(s, joinInto(siteElem.get(s), t.k == Ty.K.LIST ? elem(t) : Ty.DYN)); + escape(t); + } else { + siteElem.set(s, joinInto(siteElem.get(s), type(x, fn, sc))); + } + } + return Ty.list(new int[] {s}, false); + } + if (e instanceof NamedExpr) { + NamedExpr n = (NamedExpr) e; + Ty v = type(n.value, fn, sc); + storeName(n.target.id, v, fn, sc); + return v; + } + if (e instanceof Lambda) { + Lambda l = (Lambda) e; + for (Expr d : l.args.defaults) { + escape(type(d, fn, sc)); + } + escape(type(l.body, null, l.scope)); + return Ty.DYN; + } + if (e instanceof Comp) { + Comp c = (Comp) e; + boolean first = true; + for (Comprehension g : c.generators) { + // The first iterable is evaluated in the enclosing scope. + escape(type(g.iter, first ? fn : null, first ? sc : c.scope)); + first = false; + store(g.target, Ty.DYN, null, null, c.scope); + for (Expr x : g.ifs) { + type(x, null, c.scope); + } + } + if (c.elt != null) { + escape(type(c.elt, null, c.scope)); + } + if (c.value != null) { + escape(type(c.value, null, c.scope)); + } + return Ty.DYN; + } + anyChildren(e, fn, sc); + return Ty.DYN; + } + + /// Kinds whose comparisons always answer a bool: no user `__eq__`. + private boolean plainCompare(Ty t) { + switch (t.k) { + case BOT: + case NONE: + case BOOL: + case INT: + case FLOAT: + case STR: + return true; + case INST: + for (Layouts.Cls c = t.cls; c != null; c = c.parent) { + for (String d : new String[] {"__eq__", "__ne__", "__lt__", "__le__", "__gt__", "__ge__"}) { + if (c.bodyNames.contains(d)) { + return false; + } + } + } + return !anySubclassDefinesCompare(t.cls); + default: + return false; + } + } + + private boolean anySubclassDefinesCompare(Layouts.Cls k) { + for (Layouts.Cls c : layouts.classes.values()) { + if (isSelfOrAncestor(k, c)) { + for (String d : new String[] {"__eq__", "__ne__", "__lt__", "__le__", "__gt__", "__ge__"}) { + if (c.bodyNames.contains(d)) { + return true; + } + } + } + } + return false; + } + + Ty attrType(Ty r, String attr) { + if (r.k == Ty.K.BOT || r.k == Ty.K.NONE) { + // Nothing reached the receiver yet, or None (which raises): no value. + // Nothing reached the receiver yet: a later round decides. + return Ty.BOT; + } + if (r.k == Ty.K.INST) { + Layouts.Cls o = fieldOwner(r.cls, attr); + if (o != null) { + return fieldType(o, attr); + } + // A bound method, or something only a subclass has. + escapeMethods(attr); + return Ty.DYN; + } + escape(r); + if (r.k == Ty.K.DYN) { + escapeMethods(attr); + } + return Ty.DYN; + } + + static Ty binop(String op, Ty l, Ty r, Expr right) { + if (l.k == Ty.K.BOT || r.k == Ty.K.BOT) { + return Ty.BOT; + } + if (l.opt || r.opt) { + return Ty.DYN; + } + boolean li = l.k == Ty.K.INT || l.k == Ty.K.BOOL; + boolean ri = r.k == Ty.K.INT || r.k == Ty.K.BOOL; + boolean lf = l.k == Ty.K.FLOAT; + boolean rf = r.k == Ty.K.FLOAT; + if (li && ri) { + switch (op) { + case "+": + case "-": + case "*": + case "//": + case "%": + case "&": + case "|": + case "^": + case "<<": + case ">>": + return Ty.INT; + case "/": + return Ty.FLOAT; + case "**": + // A negative exponent answers a float. + return right instanceof Constant && ((Constant) right).kind == Constant.Kind.INT ? Ty.INT + : Ty.DYN; + default: + return Ty.DYN; + } + } + if ((lf || li) && (rf || ri)) { + switch (op) { + case "+": + case "-": + case "*": + case "/": + case "//": + case "%": + return Ty.FLOAT; + default: + // `**` of a negative float answers a complex. + return Ty.DYN; + } + } + if (l.k == Ty.K.STR && r.k == Ty.K.STR && "+".equals(op)) { + return Ty.STR; + } + return Ty.DYN; + } + + private Ty call(Call c, Fn fn, Scope sc) { + boolean simple = c.keywords.isEmpty() && !anyStarred(c.args); + if (c.func instanceof Name && sc.resolve(((Name) c.func).id) == Scope.Binding.GLOBAL) { + String n = ((Name) c.func).id; + Fn f = globalFns.get(n); + Layouts.Cls k = globalClasses.get(n); + if (f != null || k != null) { + List args = new ArrayList(); + for (Expr a : c.args) { + args.add(a instanceof Starred ? Ty.DYN : type(a, fn, sc)); + } + for (Keyword kw : c.keywords) { + escape(type(kw.value, fn, sc)); + } + if (f != null) { + callFn(f, args, 0, simple && args.size() <= f.params.length + && args.size() >= f.params.length - f.def.args.defaults.size()); + return f.ret; + } + if (open.contains(k)) { + for (Ty a : args) { + escape(a); + } + escapeClass(k); + return Ty.DYN; + } + Layouts.Cls o = Layouts.resolver(k, "__init__"); + FunctionDef init = o == null ? null : o.typable.get("__init__"); + if (o != null && init == null) { + for (Ty a : args) { + escape(a); + } + return Ty.inst(k, false); + } + if (init != null) { + Fn fi = fns.get(init); + fi.params[0] = joinInto(fi.params[0], Ty.inst(k, false)); + callFn(fi, args, 1, simple && args.size() + 1 <= fi.params.length + && args.size() + 1 >= fi.params.length - init.args.defaults.size()); + } else { + for (Ty a : args) { + escape(a); + } + } + return Ty.inst(k, false); + } + } + if (c.func instanceof Attribute) { + Attribute a = (Attribute) c.func; + if (a.value instanceof Name && globalClasses.containsKey(((Name) a.value).id) + && sc.resolve(((Name) a.value).id) == Scope.Binding.GLOBAL) { + // `Cls.method(obj, ...)`: an explicit call of a method. + Layouts.Cls k = globalClasses.get(((Name) a.value).id); + Layouts.Cls o = Layouts.resolver(k, a.attr); + FunctionDef m = o == null ? null : o.typable.get(a.attr); + List args = new ArrayList(); + for (Expr x : c.args) { + args.add(x instanceof Starred ? Ty.DYN : type(x, fn, sc)); + } + for (Keyword kw : c.keywords) { + escape(type(kw.value, fn, sc)); + } + if (m != null && !open.contains(k)) { + Fn f = fns.get(m); + callFn(f, args, 0, simple && args.size() <= f.params.length + && args.size() >= f.params.length - m.args.defaults.size()); + return f.ret; + } + for (Ty t : args) { + escape(t); + } + if (m != null) { + escapeFn(fns.get(m)); + } + return Ty.DYN; + } + Ty r = type(a.value, fn, sc); + List args = new ArrayList(); + for (Expr x : c.args) { + args.add(x instanceof Starred ? Ty.DYN : type(x, fn, sc)); + } + for (Keyword kw : c.keywords) { + escape(type(kw.value, fn, sc)); + } + if (r.k == Ty.K.BOT || r.k == Ty.K.NONE) { + return Ty.BOT; + } + if (r.k == Ty.K.INST && !open.contains(r.cls)) { + List cand = candidates(r.cls, a.attr); + if (cand != null) { + Ty ret = Ty.BOT; + for (FunctionDef m : cand) { + Fn f = fns.get(m); + callFn(f, args, 1, simple && args.size() + 1 == f.params.length); + ret = Ty.join(ret, f.ret); + } + if (Layouts.resolver(r.cls, a.attr) == null) { + // Not every instance has it: the rest raise. + ret = Ty.join(ret, Ty.BOT); + } + return ret; + } + } + if (r.k == Ty.K.LIST && !r.opt) { + return listMethod(r, a.attr, args); + } + escape(r); + for (Ty t : args) { + escape(t); + } + if (r.k == Ty.K.DYN || r.k == Ty.K.INST) { + escapeMethods(a.attr); + } + return Ty.DYN; + } + if (c.func instanceof Name && sc.resolve(((Name) c.func).id) == Scope.Binding.BUILTIN + && !moduleScope.hasStarImport) { + String n = ((Name) c.func).id; + List args = new ArrayList(); + for (Expr x : c.args) { + args.add(x instanceof Starred ? Ty.DYN : type(x, fn, sc)); + } + for (Keyword kw : c.keywords) { + escape(type(kw.value, fn, sc)); + } + if (simple && args.size() == 1 && ("len".equals(n) || "isinstance".equals(n) && false)) { + // len reads its argument and keeps nothing. + if (args.get(0).k != Ty.K.LIST) { + escape(args.get(0)); + } + return Ty.INT; + } + if (simple && "float".equals(n) && args.size() == 1 + && (args.get(0).k == Ty.K.INT || args.get(0).k == Ty.K.FLOAT) && !args.get(0).opt) { + return Ty.FLOAT; + } + if (simple && "abs".equals(n) && args.size() == 1 && !args.get(0).opt + && (args.get(0).k == Ty.K.INT || args.get(0).k == Ty.K.FLOAT)) { + return args.get(0); + } + if (simple && "list".equals(n) && args.size() == 1) { + int s = site(c); + Ty t = args.get(0); + boolean range = rangeCall(c.args.get(0), sc); + siteElem.set(s, joinInto(siteElem.get(s), t.k == Ty.K.LIST ? elem(t) : range ? Ty.INT : Ty.DYN)); + if (t.k != Ty.K.LIST && !rangeCall(c.args.get(0), sc)) { + escape(t); + } + return Ty.list(new int[] {s}, false); + } + for (Ty t : args) { + escape(t); + } + return Ty.DYN; + } + escape(type(c.func, fn, sc)); + for (Expr x : c.args) { + escape(type(x instanceof Starred ? ((Starred) x).value : x, fn, sc)); + } + for (Keyword kw : c.keywords) { + escape(type(kw.value, fn, sc)); + } + return Ty.DYN; + } + + private Ty listMethod(Ty r, String m, List args) { + switch (m) { + case "append": + if (args.size() == 1) { + storeElem(r, args.get(0)); + return Ty.NONE; + } + break; + case "pop": + if (args.size() <= 1) { + return elem(r); + } + break; + case "extend": + if (args.size() == 1) { + storeElem(r, args.get(0).k == Ty.K.LIST ? elem(args.get(0)) : Ty.DYN); + if (args.get(0).k != Ty.K.LIST) { + escape(args.get(0)); + } + return Ty.NONE; + } + break; + case "insert": + if (args.size() == 2) { + storeElem(r, args.get(1)); + return Ty.NONE; + } + break; + case "reverse": + case "clear": + return Ty.NONE; + case "copy": + return Ty.list(r.sites, false); + case "index": + case "count": + return Ty.INT; + default: + break; + } + escape(r); + for (Ty t : args) { + escape(t); + } + return Ty.DYN; + } + + /// Joins a call's arguments into `f`'s parameters, from `from` on. + private void callFn(Fn f, List args, int from, boolean exact) { + if (!exact) { + escapeFn(f); + for (Ty t : args) { + escape(t); + } + return; + } + for (int i = 0; i < args.size(); i++) { + f.params[from + i] = joinInto(f.params[from + i], args.get(i)); + } + } + + // ---- reflection-free traversal --------------------------------------- + + interface Visit { + void on(Object child); + } + + /// Calls `v` for every AST child (node or list element) of `o`. + static void children(Object o, Visit v) { + if (o instanceof List) { + for (Object x : (List) o) { + v.on(x); + } + return; + } + if (!(o instanceof Node)) { + return; + } + for (Class c = o.getClass(); c != null && c != Object.class; c = c.getSuperclass()) { + for (Field f : c.getDeclaredFields()) { + if (Modifier.isStatic(f.getModifiers())) { + continue; + } + Class t = f.getType(); + if (!Node.class.isAssignableFrom(t) && !List.class.isAssignableFrom(t)) { + continue; + } + try { + Object x = f.get(o); + if (x instanceof List) { + for (Object y : (List) x) { + v.on(y); + } + } else if (x != null) { + v.on(x); + } + } catch (IllegalAccessException e) { + throw new IllegalStateException(e); + } + } + } + } + + String dump() { + StringBuilder b = new StringBuilder(); + for (Fn f : fns.values()) { + b.append(f.cls == null ? "" : f.cls.def.name + ".").append(f.def.name).append('('); + for (int i = 0; i < f.params.length; i++) { + b.append(i == 0 ? "" : ", ").append(f.names.get(i)).append(": ").append(f.params[i]); + } + b.append(") -> ").append(f.ret).append(f.escapes ? " [escapes]" : "").append("\n ").append(f.locals) + .append('\n'); + } + for (Map.Entry> e : fields.entrySet()) { + b.append(e.getKey().def.name).append(' ').append(e.getValue()).append(" definite=") + .append(definite.get(e.getKey())).append('\n'); + } + b.append("globals ").append(globals).append("\nsites ").append(siteElem).append(" escaped ") + .append(escapedSites).append(allListsEscape ? " ALL" : "").append('\n'); + return b.toString(); + } +} diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Layouts.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Layouts.java new file mode 100644 index 00000000000..0c527bd6b1f --- /dev/null +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Layouts.java @@ -0,0 +1,307 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.transpiler.codegen; + +import com.codename1.python.transpiler.ast.Node.*; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/// Compiled layouts for the classes a module defines at its top level. +/// +/// Such a class gets a Java subclass of `PyInstance` whose attributes are +/// fields, and whose plain methods are static Java methods reached through +/// a virtual `m_` per method name and arity. Generated code then reads +/// `self.x` as an `instanceof` and a field load, and calls `t.run(p)` as an +/// `instanceof` and a virtual call, instead of going through an inline cache. +/// +/// What is planned here is only ever a fast path: every access still falls +/// back to the generic one when the receiver is not of the expected Java +/// class, a field is unset, or the class was changed after it was built +/// (`PyType.patched`). A field is a name the class's own methods store on +/// their first parameter, and never a name any compiled class binds in its +/// body, since a class attribute -- a property in particular -- must keep +/// deciding what a store to that name does. +final class Layouts { + static final class Cls { + final ClassDef def; + final String javaName; + final String typeVar; + final Cls parent; + final List fields = new ArrayList(); + /// Every field, the inherited ones first: the index order the + /// generated `fieldIndex` uses. + final List allFields = new ArrayList(); + final Set bodyNames; + final Map methods = new LinkedHashMap(); + /// `methods`, plus the plain methods with too many parameters for a + /// generic dispatch: those `Infer` and the typed code still see. + final Map typable = new LinkedHashMap(); + + Cls(ClassDef def, String javaName, String typeVar, Cls parent) { + this.def = def; + this.javaName = javaName; + this.typeVar = typeVar; + this.parent = parent; + this.bodyNames = def.scope.assigned; + } + } + + final Map classes = new LinkedHashMap(); + /// Field name -> the classes that declare it (not inherit it). + final Map> fieldOwners = new LinkedHashMap>(); + /// `name/arity` of every compiled method, arity without the receiver. + final Set dispatch = new LinkedHashSet(); + final Map staticNames = new LinkedHashMap(); + /// The class each compiled method belongs to. + final Map methodClass = new LinkedHashMap(); + + private static final Set FORBIDDEN = new HashSet(); + + static { + // A class that customises attribute access itself is left alone. + FORBIDDEN.add("__setattr__"); + FORBIDDEN.add("__getattribute__"); + FORBIDDEN.add("__delattr__"); + FORBIDDEN.add("__slots__"); + FORBIDDEN.add("__dict__"); + FORBIDDEN.add("__class__"); + FORBIDDEN.add("__new__"); + } + + /// `attrStores` is the module scope's (`Scope.attrStores`): a field the + /// module itself assigns on a class is left to the generic path, since + /// the class attribute -- a property, say -- must then decide its reads. + static Layouts plan(List body, String prefix, Set attrStores) { + Layouts l = new Layouts(); + Map byName = new LinkedHashMap(); + int n = 0; + for (Stmt s : body) { + if (!(s instanceof ClassDef)) { + continue; + } + ClassDef c = (ClassDef) s; + Cls parent = null; + boolean ok = c.keywords.isEmpty() && c.decorators.isEmpty() && c.bases.size() <= 1; + if (ok && c.bases.size() == 1) { + Expr b = c.bases.get(0); + parent = b instanceof Name ? byName.get(((Name) b).id) : null; + ok = parent != null; + } + for (String f : FORBIDDEN) { + ok &= !c.scope.assigned.contains(f); + } + if (!ok) { + byName.remove(c.name); + continue; + } + n++; + Cls k = new Cls(c, prefix + "L" + n + "_" + Emitter.mangle(c.name), "T" + n + "_" + Emitter.mangle(c.name), parent); + l.classes.put(c, k); + byName.put(c.name, k); + } + Set bodyNames = new HashSet(); + for (Cls k : l.classes.values()) { + bodyNames.addAll(k.bodyNames); + } + boolean noFields = false; + for (String st : attrStores) { + int dot = st.indexOf('.'); + if (byName.containsKey(st.substring(0, dot))) { + String a = st.substring(dot + 1); + bodyNames.add(a); + // An attribute hook added from outside sees every access. + noFields |= "*".equals(a) || FORBIDDEN.contains(a); + } + } + for (Cls k : l.classes.values()) { + Map defined = new LinkedHashMap(); + for (Stmt s : k.def.body) { + if (s instanceof FunctionDef) { + String nm = ((FunctionDef) s).name; + defined.put(nm, defined.containsKey(nm) ? defined.get(nm) + 1 : 1); + } + } + Set rebound = new HashSet(); + for (Stmt s : k.def.body) { + if (s instanceof Assign) { + for (Expr t : ((Assign) s).targets) { + names(t, rebound); + } + } else if (s instanceof AugAssign) { + names(((AugAssign) s).target, rebound); + } else if (s instanceof AnnAssign) { + names(((AnnAssign) s).target, rebound); + } else if (s instanceof For) { + names(((For) s).target, rebound); + } else if (s instanceof ClassDef) { + rebound.add(((ClassDef) s).name); + } else if (!(s instanceof FunctionDef) && !(s instanceof ExprStmt) && !(s instanceof Pass)) { + // Anything else might bind a method's name too. + rebound.addAll(k.bodyNames); + } + } + Set stored = new LinkedHashSet(); + for (Stmt s : k.def.body) { + if (!(s instanceof FunctionDef)) { + continue; + } + FunctionDef f = (FunctionDef) s; + List names = f.args.names(); + if (names.isEmpty()) { + continue; + } + selfStores(f.body, names.get(0), stored); + boolean plain = f.decorators.isEmpty() && !f.isAsync && !f.scope.hasYield && f.args.vararg == null + && f.args.kwarg == null && f.args.kwonly.isEmpty() && defined.get(f.name) == 1 + && !rebound.contains(f.name) && !hasFree(f); + if (plain && names.size() <= 8) { + k.typable.put(f.name, f); + } + if (plain && names.size() <= 4) { + k.methods.put(f.name, f); + l.dispatch.add(f.name + "/" + (names.size() - 1)); + l.staticNames.put(f, "M_" + k.javaName.substring(prefix.length()) + "_" + Emitter.mangle(f.name)); + l.methodClass.put(f, k); + } + } + if (k.parent != null) { + k.allFields.addAll(k.parent.allFields); + } + for (String f : stored) { + if (noFields || f.startsWith("__") && f.endsWith("__") || bodyNames.contains(f) || k.allFields.contains(f)) { + continue; + } + k.fields.add(f); + k.allFields.add(f); + List o = l.fieldOwners.get(f); + if (o == null) { + o = new ArrayList(); + l.fieldOwners.put(f, o); + } + o.add(k); + } + } + return l; + } + + /// Whether the method uses `super()` or `__class__`, the one way a method + /// at the top of a module captures anything: its body becomes a static + /// Java method, which can capture nothing. + private static boolean hasFree(FunctionDef f) { + return f.scope.used.contains("super") || f.scope.used.contains("__class__"); + } + + /// The attribute names `self. = ...` stores in `body`, outside + /// nested functions and classes. + private static void selfStores(List body, String self, Set out) { + for (Stmt s : body) { + if (s instanceof Assign) { + for (Expr t : ((Assign) s).targets) { + target(t, self, out); + } + } else if (s instanceof AugAssign) { + target(((AugAssign) s).target, self, out); + } else if (s instanceof AnnAssign) { + target(((AnnAssign) s).target, self, out); + } else if (s instanceof For) { + target(((For) s).target, self, out); + selfStores(((For) s).body, self, out); + selfStores(((For) s).orelse, self, out); + } else if (s instanceof While) { + selfStores(((While) s).body, self, out); + selfStores(((While) s).orelse, self, out); + } else if (s instanceof If) { + selfStores(((If) s).body, self, out); + selfStores(((If) s).orelse, self, out); + } else if (s instanceof With) { + for (WithItem w : ((With) s).items) { + if (w.vars != null) { + target(w.vars, self, out); + } + } + selfStores(((With) s).body, self, out); + } else if (s instanceof Try) { + Try t = (Try) s; + selfStores(t.body, self, out); + for (ExceptHandler h : t.handlers) { + selfStores(h.body, self, out); + } + selfStores(t.orelse, self, out); + selfStores(t.finalbody, self, out); + } + } + } + + private static void names(Expr t, Set out) { + if (t instanceof Name) { + out.add(((Name) t).id); + } else if (t instanceof TupleExpr) { + for (Expr e : ((TupleExpr) t).elts) { + names(e, out); + } + } else if (t instanceof ListExpr) { + for (Expr e : ((ListExpr) t).elts) { + names(e, out); + } + } else if (t instanceof Starred) { + names(((Starred) t).value, out); + } + } + + private static void target(Expr t, String self, Set out) { + if (t instanceof Attribute) { + Attribute a = (Attribute) t; + if (a.value instanceof Name && ((Name) a.value).id.equals(self)) { + out.add(a.attr); + } + } else if (t instanceof TupleExpr) { + for (Expr e : ((TupleExpr) t).elts) { + target(e, self, out); + } + } else if (t instanceof ListExpr) { + for (Expr e : ((ListExpr) t).elts) { + target(e, self, out); + } + } else if (t instanceof Starred) { + target(((Starred) t).value, self, out); + } + } + + /// What `name` resolves to for class `k`: the class whose body binds it + /// first along the compiled chain, or null when none does. + static Cls resolver(Cls k, String name) { + for (Cls c = k; c != null; c = c.parent) { + if (c.bodyNames.contains(name)) { + return c; + } + } + return null; + } +} diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Linearize.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Linearize.java new file mode 100644 index 00000000000..4c3dc879a1d --- /dev/null +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Linearize.java @@ -0,0 +1,772 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.transpiler.codegen; + +import com.codename1.python.transpiler.api.PythonSyntaxError; +import com.codename1.python.transpiler.ast.Node; +import com.codename1.python.transpiler.ast.Node.Assign; +import com.codename1.python.transpiler.ast.Node.Attribute; +import com.codename1.python.transpiler.ast.Node.AugAssign; +import com.codename1.python.transpiler.ast.Node.AnnAssign; +import com.codename1.python.transpiler.ast.Node.Await; +import com.codename1.python.transpiler.ast.Node.Comprehension; +import com.codename1.python.transpiler.ast.Node.BinOp; +import com.codename1.python.transpiler.ast.Node.BoolOp; +import com.codename1.python.transpiler.ast.Node.Break; +import com.codename1.python.transpiler.ast.Node.Call; +import com.codename1.python.transpiler.ast.Node.ClassDef; +import com.codename1.python.transpiler.ast.Node.Comp; +import com.codename1.python.transpiler.ast.Node.Compare; +import com.codename1.python.transpiler.ast.Node.Constant; +import com.codename1.python.transpiler.ast.Node.DictExpr; +import com.codename1.python.transpiler.ast.Node.ExceptHandler; +import com.codename1.python.transpiler.ast.Node.Expr; +import com.codename1.python.transpiler.ast.Node.ExprStmt; +import com.codename1.python.transpiler.ast.Node.For; +import com.codename1.python.transpiler.ast.Node.FormattedValue; +import com.codename1.python.transpiler.ast.Node.FunctionDef; +import com.codename1.python.transpiler.ast.Node.If; +import com.codename1.python.transpiler.ast.Node.IfExp; +import com.codename1.python.transpiler.ast.Node.JoinedStr; +import com.codename1.python.transpiler.ast.Node.Keyword; +import com.codename1.python.transpiler.ast.Node.Lambda; +import com.codename1.python.transpiler.ast.Node.ListExpr; +import com.codename1.python.transpiler.ast.Node.Match; +import com.codename1.python.transpiler.ast.Node.MatchCase; +import com.codename1.python.transpiler.ast.Node.Name; +import com.codename1.python.transpiler.ast.Node.NamedExpr; +import com.codename1.python.transpiler.ast.Node.Raise; +import com.codename1.python.transpiler.ast.Node.Return; +import com.codename1.python.transpiler.ast.Node.SetExpr; +import com.codename1.python.transpiler.ast.Node.Slice; +import com.codename1.python.transpiler.ast.Node.Starred; +import com.codename1.python.transpiler.ast.Node.Stmt; +import com.codename1.python.transpiler.ast.Node.Subscript; +import com.codename1.python.transpiler.ast.Node.Try; +import com.codename1.python.transpiler.ast.Node.TupleExpr; +import com.codename1.python.transpiler.ast.Node.UnaryOp; +import com.codename1.python.transpiler.ast.Node.While; +import com.codename1.python.transpiler.ast.Node.With; +import com.codename1.python.transpiler.ast.Node.WithItem; +import com.codename1.python.transpiler.ast.Node.Yield; +import com.codename1.python.transpiler.ast.Scope; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/// Moves every `yield` and `await` of a generator or coroutine body to the +/// top of a statement, where [GenLowering] can split the body at it. +/// +/// `x = f(a, await b)` becomes `$y1 = f; $y2 = a; $y3 = await b; +/// x = $y1($y2, $y3)`: everything evaluated before the suspension is kept in +/// a hidden local, so Python's left-to-right order survives, and `and`, `or` +/// and `if`-expressions become `if` statements so a suspension in their +/// right-hand side still happens only when Python would evaluate it. Only +/// the headers of a statement are rewritten; [GenLowering] lowers the bodies +/// statement by statement and comes back here for each. +final class Linearize { + private final String file; + private final Scope scope; + private int uid; + + Linearize(String file, Scope scope) { + this.file = file; + this.scope = scope; + } + + // ---- where the suspensions are ----------------------------------------- + + /// Whether a comprehension is asynchronous: an `async for` clause, or an + /// `await` in its element, its conditions or an inner iterable. + static boolean isAsync(Comp c) { + if (suspends(c.elt) || suspends(c.value)) { + return true; + } + for (int i = 0; i < c.generators.size(); i++) { + Comprehension g = c.generators.get(i); + if (g.isAsync || i > 0 && suspends(g.iter) || any(g.ifs)) { + return true; + } + } + return false; + } + + /// An async comprehension evaluated by awaiting it: all but a generator + /// expression, which is an async generator instead. + static boolean awaited(Comp c) { + return c.kind != Comp.Kind.GENERATOR && isAsync(c); + } + + /// Whether evaluating `e` can suspend: a yield or await outside any + /// nested function or comprehension body. + static boolean suspends(Expr e) { + if (e == null) { + return false; + } + if (e instanceof Yield || e instanceof Await) { + return true; + } + if (e instanceof Lambda) { + return false; + } + if (e instanceof Comp) { + // Only the first iterable is evaluated in the enclosing scope, + // but an async list, set or dict comprehension is awaited. + return suspends(((Comp) e).generators.get(0).iter) || awaited((Comp) e); + } + if (e instanceof NamedExpr) { + return suspends(((NamedExpr) e).value); + } + if (e instanceof BoolOp) { + return any(((BoolOp) e).values); + } + if (e instanceof IfExp) { + IfExp i = (IfExp) e; + return suspends(i.test) || suspends(i.body) || suspends(i.orelse); + } + if (e instanceof FormattedValue) { + FormattedValue f = (FormattedValue) e; + return suspends(f.value) || suspends(f.spec); + } + return any(children(e)); + } + + private static boolean any(List l) { + for (Expr x : l) { + if (suspends(x)) { + return true; + } + } + return false; + } + + /// Whether running `s` can suspend, its nested statements included. + static boolean suspends(Stmt s) { + if (s instanceof FunctionDef) { + FunctionDef f = (FunctionDef) s; + return any(f.decorators) || any(f.args.defaults) || anyNullable(f.args.kwDefaults); + } + if (s instanceof ClassDef) { + ClassDef c = (ClassDef) s; + return any(c.decorators) || any(c.bases); + } + for (Expr e : headers(s)) { + if (suspends(e)) { + return true; + } + } + for (List b : bodies(s)) { + if (suspends(b)) { + return true; + } + } + if (s instanceof For && ((For) s).isAsync) { + return true; + } + return s instanceof With && ((With) s).isAsync; + } + + static boolean suspends(List body) { + for (Stmt s : body) { + if (suspends(s)) { + return true; + } + } + return false; + } + + private static boolean anyNullable(List l) { + for (Expr x : l) { + if (x != null && suspends(x)) { + return true; + } + } + return false; + } + + /// The expressions a statement evaluates itself, not counting its body. + static List headers(Stmt s) { + List r = new ArrayList(); + if (s instanceof ExprStmt) { + r.add(((ExprStmt) s).value); + } else if (s instanceof Assign) { + r.addAll(((Assign) s).targets); + r.add(((Assign) s).value); + } else if (s instanceof AugAssign) { + r.add(((AugAssign) s).target); + r.add(((AugAssign) s).value); + } else if (s instanceof AnnAssign) { + r.add(((AnnAssign) s).target); + if (((AnnAssign) s).value != null) { + r.add(((AnnAssign) s).value); + } + } else if (s instanceof Return) { + if (((Return) s).value != null) { + r.add(((Return) s).value); + } + } else if (s instanceof If) { + r.add(((If) s).test); + } else if (s instanceof While) { + r.add(((While) s).test); + } else if (s instanceof For) { + r.add(((For) s).target); + r.add(((For) s).iter); + } else if (s instanceof With) { + for (WithItem w : ((With) s).items) { + r.add(w.context); + if (w.vars != null) { + r.add(w.vars); + } + } + } else if (s instanceof Raise) { + if (((Raise) s).exc != null) { + r.add(((Raise) s).exc); + } + if (((Raise) s).cause != null) { + r.add(((Raise) s).cause); + } + } else if (s instanceof Node.Assert) { + r.add(((Node.Assert) s).test); + if (((Node.Assert) s).msg != null) { + r.add(((Node.Assert) s).msg); + } + } else if (s instanceof Node.Delete) { + r.addAll(((Node.Delete) s).targets); + } else if (s instanceof Match) { + r.add(((Match) s).subject); + for (MatchCase c : ((Match) s).cases) { + if (c.guard != null) { + r.add(c.guard); + } + } + } else if (s instanceof Try) { + for (ExceptHandler h : ((Try) s).handlers) { + if (h.type != null) { + r.add(h.type); + } + } + } else if (s instanceof Node.TypeAlias) { + r.add(((Node.TypeAlias) s).value); + } + return r; + } + + /// The statement lists nested in `s`. + static List> bodies(Stmt s) { + List> r = new ArrayList>(); + if (s instanceof If) { + r.add(((If) s).body); + r.add(((If) s).orelse); + } else if (s instanceof While) { + r.add(((While) s).body); + r.add(((While) s).orelse); + } else if (s instanceof For) { + r.add(((For) s).body); + r.add(((For) s).orelse); + } else if (s instanceof With) { + r.add(((With) s).body); + } else if (s instanceof Try) { + Try t = (Try) s; + r.add(t.body); + for (ExceptHandler h : t.handlers) { + r.add(h.body); + } + r.add(t.orelse); + r.add(t.finalbody); + } else if (s instanceof Match) { + for (MatchCase c : ((Match) s).cases) { + r.add(c.body); + } + } + return r; + } + + /// The operands of `e` in evaluation order (nulls left out). + static List children(Expr e) { + List r = new ArrayList(); + if (e instanceof Attribute) { + r.add(((Attribute) e).value); + } else if (e instanceof Subscript) { + r.add(((Subscript) e).value); + r.add(((Subscript) e).index); + } else if (e instanceof Slice) { + Slice s = (Slice) e; + addIf(r, s.lower); + addIf(r, s.upper); + addIf(r, s.step); + } else if (e instanceof Starred) { + r.add(((Starred) e).value); + } else if (e instanceof BinOp) { + r.add(((BinOp) e).left); + r.add(((BinOp) e).right); + } else if (e instanceof UnaryOp) { + r.add(((UnaryOp) e).operand); + } else if (e instanceof Compare) { + r.add(((Compare) e).left); + r.addAll(((Compare) e).comparators); + } else if (e instanceof Call) { + Call c = (Call) e; + r.add(c.func); + r.addAll(c.args); + for (Keyword k : c.keywords) { + r.add(k.value); + } + } else if (e instanceof ListExpr) { + r.addAll(((ListExpr) e).elts); + } else if (e instanceof TupleExpr) { + r.addAll(((TupleExpr) e).elts); + } else if (e instanceof SetExpr) { + r.addAll(((SetExpr) e).elts); + } else if (e instanceof DictExpr) { + DictExpr d = (DictExpr) e; + for (int i = 0; i < d.keys.size(); i++) { + addIf(r, d.keys.get(i)); + r.add(d.values.get(i)); + } + } else if (e instanceof JoinedStr) { + r.addAll(((JoinedStr) e).parts); + } else if (e instanceof Yield) { + addIf(r, ((Yield) e).value); + } else if (e instanceof Await) { + r.add(((Await) e).value); + } + return r; + } + + private static void addIf(List r, Expr e) { + if (e != null) { + r.add(e); + } + } + + // ---- statements -------------------------------------------------------- + + /// The statements that replace `s`, so that no header of the last one + /// suspends except as the whole value of an expression statement or an + /// assignment; null when `s` is already in that form. + List stmt(Stmt s) { + if (s instanceof ExprStmt) { + Expr v = ((ExprStmt) s).value; + if (!suspends(v) || topOnly(v)) { + return null; + } + List pre = new ArrayList(); + Expr n = top(v, pre); + pre.add(at(new ExprStmt(n), s)); + return pre; + } + if (s instanceof Assign) { + Assign a = (Assign) s; + if (any(a.targets)) { + throw err(s, "await or yield inside an assignment target"); + } + if (!suspends(a.value) || topOnly(a.value)) { + return null; + } + List pre = new ArrayList(); + Expr n = top(a.value, pre); + pre.add(at(new Assign(a.targets, n), s)); + return pre; + } + if (s instanceof AnnAssign) { + AnnAssign a = (AnnAssign) s; + if (a.value == null || scope.kind == Scope.Kind.CLASS) { + return null; + } + // Inside a function an annotation is never evaluated. + List r = new ArrayList(); + r.add(at(new Assign(Collections.singletonList(a.target), a.value), s)); + return r; + } + List pre = new ArrayList(); + if (s instanceof AugAssign) { + AugAssign a = (AugAssign) s; + if (suspends(a.target)) { + throw err(s, "await or yield inside an assignment target"); + } + if (!suspends(a.value)) { + return null; + } + pre.add(at(new AugAssign(a.target, a.op, lin(a.value, pre)), s)); + return pre; + } + if (s instanceof Return) { + Return r = (Return) s; + if (r.value == null || !suspends(r.value)) { + return null; + } + pre.add(at(new Return(lin(r.value, pre)), s)); + return pre; + } + if (s instanceof If) { + If i = (If) s; + if (!suspends(i.test)) { + return null; + } + pre.add(at(new If(lin(i.test, pre), i.body, i.orelse), s)); + return pre; + } + if (s instanceof While) { + While w = (While) s; + if (!suspends(w.test)) { + return null; + } + if (!w.orelse.isEmpty()) { + throw err(s, "await or yield in the condition of a while loop with an else clause"); + } + // while T: B -> while True: (T's pre); if not T: break; B + List body = new ArrayList(); + Expr t = lin(w.test, body); + List brk = new ArrayList(); + brk.add(at(new Break(), s)); + body.add(at(new If(at(new UnaryOp("not", t), s), brk, new ArrayList()), s)); + body.addAll(w.body); + pre.add(at(new While(at(new Constant(Constant.Kind.TRUE, "True"), s), body, w.orelse), s)); + return pre; + } + if (s instanceof For) { + For f = (For) s; + if (suspends(f.target)) { + throw err(s, "await or yield inside a for target"); + } + if (!suspends(f.iter)) { + return null; + } + pre.add(at(new For(f.target, lin(f.iter, pre), f.body, f.orelse, f.isAsync), s)); + return pre; + } + if (s instanceof With) { + With w = (With) s; + if (w.items.size() > 1) { + // `with a, b: B` is `with a: with b: B`. + List inner = new ArrayList(); + inner.add(at(new With(new ArrayList(w.items.subList(1, w.items.size())), w.body, + w.isAsync), s)); + pre.add(at(new With(Collections.singletonList(w.items.get(0)), inner, w.isAsync), s)); + return pre; + } + WithItem item = w.items.get(0); + if (item.vars != null && suspends(item.vars)) { + throw err(s, "await or yield inside a with target"); + } + if (!suspends(item.context)) { + return null; + } + WithItem n = new WithItem(lin(item.context, pre), item.vars); + n.line = item.line; + n.col = item.col; + pre.add(at(new With(Collections.singletonList(n), w.body, w.isAsync), s)); + return pre; + } + if (s instanceof Raise) { + Raise r = (Raise) s; + if (!suspends(r.exc) && !suspends(r.cause)) { + return null; + } + List parts = new ArrayList(); + addIf(parts, r.exc); + addIf(parts, r.cause); + List n = seq(parts, pre); + pre.add(at(new Raise(r.exc == null ? null : n.get(0), r.cause == null ? null : n.get(n.size() - 1)), s)); + return pre; + } + if (s instanceof Node.Assert) { + Node.Assert a = (Node.Assert) s; + if (a.msg != null && suspends(a.msg)) { + throw err(s, "await or yield inside an assert message"); + } + if (!suspends(a.test)) { + return null; + } + pre.add(at(new Node.Assert(lin(a.test, pre), a.msg), s)); + return pre; + } + if (s instanceof Match) { + Match m = (Match) s; + for (MatchCase c : m.cases) { + if (c.guard != null && suspends(c.guard)) { + throw err(s, "await or yield inside a case guard"); + } + } + if (!suspends(m.subject)) { + return null; + } + pre.add(at(new Match(lin(m.subject, pre), m.cases), s)); + return pre; + } + if (s instanceof Try) { + for (ExceptHandler h : ((Try) s).handlers) { + if (h.type != null && suspends(h.type)) { + throw err(h, "await or yield inside an except clause's type"); + } + } + return null; + } + for (Expr e : headers(s)) { + if (suspends(e)) { + throw err(s, "await or yield here"); + } + } + if ((s instanceof FunctionDef || s instanceof ClassDef) && suspends(s)) { + throw err(s, "await or yield in a decorator, default or base class"); + } + return null; + } + + /// A suspension whose operand does not suspend itself. + private static boolean topOnly(Expr v) { + if (v instanceof Yield) { + return !suspends(((Yield) v).value); + } + if (!(v instanceof Await)) { + return false; + } + Expr x = ((Await) v).value; + if (x instanceof Comp && awaited((Comp) x)) { + return !suspends(((Comp) x).generators.get(0).iter); + } + return !suspends(x); + } + + /// `v` with its operands linearized, keeping a suspension at its top. + private Expr top(Expr v, List pre) { + if (v instanceof Yield) { + Yield y = (Yield) v; + return at(new Yield(lin(y.value, pre), y.from), v); + } + if (v instanceof Await) { + Expr x = ((Await) v).value; + if (x instanceof Comp && awaited((Comp) x)) { + // An async comprehension, awaited where lin put it. + return v; + } + return at(new Await(lin(x, pre)), v); + } + return lin(v, pre); + } + + // ---- expressions ------------------------------------------------------- + + /// `e` rewritten to suspend nowhere, with what it must evaluate first + /// appended to `pre`. + Expr lin(Expr e, List pre) { + if (e == null || !suspends(e)) { + return e; + } + if (e instanceof Yield || e instanceof Await) { + Name t = hidden(e); + pre.add(at(new Assign(Collections.singletonList(t), top(e, pre)), e)); + return name(t); + } + if (e instanceof BoolOp) { + return boolOp((BoolOp) e, pre); + } + if (e instanceof IfExp) { + IfExp i = (IfExp) e; + Expr test = lin(i.test, pre); + Name r = hidden(e); + List a = new ArrayList(); + a.add(assign(r, lin(i.body, a), e)); + List b = new ArrayList(); + b.add(assign(r, lin(i.orelse, b), e)); + pre.add(at(new If(test, a, b), e)); + return name(r); + } + if (e instanceof NamedExpr) { + NamedExpr n = (NamedExpr) e; + return at(new NamedExpr(n.target, lin(n.value, pre)), e); + } + if (e instanceof Comp) { + Comp c = (Comp) e; + if (suspends(c.generators.get(0).iter)) { + throw err(e, "await in a comprehension's iterable"); + } + // An async comprehension is a coroutine the enclosing one awaits. + Await w = at(new Await(c), e); + Name t = hidden(e); + pre.add(at(new Assign(Collections.singletonList(t), w), e)); + return name(t); + } + if (e instanceof Compare) { + Compare c = (Compare) e; + for (int i = 1; i < c.comparators.size(); i++) { + if (suspends(c.comparators.get(i))) { + throw err(e, "await or yield inside a chained comparison"); + } + } + } + if (e instanceof FormattedValue) { + FormattedValue f = (FormattedValue) e; + if (suspends(f.spec)) { + throw err(e, "await or yield inside a format spec"); + } + return at(new FormattedValue(lin(f.value, pre), f.conversion, f.spec), e); + } + return rebuild(e, seq(children(e), pre)); + } + + /// Linearizes operands evaluated in order: each one before the last that + /// suspends is kept in a hidden local, so it is evaluated first. + private List seq(List kids, List pre) { + int last = -1; + for (int i = 0; i < kids.size(); i++) { + if (suspends(kids.get(i))) { + last = i; + } + } + List r = new ArrayList(); + for (int i = 0; i < kids.size(); i++) { + Expr k = kids.get(i); + if (i < last) { + r.add(suspends(k) ? spill(lin(k, pre), pre) : spill(k, pre)); + } else if (i == last) { + r.add(lin(k, pre)); + } else { + r.add(k); + } + } + return r; + } + + /// `k`'s value in a hidden local, or `k` when evaluating it later is the + /// same as evaluating it now. + private Expr spill(Expr k, List pre) { + if (k instanceof Constant) { + return k; + } + if (k instanceof Starred) { + return at(new Starred(spill(((Starred) k).value, pre)), k); + } + if (k instanceof Slice) { + Slice s = (Slice) k; + return at(new Slice(s.lower == null ? null : spill(s.lower, pre), s.upper == null ? null + : spill(s.upper, pre), s.step == null ? null : spill(s.step, pre)), k); + } + Name t = hidden(k); + pre.add(assign(t, k, k)); + return name(t); + } + + private Expr boolOp(BoolOp b, List pre) { + Name r = hidden(b); + pre.add(assign(r, lin(b.values.get(0), pre), b)); + List into = pre; + for (int i = 1; i < b.values.size(); i++) { + List body = new ArrayList(); + body.add(assign(r, lin(b.values.get(i), body), b)); + Expr test = b.isAnd ? name(r) : at(new UnaryOp("not", name(r)), b); + into.add(at(new If(test, body, new ArrayList()), b)); + into = body; + } + return name(r); + } + + /// `e` with its operands (in [#children] order) replaced. + private Expr rebuild(Expr e, List k) { + int i = 0; + Expr r; + if (e instanceof Attribute) { + r = new Attribute(k.get(0), ((Attribute) e).attr); + } else if (e instanceof Subscript) { + r = new Subscript(k.get(0), k.get(1)); + } else if (e instanceof Slice) { + Slice s = (Slice) e; + Expr lo = s.lower == null ? null : k.get(i++); + Expr up = s.upper == null ? null : k.get(i++); + Expr st = s.step == null ? null : k.get(i++); + r = new Slice(lo, up, st); + } else if (e instanceof Starred) { + r = new Starred(k.get(0)); + } else if (e instanceof BinOp) { + r = new BinOp(k.get(0), ((BinOp) e).op, k.get(1)); + } else if (e instanceof UnaryOp) { + r = new UnaryOp(((UnaryOp) e).op, k.get(0)); + } else if (e instanceof Compare) { + Compare c = new Compare(k.get(0)); + c.ops.addAll(((Compare) e).ops); + c.comparators.addAll(k.subList(1, k.size())); + r = c; + } else if (e instanceof Call) { + Call o = (Call) e; + Call c = new Call(k.get(i++)); + for (int j = 0; j < o.args.size(); j++) { + c.args.add(k.get(i++)); + } + for (Keyword kw : o.keywords) { + Keyword n = new Keyword(kw.arg, k.get(i++)); + n.line = kw.line; + n.col = kw.col; + c.keywords.add(n); + } + r = c; + } else if (e instanceof ListExpr) { + r = new ListExpr(k); + } else if (e instanceof TupleExpr) { + r = new TupleExpr(k); + } else if (e instanceof SetExpr) { + r = new SetExpr(k); + } else if (e instanceof DictExpr) { + DictExpr o = (DictExpr) e; + DictExpr d = new DictExpr(); + for (int j = 0; j < o.keys.size(); j++) { + d.keys.add(o.keys.get(j) == null ? null : k.get(i++)); + d.values.add(k.get(i++)); + } + r = d; + } else if (e instanceof JoinedStr) { + JoinedStr j = new JoinedStr(); + j.parts.addAll(k); + r = j; + } else { + throw err(e, "await or yield inside " + e.getClass().getSimpleName()); + } + return at(r, e); + } + + // ---- helpers ----------------------------------------------------------- + + private Name hidden(Node at) { + String n = "$y" + (++uid); + scope.assigned.add(n); + return at(new Name(n), at); + } + + private static Name name(Name n) { + return at(new Name(n.id), n); + } + + private static Assign assign(Name t, Expr v, Node at) { + return at(new Assign(Collections.singletonList(name(t)), v), at); + } + + private static T at(T n, Node from) { + n.line = from.line; + n.col = from.col; + return n; + } + + private PythonSyntaxError err(Node n, String what) { + return new PythonSyntaxError(file, n.line, n.col, what + " is not supported yet"); + } +} diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/ListRecycling.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/ListRecycling.java new file mode 100644 index 00000000000..932d0702d4b --- /dev/null +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/ListRecycling.java @@ -0,0 +1,307 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.transpiler.codegen; + +import com.codename1.python.transpiler.ast.Node.*; +import com.codename1.python.transpiler.ast.Scope; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/// Finds the locals whose list a new slice copy may be written into, in +/// place of a new list: `perm = perm1[:]` in a loop then copies into the list +/// `perm` held before, which nothing else can still see. +/// +/// A local qualifies when every assignment to it is a slice copy +/// (`x = y[a:b]`) or `x = []`, each of which makes a list nobody else holds, +/// and every read of it is the object of a subscript (`x[i]`, `x[i] = v`, +/// `x[a:b]`), the receiver of `x.append(v)` or the argument of a literal's +/// `"sep".join(x)`, none of which hands the list itself anywhere. Its list is then reachable +/// through the local alone, so overwriting it is unobservable. Parameters +/// never qualify (the caller holds their first value), nor does a local a +/// nested scope captures. Any construct the walk does not know makes it give +/// up on the whole function. +final class ListRecycling { + private final Set bad = new HashSet(); + private final Set fresh = new HashSet(); + private boolean unknown; + + private ListRecycling() { + } + + static Set find(List body, List params, Scope fs) { + ListRecycling r = new ListRecycling(); + r.stmts(body, false); + Set out = new HashSet(); + if (r.unknown) { + return out; + } + for (String n : r.fresh) { + if (!r.bad.contains(n) && !params.contains(n) && fs.isLocal(n) && !fs.cells.contains(n)) { + out.add(n); + } + } + return out; + } + + static boolean isSliceCopy(Expr e) { + return e instanceof Subscript && ((Subscript) e).index instanceof Slice; + } + + static boolean isEmptyList(Expr e) { + return e instanceof ListExpr && ((ListExpr) e).elts.isEmpty(); + } + + /// Whether `x` in this call is used without being handed anywhere: + /// `x.append(v)`, or `"sep".join(x)` on a literal, which is always str's + /// own join and keeps nothing. + private static boolean harmlessCall(Call c) { + if (!c.keywords.isEmpty() || c.args.size() != 1 || !(c.func instanceof Attribute)) { + return false; + } + Attribute f = (Attribute) c.func; + if ("append".equals(f.attr) && f.value instanceof Name) { + return true; + } + return "join".equals(f.attr) && f.value instanceof Constant + && ((Constant) f.value).kind == Constant.Kind.STR && c.args.get(0) instanceof Name; + } + + private void stmts(List body, boolean esc) { + if (body == null) { + return; + } + for (Stmt s : body) { + stmt(s, esc); + } + } + + private void stmt(Stmt s, boolean esc) { + if (s instanceof ExprStmt) { + expr(((ExprStmt) s).value, esc); + } else if (s instanceof Assign) { + Assign a = (Assign) s; + expr(a.value, esc); + if (!esc && a.targets.size() == 1 && a.targets.get(0) instanceof Name && (isSliceCopy(a.value) || isEmptyList(a.value))) { + fresh.add(((Name) a.targets.get(0)).id); + return; + } + for (Expr t : a.targets) { + target(t, esc); + } + } else if (s instanceof AugAssign) { + AugAssign a = (AugAssign) s; + expr(a.value, esc); + target(a.target, esc); + } else if (s instanceof AnnAssign) { + AnnAssign a = (AnnAssign) s; + expr(a.value, esc); + target(a.target, esc); + } else if (s instanceof Return) { + expr(((Return) s).value, esc); + } else if (s instanceof Delete) { + for (Expr t : ((Delete) s).targets) { + target(t, esc); + } + } else if (s instanceof Raise) { + expr(((Raise) s).exc, esc); + expr(((Raise) s).cause, esc); + } else if (s instanceof Assert) { + expr(((Assert) s).test, esc); + expr(((Assert) s).msg, esc); + } else if (s instanceof Global) { + bad.addAll(((Global) s).names); + } else if (s instanceof If) { + expr(((If) s).test, esc); + stmts(((If) s).body, esc); + stmts(((If) s).orelse, esc); + } else if (s instanceof While) { + expr(((While) s).test, esc); + stmts(((While) s).body, esc); + stmts(((While) s).orelse, esc); + } else if (s instanceof For) { + For f = (For) s; + expr(f.iter, esc); + target(f.target, esc); + stmts(f.body, esc); + stmts(f.orelse, esc); + } else if (s instanceof With) { + for (WithItem w : ((With) s).items) { + expr(w.context, esc); + if (w.vars != null) { + target(w.vars, esc); + } + } + stmts(((With) s).body, esc); + } else if (s instanceof Try) { + Try t = (Try) s; + stmts(t.body, esc); + for (ExceptHandler h : t.handlers) { + expr(h.type, esc); + if (h.name != null) { + bad.add(h.name); + } + stmts(h.body, esc); + } + stmts(t.orelse, esc); + stmts(t.finalbody, esc); + } else if (s instanceof FunctionDef) { + FunctionDef f = (FunctionDef) s; + bad.add(f.name); + exprs(f.decorators, true); + exprs(f.args.defaults, true); + exprs(f.args.kwDefaults, true); + stmts(f.body, true); + } else if (s instanceof ClassDef) { + ClassDef c = (ClassDef) s; + bad.add(c.name); + exprs(c.decorators, true); + exprs(c.bases, true); + for (Keyword k : c.keywords) { + expr(k.value, true); + } + stmts(c.body, true); + } else if (!(s instanceof Pass || s instanceof Break || s instanceof Continue)) { + unknown = true; + } + } + + /// A store target: a name stored by anything but a slice copy is out; a + /// subscript stores into its object without handing it anywhere. + private void target(Expr t, boolean esc) { + if (t instanceof Name) { + bad.add(((Name) t).id); + } else if (t instanceof Subscript) { + expr(t, esc); + } else if (t instanceof TupleExpr) { + for (Expr e : ((TupleExpr) t).elts) { + target(e, esc); + } + } else if (t instanceof ListExpr) { + for (Expr e : ((ListExpr) t).elts) { + target(e, esc); + } + } else if (t instanceof Starred) { + target(((Starred) t).value, esc); + } else { + expr(t, esc); + } + } + + private void exprs(List es, boolean esc) { + if (es == null) { + return; + } + for (Expr e : es) { + expr(e, esc); + } + } + + private void expr(Expr e, boolean esc) { + if (e == null || e instanceof Constant) { + return; + } + if (e instanceof Name) { + bad.add(((Name) e).id); + } else if (e instanceof Subscript) { + Subscript s = (Subscript) e; + if (esc || !(s.value instanceof Name)) { + expr(s.value, esc); + } + expr(s.index, esc); + } else if (e instanceof Slice) { + expr(((Slice) e).lower, esc); + expr(((Slice) e).upper, esc); + expr(((Slice) e).step, esc); + } else if (e instanceof BinOp) { + expr(((BinOp) e).left, esc); + expr(((BinOp) e).right, esc); + } else if (e instanceof UnaryOp) { + expr(((UnaryOp) e).operand, esc); + } else if (e instanceof BoolOp) { + exprs(((BoolOp) e).values, esc); + } else if (e instanceof Compare) { + expr(((Compare) e).left, esc); + exprs(((Compare) e).comparators, esc); + } else if (e instanceof IfExp) { + expr(((IfExp) e).test, esc); + expr(((IfExp) e).body, esc); + expr(((IfExp) e).orelse, esc); + } else if (e instanceof Attribute) { + expr(((Attribute) e).value, esc); + } else if (e instanceof Call) { + Call c = (Call) e; + if (!esc && harmlessCall(c)) { + Attribute f = (Attribute) c.func; + expr(f.value instanceof Name ? null : f.value, esc); + expr(f.value instanceof Name ? c.args.get(0) : null, esc); + return; + } + expr(c.func, esc); + exprs(c.args, esc); + for (Keyword k : c.keywords) { + expr(k.value, esc); + } + } else if (e instanceof Starred) { + expr(((Starred) e).value, esc); + } else if (e instanceof NamedExpr) { + bad.add(((NamedExpr) e).target.id); + expr(((NamedExpr) e).value, esc); + } else if (e instanceof ListExpr) { + exprs(((ListExpr) e).elts, esc); + } else if (e instanceof TupleExpr) { + exprs(((TupleExpr) e).elts, esc); + } else if (e instanceof SetExpr) { + exprs(((SetExpr) e).elts, esc); + } else if (e instanceof DictExpr) { + exprs(((DictExpr) e).keys, esc); + exprs(((DictExpr) e).values, esc); + } else if (e instanceof JoinedStr) { + exprs(((JoinedStr) e).parts, esc); + } else if (e instanceof FormattedValue) { + expr(((FormattedValue) e).value, esc); + expr(((FormattedValue) e).spec, esc); + } else if (e instanceof Await) { + expr(((Await) e).value, esc); + } else if (e instanceof Yield) { + expr(((Yield) e).value, esc); + } else if (e instanceof Lambda) { + Lambda l = (Lambda) e; + exprs(l.args.defaults, true); + exprs(l.args.kwDefaults, true); + expr(l.body, true); + } else if (e instanceof Comp) { + Comp c = (Comp) e; + expr(c.elt, true); + expr(c.value, true); + for (Comprehension g : c.generators) { + expr(g.target, true); + expr(g.iter, true); + exprs(g.ifs, true); + } + } else { + unknown = true; + } + } +} diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/ShadowTypes.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/ShadowTypes.java new file mode 100644 index 00000000000..4f6fb86afee --- /dev/null +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/ShadowTypes.java @@ -0,0 +1,119 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.transpiler.codegen; + +import com.codename1.python.transpiler.ast.Node; +import com.codename1.python.transpiler.ast.Node.*; + +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/// Picks, for each local of a function body, the compiled class its field +/// accesses most often name: the class a typed copy of the local is kept as +/// (see `Emitter.shadows`). Nested functions, classes, lambdas and +/// comprehensions are their own scopes and are not walked. +final class ShadowTypes { + private final Layouts layouts; + private final Map> counts = new HashMap>(); + + private ShadowTypes(Layouts layouts) { + this.layouts = layouts; + } + + /// Local name -> class, for the names read for a compiled field at + /// least twice. + static Map of(List body, Layouts layouts) { + ShadowTypes t = new ShadowTypes(layouts); + for (Stmt s : body) { + t.walk(s); + } + Map r = new LinkedHashMap(); + for (Map.Entry> e : t.counts.entrySet()) { + Layouts.Cls best = null; + int n = 1; + for (Map.Entry c : e.getValue().entrySet()) { + if (c.getValue().intValue() > n) { + best = c.getKey(); + n = c.getValue().intValue(); + } + } + if (best != null) { + r.put(e.getKey(), best); + } + } + return r; + } + + private void walk(Object o) { + if (o instanceof List) { + for (Object x : (List) o) { + walk(x); + } + return; + } + if (!(o instanceof Node) || o instanceof FunctionDef || o instanceof ClassDef || o instanceof Lambda + || o instanceof Comp) { + return; + } + if (o instanceof Attribute && ((Attribute) o).value instanceof Name) { + Attribute a = (Attribute) o; + List owners = layouts.fieldOwners.get(a.attr); + if (owners != null) { + String n = ((Name) a.value).id; + Map m = counts.get(n); + if (m == null) { + m = new LinkedHashMap(); + counts.put(n, m); + } + for (Layouts.Cls c : owners) { + // A subclass reached through this access counts for the + // declaring class too: its instances pass that test. + Integer k = m.get(c); + m.put(c, Integer.valueOf(k == null ? 1 : k.intValue() + 1)); + } + } + } + for (Class c = o.getClass(); c != null && c != Object.class; c = c.getSuperclass()) { + for (Field f : c.getDeclaredFields()) { + if (Modifier.isStatic(f.getModifiers())) { + continue; + } + Class t = f.getType(); + if (!Node.class.isAssignableFrom(t) && !List.class.isAssignableFrom(t)) { + continue; + } + try { + f.setAccessible(true); + walk(f.get(o)); + } catch (IllegalAccessException e) { + throw new IllegalStateException(e); + } + } + } + } +} diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Ty.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Ty.java new file mode 100644 index 00000000000..2871cec390b --- /dev/null +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Ty.java @@ -0,0 +1,212 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.transpiler.codegen; + +import java.util.Arrays; + +/// A static type the whole-program inference (`Infer`) gives a value, and +/// the Java representation typed code keeps it in. +/// +/// The lattice is deliberately small. Every kind maps to one Java +/// representation, so a join that would need two (an int or a float, a bool +/// or None) goes to `DYN`, which is the plain `Object` the generic code uses: +/// +/// | kind | Java | None | +/// |---|---|---| +/// | `BOOL` | `boolean` | -- | +/// | `FLOAT` | `double` | -- | +/// | `INT` | `Object` (a `Long`, or a big int) | -- | +/// | `STR` | `String` | null when `opt` | +/// | `INST` | the class's layout class | null when `opt` | +/// | `LIST` | `PyList` | null when `opt` | +/// | `NONE`, `DYN`, `BOT` | `Object` | `Py.NONE` | +/// +/// An int stays an object because a Python int has no bound: overflowing a +/// `long` local would have nowhere to go without deoptimizing mid-statement. +/// On ParparVM a `Long` below 2^60 is a tagged immediate, so keeping it +/// boxed allocates nothing. +final class Ty { + enum K { BOT, NONE, BOOL, INT, FLOAT, STR, INST, LIST, DYN } + + static final Ty BOT = new Ty(K.BOT, null, false, null); + static final Ty NONE = new Ty(K.NONE, null, false, null); + static final Ty BOOL = new Ty(K.BOOL, null, false, null); + static final Ty INT = new Ty(K.INT, null, false, null); + static final Ty FLOAT = new Ty(K.FLOAT, null, false, null); + static final Ty STR = new Ty(K.STR, null, false, null); + static final Ty DYN = new Ty(K.DYN, null, false, null); + + /// The list sites a `LIST` value may have come from (see `Infer`). Past + /// this many it is any list at all. + static final int MAX_SITES = 8; + + final K k; + final Layouts.Cls cls; + final boolean opt; + /// Sorted allocation sites of a `LIST`; empty means "any list". + final int[] sites; + + private Ty(K k, Layouts.Cls cls, boolean opt, int[] sites) { + this.k = k; + this.cls = cls; + this.opt = opt; + this.sites = sites; + } + + static Ty inst(Layouts.Cls c, boolean opt) { + return new Ty(K.INST, c, opt, null); + } + + static Ty list(int[] sites, boolean opt) { + return new Ty(K.LIST, null, opt, sites); + } + + Ty orNone() { + switch (k) { + case BOT: + case NONE: + return NONE; + case STR: + case INST: + case LIST: + return opt ? this : new Ty(k, cls, true, sites); + default: + return DYN; + } + } + + /// The same type without None, for a value already tested non-None. + Ty nonNone() { + return opt ? new Ty(k, cls, false, sites) : this; + } + + /// Whether the representation can hold None (as Java null). + boolean nullable() { + return opt && (k == K.STR || k == K.INST || k == K.LIST); + } + + boolean isObject() { + return k == K.BOT || k == K.NONE || k == K.DYN || k == K.INT; + } + + static Ty join(Ty a, Ty b) { + if (a == null || a.k == K.BOT) { + return b == null ? BOT : b; + } + if (b == null || b.k == K.BOT) { + return a; + } + if (a.equals(b)) { + return a; + } + if (a.k == K.DYN || b.k == K.DYN) { + return DYN; + } + if (a.k == K.NONE) { + return b.orNone(); + } + if (b.k == K.NONE) { + return a.orNone(); + } + // A bool is an int in Python, and both are objects there. + if (a.k == K.INT && b.k == K.BOOL || a.k == K.BOOL && b.k == K.INT) { + return INT; + } + if (a.k != b.k) { + return DYN; + } + boolean opt = a.opt || b.opt; + switch (a.k) { + case INST: { + Layouts.Cls c = common(a.cls, b.cls); + return c == null ? DYN : inst(c, opt); + } + case LIST: + return list(union(a.sites, b.sites), opt); + case STR: + return opt ? STR.orNone() : STR; + default: + return a; + } + } + + private static Layouts.Cls common(Layouts.Cls a, Layouts.Cls b) { + for (Layouts.Cls x = a; x != null; x = x.parent) { + for (Layouts.Cls y = b; y != null; y = y.parent) { + if (x == y) { //NOPMD CompareObjectsWithEquals - identity is meant + return x; + } + } + } + return null; + } + + private static int[] union(int[] a, int[] b) { + if (a.length == 0 || b.length == 0) { + return new int[0]; + } + int[] r = new int[a.length + b.length]; + int n = 0; + int i = 0; + int j = 0; + while (i < a.length || j < b.length) { + int v; + if (j >= b.length || i < a.length && a[i] < b[j]) { + v = a[i++]; + } else if (i >= a.length || b[j] < a[i]) { + v = b[j++]; + } else { + v = a[i++]; + j++; + } + r[n++] = v; + } + return n > MAX_SITES ? new int[0] : Arrays.copyOf(r, n); + } + + /// Whether every value of `this` is a value of `t`. + boolean within(Ty t) { + return join(this, t).equals(t); + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof Ty)) { + return false; + } + Ty t = (Ty) o; + return k == t.k && cls == t.cls && opt == t.opt && Arrays.equals(sites, t.sites); //NOPMD CompareObjectsWithEquals - identity is meant + } + + @Override + public int hashCode() { + return k.hashCode() * 31 + (cls == null ? 0 : cls.javaName.hashCode()) + (opt ? 7 : 0); + } + + @Override + public String toString() { + String s = k == K.INST ? cls.def.name : k == K.LIST ? "list" + Arrays.toString(sites) : k.name(); + return opt ? s + "?" : s; + } +} diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/TypedEmit.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/TypedEmit.java new file mode 100644 index 00000000000..637b3811005 --- /dev/null +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/TypedEmit.java @@ -0,0 +1,2666 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.transpiler.codegen; + +import com.codename1.python.transpiler.ast.Node.*; +import com.codename1.python.transpiler.ast.Scope; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; + +/// The typed half of a closed program: a second, statically typed Java +/// method for every function `Infer` typed whose body this class can lower. +/// +/// Such a method keeps each value in the representation its type gives (see +/// `Ty`), reads and writes typed fields directly, and calls other typed +/// methods directly -- statically when one method can run, through a Java +/// virtual method (`tv_`) when several can. Nothing in it checks a type the +/// inference proved. The generic version of the function stays, and is +/// where every call from generic code lands: its first statement tests the +/// arguments against the typed signature and enters the typed method when +/// they fit, which in a closed program is always. +/// +/// A function whose body uses anything this class does not lower keeps +/// only its generic version, and so does every typed function that would +/// call it: the set of typed functions is a fixpoint. +final class TypedEmit { + /// Thrown for a construct the typed code does not lower. + static final class Unsupported extends RuntimeException { + private static final long serialVersionUID = 1L; + + Unsupported(String why) { + super(why, null, false, false); + } + } + + /// A lowered expression: Java source and the type it holds. + static final class E { + final String c; + final Ty t; + /// The same int as a Java `long` expression, cheap and free of side + /// effects, when one is at hand (a loop counter, a constant); or null. + final String lc; + + E(String c, Ty t) { + this(c, t, null); + } + + E(String c, Ty t, String lc) { + this.c = c; + this.t = t; + this.lc = lc; + } + } + + private final Emitter em; + final Infer in; + private final Layouts layouts; + /// Fields held in a typed Java slot (`t_`), per declaring class. + final Map> typedFields = new LinkedHashMap>(); + /// The functions with a typed version, and its Java name. + final Map typed = new LinkedHashMap(); + private final Map byScope = new HashMap(); + private final Set unboxers = new LinkedHashSet(); + private final Set constructors = new LinkedHashSet(); + /// `name/arity` -> the methods a `tv_` call may run. + private final Map> groups = new LinkedHashMap>(); + private final Set usedGroups = new LinkedHashSet(); + private final Map bodies = new LinkedHashMap(); + + // per function + private Infer.Fn cur; + private Scope sc; + private DefAssign da; + private Set calls; + private Set callsGroups; + private int tmp; + /// Nullable locals known not to hold None here (flow-sensitive). + private Set nonNull = new HashSet(); + /// Int locals whose value a Java long holds right now, and that long. + private Map shadow = new HashMap(); + /// Shadowed locals with no Object variable at all: read boxed only on a + /// slow path. + private Set longOnly = new HashSet(); + /// Locals whose previous list a new one may be written into (see + /// ListRecycling). + private Set recycled = new HashSet(); + /// What `name` answers for a local instead, inside an outlined slow path. + private Map subst; + /// The outlined slow paths of hoisted arithmetic. + private final StringBuilder slow = new StringBuilder(); //NOPMD AvoidStringBufferField - the generator's output + private int slowCount; + /// The function being emitted is its `TL_` variant: its unassigned int + /// parameters arrive as longs. + private boolean longParams; + /// Typed functions some call site wants a `TL_` variant of. + private final Set wantLong = new LinkedHashSet(); + private final Map longBodies = new LinkedHashMap(); + + TypedEmit(Emitter em, Infer in, Layouts layouts) { + this.em = em; + this.in = in; + this.layouts = layouts; + } + + // ---- planning -------------------------------------------------------- + + /// Decides the typed fields and functions; call before the generic code + /// is emitted, which must then leave the typed fields alone. + /// Module functions that store into nothing but their own locals and + /// call only such functions: a call of one cannot change a list or a + /// field, so reads around it may move across it. + private final Set pure = new HashSet(); + private static final Set PURE_BUILTINS = new HashSet(Arrays.asList("len", "float", "abs", + "int", "bool", "range", "min", "max")); + /// Every call in the statement being emitted is of a pure function. + private boolean stmtPure; + /// How many enclosing operands run only conditionally (`and`, `or`, an + /// if-expression branch): nothing there may be hoisted to run always. + private int condDepth; + + private void findPure() { + for (Map.Entry e : in.globalFns.entrySet()) { + pure.add(e.getValue()); + } + boolean changed = true; + while (changed) { + changed = false; + for (Infer.Fn f : new ArrayList(pure)) { + for (Stmt st : f.def.body) { + if (!pureNode(st)) { + pure.remove(f); + changed = true; + break; + } + } + } + } + } + + private boolean pureNode(Object o) { + if (o instanceof Call) { + if (!pureCall((Call) o, null)) { + return false; + } + } else if (o instanceof Assign) { + for (Expr t : ((Assign) o).targets) { + if (!localTarget(t)) { + return false; + } + } + } else if (o instanceof AugAssign) { + if (!(((AugAssign) o).target instanceof Name)) { + return false; + } + } else if (o instanceof For) { + if (!localTarget(((For) o).target)) { + return false; + } + } else if (!(o instanceof Name || o instanceof Constant || o instanceof BinOp || o instanceof UnaryOp + || o instanceof BoolOp || o instanceof Compare || o instanceof IfExp || o instanceof Subscript + || o instanceof Slice || o instanceof TupleExpr || o instanceof ListExpr || o instanceof Attribute + || o instanceof ExprStmt || o instanceof Return || o instanceof Pass || o instanceof Break + || o instanceof Continue || o instanceof If || o instanceof While || o instanceof Expr + && !(o instanceof NamedExpr || o instanceof Lambda || o instanceof Comp || o instanceof Yield + || o instanceof Await || o instanceof Starred))) { + return false; + } + if (o instanceof Expr && !(o instanceof Name || o instanceof Constant || o instanceof BinOp + || o instanceof UnaryOp || o instanceof BoolOp || o instanceof Compare || o instanceof IfExp + || o instanceof Subscript || o instanceof Slice || o instanceof TupleExpr || o instanceof ListExpr + || o instanceof Attribute || o instanceof Call)) { + return false; + } + boolean[] ok = {true}; + Infer.children(o, x -> ok[0] &= pureNode(x)); + return ok[0]; + } + + private static boolean localTarget(Expr t) { + if (t instanceof Name) { + return true; + } + if (t instanceof TupleExpr) { + for (Expr e : ((TupleExpr) t).elts) { + if (!localTarget(e)) { + return false; + } + } + return true; + } + return false; + } + + /// Whether `c` calls a pure function; `scope` resolves its name (null: + /// the module's view, for `findPure`). + private boolean pureCall(Call c, Scope scope) { + if (!(c.func instanceof Name) || !c.keywords.isEmpty()) { + return false; + } + String n = ((Name) c.func).id; + Scope.Binding b = scope == null ? null : scope.resolve(n); + Infer.Fn f = in.globalFns.get(n); + if (f != null && (b == null || b == Scope.Binding.GLOBAL)) { + return pure.contains(f); + } + return (b == null || b == Scope.Binding.BUILTIN) && PURE_BUILTINS.contains(n) + && !em.moduleScopeHasStarImport(); + } + + private boolean callsArePure(Object o) { + if (o instanceof Call && !pureCall((Call) o, sc)) { + return false; + } + boolean[] ok = {true}; + Infer.children(o, x -> ok[0] &= callsArePure(x)); + return ok[0]; + } + + void plan() { + findPure(); + if (!in.open.isEmpty()) { + return; + } + for (Layouts.Cls k : layouts.classes.values()) { + Map m = new LinkedHashMap(); + for (String f : k.fields) { + Ty t = in.fieldType(k, f); + if (representable(t) && in.definitelySet(k, f)) { + m.put(f, t); + } + } + typedFields.put(k, m); + } + for (Infer.Fn f : in.fns.values()) { + byScope.put(f.def.scope, f); + if (f.cls != null && !f.def.name.equals("__init__")) { + String key = f.def.name + "/" + (f.params.length - 1); + List g = groups.get(key); + if (g == null) { + g = new ArrayList(); + groups.put(key, g); + } + g.add(f); + } + } + Set ok = new LinkedHashSet(in.fns.values()); + Map> deps = new HashMap>(); + Map> gdeps = new HashMap>(); + for (Infer.Fn f : in.fns.values()) { + typed.put(f, typedName(f)); + } + boolean changed = true; + while (changed) { + changed = false; + for (Infer.Fn f : new ArrayList(ok)) { + if (bodies.containsKey(f)) { + continue; + } + try { + bodies.put(f, function(f)); + deps.put(f, calls); + gdeps.put(f, callsGroups); + } catch (Unsupported u) { + if (System.getProperty("cn1.python.dumpTypes") != null) { + System.err.println("untyped " + f.def.name + ": " + u.getMessage()); + } + ok.remove(f); + typed.remove(f); + changed = true; + } + } + for (Infer.Fn f : new ArrayList(ok)) { + boolean good = ok.containsAll(deps.get(f)); + for (String g : gdeps.get(f)) { + good &= ok.containsAll(groups.get(g)); + } + if (!good) { + ok.remove(f); + typed.remove(f); + bodies.remove(f); + changed = true; + } + } + } + for (Set g : gdeps.values()) { + if (g != null) { + usedGroups.addAll(g); + } + } + usedGroups.retainAll(liveGroups(ok)); + // A typed field no generic code may reach directly: the generic + // code goes through getField/setField, which box. + for (Map.Entry> e : typedFields.entrySet()) { + for (String f : e.getValue().keySet()) { + List o = layouts.fieldOwners.get(f); + o.remove(e.getKey()); + if (o.isEmpty()) { + layouts.fieldOwners.remove(f); + } + } + } + } + + private Set liveGroups(Set ok) { + Set r = new HashSet(); + for (Map.Entry> e : groups.entrySet()) { + if (ok.containsAll(e.getValue())) { + r.add(e.getKey()); + } + } + return r; + } + + private static boolean representable(Ty t) { + switch (t.k) { + case BOOL: + case FLOAT: + case STR: + case INST: + case LIST: + return true; + default: + return false; + } + } + + private String typedName(Infer.Fn f) { + if (f.cls != null) { + return "TM_" + f.cls.javaName + "_" + Emitter.mangle(f.def.name); + } + return "TF_" + Emitter.mangle(f.def.name); + } + + Ty typedField(Layouts.Cls owner, String f) { + Map m = typedFields.get(owner); + return m == null ? null : m.get(f); + } + + // ---- Java types ------------------------------------------------------ + + static String jt(Ty t) { + switch (t.k) { + case BOOL: + return "boolean"; + case FLOAT: + return "double"; + case STR: + return "String"; + case INST: + return t.cls.javaName; + case LIST: + return "PyList"; + default: + return "Object"; + } + } + + static boolean objectRet(Ty t) { + return "Object".equals(jt(t)); + } + + static String retJ(Infer.Fn f) { + return jt(f.ret); + } + + private static String zero(Ty t) { + switch (t.k) { + case BOOL: + return "false"; + case FLOAT: + return "0.0"; + default: + return "null"; + } + } + + // ---- conversions ----------------------------------------------------- + + String box(E e) { + switch (e.t.k) { + case BOOL: + return "Py.bool(" + e.c + ")"; + case FLOAT: + return "Py.of(" + e.c + ")"; + case STR: + case INST: + case LIST: + return e.t.opt ? "Typed.box(" + e.c + ")" : e.c; + default: + return e.c; + } + } + + /// `c`, an object the inference proved to be of type `to`, as `to`. + String unbox(String c, Ty to) { + switch (to.k) { + case BOOL: + return "Typed.asBool(" + c + ")"; + case FLOAT: + return "Typed.asFloat(" + c + ")"; + case STR: + return "Typed.asStr(" + c + ")"; + case LIST: + return "Typed.asList(" + c + ")"; + case INST: + unboxers.add(to.cls); + return "u_" + to.cls.javaName + "(" + c + ")"; + default: + return c; + } + } + + String conv(E e, Ty to) { + String tj = jt(to); + if ("Object".equals(tj)) { + return box(e); + } + if (e.t.k == Ty.K.NONE) { + if (to.nullable()) { + return "Py.NONE".equals(e.c) ? "null" : "Typed.<" + tj + ">none(" + e.c + ")"; + } + throw new Unsupported("None as " + to); + } + if ("Object".equals(jt(e.t))) { + return unbox(e.c, to); + } + if (e.t.k == to.k) { + if (e.t.k == Ty.K.INST && !Infer.isSelfOrAncestor(to.cls, e.t.cls)) { + throw new Unsupported("downcast " + e.t + " to " + to); + } + return e.c; + } + throw new Unsupported("convert " + e.t + " to " + to); + } + + String toD(E e) { + switch (e.t.k) { + case FLOAT: + return e.c; + case INT: + return "h_toFloat(" + e.c + ")"; + case BOOL: + return "(" + e.c + " ? 1.0 : 0.0)"; + default: + throw new Unsupported("float of " + e.t); + } + } + + private boolean customTruth(Layouts.Cls k) { + for (Layouts.Cls c : layouts.classes.values()) { + if (Infer.isSelfOrAncestor(k, c) || Infer.isSelfOrAncestor(c, k)) { + if (c.bodyNames.contains("__bool__") || c.bodyNames.contains("__len__")) { + return true; + } + } + } + return false; + } + + String test(E e) { + switch (e.t.k) { + case BOOL: + return e.c; + case FLOAT: + return "(" + e.c + " != 0.0)"; + case INST: + if (customTruth(e.t.cls)) { + return "Py.isTrue(" + box(e) + ")"; + } + return "(" + e.c + " != null)"; + case STR: + case LIST: + return "Typed.truthy(" + e.c + ")"; + default: + return "Py.isTrue(" + e.c + ")"; + } + } + + // ---- functions ------------------------------------------------------- + + private String var(String n) { + return "v_" + Emitter.mangle(n); + } + + private Ty localType(String n) { + Ty t = cur.locals.get(n); + return t == null ? Ty.BOT : t; + } + + /// Whether local `n` is read anywhere in the function but in the body + /// of loop `f` (whose target it is). + private boolean readOutside(String n, For f) { + int[] c = new int[1]; + countReads(cur.def.body, n, f, c); + return c[0] > 0; + } + + private static void countReads(Object o, String n, For skip, int[] c) { + if (o == skip) { //NOPMD CompareObjectsWithEquals - identity is meant + countReads(skip.iter, n, null, c); + return; + } + if (o instanceof Name && ((Name) o).id.equals(n)) { + c[0]++; + } + Infer.children(o, x -> countReads(x, n, skip, c)); + } + + /// Whether parameter `i` of `f` can arrive as a long: an int nothing in + /// the body stores into. + private static boolean longParam(Infer.Fn f, int i, Set stored) { + return f.params[i].k == Ty.K.INT && !stored.contains(f.names.get(i)); + } + + private static boolean hasLongParam(Infer.Fn f) { + Set stored = assigned(f.def.body); + for (int i = 0; i < f.names.size(); i++) { + if (longParam(f, i, stored)) { + return true; + } + } + return false; + } + + private String function(Infer.Fn f) { + cur = f; + sc = f.def.scope; + calls = new LinkedHashSet(); + callsGroups = new LinkedHashSet(); + tmp = 0; + nonNull = new HashSet(); + shadow = new HashMap(); + longOnly = new HashSet(); + if (!f.def.scope.cells.isEmpty() || f.def.scope.hasYield || f.def.isAsync || !f.def.decorators.isEmpty() + || f.def.args.vararg != null || f.def.args.kwarg != null || !f.def.args.kwonly.isEmpty()) { + throw new Unsupported("signature"); + } + for (String n : f.locals.keySet()) { + if (sc.resolve(n) != Scope.Binding.LOCAL) { + throw new Unsupported("non-local " + n); + } + } + da = new DefAssign(); + da.function(f.def.body, sc.params); + recycled = ListRecycling.find(f.def.body, f.names, sc); + StringBuilder b = new StringBuilder(); + String fname = longParams ? "TL" + typed.get(f).substring(2) : typed.get(f); + b.append(" static ").append(retJ(f)).append(' ').append(fname).append('('); + Set declared = new HashSet(f.names); + Set stored = assigned(f.def.body); + for (int i = 0; i < f.names.size(); i++) { + String n = f.names.get(i); + if (longParams && longParam(f, i, stored)) { + String q = "q_" + Emitter.mangle(n); + b.append(i == 0 ? "" : ", ").append("long ").append(q); + shadow.put(n, q); + longOnly.add(n); + } else { + b.append(i == 0 ? "" : ", ").append(jt(f.params[i])).append(' ').append(var(n)); + } + } + b.append(") {\n"); + for (Map.Entry e : f.locals.entrySet()) { + if (declared.add(e.getKey())) { + b.append(" ").append(jt(e.getValue())).append(' ').append(var(e.getKey())).append(" = ") + .append(zero(e.getValue())).append(";\n"); + } + } + boolean abrupt = block(f.def.body, b, 2); + if (!abrupt) { + if (objectRet(f.ret)) { + b.append(" return Py.NONE;\n"); + } else if (f.ret.nullable()) { + b.append(" return null;\n"); + } else { + throw new Unsupported("falls off the end returning " + f.ret); + } + } + b.append(" }\n"); + return b.toString(); + } + + // ---- statements ------------------------------------------------------ + + private static void indent(StringBuilder b, int ind) { + for (int i = 0; i < ind; i++) { + b.append(" "); + } + } + + /// Emits `body`; true when it cannot complete normally (javac agrees: + /// nothing is emitted after such a statement). + private boolean block(List body, StringBuilder b, int ind) { + for (Stmt s : body) { + if (stmt(s, b, ind)) { + return true; + } + } + return false; + } + + private String fresh() { + return "t" + (++tmp); + } + + /// Statements that hoisted int arithmetic (`hoistInt`) runs before, or + /// null where nothing may run first (a loop header). + private StringBuilder pre; //NOPMD AvoidStringBufferField - the generator's output + private int preInd; + + private boolean stmt(Stmt s, StringBuilder b, int ind) { + StringBuilder savedPre = pre; + int savedInd = preInd; + boolean hoist = (s instanceof ExprStmt || s instanceof Assign || s instanceof AugAssign + || s instanceof Return || s instanceof If) && !hasWalrus(s); + pre = hoist ? new StringBuilder() : null; + preInd = ind; + boolean savedPure = stmtPure; + stmtPure = hoist && callsArePure(s instanceof If ? ((If) s).test : s); + StringBuilder sb = new StringBuilder(); + try { + boolean r = stmt0(s, sb, ind); + if (pre != null) { + b.append(pre); + } + b.append(sb); + return r; + } finally { + pre = savedPre; + preInd = savedInd; + stmtPure = savedPure; + } + } + + private static boolean hasWalrus(Object o) { + if (o instanceof NamedExpr) { + return true; + } + boolean[] f = new boolean[1]; + Infer.children(o, x -> f[0] |= hasWalrus(x)); + return f[0]; + } + + private boolean stmt0(Stmt s, StringBuilder b, int ind) { + if (s instanceof Pass) { + return false; + } + if (s instanceof ExprStmt) { + Expr v = ((ExprStmt) s).value; + if (v instanceof Constant) { + return false; + } + E e = expr(v); + indent(b, ind); + if (v instanceof Call && e.c.matches("[A-Za-z_][A-Za-z0-9_.<>]*\\(.*\\)")) { + b.append(e.c).append(";\n"); + } else { + b.append("Typed.drop(").append(box(e)).append(");\n"); + } + return false; + } + if (s instanceof Assign && ((Assign) s).targets.size() == 1 && sliceOf(((Assign) s).targets.get(0)) + && sliceOf(((Assign) s).value)) { + // `o[a:b] = v[d:e]`: copied without the intermediate list, the + // source evaluated first as in Python. + Subscript src = (Subscript) ((Assign) s).value; + Subscript dst = (Subscript) ((Assign) s).targets.get(0); + E v = expr(src.value); + String vp = sliceParts((Slice) src.index); + E o = expr(dst.value); + String op = sliceParts((Slice) dst.index); + if (v.t.k == Ty.K.LIST && o.t.k == Ty.K.LIST) { + indent(b, ind); + b.append("Py.setsliceFrom(").append(v.c).append(", ").append(vp).append(", ").append(o.c) + .append(", ").append(op).append(");\n"); + return false; + } + throw new Unsupported("slice copy between " + v.t + " and " + o.t); + } + if (s instanceof Assign && ((Assign) s).targets.size() == 1 && ((Assign) s).targets.get(0) instanceof Name + && recycled.contains(((Name) ((Assign) s).targets.get(0)).id) + && localType(((Name) ((Assign) s).targets.get(0)).id).k == Ty.K.LIST + && (ListRecycling.isSliceCopy(((Assign) s).value) || ListRecycling.isEmptyList(((Assign) s).value))) { + // Nothing but this local can see its previous list, so the new + // one reuses its array, as the generic code does. + Name n = (Name) ((Assign) s).targets.get(0); + Expr v = ((Assign) s).value; + String old = var(n.id); + String c; + if (v instanceof ListExpr) { + c = "Py.recycleList(" + old + ")"; + } else { + E src = expr(((Subscript) v).value); + if (src.t.k != Ty.K.LIST) { + throw new Unsupported("slice copy of " + src.t); + } + c = "Typed.asList(Py.recycleSlice(" + old + ", " + src.c + ", " + sliceParts((Slice) ((Subscript) v).index) + + "))"; + } + store(n, new E(c, localType(n.id).nonNone()), b, ind); + return false; + } + if (s instanceof Assign) { + Assign a = (Assign) s; + if (a.targets.size() == 1 && a.targets.get(0) instanceof TupleExpr && a.value instanceof TupleExpr) { + List ts = ((TupleExpr) a.targets.get(0)).elts; + List vs = ((TupleExpr) a.value).elts; + if (ts.size() != vs.size()) { + throw new Unsupported("unpack length"); + } + List tv = new ArrayList(); + for (Expr v : vs) { + if (v instanceof Starred) { + throw new Unsupported("starred"); + } + E e = expr(v); + String t = fresh(); + indent(b, ind); + b.append(jt(e.t)).append(' ').append(t).append(" = ").append(e.c).append(";\n"); + tv.add(new E(t, e.t)); + } + for (int i = 0; i < ts.size(); i++) { + store(ts.get(i), tv.get(i), b, ind); + } + return false; + } + E v = expr(a.value); + if (a.targets.size() > 1) { + String t = fresh(); + indent(b, ind); + b.append(jt(v.t)).append(' ').append(t).append(" = ").append(v.c).append(";\n"); + v = new E(t, v.t); + } + for (Expr t : a.targets) { + store(t, v, b, ind); + } + return false; + } + if (s instanceof AugAssign) { + augAssign((AugAssign) s, b, ind); + return false; + } + if (s instanceof Return) { + Expr v = ((Return) s).value; + indent(b, ind); + if (v == null) { + if (objectRet(cur.ret)) { + b.append("return Py.NONE;\n"); + } else if (cur.ret.nullable()) { + b.append("return null;\n"); + } else { + throw new Unsupported("return None as " + cur.ret); + } + } else { + b.append("return ").append(conv(expr(v), cur.ret)).append(";\n"); + } + return true; + } + if (s instanceof If) { + If i = (If) s; + Set before = new HashSet(nonNull); + indent(b, ind); + b.append("if (").append(testExpr(i.test)).append(") {\n"); + nonNull = new HashSet(before); + nonNull.addAll(whenTrue(i.test)); + boolean a1 = block(i.body, b, ind + 1); + Set after1 = nonNull; + nonNull = new HashSet(before); + nonNull.addAll(whenFalse(i.test)); + indent(b, ind); + boolean a2 = false; + if (i.orelse.isEmpty()) { + b.append("}\n"); + } else { + b.append("} else {\n"); + a2 = block(i.orelse, b, ind + 1); + indent(b, ind); + b.append("}\n"); + } + // When the if branch always leaves, what holds is what the else + // branch left, which nonNull already is. + Set after2 = nonNull; + if (!a1) { + if (!a2) { + after1.retainAll(after2); + } + nonNull = after1; + } + return a1 && a2; + } + if (s instanceof While) { + While w = (While) s; + if (!w.orelse.isEmpty()) { + throw new Unsupported("while-else"); + } + boolean forever = w.test instanceof Constant && ((Constant) w.test).kind == Constant.Kind.TRUE; + nonNull.removeAll(assigned(w.body)); + shadow.keySet().removeAll(assigned(w.body)); + forgetPaths(); + Set inv = new HashSet(nonNull); + indent(b, ind); + b.append("while (").append(forever ? "true" : testExpr(w.test)).append(") {\n"); + nonNull.addAll(whenTrue(w.test)); + block(w.body, b, ind + 1); + nonNull = inv; + if (!Infer.hasBreak(w.body)) { + nonNull.addAll(whenFalse(w.test)); + } + indent(b, ind); + b.append("}\n"); + return forever && !Infer.hasBreak(w.body); + } + if (s instanceof For) { + forLoop((For) s, b, ind); + return false; + } + if (s instanceof Break) { + indent(b, ind); + b.append("break;\n"); + return true; + } + if (s instanceof Continue) { + indent(b, ind); + b.append("continue;\n"); + return true; + } + throw new Unsupported(s.getClass().getSimpleName()); + } + + private void forLoop(For f, StringBuilder b, int ind) { + if (!f.orelse.isEmpty()) { + throw new Unsupported("for-else"); + } + Set inBody = assigned(f.body); + nonNull.removeAll(inBody); + shadow.keySet().removeAll(inBody); + forgetPaths(); + if (f.target instanceof Name) { + nonNull.remove(((Name) f.target).id); + } + if (!(f.target instanceof Name) || sc.resolve(((Name) f.target).id) != Scope.Binding.LOCAL) { + throw new Unsupported("for target"); + } + String v = var(((Name) f.target).id); + Ty vt = localType(((Name) f.target).id); + if (in.rangeCall(f.iter, sc)) { + List a = ((Call) f.iter).args; + String start = "0L"; + String stop; + long step = 1; + if (a.size() == 1) { + stop = "Typed.bound(" + box(expr(a.get(0))) + ")"; + } else { + start = "Typed.bound(" + box(expr(a.get(0))) + ")"; + stop = "Typed.bound(" + box(expr(a.get(1))) + ")"; + if (a.size() == 3) { + Long c = intConstant(a.get(2)); + if (c == null || c.longValue() == 0) { + throw new Unsupported("range step"); + } + step = c.longValue(); + } + } + String i = fresh(); + String e = fresh(); + indent(b, ind); + b.append("{\n"); + indent(b, ind + 1); + // Bounds are evaluated in order, once, before the loop. + String s0 = fresh(); + b.append("long ").append(s0).append(" = ").append(start).append(";\n"); + indent(b, ind + 1); + b.append("long ").append(e).append(" = ").append(stop).append(";\n"); + indent(b, ind + 1); + b.append("for (long ").append(i).append(" = ").append(s0).append("; ").append(i) + .append(step > 0 ? " < " : " > ").append(e).append("; ").append(i).append(" += ").append(step) + .append("L) {\n"); + String tn = ((Name) f.target).id; + boolean counter = vt.k == Ty.K.INT && !inBody.contains(tn); + boolean unboxed = counter && !longOnly.contains(tn) && !readOutside(tn, f); + if (unboxed) { + // Read nowhere but in this body: no boxed copy at all. + longOnly.add(tn); + } else if (longOnly.contains(tn)) { + throw new Unsupported("long-only " + tn + " reused"); + } else { + indent(b, ind + 2); + b.append(v).append(" = ").append(conv(new E("Long.valueOf(" + i + ")", Ty.INT), vt)).append(";\n"); + } + if (counter) { + shadow.put(tn, i); + } + block(f.body, b, ind + 2); + shadow.remove(tn); + if (unboxed) { + longOnly.remove(tn); + } + indent(b, ind + 1); + b.append("}\n"); + indent(b, ind); + b.append("}\n"); + return; + } + E it = expr(f.iter); + if (it.t.k != Ty.K.LIST) { + throw new Unsupported("for over " + it.t); + } + Ty el = in.elem(it.t); + String l = fresh(); + String k = fresh(); + indent(b, ind); + b.append("{\n"); + indent(b, ind + 1); + b.append("PyList ").append(l).append(" = Typed.iterable(").append(it.c).append(");\n"); + indent(b, ind + 1); + b.append("for (int ").append(k).append(" = 0; ").append(k).append(" < ").append(l).append(".size(); ") + .append(k).append("++) {\n"); + indent(b, ind + 2); + E item = el.k == Ty.K.FLOAT ? new E("Typed.atF(" + l + ", " + k + ")", Ty.FLOAT) + : new E(unbox(l + ".get(" + k + ")", el), el); + b.append(v).append(" = ").append(conv(item, vt)).append(";\n"); + block(f.body, b, ind + 2); + indent(b, ind + 1); + b.append("}\n"); + indent(b, ind); + b.append("}\n"); + } + + private static Long intConstant(Expr e) { + if (e instanceof Constant && ((Constant) e).kind == Constant.Kind.INT) { + try { + return Long.valueOf(Long.parseLong(((Constant) e).text.replace("_", ""))); + } catch (NumberFormatException x) { + return null; + } + } + if (e instanceof UnaryOp && "-".equals(((UnaryOp) e).op)) { + Long v = intConstant(((UnaryOp) e).operand); + return v == null ? null : Long.valueOf(-v.longValue()); + } + return null; + } + + /// Whether `c` is a plain Java local: evaluating it twice, or after + /// something else, is the same as once. + private static boolean plain(String c) { + return c.matches("[A-Za-z_][A-Za-z0-9_]*"); + } + + private void store(Expr t, E v, StringBuilder b, int ind) { + if (t instanceof Name) { + String n = ((Name) t).id; + switch (sc.resolve(n)) { + case LOCAL: + indent(b, ind); + b.append(var(n)).append(" = ").append(conv(v, localType(n))).append(";\n"); + forgetPathsOf(n); + if (longOnly.contains(n)) { + throw new Unsupported("store to a long-only " + n); + } + shadow.remove(n); + if (v.t.k == localType(n).k && !v.t.opt && v.t.k != Ty.K.NONE) { + nonNull.add(n); + } else { + nonNull.remove(n); + } + return; + case GLOBAL: + indent(b, ind); + b.append("G[").append(em.slot(n)).append("] = ").append(box(v)).append(";\n"); + return; + default: + throw new Unsupported("store " + n); + } + } + if (t instanceof Attribute) { + Attribute a = (Attribute) t; + E r = expr(a.value); + if (!plain(v.c) && !plain(r.c)) { + // Python evaluates the value before the receiver. + String tv = fresh(); + indent(b, ind); + b.append(jt(v.t)).append(' ').append(tv).append(" = ").append(v.c).append(";\n"); + v = new E(tv, v.t); + } + indent(b, ind); + b.append(fieldRef(r, a.attr)).append(" = ").append(conv(v, fieldTy(r, a.attr))).append(";\n"); + forgetAttr(a.attr); + String p = path(a); + if (p != null && !v.t.opt && v.t.k != Ty.K.NONE && v.t.k == fieldTy(r, a.attr).k) { + nonNull.add(p); + } + return; + } + if (t instanceof Subscript) { + Subscript s = (Subscript) t; + if (s.index instanceof Slice) { + E l = expr(s.value); + if (l.t.k != Ty.K.LIST) { + throw new Unsupported("slice store on " + l.t); + } + String p = sliceParts((Slice) s.index); + indent(b, ind); + b.append("Py.setslice(").append(l.c).append(", ").append(p).append(", ").append(box(v)).append(");\n"); + return; + } + E l = expr(s.value); + E i = expr(s.index); + if (l.t.k != Ty.K.LIST || !(i.t.k == Ty.K.INT || i.t.k == Ty.K.BOOL)) { + throw new Unsupported("subscript store on " + l.t); + } + if (!plain(v.c)) { + String tv = fresh(); + indent(b, ind); + b.append(jt(v.t)).append(' ').append(tv).append(" = ").append(v.c).append(";\n"); + v = new E(tv, v.t); + } + Ty el = in.elem(l.t); + indent(b, ind); + if (i.lc != null && plain(l.c)) { + String in = inRange(l, i.lc); + if (el.k == Ty.K.FLOAT) { + b.append("if (").append(in).append(" && ").append(l.c).append(".floats != null) {\n"); + indent(b, ind + 1); + b.append(l.c).append(".floats[(int) ").append(i.lc).append("] = ").append(conv(v, el)).append(";\n"); + } else { + b.append("if (").append(in).append(" && ").append(l.c).append(".ints == null && ").append(l.c) + .append(".floats == null) {\n"); + indent(b, ind + 1); + b.append(l.c).append(".items[(int) ").append(i.lc).append("] = ").append(box(v)).append(";\n"); + } + indent(b, ind); + b.append("} else {\n"); + indent(b, ind + 1); + b.append(el.k == Ty.K.FLOAT ? "h_setF(" : "h_set(").append(l.c).append(", ").append(box(i)) + .append(", ").append(el.k == Ty.K.FLOAT ? conv(v, el) : box(v)).append(");\n"); + indent(b, ind); + b.append("}\n"); + return; + } + if (el.k == Ty.K.FLOAT) { + b.append("h_setF(").append(l.c).append(", ").append(box(i)).append(", ").append(conv(v, el)) + .append(");\n"); + } else { + b.append("h_set(").append(l.c).append(", ").append(box(i)).append(", ").append(box(v)) + .append(");\n"); + } + return; + } + throw new Unsupported("store target " + t.getClass().getSimpleName()); + } + + private void augAssign(AugAssign a, StringBuilder b, int ind) { + Expr t = a.target; + if (t instanceof Name) { + E cur0 = expr(t); + E r = binop(a.op, cur0, expr(a.value), a.value); + store(t, r, b, ind); + return; + } + if (t instanceof Attribute) { + Attribute at = (Attribute) t; + E r = expr(at.value); + String rv = r.c; + if (!plain(rv)) { + rv = fresh(); + indent(b, ind); + b.append(jt(r.t)).append(' ').append(rv).append(" = ").append(r.c).append(";\n"); + } + E recv = new E(rv, r.t); + String ref = fieldRef(recv, at.attr); + E old = new E(ref, fieldTy(recv, at.attr)); + E v = binop(a.op, old, expr(a.value), a.value); + indent(b, ind); + b.append(fieldRef(recv, at.attr)).append(" = ").append(conv(v, fieldTy(recv, at.attr))) + .append(";\n"); + return; + } + if (t instanceof Subscript && !(((Subscript) t).index instanceof Slice)) { + Subscript s = (Subscript) t; + E l = expr(s.value); + E i = expr(s.index); + if (l.t.k != Ty.K.LIST || !(i.t.k == Ty.K.INT || i.t.k == Ty.K.BOOL)) { + throw new Unsupported("subscript aug on " + l.t); + } + Ty el0 = in.elem(l.t); + if (el0.k == Ty.K.FLOAT && i.lc != null && plain(l.c) && !hasCall(a.value)) { + // No call in the value: nothing it evaluates can change the + // list, so the element is read after it. + E rv = expr(a.value); + if (rv.t.k == Ty.K.FLOAT || rv.t.k == Ty.K.INT || rv.t.k == Ty.K.BOOL) { + String tv = fresh(); + indent(b, ind); + b.append(jt(rv.t)).append(' ').append(tv).append(" = ").append(rv.c).append(";\n"); + E rvE = new E(tv, rv.t, rv.lc); + String fa = l.c + ".floats[(int) " + i.lc + "]"; + indent(b, ind); + b.append("if (").append(inRange(l, i.lc)).append(" && ").append(l.c).append(".floats != null) {\n"); + E nv = binop(a.op, new E(fa, el0), rvE, a.value); + indent(b, ind + 1); + b.append(fa).append(" = ").append(conv(nv, el0)).append(";\n"); + indent(b, ind); + b.append("} else {\n"); + E ov = new E("h_getF(" + l.c + ", " + box(i) + ")", el0); + E sv = binop(a.op, ov, rvE, a.value); + indent(b, ind + 1); + b.append("h_setF(").append(l.c).append(", ").append(box(i)).append(", ").append(conv(sv, el0)) + .append(");\n"); + indent(b, ind); + b.append("}\n"); + return; + } + throw new Unsupported("float aug with " + rv.t); + } + String lv = fresh(); + String iv = fresh(); + indent(b, ind); + b.append("PyList ").append(lv).append(" = ").append(l.c).append(";\n"); + indent(b, ind); + b.append("Object ").append(iv).append(" = ").append(box(i)).append(";\n"); + Ty el = in.elem(l.t); + E old = el.k == Ty.K.FLOAT ? new E("h_getF(" + lv + ", " + iv + ")", el) + : new E(unbox("h_get(" + lv + ", " + iv + ")", el), el); + E v = binop(a.op, old, expr(a.value), a.value); + indent(b, ind); + if (el.k == Ty.K.FLOAT) { + b.append("h_setF(").append(lv).append(", ").append(iv).append(", ").append(conv(v, el)) + .append(");\n"); + } else { + b.append("h_set(").append(lv).append(", ").append(iv).append(", ").append(box(v)).append(");\n"); + } + return; + } + throw new Unsupported("augmented target"); + } + + // ---- None narrowing -------------------------------------------------- + + /// Locals a true `test` proves are not None. + private Set whenTrue(Expr test) { + Set r = new HashSet(); + if (test instanceof BoolOp && ((BoolOp) test).isAnd) { + for (Expr v : ((BoolOp) test).values) { + r.addAll(whenTrue(v)); + } + } else if (test instanceof UnaryOp && "not".equals(((UnaryOp) test).op)) { + r.addAll(whenFalse(((UnaryOp) test).operand)); + } else if (test instanceof Compare) { + String n = noneTest((Compare) test); + if (n != null && ((Compare) test).ops.get(0).equals("is not")) { + r.add(n); + } + } else if (test instanceof Name && narrowable(((Name) test).id)) { + Ty t = localType(((Name) test).id); + if (t.k == Ty.K.INST && !customTruth(t.cls)) { + r.add(((Name) test).id); + } + } + return r; + } + + /// Locals a false `test` proves are not None. + private Set whenFalse(Expr test) { + Set r = new HashSet(); + if (test instanceof BoolOp && !((BoolOp) test).isAnd) { + for (Expr v : ((BoolOp) test).values) { + r.addAll(whenFalse(v)); + } + } else if (test instanceof UnaryOp && "not".equals(((UnaryOp) test).op)) { + r.addAll(whenTrue(((UnaryOp) test).operand)); + } else if (test instanceof Compare) { + String n = noneTest((Compare) test); + if (n != null && ((Compare) test).ops.get(0).equals("is")) { + r.add(n); + } + } + return r; + } + + /// `x.attr` of a local `x`, the key narrowing keeps for that field. + private String path(Attribute a) { + if (a.value instanceof Name && sc.resolve(((Name) a.value).id) == Scope.Binding.LOCAL) { + return ((Name) a.value).id + "." + a.attr; + } + return null; + } + + private void forgetAttr(String attr) { + String suffix = "." + attr; + nonNull.removeIf(k -> k.endsWith(suffix)); + } + + private void forgetPathsOf(String local) { + String prefix = local + "."; + nonNull.removeIf(k -> k.startsWith(prefix)); + } + + private void forgetPaths() { + nonNull.removeIf(k -> k.indexOf('.') >= 0); + } + + private static boolean sliceOf(Expr e) { + return e instanceof Subscript && ((Subscript) e).index instanceof Slice; + } + + /// The three parts of a slice, boxed, `Py.NONE` for a missing one. + private String sliceParts(Slice s) { + Expr[] parts = {s.lower, s.upper, s.step}; + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < 3; i++) { + sb.append(i == 0 ? "" : ", ").append(parts[i] == null ? "Py.NONE" : box(expr(parts[i]))); + } + return sb.toString(); + } + + private boolean narrowable(String n) { + return sc.resolve(n) == Scope.Binding.LOCAL && localType(n).nullable(); + } + + /// The local `x` of `x is None` / `x is not None`, or null. + private String noneTest(Compare c) { + if (c.ops.size() != 1 || !c.ops.get(0).startsWith("is")) { + return null; + } + Expr r = c.comparators.get(0); + if (!(r instanceof Constant) || ((Constant) r).kind != Constant.Kind.NONE) { + return null; + } + if (c.left instanceof Name && narrowable(((Name) c.left).id)) { + return ((Name) c.left).id; + } + return c.left instanceof Attribute ? path((Attribute) c.left) : null; + } + + /// Every local `body` may store into. + private static Set assigned(List body) { + List out = new ArrayList(); + for (Stmt s : body) { + collectStores(s, out); + } + return new HashSet(out); + } + + private static void collectStores(Object o, List out) { + if (o instanceof Assign) { + for (Expr t : ((Assign) o).targets) { + names(t, out); + } + } else if (o instanceof AugAssign) { + names(((AugAssign) o).target, out); + } else if (o instanceof For) { + names(((For) o).target, out); + } else if (o instanceof NamedExpr) { + out.add(((NamedExpr) o).target.id); + } + Infer.children(o, x -> collectStores(x, out)); + } + + private static void names(Expr t, List out) { + if (t instanceof Name) { + out.add(((Name) t).id); + } else if (t instanceof TupleExpr) { + for (Expr e : ((TupleExpr) t).elts) { + names(e, out); + } + } else if (t instanceof ListExpr) { + for (Expr e : ((ListExpr) t).elts) { + names(e, out); + } + } else if (t instanceof Starred) { + names(((Starred) t).value, out); + } + } + + // ---- fields ---------------------------------------------------------- + + private Layouts.Cls owner(E r, String attr) { + if (r.t.k != Ty.K.INST) { + throw new Unsupported("attribute of " + r.t); + } + Layouts.Cls o = Infer.fieldOwner(r.t.cls, attr); + if (o == null) { + throw new Unsupported("not a field: " + attr); + } + return o; + } + + private Ty fieldTy(E r, String attr) { + Layouts.Cls o = owner(r, attr); + Ty t = typedField(o, attr); + return t != null ? t : in.fieldType(o, attr); + } + + /// `r` checked not to be None before reaching its `attr`. + private String nz(E r, String attr) { + if (!r.t.opt) { + return r.c; + } + if (plain(r.c)) { + // Inline: the throwing path is the only call. + return "(" + r.c + " != null ? " + r.c + " : Typed.<" + jt(r.t) + ">nzf(" + Emitter.javaString(attr) + + "))"; + } + return "Typed.nz(" + r.c + ", " + Emitter.javaString(attr) + ")"; + } + + /// The Java lvalue of field `attr` of `r`. + private String fieldRef(E r, String attr) { + Layouts.Cls o = owner(r, attr); + String recv = nz(r, attr); + if (typedField(o, attr) != null) { + return recv + ".t_" + Emitter.mangle(attr); + } + if (!in.definitelySet(o, attr)) { + throw new Unsupported("field may be unset: " + attr); + } + return recv + ".f_" + Emitter.mangle(attr); + } + + // ---- expressions ----------------------------------------------------- + + /// `e` as a Java boolean, for a test: `and`, `or` and `not` need no + /// Python value there. + String testExpr(Expr e) { + if (e instanceof BoolOp) { + BoolOp o = (BoolOp) e; + StringBuilder sb = new StringBuilder("("); + String j = o.isAnd ? " && " : " || "; + Set saved = new HashSet(nonNull); + for (int i = 0; i < o.values.size(); i++) { + Expr v = o.values.get(i); + condDepth += i == 0 ? 0 : 1; + try { + sb.append(i == 0 ? "" : j).append(testExpr(v)); + } finally { + condDepth -= i == 0 ? 0 : 1; + } + // A later operand runs only when the earlier ones were true + // (and) or false (or). + nonNull.addAll(o.isAnd ? whenTrue(v) : whenFalse(v)); + } + nonNull = saved; + return sb.append(')').toString(); + } + if (e instanceof UnaryOp && "not".equals(((UnaryOp) e).op)) { + return "!" + testExpr(((UnaryOp) e).operand); + } + return test(expr(e)); + } + + E expr(Expr e) { + if (e instanceof Constant) { + Constant c = (Constant) e; + switch (c.kind) { + case NONE: + return new E("Py.NONE", Ty.NONE); + case TRUE: + return new E("true", Ty.BOOL); + case FALSE: + return new E("false", Ty.BOOL); + case INT: { + Long v = intConstant(c); + return new E(em.constant(c), Ty.INT, + v != null && Math.abs(v.longValue()) < (1L << 60) ? v + "L" : null); + } + case FLOAT: { + if (c.text.endsWith("j") || c.text.endsWith("J")) { + throw new Unsupported("complex"); + } + double d = Double.parseDouble(c.text.replace("_", "")); + return new E(Double.isInfinite(d) ? "Double.POSITIVE_INFINITY" : Double.toString(d), Ty.FLOAT); + } + case STR: + return new E(em.constant(c), Ty.STR); + default: + throw new Unsupported("constant"); + } + } + if (e instanceof Name) { + return name((Name) e); + } + if (e instanceof Attribute) { + Attribute a = (Attribute) e; + E r = expr(a.value); + Ty ft = fieldTy(r, a.attr); + String p = path(a); + return new E(fieldRef(r, a.attr), p != null && nonNull.contains(p) ? ft.nonNone() : ft); + } + if (e instanceof BinOp) { + if (pre != null && (pureInt(e) || pureFloat(e))) { + return hoistInt(e); + } + BinOp b = (BinOp) e; + return binop(b.op, expr(b.left), expr(b.right), b.right); + } + if (e instanceof UnaryOp) { + UnaryOp u = (UnaryOp) e; + if ("not".equals(u.op)) { + return new E("!" + testExpr(u.operand), Ty.BOOL); + } + E o = expr(u.operand); + if (o.t.k == Ty.K.FLOAT) { + return new E("-".equals(u.op) ? "(-" + o.c + ")" : o.c, Ty.FLOAT); + } + if (o.t.k == Ty.K.INT || o.t.k == Ty.K.BOOL) { + switch (u.op) { + case "-": + return new E("Py.neg(" + box(o) + ")", Ty.INT); + case "~": + return new E("Py.invert(" + box(o) + ")", Ty.INT); + default: + if (o.t.k == Ty.K.INT) { + return o; + } + } + } + throw new Unsupported("unary " + u.op + " " + o.t); + } + if (e instanceof BoolOp) { + BoolOp o = (BoolOp) e; + StringBuilder sb = new StringBuilder("("); + String j = o.isAnd ? " && " : " || "; + for (int i = 0; i < o.values.size(); i++) { + condDepth += i == 0 ? 0 : 1; + E v; + try { + v = expr(o.values.get(i)); + } finally { + condDepth -= i == 0 ? 0 : 1; + } + if (v.t.k != Ty.K.BOOL) { + throw new Unsupported("and/or of " + v.t); + } + sb.append(i == 0 ? "" : j).append(v.c); + } + return new E(sb.append(')').toString(), Ty.BOOL); + } + if (e instanceof Compare) { + if (pre != null && intCompare((Compare) e)) { + return hoistInt(e); + } + return compare((Compare) e); + } + if (e instanceof IfExp) { + IfExp i = (IfExp) e; + String tc = testExpr(i.test); + Set saved = new HashSet(nonNull); + nonNull.addAll(whenTrue(i.test)); + condDepth++; + E a; + E b; + try { + a = expr(i.body); + nonNull = new HashSet(saved); + nonNull.addAll(whenFalse(i.test)); + b = expr(i.orelse); + } finally { + condDepth--; + } + nonNull = saved; + Ty t = Ty.join(a.t, b.t); + if (t.k == Ty.K.BOT) { + throw new Unsupported("if-expression of nothing"); + } + return new E("(" + tc + " ? " + conv(a, t) + " : " + conv(b, t) + ")", t); + } + if (e instanceof Subscript) { + Subscript s = (Subscript) e; + if (s.index instanceof Slice) { + E l = expr(s.value); + if (l.t.k != Ty.K.LIST) { + throw new Unsupported("slice of " + l.t); + } + return new E("Typed.asList(Py.getslice(" + l.c + ", " + sliceParts((Slice) s.index) + "))", + l.t.nonNone()); + } + E l = expr(s.value); + E i = expr(s.index); + if (l.t.k != Ty.K.LIST || !(i.t.k == Ty.K.INT || i.t.k == Ty.K.BOOL)) { + throw new Unsupported("subscript of " + l.t); + } + Ty el = in.elem(l.t); + if (el.k == Ty.K.BOT) { + throw new Unsupported("element of nothing"); + } + String slow = el.k == Ty.K.FLOAT ? "h_getF(" + l.c + ", " + box(i) + ")" : "h_get(" + l.c + ", " + box(i) + ")"; + String fast = slow; + if (i.lc != null && plain(l.c) && pre != null && stmtPure && condDepth == 0) { + // Read into a local before the statement: no call in it can + // change the list, and a Java local, unlike an operand of a + // conditional expression, stays out of ParparVM's stack. + String r = fresh(); + String in = inRange(l, i.lc); + indent(pre, preInd); + pre.append(el.k == Ty.K.FLOAT ? "double " : "Object ").append(r).append(";\n"); + indent(pre, preInd); + if (el.k == Ty.K.FLOAT) { + pre.append("if (").append(in).append(" && ").append(l.c).append(".floats != null) {\n"); + indent(pre, preInd + 1); + pre.append(r).append(" = ").append(l.c).append(".floats[(int) ").append(i.lc).append("];\n"); + } else { + pre.append("if (").append(in).append(" && ").append(l.c).append(".ints == null && ") + .append(l.c).append(".floats == null) {\n"); + indent(pre, preInd + 1); + pre.append(r).append(" = ").append(l.c).append(".items[(int) ").append(i.lc).append("];\n"); + } + indent(pre, preInd); + pre.append("} else {\n"); + indent(pre, preInd + 1); + pre.append(r).append(" = ").append(slow).append(";\n"); + indent(pre, preInd); + pre.append("}\n"); + return el.k == Ty.K.FLOAT ? new E(r, el) : new E(unbox(r, el), el); + } + if (i.lc != null && plain(l.c)) { + // The common case inline, no call: ParparVM inlines none. + String in = inRange(l, i.lc); + fast = el.k == Ty.K.FLOAT + ? "(" + in + " && " + l.c + ".floats != null ? " + l.c + ".floats[(int) " + i.lc + "] : " + slow + ")" + : "(" + in + " ? (" + l.c + ".ints != null ? (Object) Long.valueOf(" + l.c + ".ints[(int) " + i.lc + + "]) : " + l.c + ".floats == null ? " + l.c + ".items[(int) " + i.lc + "] : " + slow + ") : " + + slow + ")"; + } + return el.k == Ty.K.FLOAT ? new E(fast, el) : new E(unbox(fast, el), el); + } + if (e instanceof TupleExpr) { + StringBuilder sb = new StringBuilder("Py.tuple("); + List el = ((TupleExpr) e).elts; + for (int i = 0; i < el.size(); i++) { + if (el.get(i) instanceof Starred) { + throw new Unsupported("starred"); + } + sb.append(i == 0 ? "" : ", ").append(box(expr(el.get(i)))); + } + return new E(sb.append(')').toString(), Ty.DYN); + } + if (e instanceof ListExpr) { + ListExpr l = (ListExpr) e; + Integer site = in.siteOf.get(l); + if (site == null) { + throw new Unsupported("list site"); + } + StringBuilder sb = new StringBuilder("Typed.list(new Object[] {"); + for (int i = 0; i < l.elts.size(); i++) { + if (l.elts.get(i) instanceof Starred) { + throw new Unsupported("starred"); + } + sb.append(i == 0 ? "" : ", ").append(box(expr(l.elts.get(i)))); + } + return new E(sb.append("})").toString(), Ty.list(new int[] {site.intValue()}, false)); + } + if (e instanceof Call) { + return call((Call) e); + } + throw new Unsupported(e.getClass().getSimpleName()); + } + + private E name(Name nm) { + String n = nm.id; + if (subst != null && subst.containsKey(n)) { + return subst.get(n); + } + switch (sc.resolve(n)) { + case LOCAL: { + if (!sc.params.contains(n) && !da.safe.contains(nm)) { + throw new Unsupported("maybe unbound " + n); + } + Ty t = localType(n); + if (sc.deleted.contains(n)) { + throw new Unsupported("deleted " + n); + } + String l = shadow.get(n); + if (l != null) { + return new E(longOnly.contains(n) ? "Long.valueOf(" + l + ")" : var(n), t, l); + } + return new E(var(n), nonNull.contains(n) ? t.nonNone() : t); + } + case GLOBAL: { + if (in.globalFns.containsKey(n) || in.globalClasses.containsKey(n)) { + throw new Unsupported("function as a value"); + } + Ty t = in.globals.get(n); + if (t == null || t.k == Ty.K.BOT) { + throw new Unsupported("global " + n); + } + return new E(unbox("g(" + em.slot(n) + ")", t), t); + } + default: + throw new Unsupported("name " + n); + } + } + + private static String pyOp(String op) { + switch (op) { + case "+": + return "add"; + case "-": + return "sub"; + case "*": + return "mul"; + case "/": + return "truediv"; + case "//": + return "floordiv"; + case "%": + return "mod"; + case "**": + return "pow"; + case "<<": + return "lshift"; + case ">>": + return "rshift"; + case "&": + return "and"; + case "|": + return "or"; + case "^": + return "xor"; + default: + throw new Unsupported("operator " + op); + } + } + + /// The list and float operations typed code uses, emitted into every + /// module that has typed code: ParparVM does not inline a call into + /// another class, and these run once per element. Each falls back to + /// `Typed` (or the list's own method) for every case but the common one. + private static final String LOCAL_HELPERS = String.join("\n", new String[] { + " static Object h_get(PyList l, Object i) {", + " if (l != null && i instanceof Long) {", + " long k = ((Long) i).longValue();", + " if (k >= 0 && k < l.size) {", + " long[] n = l.ints;", + " if (n != null) {", + " return Long.valueOf(n[(int) k]);", + " }", + " if (l.floats == null) {", + " return l.items[(int) k];", + " }", + " }", + " }", + " return Typed.get(l, i);", + " }", + " static double h_getF(PyList l, Object i) {", + " if (l != null && i instanceof Long) {", + " long k = ((Long) i).longValue();", + " double[] f = l.floats;", + " if (f != null && k >= 0 && k < l.size) {", + " return f[(int) k];", + " }", + " }", + " return Typed.getF(l, i);", + " }", + " static void h_set(PyList l, Object i, Object v) {", + " if (l != null && i instanceof Long && l.ints == null && l.floats == null) {", + " long k = ((Long) i).longValue();", + " if (k >= 0 && k < l.size) {", + " l.items[(int) k] = v;", + " return;", + " }", + " }", + " Typed.set(l, i, v);", + " }", + " static void h_setF(PyList l, Object i, double v) {", + " if (l != null && i instanceof Long) {", + " long k = ((Long) i).longValue();", + " double[] f = l.floats;", + " if (f != null && k >= 0 && k < l.size) {", + " f[(int) k] = v;", + " return;", + " }", + " }", + " Typed.setF(l, i, v);", + " }", + " static Object h_len(PyList l) {", + " return l != null ? Long.valueOf(l.size) : Typed.len(l);", + " }", + " static double h_toFloat(Object i) {", + " return i instanceof Long ? (double) ((Long) i).longValue() : Typed.toFloat(i);", + " }", + " static double h_div(double a, double b) {", + " return b != 0 ? a / b : Typed.div(a, b);", + " }", + "" + }); + + // ---- inline int arithmetic ------------------------------------------- + + /// Whether `e` is int arithmetic over int locals and constants that + /// cannot raise: evaluating it early, or twice, changes nothing. + private boolean pureInt(Expr e) { + if (e instanceof Constant) { + Long c = intConstant(e); + return c != null && Math.abs(c.longValue()) < (1L << 60); + } + if (e instanceof Name) { + String n = ((Name) e).id; + return sc.resolve(n) == Scope.Binding.LOCAL && localType(n).k == Ty.K.INT; + } + if (e instanceof UnaryOp) { + UnaryOp u = (UnaryOp) e; + return "-".equals(u.op) && pureInt(u.operand); + } + if (e instanceof BinOp) { + BinOp b = (BinOp) e; + switch (b.op) { + case "+": + case "-": + case "*": + case "&": + case "|": + case "^": + return pureInt(b.left) && pureInt(b.right); + case ">>": { + Long c = intConstant(b.right); + return c != null && c.longValue() >= 0 && c.longValue() < 64 && pureInt(b.left); + } + default: + return false; + } + } + return false; + } + + /// Float arithmetic over float and int locals and constants, as + /// `pureInt` is for ints; every step here is IEEE arithmetic, as in + /// Python, except a division by zero, which takes the generic path. + private boolean pureFloat(Expr e) { + if (e instanceof Constant) { + return ((Constant) e).kind == Constant.Kind.FLOAT && !((Constant) e).text.matches(".*[jJ]"); + } + if (e instanceof Name) { + String n = ((Name) e).id; + return sc.resolve(n) == Scope.Binding.LOCAL && localType(n).k == Ty.K.FLOAT; + } + if (e instanceof UnaryOp) { + UnaryOp u = (UnaryOp) e; + return "-".equals(u.op) && pureFloat(u.operand); + } + if (e instanceof BinOp) { + BinOp b = (BinOp) e; + switch (b.op) { + case "+": + case "-": + case "*": + return pureFloat(b.left) && (pureFloat(b.right) || pureInt(b.right)) + || pureInt(b.left) && pureFloat(b.right); + case "/": + return (pureFloat(b.left) || pureInt(b.left)) && (pureFloat(b.right) || pureInt(b.right)); + default: + return false; + } + } + return false; + } + + private boolean intCompare(Compare c) { + if (c.ops.size() != 1) { + return false; + } + switch (c.ops.get(0)) { + case "<": + case "<=": + case ">": + case ">=": + case "==": + case "!=": + return pureInt(c.left) && pureInt(c.comparators.get(0)); + default: + return false; + } + } + + /// `e` computed on Java longs in a block before the statement, falling + /// back to the generic operations when an operand is a big int or a + /// step overflows. ParparVM inlines no call, not even to a helper of + /// the same class, so each operation is spelled out here. + private E hoistInt(Expr e) { + StringBuilder saved = pre; + String generic; + Ty t; + // The generic path goes into a method of its own: with it inline, a + // function made only of such arithmetic would call out, and a + // ParparVM function that calls out gets a stack guard and is no + // longer inlined by the C compiler. + Map nodes = new LinkedHashMap(); + for (Name nm : namesIn(e)) { + nodes.putIfAbsent(nm.id, nm); + } + List names = new ArrayList(nodes.keySet()); + Map sub = new HashMap(); + StringBuilder params = new StringBuilder(); + StringBuilder args = new StringBuilder(); + for (int k = 0; k < names.size(); k++) { + E ne = name(nodes.get(names.get(k))); + String pn = "p" + k; + String sep = k == 0 ? "" : ", "; + if (ne.lc != null && longOnly.contains(names.get(k))) { + params.append(sep).append("long ").append(pn); + args.append(sep).append(ne.lc); + sub.put(names.get(k), new E("Long.valueOf(" + pn + ")", ne.t, pn)); + } else { + params.append(sep).append(jt(ne.t)).append(' ').append(pn); + args.append(sep).append(ne.c); + sub.put(names.get(k), new E(pn, ne.t)); + } + } + Map savedSub = subst; + pre = null; + subst = sub; + try { + E g = e instanceof Compare ? compare((Compare) e) : expr(e); + t = g.t; + String m = "S" + (slowCount++); + slow.append(" static ").append(jt(t)).append(' ').append(m).append('(').append(params) + .append(") {\n return ").append(g.c).append(";\n }\n"); + generic = m + "(" + args + ")"; + } finally { + pre = saved; + subst = savedSub; + } + String r = fresh(); + String done = fresh(); + String fast = fresh(); + StringBuilder f = new StringBuilder(); + Map leaves = new HashMap(); + String v; + if (e instanceof Compare) { + Compare c = (Compare) e; + String a = longOf(c.left, f, leaves, fast); + String b2 = longOf(c.comparators.get(0), f, leaves, fast); + v = a + " " + c.ops.get(0) + " " + b2; + } else if (t.k == Ty.K.FLOAT) { + v = doubleOf(e, f, leaves, fast); + } else { + v = "Long.valueOf(" + longOf(e, f, leaves, fast) + ")"; + } + int ind = preInd; + indent(pre, ind); + pre.append(jt(t)).append(' ').append(r).append(";\n"); + indent(pre, ind); + pre.append(done).append(": {\n"); + indent(pre, ind + 1); + pre.append(fast).append(": {\n"); + for (String line : f.toString().split("\n")) { + indent(pre, ind + 2); + pre.append(line).append('\n'); + } + indent(pre, ind + 2); + pre.append(r).append(" = ").append(v).append(";\n"); + indent(pre, ind + 2); + pre.append("break ").append(done).append(";\n"); + indent(pre, ind + 1); + pre.append("}\n"); + indent(pre, ind + 1); + pre.append(r).append(" = ").append(generic).append(";\n"); + indent(pre, ind); + pre.append("}\n"); + return new E(r, t); + } + + private static List namesIn(Object o) { + List out = new ArrayList(); + if (o instanceof Name) { + out.add((Name) o); + } + Infer.children(o, x -> out.addAll(namesIn(x))); + return out; + } + + /// Statements into `f` computing `e` (`pureFloat`, or `pureInt` to be + /// converted) as a double; the expression holding it. + private String doubleOf(Expr e, StringBuilder f, Map leaves, String fast) { + if (pureInt(e)) { + String x = longOf(e, f, leaves, fast); + String d = fresh(); + // Exact below 2^53; Python rounds a wider int the same way, but + // the generic path is the one that knows. + f.append("if (").append(x).append(" > 9007199254740992L || ").append(x) + .append(" < -9007199254740992L) {\n break ").append(fast).append(";\n}\ndouble ").append(d) + .append(" = (double) ").append(x).append(";\n"); + return d; + } + if (e instanceof Constant || e instanceof Name) { + return expr(e).c; + } + String d = fresh(); + if (e instanceof UnaryOp) { + f.append("double ").append(d).append(" = -").append(doubleOf(((UnaryOp) e).operand, f, leaves, fast)) + .append(";\n"); + return d; + } + BinOp b = (BinOp) e; + String x = doubleOf(b.left, f, leaves, fast); + String y = doubleOf(b.right, f, leaves, fast); + if ("/".equals(b.op)) { + f.append("if (").append(y).append(" == 0) {\n break ").append(fast).append(";\n}\n"); + } + f.append("double ").append(d).append(" = ").append(x).append(' ').append(b.op).append(' ').append(y) + .append(";\n"); + return d; + } + + /// Statements into `f` computing `e` as a long; the name holding it. + private String longOf(Expr e, StringBuilder f, Map leaves, String fast) { + if (e instanceof Constant) { + return intConstant(e) + "L"; + } + if (e instanceof Name) { + E ne = name((Name) e); + if (ne.lc != null) { + return ne.lc; + } + String c = ne.c; + String d = leaves.get(c); + if (d == null) { + d = fresh(); + leaves.put(c, d); + f.append("if (!(").append(c).append(" instanceof Long)) {\n break ").append(fast) + .append(";\n}\nlong ").append(d).append(" = ((Long) ").append(c).append(").longValue();\n"); + } + return d; + } + String d = fresh(); + if (e instanceof UnaryOp) { + String x = longOf(((UnaryOp) e).operand, f, leaves, fast); + f.append("if (").append(x).append(" == Long.MIN_VALUE) {\n break ").append(fast).append(";\n}\n") + .append("long ").append(d).append(" = -").append(x).append(";\n"); + return d; + } + BinOp b = (BinOp) e; + String x = longOf(b.left, f, leaves, fast); + String y = longOf(b.right, f, leaves, fast); + switch (b.op) { + case "+": + f.append("long ").append(d).append(" = ").append(x).append(" + ").append(y).append(";\n") + .append("if (((").append(x).append(" ^ ").append(d).append(") & (").append(y).append(" ^ ") + .append(d).append(")) < 0) {\n break ").append(fast).append(";\n}\n"); + break; + case "-": + f.append("long ").append(d).append(" = ").append(x).append(" - ").append(y).append(";\n") + .append("if (((").append(x).append(" ^ ").append(y).append(") & (").append(x).append(" ^ ") + .append(d).append(")) < 0) {\n break ").append(fast).append(";\n}\n"); + break; + case "*": + // Exact when both fit 31 bits; anything wider takes the runtime. + f.append("if (((").append(x).append(" + 0x80000000L) | (").append(y) + .append(" + 0x80000000L)) >>> 32 != 0) {\n break ").append(fast).append(";\n}\n") + .append("long ").append(d).append(" = ").append(x).append(" * ").append(y).append(";\n"); + break; + default: + f.append("long ").append(d).append(" = ").append(x).append(' ').append(b.op).append(' ').append(y) + .append(";\n"); + break; + } + return d; + } + + /// Helpers emitted into this module: an int operation with the `Long` + /// fast path inline, so the common case costs no call into the runtime. + private final Set intHelpers = new TreeSet(); + + private static final Set INT_HELPED = new HashSet(Arrays.asList("add", "sub", "mul", "and", "or", + "xor", "rshift", "lt", "le", "gt", "ge", "eq")); + + /// `f` (a `Py` operation, or a comparison) of two boxed ints. + private String intOp(String f, String a, String b, Expr right) { + if (!INT_HELPED.contains(f)) { + return "Py." + f + "(" + a + ", " + b + ")"; + } + Long c = intConstant(right); + if (c != null && (!f.equals("rshift") || c.longValue() >= 0 && c.longValue() < 64)) { + intHelpers.add(f + "c"); + return "i_" + f + "c(" + a + ", " + c + "L)"; + } + if ("rshift".equals(f)) { + return "Py.rshift(" + a + ", " + b + ")"; + } + intHelpers.add(f); + return "i_" + f + "(" + a + ", " + b + ")"; + } + + private static String intHelper(String h) { + boolean c = h.endsWith("c") && !"c".equals(h); + String f = c ? h.substring(0, h.length() - 1) : h; + boolean cmp = "lt".equals(f) || "le".equals(f) || "gt".equals(f) || "ge".equals(f) || "eq".equals(f); + String y = c ? "y" : "((Long) b).longValue()"; + String guard = c ? "a instanceof Long" : "a instanceof Long && b instanceof Long"; + String bo = c ? "Long.valueOf(y)" : "b"; + StringBuilder o = new StringBuilder(" static ").append(cmp ? "boolean" : "Object").append(" i_").append(h) + .append("(Object a, ").append(c ? "long y" : "Object b").append(") {\n if (").append(guard) + .append(") {\n long x = ((Long) a).longValue();\n"); + if (!c) { + o.append(" long y = ").append(y).append(";\n"); + } + switch (f) { + case "add": + o.append(" long s = x + y;\n if (((x ^ s) & (y ^ s)) >= 0) {\n") + .append(" return Long.valueOf(s);\n }\n"); + break; + case "sub": + o.append(" long s = x - y;\n if (((x ^ y) & (x ^ s)) >= 0) {\n") + .append(" return Long.valueOf(s);\n }\n"); + break; + case "mul": + // Exact when both fit 31 bits; otherwise the runtime decides. + o.append(" if (((x + 0x80000000L) | (y + 0x80000000L)) >>> 32 == 0) {\n") + .append(" return Long.valueOf(x * y);\n }\n"); + break; + case "and": + o.append(" return Long.valueOf(x & y);\n"); + break; + case "or": + o.append(" return Long.valueOf(x | y);\n"); + break; + case "xor": + o.append(" return Long.valueOf(x ^ y);\n"); + break; + case "rshift": + o.append(" return Long.valueOf(x >> y);\n"); + break; + default: { + String jop = "lt".equals(f) ? "<" : "le".equals(f) ? "<=" : "gt".equals(f) ? ">" : "ge".equals(f) + ? ">=" : "=="; + o.append(" return x ").append(jop).append(" y;\n"); + } + } + o.append(" }\n return "); + if (cmp) { + o.append("eq".equals(f) ? "Py.eqBool(a, " + bo + ")" : "Py.isTrue(Py." + f + "(a, " + bo + "))"); + } else { + o.append("Py.").append(f).append("(a, ").append(bo).append(")"); + } + return o.append(";\n }\n").toString(); + } + + E binop(String op, E l, E r, Expr right) { + Ty t = Infer.binop(op, l.t, r.t, right); + if (l.t.k == Ty.K.LIST && "*".equals(op) && (r.t.k == Ty.K.INT || r.t.k == Ty.K.BOOL)) { + // `[x] * n`: Infer gave the product the sites of the literal. + return new E("Typed.asList(Py.mul(" + l.c + ", " + box(r) + "))", Ty.list(l.t.sites, false)); + } + switch (t.k) { + case FLOAT: { + if ((l.t.k == Ty.K.INT || l.t.k == Ty.K.BOOL) && (r.t.k == Ty.K.INT || r.t.k == Ty.K.BOOL)) { + return new E("Typed.truediv(" + box(l) + ", " + box(r) + ")", Ty.FLOAT); + } + String a = toD(l); + String b = toD(r); + switch (op) { + case "+": + case "-": + case "*": + return new E("(" + a + " " + op + " " + b + ")", Ty.FLOAT); + case "/": + return new E("h_div(" + a + ", " + b + ")", Ty.FLOAT); + case "//": + return new E("Typed.floordiv(" + a + ", " + b + ")", Ty.FLOAT); + case "%": + return new E("Typed.mod(" + a + ", " + b + ")", Ty.FLOAT); + default: + throw new Unsupported("float " + op); + } + } + case INT: + return new E(intOp(pyOp(op), box(l), box(r), right), Ty.INT); + case STR: + return new E("Typed.concat(" + l.c + ", " + r.c + ")", Ty.STR); + case BOT: + throw new Unsupported("operation on nothing"); + default: + return new E("Py." + pyOp(op) + "(" + box(l) + ", " + box(r) + ")", Ty.DYN); + } + } + + private static boolean isNone(E e) { + return e.t.k == Ty.K.NONE; + } + + private String isNoneTest(E x) { + if (x.t.nullable()) { + return "(" + x.c + " == null)"; + } + if ("Object".equals(jt(x.t))) { + return "(" + x.c + " == Py.NONE)"; + } + if (plain(x.c)) { + return "false"; + } + throw new Unsupported("is None of " + x.t); + } + + private E compare(Compare c) { + if (c.ops.size() != 1) { + throw new Unsupported("chained comparison"); + } + String op = c.ops.get(0); + E l = expr(c.left); + E r = expr(c.comparators.get(0)); + if ("is".equals(op) || "is not".equals(op) || isNone(l) || isNone(r)) { + boolean neg = "is not".equals(op) || "!=".equals(op); + if (!("is".equals(op) || "is not".equals(op) || "==".equals(op) || "!=".equals(op))) { + throw new Unsupported("ordering against None"); + } + String s; + if (isNone(r)) { + s = isNoneTest(l); + } else if (isNone(l)) { + s = isNoneTest(r); + } else if (l.t.k == Ty.K.INST && r.t.k == Ty.K.INST || l.t.k == Ty.K.LIST && r.t.k == Ty.K.LIST) { + s = "(" + l.c + " == " + r.c + ")"; + } else if (op.startsWith("is")) { + s = "(" + box(l) + " == " + box(r) + ")"; + } else { + throw new Unsupported("== on " + l.t + ", " + r.t); + } + return new E(neg ? "!" + s : s, Ty.BOOL); + } + boolean li = l.t.k == Ty.K.INT || l.t.k == Ty.K.BOOL; + boolean ri = r.t.k == Ty.K.INT || r.t.k == Ty.K.BOOL; + if ("in".equals(op) || "not in".equals(op)) { + if (r.t.k == Ty.K.LIST) { + String s = "Typed.nzl(" + r.c + ").contains(" + box(l) + ")"; + return new E("in".equals(op) ? s : "!" + s, Ty.BOOL); + } + throw new Unsupported("in " + r.t); + } + if (l.t.k == Ty.K.BOOL && r.t.k == Ty.K.BOOL && ("==".equals(op) || "!=".equals(op))) { + return new E("(" + l.c + " " + op + " " + r.c + ")", Ty.BOOL); + } + if (li && ri) { + String f; + switch (op) { + case "<": + f = "lt"; + break; + case "<=": + f = "le"; + break; + case ">": + f = "gt"; + break; + case ">=": + f = "ge"; + break; + case "==": + f = "eq"; + break; + case "!=": + return new E("!" + intOp("eq", box(l), box(r), c.comparators.get(0)), Ty.BOOL); + default: + throw new Unsupported("compare " + op); + } + return new E(intOp(f, box(l), box(r), c.comparators.get(0)), Ty.BOOL); + } + if ((l.t.k == Ty.K.FLOAT || li) && (r.t.k == Ty.K.FLOAT || ri)) { + return new E("(" + toD(l) + " " + op + " " + toD(r) + ")", Ty.BOOL); + } + if (l.t.k == Ty.K.STR && r.t.k == Ty.K.STR && !l.t.opt && !r.t.opt + && ("==".equals(op) || "!=".equals(op))) { + String s = l.c + ".equals(" + r.c + ")"; + return new E("==".equals(op) ? s : "!" + s, Ty.BOOL); + } + if (l.t.k == Ty.K.INST && r.t.k == Ty.K.INST && ("==".equals(op) || "!=".equals(op)) + && !customCompare(l.t.cls) && !customCompare(r.t.cls)) { + return new E("(" + l.c + " " + op + " " + r.c + ")", Ty.BOOL); + } + String f; + switch (op) { + case "==": + f = "eq"; + break; + case "!=": + f = "ne"; + break; + case "<": + f = "lt"; + break; + case "<=": + f = "le"; + break; + case ">": + f = "gt"; + break; + case ">=": + f = "ge"; + break; + default: + throw new Unsupported("compare " + op); + } + // Infer types this DYN too: a user comparison may answer anything. + return new E("Py." + f + "(" + box(l) + ", " + box(r) + ")", Ty.DYN); + } + + private boolean customCompare(Layouts.Cls k) { + for (Layouts.Cls c : layouts.classes.values()) { + if (Infer.isSelfOrAncestor(k, c) || Infer.isSelfOrAncestor(c, k)) { + if (c.bodyNames.contains("__eq__") || c.bodyNames.contains("__ne__")) { + return true; + } + } + } + return false; + } + + // ---- calls ----------------------------------------------------------- + + private List args(Call c) { + if (!c.keywords.isEmpty()) { + throw new Unsupported("keyword arguments"); + } + List r = new ArrayList(); + for (Expr a : c.args) { + if (a instanceof Starred) { + throw new Unsupported("starred argument"); + } + r.add(expr(a)); + } + return r; + } + + /// The Java argument list for `f`, from its parameter `from` on; + /// missing trailing arguments take their constant defaults. + private String argList(Infer.Fn f, int from, List args) { + int n = f.params.length - from; + if (args.size() > n) { + throw new Unsupported("too many arguments"); + } + StringBuilder sb = new StringBuilder(); + List defs = f.def.args.defaults; + for (int i = 0; i < n; i++) { + E a; + if (i < args.size()) { + a = args.get(i); + } else { + int d = defs.size() - (f.params.length - (from + i)); + if (d < 0 || !(defs.get(d) instanceof Constant)) { + throw new Unsupported("missing argument"); + } + a = expr(defs.get(d)); + } + sb.append(i == 0 ? "" : ", ").append(conv(a, f.params[from + i])); + } + return sb.toString(); + } + + /// `0 <= lc < size` of list `l` (and `l` not None). + private static String inRange(E l, String lc) { + String r = lc + " >= 0 && " + lc + " < " + l.c + ".size"; + return l.t.opt ? l.c + " != null && " + r : r; + } + + private static boolean hasCall(Object o) { + if (o instanceof Call) { + return true; + } + boolean[] f = new boolean[1]; + Infer.children(o, x -> f[0] |= hasCall(x)); + return f[0]; + } + + /// A call of `f`'s `TL_` variant when every argument for one of its + /// long parameters has a long form; else null. + private String longCall(Infer.Fn f, List args) { + if (args.size() != f.params.length || !hasLongParam(f)) { + return null; + } + Set stored = assigned(f.def.body); + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < args.size(); i++) { + E a = args.get(i); + String v; + if (longParam(f, i, stored)) { + if (a.lc == null) { + return null; + } + v = a.lc; + } else { + v = conv(a, f.params[i]); + } + sb.append(i == 0 ? "" : ", ").append(v); + } + wantLong.add(f); + return "TL" + typed.get(f).substring(2) + "(" + sb + ")"; + } + + private E result(Infer.Fn f, String call) { + return new E(call, f.ret.k == Ty.K.BOT ? Ty.DYN : f.ret); + } + + private E call(Call c) { + E r = call0(c); + if (pre != null && stmtPure && condDepth == 0 && (r.t.k == Ty.K.FLOAT || r.t.k == Ty.K.BOOL)) { + // A pure call's result into a local, in evaluation order (see + // the subscript read). + String t = fresh(); + indent(pre, preInd); + pre.append(jt(r.t)).append(' ').append(t).append(" = ").append(r.c).append(";\n"); + r = new E(t, r.t); + } + // The callee may store into any field. + forgetPaths(); + return r; + } + + private E call0(Call c) { + if (c.func instanceof Name) { + String n = ((Name) c.func).id; + Scope.Binding b = sc.resolve(n); + if (b == Scope.Binding.GLOBAL) { + Infer.Fn f = in.globalFns.get(n); + if (f != null) { + List a = args(c); + calls.add(f); + String lcall = longCall(f, a); + if (lcall != null) { + return result(f, lcall); + } + return result(f, typed.get(f) + "(" + argList(f, 0, a) + ")"); + } + Layouts.Cls k = in.globalClasses.get(n); + if (k != null) { + return construct(k, args(c)); + } + } + if (b == Scope.Binding.BUILTIN && !em.moduleScopeHasStarImport() && "list".equals(n) + && c.args.size() == 1 && c.keywords.isEmpty() && in.siteOf.get(c) != null) { + Ty lt = Ty.list(new int[] {in.siteOf.get(c).intValue()}, false); + Expr a0 = c.args.get(0); + if (in.rangeCall(a0, sc)) { + StringBuilder sb = new StringBuilder("Typed.rangeList("); + List ra = ((Call) a0).args; + for (int i = 0; i < ra.size(); i++) { + sb.append(i == 0 ? "" : ", ").append(box(expr(ra.get(i)))); + } + return new E(sb.append(')').toString(), lt); + } + E x = expr(a0); + if (x.t.k == Ty.K.LIST) { + return new E("Typed.copy(" + x.c + ")", lt); + } + throw new Unsupported("list of " + x.t); + } + if (b == Scope.Binding.BUILTIN && !em.moduleScopeHasStarImport()) { + return builtin(n, args(c)); + } + throw new Unsupported("call of " + n); + } + if (c.func instanceof Attribute) { + Attribute a = (Attribute) c.func; + if (a.value instanceof Name && in.globalClasses.containsKey(((Name) a.value).id) + && sc.resolve(((Name) a.value).id) == Scope.Binding.GLOBAL) { + Layouts.Cls k = in.globalClasses.get(((Name) a.value).id); + Layouts.Cls o = Layouts.resolver(k, a.attr); + FunctionDef m = o == null ? null : o.typable.get(a.attr); + if (m == null) { + throw new Unsupported("class attribute call"); + } + Infer.Fn f = in.fns.get(m); + calls.add(f); + return result(f, typed.get(f) + "(" + argList(f, 0, args(c)) + ")"); + } + E r = expr(a.value); + List args = args(c); + if (r.t.k == Ty.K.LIST && !r.t.opt) { + return listMethod(r, a.attr, args); + } + if (r.t.k != Ty.K.INST) { + throw new Unsupported("method of " + r.t); + } + List cand = in.candidates(r.t.cls, a.attr); + if (cand == null) { + throw new Unsupported("method " + a.attr); + } + String recv = nz(r, a.attr); + Layouts.Cls res = Layouts.resolver(r.t.cls, a.attr); + if (cand.size() == 1 && res != null && res.typable.get(a.attr) == cand.get(0)) { //NOPMD CompareObjectsWithEquals - identity is meant + Infer.Fn f = in.fns.get(cand.get(0)); + calls.add(f); + String al = argList(f, 1, args); + return result(f, typed.get(f) + "(" + recv + (al.isEmpty() ? "" : ", " + al) + ")"); + } + String key = a.attr + "/" + args.size(); + List g = groups.get(key); + if (g == null) { + throw new Unsupported("no group " + key); + } + for (FunctionDef m : cand) { + if (!g.contains(in.fns.get(m))) { + throw new Unsupported("arity mismatch " + key); + } + } + callsGroups.add(key); + Infer.Fn f = g.get(0); + String al = argList(f, 1, args); + return result(f, recv + ".tv_" + Emitter.mangle(a.attr) + "_" + args.size() + "(" + al + ")"); + } + throw new Unsupported("call"); + } + + private E construct(Layouts.Cls k, List args) { + Layouts.Cls o = Layouts.resolver(k, "__init__"); + if (o == null) { + if (!args.isEmpty()) { + throw new Unsupported("arguments to object()"); + } + return new E("new " + k.javaName + "(" + k.typeVar + ")", Ty.inst(k, false)); + } + FunctionDef init = o.typable.get("__init__"); + if (init == null) { + throw new Unsupported("__init__"); + } + Infer.Fn f = in.fns.get(init); + calls.add(f); + constructors.add(k); + String al = argList(f, 1, args); + return new E("n_" + k.javaName + "(" + al + ")", Ty.inst(k, false)); + } + + private E builtin(String n, List a) { + switch (n) { + case "len": + if (a.size() == 1 && a.get(0).t.k == Ty.K.LIST) { + return new E("h_len(" + a.get(0).c + ")", Ty.INT); + } + break; + case "float": + if (a.size() == 1 && !a.get(0).t.opt + && (a.get(0).t.k == Ty.K.INT || a.get(0).t.k == Ty.K.FLOAT)) { + return new E(toD(a.get(0)), Ty.FLOAT); + } + break; + case "abs": + if (a.size() == 1 && a.get(0).t.k == Ty.K.FLOAT) { + return new E("Math.abs(" + a.get(0).c + ")", Ty.FLOAT); + } + if (a.size() == 1 && a.get(0).t.k == Ty.K.INT) { + return new E("Typed.abs(" + a.get(0).c + ")", Ty.INT); + } + break; + default: + break; + } + throw new Unsupported("builtin " + n); + } + + private E listMethod(E l, String m, List a) { + switch (m) { + case "append": + if (a.size() == 1) { + return new E("Typed.append(" + l.c + ", " + box(a.get(0)) + ")", Ty.NONE); + } + break; + case "pop": + if (a.size() <= 1) { + Ty el = in.elem(l.t); + String c = l.c + ".pop(" + (a.isEmpty() ? "-1L" : "Py.asLong(" + box(a.get(0)) + ")") + ")"; + return new E(unbox(c, el), el.k == Ty.K.BOT ? Ty.DYN : el); + } + break; + default: + break; + } + throw new Unsupported("list." + m); + } + + // ---- output ---------------------------------------------------------- + + /// The typed fn whose generic body is `fs`, or null. + Infer.Fn fnFor(Scope fs) { + Infer.Fn f = byScope.get(fs); + return f != null && typed.containsKey(f) ? f : null; + } + + /// The statement the generic body of `f` starts with: enter the typed + /// version when the arguments, held in `locals`, fit it. + String entry(Infer.Fn f, List locals) { + StringBuilder test = new StringBuilder(); + StringBuilder args = new StringBuilder(); + for (int i = 0; i < f.params.length; i++) { + Ty t = f.params[i]; + String v = locals.get(i); + String check = null; + String arg; + switch (t.k) { + case BOOL: + check = "(" + v + " == Py.TRUE || " + v + " == Py.FALSE)"; + arg = v + " == Py.TRUE"; + break; + case FLOAT: + check = v + " instanceof Double"; + arg = "((Double) " + v + ").doubleValue()"; + break; + case STR: + check = t.opt ? "(" + v + " instanceof String || " + v + " == Py.NONE)" : v + " instanceof String"; + arg = "Typed.asStr(" + v + ")"; + break; + case LIST: + check = t.opt ? "(" + v + " instanceof PyList || " + v + " == Py.NONE)" : v + " instanceof PyList"; + arg = "Typed.asList(" + v + ")"; + break; + case INST: + check = t.opt ? "(" + v + " instanceof " + t.cls.javaName + " || " + v + " == Py.NONE)" + : v + " instanceof " + t.cls.javaName; + arg = unbox(v, t); + break; + case INT: + // A guessed parameter (Infer.guess) is checked like any other. + check = "(" + v + " instanceof Long || " + v + " instanceof PyInt)"; + arg = v; + break; + case NONE: + check = v + " == Py.NONE"; + arg = v; + break; + default: + arg = v; + break; + } + if (check != null) { + test.append(test.length() == 0 ? "" : " && ").append(check); + } + args.append(i == 0 ? "" : ", ").append(arg); + } + String call = box(new E(typed.get(f) + "(" + args + ")", f.ret)); + if (test.length() == 0) { + // Always taken; javac must still see the generic body as reachable. + test.append("Typed.yes"); + } + return "if (" + test + ") {\n return " + call + ";\n}\n"; + } + + /// The Java declaration of a typed field, or null for an object one. + String fieldDecl(Layouts.Cls k, String f) { + Ty t = typedField(k, f); + return t == null ? null : jt(t) + " t_" + Emitter.mangle(f) + ";"; + } + + String fieldGetCase(Layouts.Cls k, String f) { + Ty t = typedField(k, f); + return box(new E("t_" + Emitter.mangle(f), t)); + } + + String fieldSetCase(Layouts.Cls k, String f, String v) { + Ty t = typedField(k, f); + return "t_" + Emitter.mangle(f) + " = " + unbox(v, t) + ";"; + } + + /// Members of layout class `k` (`tv_` overrides) and of the base `LB` + /// (`k` null: the `tv_` declarations). + String layoutMembers(Layouts.Cls k) { + StringBuilder o = new StringBuilder(); + for (String key : usedGroups) { + List g = groups.get(key); + Infer.Fn any = g.get(0); + String name = any.def.name; + int ar = any.params.length - 1; + StringBuilder sig = new StringBuilder(); + StringBuilder pass = new StringBuilder(); + for (int i = 1; i <= ar; i++) { + sig.append(i == 1 ? "" : ", ").append(jt(any.params[i])).append(" a").append(i); + pass.append(", a").append(i); + } + String head = " " + retJ(any) + " tv_" + Emitter.mangle(name) + "_" + ar + "(" + sig + ") {\n"; + if (k == null) { + o.append(head).append(" throw Py.attributeError(\"'\" + cls.name + \"' object has no attribute ") + .append(Emitter.javaString(name).replace("\"", "'")).append("\");\n }\n"); + continue; + } + FunctionDef m = k.typable.get(name); + if (m == null || !k.bodyNames.contains(name)) { + continue; + } + Infer.Fn f = in.fns.get(m); + if (!g.contains(f)) { + continue; + } + o.append(" @Override\n").append(head).append(" return ").append(typed.get(f)) + .append("(this").append(pass).append(");\n }\n"); + } + return o.toString(); + } + + /// The typed methods and their helpers, as static members of the + /// module class. + String members() { + StringBuilder o = new StringBuilder(); + for (String b : bodies.values()) { + o.append(b); + } + // A TL_ body may want further variants; emit until none is new. + while (!longBodies.keySet().containsAll(wantLong)) { + for (Infer.Fn f : new ArrayList(wantLong)) { + if (!bodies.containsKey(f)) { + // Its caller turned out untyped too; nothing calls it. + longBodies.put(f, ""); + } else if (!longBodies.containsKey(f)) { + longParams = true; + try { + longBodies.put(f, function(f)); + } finally { + longParams = false; + } + o.append(longBodies.get(f)); + } + } + } + for (Layouts.Cls k : constructors) { + Layouts.Cls ow = Layouts.resolver(k, "__init__"); + Infer.Fn f = in.fns.get(ow.typable.get("__init__")); + StringBuilder sig = new StringBuilder(); + StringBuilder pass = new StringBuilder(); + for (int i = 1; i < f.params.length; i++) { + sig.append(i == 1 ? "" : ", ").append(jt(f.params[i])).append(" a").append(i); + pass.append(", a").append(i); + } + o.append(" static ").append(k.javaName).append(" n_").append(k.javaName).append('(').append(sig) + .append(") {\n ").append(k.javaName).append(" o = new ").append(k.javaName).append('(') + .append(k.typeVar).append(");\n ").append(typed.get(f)).append("(o").append(pass) + .append(");\n return o;\n }\n"); + } + if (!bodies.isEmpty()) { + o.append(LOCAL_HELPERS); + } + o.append(slow); + for (String h : intHelpers) { + o.append(intHelper(h)); + } + for (Layouts.Cls k : unboxers) { + o.append(" static ").append(k.javaName).append(" u_").append(k.javaName) + .append("(Object o) {\n if (o instanceof ").append(k.javaName).append(") {\n return (") + .append(k.javaName).append(") o;\n }\n if (o == Py.NONE) {\n return null;\n") + .append(" }\n throw Typed.mismatch(o);\n }\n"); + } + return o.toString(); + } +} diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/parser/Lexer.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/parser/Lexer.java new file mode 100644 index 00000000000..a623aff46ac --- /dev/null +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/parser/Lexer.java @@ -0,0 +1,533 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.transpiler.parser; + +import com.codename1.python.transpiler.api.PythonSyntaxError; + +import java.util.ArrayList; +import java.util.List; + +/// Python's tokenizer: logical lines, INDENT/DEDENT, implicit joining inside +/// brackets, explicit `\` continuation, and every string literal form +/// including PEP 701 f-strings (which may nest quotes of their own kind). +/// +/// An f-string is returned as one STRING token whose `value` is its raw body; +/// the parser splits the replacement fields and lexes each expression again. +public final class Lexer { + private static final String[] OPS3 = {"**=", "//=", ">>=", "<<=", "...", "!="}; + private static final String[] OPS2 = {"**", "//", ">>", "<<", "<=", ">=", "==", "!=", "->", "+=", "-=", "*=", + "/=", "%=", "&=", "|=", "^=", "@=", ":="}; + private static final String OPS1 = "+-*/%@&|^~<>()[]{},:.;="; + + private final String src; + private final String file; + private final List out = new ArrayList(); + private final List indents = new ArrayList(); + private int pos; + private int line = 1; + private int lineStart; + private int depth; + + public Lexer(String src, String file) { + this.src = src.replace("\r\n", "\n").replace('\r', '\n'); + this.file = file; + indents.add(Integer.valueOf(0)); + } + + private PythonSyntaxError error(String msg) { + return new PythonSyntaxError(file, line, pos - lineStart + 1, msg); + } + + private void add(Token.Kind k, String text, int startCol) { + out.add(new Token(k, text, line, startCol, null, null)); + } + + public List tokenize() { + boolean atLineStart = true; + int n = src.length(); + if (n > 0 && src.charAt(0) == '\uFEFF') { + pos = 1; + } + while (pos < n) { + if (atLineStart && depth == 0) { + int col = 0; + int p = pos; + while (p < n && (src.charAt(p) == ' ' || src.charAt(p) == '\t' || src.charAt(p) == '\f')) { + col = src.charAt(p) == '\t' ? (col / 8 + 1) * 8 : src.charAt(p) == '\f' ? 0 : col + 1; + p++; + } + if (p >= n) { + pos = p; + break; + } + char c = src.charAt(p); + if (c == '\n' || c == '#' || c == '\\' && p + 1 < n && src.charAt(p + 1) == '\n') { + // Blank or comment-only line: no indentation change. + pos = p; + if (c == '#') { + while (pos < n && src.charAt(pos) != '\n') { + pos++; + } + } + if (pos < n && src.charAt(pos) == '\\') { + pos++; + } + if (pos < n) { + pos++; + line++; + lineStart = pos; + } + continue; + } + pos = p; + int top = indents.get(indents.size() - 1).intValue(); + if (col > top) { + indents.add(Integer.valueOf(col)); + add(Token.Kind.INDENT, "", 1); + } else { + while (col < indents.get(indents.size() - 1).intValue()) { + indents.remove(indents.size() - 1); + add(Token.Kind.DEDENT, "", 1); + } + if (col != indents.get(indents.size() - 1).intValue()) { + throw error("unindent does not match any outer indentation level"); + } + } + atLineStart = false; + } + char c = src.charAt(pos); + int startCol = pos - lineStart + 1; + if (c == '\n') { + if (depth == 0) { + add(Token.Kind.NEWLINE, "", startCol); + atLineStart = true; + } + pos++; + line++; + lineStart = pos; + continue; + } + if (c == ' ' || c == '\t' || c == '\f') { + pos++; + continue; + } + if (c == '#') { + while (pos < n && src.charAt(pos) != '\n') { + pos++; + } + continue; + } + if (c == '\\') { + if (pos + 1 < n && src.charAt(pos + 1) == '\n') { + pos += 2; + line++; + lineStart = pos; + continue; + } + throw error("unexpected character after line continuation character"); + } + if (isIdStart(src.codePointAt(pos))) { + int s = pos; + while (pos < n && isIdPart(src.codePointAt(pos))) { + pos += Character.charCount(src.codePointAt(pos)); + } + String word = src.substring(s, pos); + if (pos < n && (src.charAt(pos) == '"' || src.charAt(pos) == '\'') && isTemplatePrefix(word)) { + throw error("t-strings are not supported yet"); + } + if (pos < n && (src.charAt(pos) == '"' || src.charAt(pos) == '\'') && isPrefix(word)) { + string(word, s, startCol); + continue; + } + add(Token.Kind.NAME, word, startCol); + continue; + } + if (c == '"' || c == '\'') { + string("", pos, startCol); + continue; + } + if (Character.isDigit(c) || c == '.' && pos + 1 < n && Character.isDigit(src.charAt(pos + 1))) { + number(startCol); + continue; + } + String op = null; + for (int i = 0; i < OPS3.length && op == null; i++) { + if (src.startsWith(OPS3[i], pos)) { + op = OPS3[i]; + } + } + for (int i = 0; i < OPS2.length && op == null; i++) { + if (src.startsWith(OPS2[i], pos)) { + op = OPS2[i]; + } + } + if (op == null && OPS1.indexOf(c) >= 0) { + op = String.valueOf(c); + } + if (op == null) { + throw error("invalid character '" + c + "' (U+" + Integer.toHexString(c) + ")"); + } + if ("([{".indexOf(op.charAt(0)) >= 0 && op.length() == 1) { + depth++; + } else if (")]}".indexOf(op.charAt(0)) >= 0 && op.length() == 1) { + depth = Math.max(0, depth - 1); + } + pos += op.length(); + add(Token.Kind.OP, op, startCol); + } + if (!out.isEmpty() && out.get(out.size() - 1).kind != Token.Kind.NEWLINE + && out.get(out.size() - 1).kind != Token.Kind.DEDENT) { + add(Token.Kind.NEWLINE, "", 1); + } + while (indents.size() > 1) { + indents.remove(indents.size() - 1); + add(Token.Kind.DEDENT, "", 1); + } + add(Token.Kind.EOF, "", 1); + return out; + } + + /// Python identifiers are XID_Start / XID_Continue; Java's Unicode + /// identifier classes are the same sets apart from the ignorable format + /// characters, which Python rejects. Code points, not chars, so a + /// supplementary-plane letter or variation selector is one character. + private static boolean isIdStart(int c) { + return c == '_' || Character.isUnicodeIdentifierStart(c); + } + + private static boolean isIdPart(int c) { + return c == '_' || (Character.isUnicodeIdentifierPart(c) && !Character.isIdentifierIgnorable(c)); + } + + private static boolean isTemplatePrefix(String w) { + String l = w.toLowerCase(java.util.Locale.ROOT); + return "t".equals(l) || "rt".equals(l) || "tr".equals(l); + } + + private static boolean isPrefix(String w) { + String l = w.toLowerCase(java.util.Locale.ROOT); + return "r".equals(l) || "u".equals(l) || "b".equals(l) || "f".equals(l) || "rb".equals(l) || "br".equals(l) + || "fr".equals(l) || "rf".equals(l); + } + + private void number(int startCol) { + int s = pos; + int n = src.length(); + if (src.charAt(pos) == '0' && pos + 1 < n && "xXoObB".indexOf(src.charAt(pos + 1)) >= 0) { + pos += 2; + while (pos < n && (Character.isLetterOrDigit(src.charAt(pos)) || src.charAt(pos) == '_')) { + pos++; + } + } else { + while (pos < n && (Character.isDigit(src.charAt(pos)) || src.charAt(pos) == '_')) { + pos++; + } + if (pos < n && src.charAt(pos) == '.') { + pos++; + while (pos < n && (Character.isDigit(src.charAt(pos)) || src.charAt(pos) == '_')) { + pos++; + } + } + if (pos < n && (src.charAt(pos) == 'e' || src.charAt(pos) == 'E')) { + int save = pos; + pos++; + if (pos < n && (src.charAt(pos) == '+' || src.charAt(pos) == '-')) { + pos++; + } + if (pos < n && Character.isDigit(src.charAt(pos))) { + while (pos < n && (Character.isDigit(src.charAt(pos)) || src.charAt(pos) == '_')) { + pos++; + } + } else { + pos = save; + } + } + if (pos < n && (src.charAt(pos) == 'j' || src.charAt(pos) == 'J')) { + pos++; + } + } + out.add(new Token(Token.Kind.NUMBER, src.substring(s, pos), line, startCol, null, null)); + } + + private void string(String prefix, int start, int startCol) { + String p = prefix.toLowerCase(java.util.Locale.ROOT); + boolean raw = p.indexOf('r') >= 0; + boolean fstr = p.indexOf('f') >= 0; + int startLine = line; + char q = src.charAt(pos); + boolean triple = src.startsWith("" + q + q + q, pos); + pos += triple ? 3 : 1; + int bodyStart = pos; + int n = src.length(); + int braces = 0; + int parens = 0; + // Bit k: the replacement field at nesting level k is in its format spec. + long specLevels = 0; + while (true) { + if (pos >= n) { + throw new PythonSyntaxError(file, startLine, startCol, "unterminated string literal"); + } + char c = src.charAt(pos); + if (fstr && braces > 0) { + if (c == '{') { + braces++; + pos++; + continue; + } + if (c == '}') { + if (braces < 63) { + specLevels &= ~(1L << braces); + } + braces--; + pos++; + continue; + } + if (c == '(' || c == '[') { + parens++; + } else if ((c == ')' || c == ']') && parens > 0) { + parens--; + } else if (c == ':' && parens == 0 && braces < 63 && (specLevels & (1L << braces)) == 0 + && (pos + 1 >= n || src.charAt(pos + 1) != '=')) { + specLevels |= 1L << braces; + } + if (braces < 63 && (specLevels & (1L << braces)) != 0) { + if (c == '\n') { + line++; + lineStart = pos + 1; + } + pos++; + continue; + } + if (c == '\'' || c == '"') { + skipNestedString(); + continue; + } + if (c == '#') { + // PEP 701: a comment inside a multi-line replacement field. + while (pos < n && src.charAt(pos) != '\n') { + pos++; + } + continue; + } + if (c == '\n') { + line++; + lineStart = pos + 1; + } + pos++; + continue; + } + if (c == '\\') { + if (pos + 1 < n && src.charAt(pos + 1) == '\n') { + line++; + lineStart = pos + 2; + } + pos += 2; + continue; + } + if (fstr && c == '{') { + if (pos + 1 < n && src.charAt(pos + 1) == '{') { + pos += 2; + continue; + } + braces = 1; + parens = 0; + specLevels = 0; + pos++; + continue; + } + if (c == '\n') { + if (!triple) { + throw new PythonSyntaxError(file, startLine, startCol, "unterminated string literal"); + } + line++; + lineStart = pos + 1; + pos++; + continue; + } + if (c == q && (!triple || src.startsWith("" + q + q + q, pos))) { + break; + } + pos++; + } + String body = src.substring(bodyStart, pos); + pos += triple ? 3 : 1; + String value = fstr ? body : (raw ? body : decode(body, p.indexOf('b') >= 0, startLine, startCol)); + out.add(new Token(Token.Kind.STRING, src.substring(start, pos), startLine, startCol, value, p)); + } + + /// Skips a string literal nested inside an f-string replacement field. + private void skipNestedString() { + int n = src.length(); + int s = pos; + int ps = s; + while (ps > 0 && Character.isLetter(src.charAt(ps - 1))) { + ps--; + } + boolean nestedF = src.substring(ps, s).toLowerCase(java.util.Locale.ROOT).indexOf('f') >= 0; + char q = src.charAt(pos); + boolean triple = src.startsWith("" + q + q + q, pos); + pos += triple ? 3 : 1; + int braces = 0; + while (pos < n) { + char c = src.charAt(pos); + if (nestedF && braces > 0) { + if (c == '{') { + braces++; + } else if (c == '}') { + braces--; + } else if (c == '\'' || c == '"') { + skipNestedString(); + continue; + } + pos++; + continue; + } + if (c == '\\') { + pos += 2; + continue; + } + if (nestedF && c == '{') { + if (pos + 1 < n && src.charAt(pos + 1) == '{') { + pos += 2; + continue; + } + braces = 1; + pos++; + continue; + } + if (c == q && (!triple || src.startsWith("" + q + q + q, pos))) { + pos += triple ? 3 : 1; + return; + } + if (c == '\n') { + line++; + lineStart = pos + 1; + } + pos++; + } + throw error("unterminated string literal"); + } + + /// Decodes backslash escapes. Bytes literals keep each byte as a char. + public static String decode(String body, boolean bytes, int line, int col) { + if (body.indexOf('\\') < 0) { + return body; + } + StringBuilder sb = new StringBuilder(body.length()); + int n = body.length(); + for (int i = 0; i < n; i++) { + char c = body.charAt(i); + if (c != '\\' || i + 1 >= n) { + sb.append(c); + continue; + } + char e = body.charAt(++i); + switch (e) { + case '\n': + break; + case '\\': + sb.append('\\'); + break; + case '\'': + sb.append('\''); + break; + case '"': + sb.append('"'); + break; + case 'a': + sb.append((char) 7); + break; + case 'b': + sb.append('\b'); + break; + case 'f': + sb.append('\f'); + break; + case 'n': + sb.append('\n'); + break; + case 'r': + sb.append('\r'); + break; + case 't': + sb.append('\t'); + break; + case 'v': + sb.append((char) 11); + break; + case 'x': + sb.append((char) Integer.parseInt(body.substring(i + 1, i + 3), 16)); + i += 2; + break; + case 'u': + if (bytes) { + sb.append('\\').append(e); + break; + } + sb.append((char) Integer.parseInt(body.substring(i + 1, i + 5), 16)); + i += 4; + break; + case 'U': + if (bytes) { + sb.append('\\').append(e); + break; + } + sb.appendCodePoint(Integer.parseInt(body.substring(i + 1, i + 9), 16)); + i += 8; + break; + case 'N': { + if (bytes) { + sb.append('\\').append(e); + break; + } + int close = i + 1 < n && body.charAt(i + 1) == '{' ? body.indexOf('}', i + 2) : -1; + if (close < 0) { + throw new PythonSyntaxError("", line, col, "malformed \\N character escape"); + } + int cp = UnicodeNames.lookup(body.substring(i + 2, close)); + if (cp < 0) { + throw new PythonSyntaxError("", line, col, "unknown Unicode character name"); + } + sb.appendCodePoint(cp); + i = close; + break; + } + default: + if (e >= '0' && e <= '7') { + int v = e - '0'; + int k = 1; + while (k < 3 && i + 1 < n && body.charAt(i + 1) >= '0' && body.charAt(i + 1) <= '7') { + v = v * 8 + (body.charAt(++i) - '0'); + k++; + } + sb.append((char) v); + } else { + sb.append('\\').append(e); + } + break; + } + } + return sb.toString(); + } +} diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/parser/Parser.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/parser/Parser.java new file mode 100644 index 00000000000..298fb5b6fcd --- /dev/null +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/parser/Parser.java @@ -0,0 +1,1604 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.transpiler.parser; + +import com.codename1.python.transpiler.api.PythonSyntaxError; +import com.codename1.python.transpiler.ast.Node; +import com.codename1.python.transpiler.ast.Node.*; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/// A recursive-descent parser for Python 3.13, following the precedence and +/// statement structure of the language reference's grammar. +/// +/// Soft keywords (`match`, `case`, `type`, `_`) are recognised by context, as +/// CPython's PEG parser does, so they stay usable as ordinary names. +public final class Parser { + private static final Set KEYWORDS = new HashSet(Arrays.asList( + "False", "None", "True", "and", "as", "assert", "async", "await", "break", "class", "continue", + "def", "del", "elif", "else", "except", "finally", "for", "from", "global", "if", "import", "in", + "is", "lambda", "nonlocal", "not", "or", "pass", "raise", "return", "try", "while", "with", "yield")); + private static final Set AUG_OPS = new HashSet(Arrays.asList( + "+=", "-=", "*=", "/=", "//=", "%=", "**=", ">>=", "<<=", "&=", "|=", "^=", "@=")); + + private final List toks; + private final String file; + private int p; + + public Parser(List toks, String file) { + this.toks = toks; + this.file = file; + } + + public static Node.Module parse(String source, String file) { + return new Parser(new Lexer(source, file).tokenize(), file).module(); + } + + // ---- token helpers --------------------------------------------------- + + private Token peek() { + return toks.get(p); + } + + private Token peek(int k) { + return toks.get(Math.min(p + k, toks.size() - 1)); + } + + private Token next() { + return toks.get(p++); + } + + private boolean at(String s) { + Token t = peek(); + return (t.kind == Token.Kind.OP || t.kind == Token.Kind.NAME) && t.text.equals(s); + } + + private boolean atKind(Token.Kind k) { + return peek().kind == k; + } + + private boolean accept(String s) { + if (at(s)) { + p++; + return true; + } + return false; + } + + private Token expect(String s) { + if (!at(s)) { + throw err(peek(), "expected '" + s + "'"); + } + return next(); + } + + private void expectKind(Token.Kind k) { + if (peek().kind != k) { + throw err(peek(), "expected " + k.name().toLowerCase(java.util.Locale.ROOT)); + } + p++; + } + + private PythonSyntaxError err(Token t, String msg) { + String near = t.kind == Token.Kind.EOF ? "end of file" : t.kind == Token.Kind.NEWLINE ? "end of line" + : "'" + t.text + "'"; + return new PythonSyntaxError(file, t.line, t.col, "invalid syntax: " + msg + " near " + near); + } + + private String name() { + Token t = peek(); + if (t.kind != Token.Kind.NAME || KEYWORDS.contains(t.text)) { + throw err(t, "expected a name"); + } + p++; + return t.text; + } + + private boolean atName() { + Token t = peek(); + return t.kind == Token.Kind.NAME && !KEYWORDS.contains(t.text); + } + + // ---- module and blocks ----------------------------------------------- + + public Node.Module module() { + List body = new ArrayList(); + while (!atKind(Token.Kind.EOF)) { + if (atKind(Token.Kind.NEWLINE)) { + p++; + continue; + } + statement(body); + } + return new Node.Module(body).at(1, 1); + } + + private List block() { + List body = new ArrayList(); + if (atKind(Token.Kind.NEWLINE)) { + p++; + if (!atKind(Token.Kind.INDENT)) { + throw err(peek(), "expected an indented block"); + } + p++; + while (!atKind(Token.Kind.DEDENT) && !atKind(Token.Kind.EOF)) { + if (atKind(Token.Kind.NEWLINE)) { + p++; + continue; + } + statement(body); + } + accept0(Token.Kind.DEDENT); + } else { + simpleStatements(body); + } + return body; + } + + private void accept0(Token.Kind k) { + if (atKind(k)) { + p++; + } + } + + private void statement(List out) { + Token t = peek(); + if (t.kind == Token.Kind.NAME) { + switch (t.text) { + case "def": + out.add(funcDef(new ArrayList(), false)); + return; + case "class": + out.add(classDef(new ArrayList())); + return; + case "if": + out.add(ifStmt()); + return; + case "while": + out.add(whileStmt()); + return; + case "for": + out.add(forStmt(false)); + return; + case "try": + out.add(tryStmt()); + return; + case "with": + out.add(withStmt(false)); + return; + case "async": { + p++; + Token k = peek(); + if (k.is("def")) { + out.add(funcDef(new ArrayList(), true)); + } else if (k.is("for")) { + out.add(forStmt(true)); + } else if (k.is("with")) { + out.add(withStmt(true)); + } else { + throw err(k, "expected def, for or with after async"); + } + return; + } + case "match": + if (isMatchStatement()) { + out.add(matchStmt()); + return; + } + break; + default: + break; + } + } + if (t.is("@")) { + List decos = new ArrayList(); + while (accept("@")) { + decos.add(namedExprTest()); + expectKind(Token.Kind.NEWLINE); + } + if (at("class")) { + out.add(classDef(decos)); + } else if (at("async")) { + p++; + out.add(funcDef(decos, true)); + } else { + out.add(funcDef(decos, false)); + } + return; + } + simpleStatements(out); + } + + private void simpleStatements(List out) { + out.add(simpleStatement()); + while (accept(";")) { + if (atKind(Token.Kind.NEWLINE)) { + break; + } + out.add(simpleStatement()); + } + expectKind(Token.Kind.NEWLINE); + } + + private Stmt simpleStatement() { + Token t = peek(); + int l = t.line; + int c = t.col; + if (t.kind == Token.Kind.NAME) { + switch (t.text) { + case "pass": + p++; + return new Pass().at(l, c); + case "break": + p++; + return new Break().at(l, c); + case "continue": + p++; + return new Continue().at(l, c); + case "return": { + p++; + Expr v = atStmtEnd() ? null : starExpressions(); + return new Return(v).at(l, c); + } + case "raise": { + p++; + if (atStmtEnd()) { + return new Raise(null, null).at(l, c); + } + Expr e = test(); + Expr cause = accept("from") ? test() : null; + return new Raise(e, cause).at(l, c); + } + case "global": + case "nonlocal": { + p++; + List names = new ArrayList(); + names.add(name()); + while (accept(",")) { + names.add(name()); + } + return new Global(names, "nonlocal".equals(t.text)).at(l, c); + } + case "del": { + p++; + List targets = new ArrayList(); + targets.add(bitOr()); + while (accept(",")) { + if (atStmtEnd()) { + break; + } + targets.add(bitOr()); + } + return new Delete(targets).at(l, c); + } + case "assert": { + p++; + Expr test = test(); + Expr msg = accept(",") ? test() : null; + return new Assert(test, msg).at(l, c); + } + case "import": + return importStmt(); + case "from": + return fromImport(); + case "type": + if (peek(1).kind == Token.Kind.NAME && (peek(2).is("=") || peek(2).is("["))) { + p++; + String n = name(); + if (accept("[")) { + int depth = 1; + while (depth > 0) { + Token k = next(); + if (k.is("[")) { + depth++; + } else if (k.is("]")) { + depth--; + } + } + } + expect("="); + return new TypeAlias(n, test()).at(l, c); + } + break; + default: + break; + } + } + return exprStatement(); + } + + private boolean atStmtEnd() { + return atKind(Token.Kind.NEWLINE) || at(";") || atKind(Token.Kind.EOF); + } + + private Stmt exprStatement() { + Token t = peek(); + Expr first = at("yield") ? yieldExpr() : starExpressions(); + if (at(":") ) { + p++; + Expr ann = test(); + Expr value = null; + if (accept("=")) { + value = at("yield") ? yieldExpr() : starExpressions(); + } + checkTarget(first, t); + return new AnnAssign(first, ann, value).at(t.line, t.col); + } + if (peek().kind == Token.Kind.OP && AUG_OPS.contains(peek().text)) { + String op = next().text; + op = op.substring(0, op.length() - 1); + Expr value = at("yield") ? yieldExpr() : starExpressions(); + if (!(first instanceof Name || first instanceof Attribute || first instanceof Subscript)) { + throw err(t, "illegal expression for augmented assignment"); + } + return new AugAssign(first, op, value).at(t.line, t.col); + } + if (at("=")) { + List targets = new ArrayList(); + targets.add(first); + Expr value = null; + while (accept("=")) { + value = at("yield") ? yieldExpr() : starExpressions(); + targets.add(value); + } + targets.remove(targets.size() - 1); + for (Expr e : targets) { + checkTarget(e, t); + } + return new Assign(targets, value).at(t.line, t.col); + } + return new ExprStmt(first).at(t.line, t.col); + } + + private void checkTarget(Expr e, Token t) { + if (e instanceof Name || e instanceof Attribute || e instanceof Subscript) { + return; + } + if (e instanceof Starred) { + checkTarget(((Starred) e).value, t); + return; + } + if (e instanceof TupleExpr || e instanceof ListExpr) { + List elts = e instanceof TupleExpr ? ((TupleExpr) e).elts : ((ListExpr) e).elts; + for (Expr x : elts) { + checkTarget(x, t); + } + return; + } + throw new PythonSyntaxError(file, e.line, e.col, "cannot assign to expression"); + } + + // ---- imports --------------------------------------------------------- + + private String dottedName() { + StringBuilder sb = new StringBuilder(name()); + while (accept(".")) { + sb.append('.').append(name()); + } + return sb.toString(); + } + + private Stmt importStmt() { + Token t = next(); + List names = new ArrayList(); + do { + Token a = peek(); + String n = dottedName(); + String as = accept("as") ? name() : null; + names.add(new Alias(n, as).at(a.line, a.col)); + } while (accept(",")); + return new Import(names).at(t.line, t.col); + } + + private Stmt fromImport() { + Token t = next(); + int level = 0; + while (at(".") || at("...")) { + level += next().text.length(); + } + String module = null; + if (!at("import")) { + module = dottedName(); + } + expect("import"); + List names = new ArrayList(); + if (accept("*")) { + names.add(new Alias("*", null).at(t.line, t.col)); + } else { + boolean paren = accept("("); + do { + if (paren && at(")")) { + break; + } + Token a = peek(); + String n = name(); + String as = accept("as") ? name() : null; + names.add(new Alias(n, as).at(a.line, a.col)); + } while (accept(",")); + if (paren) { + expect(")"); + } + } + return new ImportFrom(module, level, names).at(t.line, t.col); + } + + // ---- compound statements --------------------------------------------- + + private Stmt ifStmt() { + Token t = next(); + Expr test = namedExprTest(); + expect(":"); + List body = block(); + List orelse = new ArrayList(); + if (at("elif")) { + orelse.add(ifStmt()); + } else if (accept("else")) { + expect(":"); + orelse = block(); + } + return new If(test, body, orelse).at(t.line, t.col); + } + + private Stmt whileStmt() { + Token t = next(); + Expr test = namedExprTest(); + expect(":"); + List body = block(); + List orelse = new ArrayList(); + if (accept("else")) { + expect(":"); + orelse = block(); + } + return new While(test, body, orelse).at(t.line, t.col); + } + + private Stmt forStmt(boolean isAsync) { + Token t = next(); + Expr target = targetList(); + expect("in"); + Expr iter = starExpressions(); + expect(":"); + List body = block(); + List orelse = new ArrayList(); + if (accept("else")) { + expect(":"); + orelse = block(); + } + return new For(target, iter, body, orelse, isAsync).at(t.line, t.col); + } + + /// The target of `for` and comprehensions: stops before `in`. + private Expr targetList() { + Token t = peek(); + List elts = new ArrayList(); + boolean tuple = false; + elts.add(targetAtom()); + while (accept(",")) { + tuple = true; + if (at("in") || at("=")) { + break; + } + elts.add(targetAtom()); + } + Expr r = tuple ? new TupleExpr(elts).at(t.line, t.col) : elts.get(0); + checkTarget(r, t); + return r; + } + + private Expr targetAtom() { + if (at("*")) { + Token s = next(); + return new Starred(bitOr()).at(s.line, s.col); + } + return bitOr(); + } + + private Stmt tryStmt() { + Token t = next(); + expect(":"); + List body = block(); + List handlers = new ArrayList(); + boolean star = false; + while (at("except")) { + Token h = next(); + if (accept("*")) { + star = true; + } + Expr type = null; + String name = null; + if (!at(":")) { + type = test(); + if (at(",")) { + List elts = new ArrayList(); + elts.add(type); + while (accept(",")) { + elts.add(test()); + } + type = new TupleExpr(elts).at(type.line, type.col); + } + if (accept("as")) { + name = name(); + } + } + expect(":"); + handlers.add(new ExceptHandler(type, name, block()).at(h.line, h.col)); + } + List orelse = new ArrayList(); + List fin = new ArrayList(); + if (accept("else")) { + expect(":"); + orelse = block(); + } + if (accept("finally")) { + expect(":"); + fin = block(); + } + if (handlers.isEmpty() && fin.isEmpty()) { + throw err(peek(), "expected 'except' or 'finally' block"); + } + return new Try(body, handlers, orelse, fin, star).at(t.line, t.col); + } + + private Stmt withStmt(boolean isAsync) { + Token t = next(); + List items = new ArrayList(); + // `with (a as b, c as d):` - parenthesised items (3.10+). + boolean paren = false; + if (at("(")) { + int save = p; + int depth = 0; + int q = p; + boolean hasAs = false; + for (; q < toks.size(); q++) { + Token k = toks.get(q); + if (k.is("(") || k.is("[") || k.is("{")) { + depth++; + } else if (k.is(")") || k.is("]") || k.is("}")) { + depth--; + if (depth == 0) { + break; + } + } else if (depth == 1 && k.is("as")) { + hasAs = true; + } + } + paren = hasAs && q + 1 < toks.size() && toks.get(q + 1).is(":"); + p = save; + if (paren) { + p++; + } + } + do { + if (paren && at(")")) { + break; + } + Token it = peek(); + Expr ctx = test(); + Expr vars = null; + if (accept("as")) { + vars = targetAtom(); + checkTarget(vars, it); + } + items.add(new WithItem(ctx, vars).at(it.line, it.col)); + } while (accept(",")); + if (paren) { + expect(")"); + } + expect(":"); + return new With(items, block(), isAsync).at(t.line, t.col); + } + + private Stmt funcDef(List decorators, boolean isAsync) { + Token t = expect("def"); + String n = name(); + if (at("[")) { + skipTypeParams(); + } + expect("("); + Arguments args = parameters(")", true); + expect(")"); + Expr returns = accept("->") ? test() : null; + expect(":"); + List body = block(); + return new FunctionDef(n, args, body, decorators, returns, isAsync).at(t.line, t.col); + } + + private void skipTypeParams() { + expect("["); + int depth = 1; + while (depth > 0) { + Token k = next(); + if (k.is("[")) { + depth++; + } else if (k.is("]")) { + depth--; + } else if (k.kind == Token.Kind.EOF) { + throw err(k, "unterminated type parameter list"); + } + } + } + + private Arguments parameters(String end, boolean annotations) { + Arguments a = new Arguments(); + boolean kwOnly = false; + while (!at(end)) { + Token t = peek(); + if (accept("/")) { + a.positionalOnly = a.positional.size(); + } else if (accept("**")) { + a.kwarg = param(annotations, true); + } else if (accept("*")) { + kwOnly = true; + if (!at(",") && !at(end)) { + a.vararg = param(annotations, true); + } + } else { + Arg arg = param(annotations, false); + Expr def = accept("=") ? test() : null; + if (kwOnly) { + a.kwonly.add(arg); + a.kwDefaults.add(def); + } else { + a.positional.add(arg); + if (def != null) { + a.defaults.add(def); + } else if (!a.defaults.isEmpty()) { + throw err(t, "parameter without a default follows parameter with a default"); + } + } + } + if (!accept(",")) { + break; + } + } + return a; + } + + private Arg param(boolean annotations, boolean star) { + Token t = peek(); + String n = name(); + Expr ann = null; + if (annotations && accept(":")) { + ann = star && at("*") ? starredAnnotation() : test(); + } + return new Arg(n, ann).at(t.line, t.col); + } + + private Expr starredAnnotation() { + Token s = next(); + return new Starred(test()).at(s.line, s.col); + } + + private Stmt classDef(List decorators) { + Token t = expect("class"); + String n = name(); + if (at("[")) { + skipTypeParams(); + } + List bases = new ArrayList(); + List kws = new ArrayList(); + if (accept("(")) { + Call c = new Call(null); + callArgs(c); + expect(")"); + bases.addAll(c.args); + kws.addAll(c.keywords); + } + expect(":"); + return new ClassDef(n, bases, kws, block(), decorators).at(t.line, t.col); + } + + // ---- match ----------------------------------------------------------- + + private boolean isMatchStatement() { + // `match` is a statement only when followed by an expression and ':' NEWLINE. + Token n1 = peek(1); + if (n1.kind == Token.Kind.NEWLINE || n1.is("=") || n1.is(".") + || n1.kind == Token.Kind.OP && AUG_OPS.contains(n1.text) || n1.is(":") || n1.is(",") + || n1.is(")")) { + return false; + } + int depth = 0; + for (int q = p + 1; q < toks.size(); q++) { + Token k = toks.get(q); + if (k.is("(") || k.is("[") || k.is("{")) { + depth++; + } else if (k.is(")") || k.is("]") || k.is("}")) { + depth--; + } else if (depth == 0 && k.is(":")) { + return toks.get(q + 1).kind == Token.Kind.NEWLINE; + } else if (k.kind == Token.Kind.NEWLINE || depth == 0 && k.is("=")) { + return false; + } + } + return false; + } + + private Stmt matchStmt() { + Token t = next(); + Expr subject = starExpressions(); + expect(":"); + expectKind(Token.Kind.NEWLINE); + expectKind(Token.Kind.INDENT); + List cases = new ArrayList(); + while (!atKind(Token.Kind.DEDENT) && !atKind(Token.Kind.EOF)) { + Token c = peek(); + if (!c.is("case")) { + throw err(c, "expected 'case'"); + } + p++; + Pattern pat = openSequencePattern(); + Expr guard = accept("if") ? namedExprTest() : null; + expect(":"); + cases.add(new MatchCase(pat, guard, block()).at(c.line, c.col)); + } + accept0(Token.Kind.DEDENT); + return new Match(subject, cases).at(t.line, t.col); + } + + private Pattern openSequencePattern() { + Token t = peek(); + Pattern first = asPattern(); + if (!at(",")) { + return first; + } + Pattern seq = new Pattern(Pattern.Kind.SEQUENCE).at(t.line, t.col); + seq.patterns.add(first); + while (accept(",")) { + if (at(":") || at("if")) { + break; + } + seq.patterns.add(asPattern()); + } + return seq; + } + + private Pattern asPattern() { + Token t = peek(); + Pattern pat = orPattern(); + if (accept("as")) { + Pattern as = new Pattern(Pattern.Kind.AS).at(t.line, t.col); + as.patterns.add(pat); + as.name = name(); + return as; + } + return pat; + } + + private Pattern orPattern() { + Token t = peek(); + Pattern first = closedPattern(); + if (!at("|")) { + return first; + } + Pattern or = new Pattern(Pattern.Kind.OR).at(t.line, t.col); + or.patterns.add(first); + while (accept("|")) { + or.patterns.add(closedPattern()); + } + return or; + } + + private Pattern closedPattern() { + Token t = peek(); + if (accept("*")) { + Pattern s = new Pattern(Pattern.Kind.STAR).at(t.line, t.col); + String n = name(); + s.name = "_".equals(n) ? null : n; + return s; + } + if (at("(") || at("[")) { + String close = next().is("(") ? ")" : "]"; + Pattern seq = new Pattern(Pattern.Kind.SEQUENCE).at(t.line, t.col); + boolean comma = false; + while (!at(close)) { + seq.patterns.add(asPattern()); + if (!accept(",")) { + break; + } + comma = true; + } + expect(close); + if (")".equals(close) && seq.patterns.size() == 1 && !comma) { + return seq.patterns.get(0); + } + return seq; + } + if (at("{")) { + p++; + Pattern m = new Pattern(Pattern.Kind.MAPPING).at(t.line, t.col); + while (!at("}")) { + if (accept("**")) { + m.name = name(); + } else { + m.keys.add(patternValue()); + expect(":"); + m.patterns.add(asPattern()); + } + if (!accept(",")) { + break; + } + } + expect("}"); + return m; + } + if (t.kind == Token.Kind.NAME && !KEYWORDS.contains(t.text) && !peek(1).is(".") && !peek(1).is("(")) { + p++; + if ("_".equals(t.text)) { + return new Pattern(Pattern.Kind.WILDCARD).at(t.line, t.col); + } + Pattern c = new Pattern(Pattern.Kind.CAPTURE).at(t.line, t.col); + c.name = t.text; + return c; + } + if (t.is("None") || t.is("True") || t.is("False")) { + Pattern s = new Pattern(Pattern.Kind.SINGLETON).at(t.line, t.col); + s.value = atom(); + return s; + } + if (t.kind == Token.Kind.NAME && !KEYWORDS.contains(t.text)) { + Expr v = new Name(name()).at(t.line, t.col); + while (accept(".")) { + Token a = peek(); + v = new Attribute(v, name()).at(a.line, a.col); + } + if (accept("(")) { + Pattern c = new Pattern(Pattern.Kind.CLASS).at(t.line, t.col); + c.value = v; + while (!at(")")) { + if (peek().kind == Token.Kind.NAME && peek(1).is("=")) { + c.kwdAttrs.add(name()); + expect("="); + c.kwdPatterns.add(asPattern()); + } else { + c.patterns.add(asPattern()); + } + if (!accept(",")) { + break; + } + } + expect(")"); + return c; + } + Pattern val = new Pattern(Pattern.Kind.VALUE).at(t.line, t.col); + val.value = v; + return val; + } + Pattern val = new Pattern(Pattern.Kind.VALUE).at(t.line, t.col); + val.value = patternValue(); + return val; + } + + /// A literal in a pattern: numbers (optionally signed, or `a + bj`), strings. + private Expr patternValue() { + return sum(); + } + + // ---- expressions ----------------------------------------------------- + + /// `a, *b, c` as an expression statement / right-hand side. + private Expr starExpressions() { + Token t = peek(); + Expr first = starExpr(); + if (!at(",")) { + return first; + } + List elts = new ArrayList(); + elts.add(first); + while (accept(",")) { + if (atStmtEnd() || at("=") || at(")") || at(":") || peek().kind == Token.Kind.OP + && AUG_OPS.contains(peek().text)) { + break; + } + elts.add(starExpr()); + } + return new TupleExpr(elts).at(t.line, t.col); + } + + private Expr starExpr() { + if (at("*")) { + Token s = next(); + return new Starred(bitOr()).at(s.line, s.col); + } + return namedExprTest(); + } + + private Expr namedExprTest() { + if (atName() && peek(1).is(":=")) { + Token t = next(); + p++; + Name target = new Name(t.text).at(t.line, t.col); + return new NamedExpr(target, test()).at(t.line, t.col); + } + return test(); + } + + /// `expression` in the reference grammar. + public Expr test() { + if (at("lambda")) { + return lambda(false); + } + Token t = peek(); + Expr e = orTest(); + if (at("if") && !atComprehensionIf()) { + p++; + Expr cond = orTest(); + expect("else"); + Expr other = test(); + return new IfExp(cond, e, other).at(t.line, t.col); + } + return e; + } + + private boolean atComprehensionIf() { + return false; + } + + private Expr testNoCond() { + if (at("lambda")) { + return lambda(true); + } + return orTest(); + } + + private Expr lambda(boolean noCond) { + Token t = expect("lambda"); + Arguments args = parameters(":", false); + expect(":"); + Expr body = noCond ? testNoCond() : test(); + return new Lambda(args, body).at(t.line, t.col); + } + + private Expr orTest() { + Token t = peek(); + Expr e = andTest(); + if (!at("or")) { + return e; + } + List vals = new ArrayList(); + vals.add(e); + while (accept("or")) { + vals.add(andTest()); + } + return new BoolOp(false, vals).at(t.line, t.col); + } + + private Expr andTest() { + Token t = peek(); + Expr e = notTest(); + if (!at("and")) { + return e; + } + List vals = new ArrayList(); + vals.add(e); + while (accept("and")) { + vals.add(notTest()); + } + return new BoolOp(true, vals).at(t.line, t.col); + } + + private Expr notTest() { + if (at("not")) { + Token t = next(); + return new UnaryOp("not", notTest()).at(t.line, t.col); + } + return comparison(); + } + + private String compOp() { + Token t = peek(); + if (t.kind == Token.Kind.OP) { + switch (t.text) { + case "<": + case ">": + case "==": + case ">=": + case "<=": + case "!=": + p++; + return t.text; + default: + return null; + } + } + if (t.is("in")) { + p++; + return "in"; + } + if (t.is("not") && peek(1).is("in")) { + p += 2; + return "not in"; + } + if (t.is("is")) { + p++; + if (accept("not")) { + return "is not"; + } + return "is"; + } + return null; + } + + private Expr comparison() { + Token t = peek(); + Expr e = bitOr(); + String op = compOp(); + if (op == null) { + return e; + } + Compare c = new Compare(e).at(t.line, t.col); + while (op != null) { + c.ops.add(op); + c.comparators.add(bitOr()); + op = compOp(); + } + return c; + } + + private Expr bitOr() { + Expr e = bitXor(); + while (at("|")) { + Token t = next(); + e = new BinOp(e, "|", bitXor()).at(t.line, t.col); + } + return e; + } + + private Expr bitXor() { + Expr e = bitAnd(); + while (at("^")) { + Token t = next(); + e = new BinOp(e, "^", bitAnd()).at(t.line, t.col); + } + return e; + } + + private Expr bitAnd() { + Expr e = shift(); + while (at("&")) { + Token t = next(); + e = new BinOp(e, "&", shift()).at(t.line, t.col); + } + return e; + } + + private Expr shift() { + Expr e = sum(); + while (at("<<") || at(">>")) { + Token t = next(); + e = new BinOp(e, t.text, sum()).at(t.line, t.col); + } + return e; + } + + private Expr sum() { + Expr e = term(); + while (at("+") || at("-")) { + Token t = next(); + e = new BinOp(e, t.text, term()).at(t.line, t.col); + } + return e; + } + + private Expr term() { + Expr e = factor(); + while (at("*") || at("/") || at("//") || at("%") || at("@")) { + Token t = next(); + e = new BinOp(e, t.text, factor()).at(t.line, t.col); + } + return e; + } + + private Expr factor() { + if (at("+") || at("-") || at("~")) { + Token t = next(); + return new UnaryOp(t.text, factor()).at(t.line, t.col); + } + return power(); + } + + private Expr power() { + Expr e = awaitPrimary(); + if (at("**")) { + Token t = next(); + return new BinOp(e, "**", factor()).at(t.line, t.col); + } + return e; + } + + private Expr awaitPrimary() { + if (at("await")) { + Token t = next(); + return new Await(primary()).at(t.line, t.col); + } + return primary(); + } + + private Expr primary() { + Expr e = atom(); + while (true) { + Token t = peek(); + if (accept(".")) { + e = new Attribute(e, name()).at(t.line, t.col); + } else if (accept("(")) { + Call c = new Call(e).at(t.line, t.col); + callArgs(c); + expect(")"); + e = c; + } else if (accept("[")) { + e = new Subscript(e, slices()).at(t.line, t.col); + expect("]"); + } else { + return e; + } + } + } + + private Expr slices() { + Token t = peek(); + Expr first = sliceItem(); + if (!at(",")) { + return first; + } + List elts = new ArrayList(); + elts.add(first); + while (accept(",")) { + if (at("]")) { + break; + } + elts.add(sliceItem()); + } + return new TupleExpr(elts).at(t.line, t.col); + } + + private Expr sliceItem() { + Token t = peek(); + Expr lower = null; + if (!at(":")) { + lower = starExpr(); + if (!at(":")) { + return lower; + } + } + expect(":"); + Expr upper = at(":") || at("]") || at(",") ? null : test(); + Expr step = null; + if (accept(":")) { + step = at("]") || at(",") ? null : test(); + } + return new Slice(lower, upper, step).at(t.line, t.col); + } + + private void callArgs(Call c) { + while (!at(")")) { + Token t = peek(); + if (accept("**")) { + c.keywords.add(new Keyword(null, test()).at(t.line, t.col)); + } else if (accept("*")) { + c.args.add(new Starred(test()).at(t.line, t.col)); + } else if (atName() && peek(1).is("=")) { + String n = name(); + p++; + c.keywords.add(new Keyword(n, test()).at(t.line, t.col)); + } else { + Expr e = namedExprTest(); + if (at("for") || at("async")) { + Comp g = new Comp(Comp.Kind.GENERATOR, e, null).at(t.line, t.col); + compFor(g); + e = g; + } + if (!c.keywords.isEmpty() && c.keywords.get(c.keywords.size() - 1).arg != null) { + throw err(t, "positional argument follows keyword argument"); + } + c.args.add(e); + } + if (!accept(",")) { + break; + } + } + } + + private void compFor(Comp comp) { + while (at("for") || at("async")) { + boolean isAsync = accept("async"); + Token t = expect("for"); + Expr target = targetList(); + expect("in"); + Expr iter = orTest(); + Comprehension g = new Comprehension(target, iter, isAsync).at(t.line, t.col); + while (at("if")) { + p++; + g.ifs.add(testNoCond()); + } + comp.generators.add(g); + } + } + + private Expr yieldExpr() { + Token t = expect("yield"); + if (accept("from")) { + return new Yield(test(), true).at(t.line, t.col); + } + if (atStmtEnd() || at(")") || at("]") || at("}")) { + return new Yield(null, false).at(t.line, t.col); + } + return new Yield(starExpressions(), false).at(t.line, t.col); + } + + private Expr atom() { + Token t = peek(); + switch (t.kind) { + case NUMBER: { + p++; + String s = t.text; + String l = s.toLowerCase(java.util.Locale.ROOT); + // An imaginary literal stays FLOAT with its `j`; the code + // generators check the suffix. + boolean isFloat = !l.startsWith("0x") && (l.indexOf('.') >= 0 || l.indexOf('e') >= 0 + || l.endsWith("j")); + return new Constant(isFloat ? Constant.Kind.FLOAT : Constant.Kind.INT, s).at(t.line, t.col); + } + case STRING: + return strings(); + case NAME: { + switch (t.text) { + case "None": + p++; + return new Constant(Constant.Kind.NONE, null).at(t.line, t.col); + case "True": + p++; + return new Constant(Constant.Kind.TRUE, null).at(t.line, t.col); + case "False": + p++; + return new Constant(Constant.Kind.FALSE, null).at(t.line, t.col); + default: + return new Name(name()).at(t.line, t.col); + } + } + case OP: + break; + default: + throw err(t, "expected an expression"); + } + if (accept("...")) { + return new Constant(Constant.Kind.ELLIPSIS, null).at(t.line, t.col); + } + if (accept("(")) { + if (accept(")")) { + return new TupleExpr(new ArrayList()).at(t.line, t.col); + } + if (at("yield")) { + Expr y = yieldExpr(); + expect(")"); + return y; + } + Expr first = starExpr(); + if (at("for") || at("async")) { + Comp g = new Comp(Comp.Kind.GENERATOR, first, null).at(t.line, t.col); + compFor(g); + expect(")"); + return g; + } + if (!at(",")) { + expect(")"); + return first; + } + List elts = new ArrayList(); + elts.add(first); + while (accept(",")) { + if (at(")")) { + break; + } + elts.add(starExpr()); + } + expect(")"); + return new TupleExpr(elts).at(t.line, t.col); + } + if (accept("[")) { + List elts = new ArrayList(); + if (accept("]")) { + return new ListExpr(elts).at(t.line, t.col); + } + Expr first = starExpr(); + if (at("for") || at("async")) { + Comp g = new Comp(Comp.Kind.LIST, first, null).at(t.line, t.col); + compFor(g); + expect("]"); + return g; + } + elts.add(first); + while (accept(",")) { + if (at("]")) { + break; + } + elts.add(starExpr()); + } + expect("]"); + return new ListExpr(elts).at(t.line, t.col); + } + if (accept("{")) { + if (accept("}")) { + return new DictExpr().at(t.line, t.col); + } + if (accept("**")) { + DictExpr d = new DictExpr().at(t.line, t.col); + d.keys.add(null); + d.values.add(bitOr()); + return dictRest(d); + } + Expr first = starExpr(); + if (accept(":")) { + Expr v = test(); + if (at("for") || at("async")) { + Comp g = new Comp(Comp.Kind.DICT, first, v).at(t.line, t.col); + compFor(g); + expect("}"); + return g; + } + DictExpr d = new DictExpr().at(t.line, t.col); + d.keys.add(first); + d.values.add(v); + return dictRest(d); + } + if (at("for") || at("async")) { + Comp g = new Comp(Comp.Kind.SET, first, null).at(t.line, t.col); + compFor(g); + expect("}"); + return g; + } + List elts = new ArrayList(); + elts.add(first); + while (accept(",")) { + if (at("}")) { + break; + } + elts.add(starExpr()); + } + expect("}"); + return new SetExpr(elts).at(t.line, t.col); + } + throw err(t, "expected an expression"); + } + + private Expr dictRest(DictExpr d) { + while (accept(",")) { + if (at("}")) { + break; + } + if (accept("**")) { + d.keys.add(null); + d.values.add(bitOr()); + } else { + d.keys.add(test()); + expect(":"); + d.values.add(test()); + } + } + expect("}"); + return d; + } + + /// Adjacent string literals concatenate; any f-string makes the whole a + /// JoinedStr. Mixing bytes and str is an error, as in CPython. + private Expr strings() { + Token first = peek(); + boolean bytes = first.prefix.indexOf('b') >= 0; + JoinedStr joined = new JoinedStr().at(first.line, first.col); + boolean anyF = false; + StringBuilder plain = new StringBuilder(); + while (atKind(Token.Kind.STRING)) { + Token t = next(); + if ((t.prefix.indexOf('b') >= 0) != bytes) { + throw err(t, "cannot mix bytes and nonbytes literals"); + } + if (t.prefix.indexOf('f') >= 0) { + if (plain.length() > 0) { + joined.parts.add(new Constant(Constant.Kind.STR, plain.toString()).at(t.line, t.col)); + plain.setLength(0); + } + anyF = true; + fstring(t, joined); + } else { + plain.append(t.value); + } + } + if (!anyF) { + return new Constant(bytes ? Constant.Kind.BYTES : Constant.Kind.STR, plain.toString()) + .at(first.line, first.col); + } + if (plain.length() > 0) { + joined.parts.add(new Constant(Constant.Kind.STR, plain.toString()).at(first.line, first.col)); + } + return joined; + } + + private void fstring(Token t, JoinedStr out) { + boolean raw = t.prefix.indexOf('r') >= 0; + fstringBody(t.value, raw, t, out); + } + + private void fstringBody(String body, boolean raw, Token t, JoinedStr out) { + StringBuilder lit = new StringBuilder(); + int n = body.length(); + int i = 0; + while (i < n) { + char c = body.charAt(i); + if (c == '{' && i + 1 < n && body.charAt(i + 1) == '{') { + lit.append('{'); + i += 2; + continue; + } + if (c == '}' && i + 1 < n && body.charAt(i + 1) == '}') { + lit.append('}'); + i += 2; + continue; + } + if (c == '}') { + throw new PythonSyntaxError(file, t.line, t.col, "f-string: single '}' is not allowed"); + } + if (c != '{') { + if (c == '\\' && !raw && i + 1 < n) { + int j = i + 2; + char e = body.charAt(i + 1); + if (e == 'x') { + j = i + 4; + } else if (e == 'u') { + j = i + 6; + } else if (e == 'U') { + j = i + 10; + } else if (e >= '0' && e <= '7') { + j = i + 2; + while (j < n && j < i + 4 && body.charAt(j) >= '0' && body.charAt(j) <= '7') { + j++; + } + } + lit.append(Lexer.decode(body.substring(i, Math.min(j, n)), false, t.line, t.col)); + i = Math.min(j, n); + continue; + } + lit.append(c); + i++; + continue; + } + if (lit.length() > 0) { + out.parts.add(new Constant(Constant.Kind.STR, lit.toString()).at(t.line, t.col)); + lit.setLength(0); + } + // Find the end of the replacement field and its top-level '!' / ':'. + int depth = 0; + int j = i + 1; + int exprEnd = -1; + int convAt = -1; + int specAt = -1; + while (j < n) { + char d = body.charAt(j); + if (d == '\'' || d == '"') { + j = skipString(body, j); + continue; + } + if (d == '#' && specAt < 0) { + while (j < n && body.charAt(j) != '\n') { + j++; + } + continue; + } + if (d == '(' || d == '[' || d == '{') { + depth++; + } else if (d == ')' || d == ']') { + depth--; + } else if (d == '}') { + if (depth == 0) { + break; + } + depth--; + } else if (depth == 0 && specAt < 0 && d == '!' && j + 1 < n && body.charAt(j + 1) != '=') { + if (exprEnd < 0) { + exprEnd = j; + } + convAt = j; + } else if (depth == 0 && d == ':' && specAt < 0) { + if (exprEnd < 0) { + exprEnd = j; + } + specAt = j; + } + j++; + } + if (j >= n) { + throw new PythonSyntaxError(file, t.line, t.col, "f-string: expecting '}'"); + } + if (exprEnd < 0) { + exprEnd = j; + } + String exprText = body.substring(i + 1, exprEnd); + String trimmed = exprText.trim(); + boolean debug = false; + if (trimmed.endsWith("=") && !trimmed.endsWith("==") && !trimmed.endsWith("!=") + && !trimmed.endsWith("<=") && !trimmed.endsWith(">=")) { + debug = true; + out.parts.add(new Constant(Constant.Kind.STR, exprText).at(t.line, t.col)); + exprText = exprText.substring(0, exprText.lastIndexOf('=')); + } + Expr value = subExpression(exprText, t); + char conv = 0; + if (convAt >= 0) { + conv = body.charAt(convAt + 1); + if (conv != 'r' && conv != 's' && conv != 'a') { + throw new PythonSyntaxError(file, t.line, t.col, "f-string: invalid conversion character"); + } + } else if (debug && specAt < 0) { + conv = 'r'; + } + JoinedStr spec = null; + if (specAt >= 0) { + spec = new JoinedStr().at(t.line, t.col); + fstringBody(body.substring(specAt + 1, j), true, t, spec); + } + out.parts.add(new FormattedValue(value, conv, spec).at(t.line, t.col)); + i = j + 1; + } + if (lit.length() > 0) { + out.parts.add(new Constant(Constant.Kind.STR, lit.toString()).at(t.line, t.col)); + } + } + + private static int skipString(String s, int i) { + char q = s.charAt(i); + boolean triple = s.startsWith("" + q + q + q, i); + int j = i + (triple ? 3 : 1); + while (j < s.length()) { + char c = s.charAt(j); + if (c == '\\') { + j += 2; + continue; + } + if (c == q && (!triple || s.startsWith("" + q + q + q, j))) { + return j + (triple ? 3 : 1); + } + j++; + } + return j; + } + + private Expr subExpression(String text, Token t) { + List sub = new Lexer("(" + text + ")", file).tokenize(); + List fixed = new ArrayList(); + for (Token k : sub) { + fixed.add(new Token(k.kind, k.text, t.line + k.line - 1, k.line == 1 ? t.col : k.col, k.value, k.prefix)); + } + Parser ps = new Parser(fixed, file); + Expr e = ps.starExpressions(); + ps.expectKind(Token.Kind.NEWLINE); + return e; + } + + /// Unused positional helper for future grammar extensions. + static Node unused() { + return null; + } +} diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/parser/Token.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/parser/Token.java new file mode 100644 index 00000000000..3e596598b7d --- /dev/null +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/parser/Token.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.transpiler.parser; + +/// One lexical token of Python source. +public final class Token { + + /// Token kinds. Operators and keywords are `OP` and `NAME` with their text. + public enum Kind { + NAME, NUMBER, STRING, OP, NEWLINE, INDENT, DEDENT, EOF + } + + public final Kind kind; + public final String text; + public final int line; + public final int col; + /// For STRING: the decoded value, or the raw body of an f-string. + public final String value; + /// For STRING: the lower-cased prefix letters (`f`, `rb`, ...). + public final String prefix; + + public Token(Kind kind, String text, int line, int col, String value, String prefix) { + this.kind = kind; + this.text = text; + this.line = line; + this.col = col; + this.value = value; + this.prefix = prefix; + } + + public boolean is(String op) { + return (kind == Kind.OP || kind == Kind.NAME) && text.equals(op); + } + + @Override + public String toString() { + return kind + "(" + text + ")@" + line + ":" + col; + } +} diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/parser/UnicodeNames.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/parser/UnicodeNames.java new file mode 100644 index 00000000000..9cd4d02dfa6 --- /dev/null +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/parser/UnicodeNames.java @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.transpiler.parser; + +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; + +/// Resolves the name in a `\N{...}` escape at transpile time, so the +/// generated source holds the character itself and the runtime needs no +/// name database. +/// +/// The table is the JDK's `Character.getName`, read backwards and built only +/// the first time a source uses the escape. The names the Unicode standard +/// derives by algorithm are computed instead, because the JDK spells them +/// differently from Python ("CJK UNIFIED IDEOGRAPHS 4E00" rather than +/// "CJK UNIFIED IDEOGRAPH-4E00"). Python's name aliases are not covered. +final class UnicodeNames { + private static final String[] JAMO_L = {"G", "GG", "N", "D", "DD", "R", "M", "B", "BB", "S", "SS", "", "J", "JJ", + "C", "K", "T", "P", "H"}; + private static final String[] JAMO_V = {"A", "AE", "YA", "YAE", "EO", "E", "YEO", "YE", "O", "WA", "WAE", "OE", + "YO", "U", "WEO", "WE", "WI", "YU", "EU", "YI", "I"}; + private static final String[] JAMO_T = {"", "G", "GG", "GS", "N", "NJ", "NH", "D", "L", "LG", "LM", "LB", "LS", + "LT", "LP", "LH", "M", "B", "BS", "S", "SS", "NG", "J", "C", "K", "T", "P", "H"}; + + private static Map table; + + private UnicodeNames() { + } + + /// The code point named `name`, case-insensitively, or -1. + static int lookup(String name) { + String key = name.toUpperCase(Locale.ROOT); + int cp = derived(key); + if (cp >= 0) { + return cp; + } + Integer v = table().get(key); + return v == null ? -1 : v.intValue(); + } + + private static int derived(String key) { + String[] hexPrefixes = {"CJK UNIFIED IDEOGRAPH-", "CJK COMPATIBILITY IDEOGRAPH-", "TANGUT IDEOGRAPH-", + "KHITAN SMALL SCRIPT CHARACTER-", "NUSHU CHARACTER-"}; + for (String p : hexPrefixes) { + if (key.startsWith(p) && key.length() > p.length()) { + try { + int cp = Integer.parseInt(key.substring(p.length()), 16); + return cp <= Character.MAX_CODE_POINT && Character.isDefined(cp) ? cp : -1; + } catch (NumberFormatException e) { + return -1; + } + } + } + String hangul = "HANGUL SYLLABLE "; + if (key.startsWith(hangul)) { + String rest = key.substring(hangul.length()); + for (int l = 0; l < JAMO_L.length; l++) { + for (int v = 0; v < JAMO_V.length; v++) { + for (int t = 0; t < JAMO_T.length; t++) { + if (rest.equals(JAMO_L[l] + JAMO_V[v] + JAMO_T[t])) { + return 0xAC00 + (l * JAMO_V.length + v) * JAMO_T.length + t; + } + } + } + } + } + return -1; + } + + private static synchronized Map table() { + if (table == null) { + Map m = new HashMap(); + for (int cp = 0; cp <= Character.MAX_CODE_POINT; cp++) { + String n = Character.getName(cp); + if (n != null && !m.containsKey(n)) { + m.put(n, Integer.valueOf(cp)); + } + } + table = m; + } + return table; + } +} diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/_strptime.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/_strptime.py new file mode 100644 index 00000000000..4d19d33293f --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/_strptime.py @@ -0,0 +1,217 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""strptime for time and datetime: a format becomes a regular expression.""" + +import re +import time as _t + +_FIELDS = { + "d": r"(?P3[01]|[12]\d|0[1-9]|[1-9]| [1-9])", + "f": r"(?P[0-9]{1,6})", + "H": r"(?P2[0-3]|[0-1]\d|\d)", + "I": r"(?P1[0-2]|0[1-9]|[1-9]| [1-9])", + "G": r"(?P\d\d\d\d)", + "j": r"(?P36[0-6]|3[0-5]\d|[12]\d\d|0[1-9]\d|00[1-9]|[1-9]\d|0[1-9]|[1-9])", + "m": r"(?P1[0-2]|0[1-9]|[1-9])", + "M": r"(?P[0-5]\d|\d)", + "S": r"(?P6[0-1]|[0-5]\d|\d)", + "U": r"(?P5[0-3]|[0-4]\d|\d)", + "w": r"(?P[0-6])", + "u": r"(?P[1-7])", + "V": r"(?P5[0-3]|0[1-9]|[1-4]\d|\d)", + "y": r"(?P\d\d)", + "Y": r"(?P\d\d\d\d)", + "z": r"(?P[+-]\d\d:?[0-5]\d(:?[0-5]\d(\.\d{1,6})?)?|(?-i:Z))", + "A": r"(?Pmonday|tuesday|wednesday|thursday|friday|saturday|sunday)", + "a": r"(?Pmon|tue|wed|thu|fri|sat|sun)", + "B": r"(?Pjanuary|february|march|april|may|june|july|august|september|october|november|december)", + "b": r"(?Pjan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)", + "p": r"(?P

am|pm)", + "Z": r"(?Putc|gmt|[a-z]{3,5}|[+-]\d\d(\d\d)?)", + "%": "%", +} +_ALIASES = {"W": "U", "h": "b", "e": "d"} +_EXPAND = {"c": "%a %b %d %H:%M:%S %Y", "x": "%m/%d/%y", "X": "%H:%M:%S", "D": "%m/%d/%y", + "T": "%H:%M:%S", "R": "%H:%M", "F": "%Y-%m-%d", "r": "%I:%M:%S %p"} + +_cache = {} + + +def _pattern(fmt): + p = _cache.get(fmt) + if p is not None: + return p + out = [] + seen = set() + i = 0 + while i < len(fmt): + c = fmt[i] + if c == "%" and i + 1 < len(fmt): + k = fmt[i + 1] + i += 2 + if k in _EXPAND: + out.append(_pattern(_EXPAND[k]).pattern[4:-2]) + continue + name = _ALIASES.get(k, k) + body = _FIELDS.get(name) + if body is None: + raise ValueError("'%s' is a bad directive in format '%s'" % (k, fmt)) + if k == "W": + body = body.replace("?P", "?P") + if name in seen and body.startswith("(?P<"): + body = "(?P=" + body[4:body.index(">")] + ")" + seen.add(name) + out.append(body) + elif c == "%": + raise ValueError("stray % in format '%s'" % fmt) + elif c.isspace(): + while i < len(fmt) and fmt[i].isspace(): + i += 1 + out.append(r"\s+") + continue + else: + out.append(re.escape(c)) + i += 1 + p = re.compile("(?i:" + "".join(out) + ")", re.IGNORECASE) + _cache[fmt] = p + return p + + +_MON = ["jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec"] +_DAY = ["mon", "tue", "wed", "thu", "fri", "sat", "sun"] + + +def _strptime(data, fmt): + if not isinstance(data, str): + raise TypeError("strptime() argument 1 must be str, not %s" % type(data).__name__) + if not isinstance(fmt, str): + raise TypeError("strptime() argument 2 must be str, not %s" % type(fmt).__name__) + m = _pattern(fmt).match(data) + if m is None: + raise ValueError("time data %r does not match format %r" % (data, fmt)) + if m.end() != len(data): + raise ValueError("unconverted data remains: %s" % data[m.end():]) + g = m.groupdict() + year = None + month = day = 1 + hour = minute = second = fraction = 0 + tz = -1 + gmtoff = None + gmtoff_fraction = 0 + weekday = julian = None + week = None + week_start = None + iso_year = iso_week = None + for k, v in g.items(): + if v is None: + continue + if k == "y": + year = int(v) + year += 2000 if year <= 68 else 1900 + elif k == "Y": + year = int(v) + elif k == "G": + iso_year = int(v) + elif k == "m": + month = int(v) + elif k == "B" or k == "b": + month = _MON.index(v[:3].lower()) + 1 + elif k == "d": + day = int(v) + elif k == "H": + hour = int(v) + elif k == "I": + hour = int(v) + ap = (g.get("p") or "").lower() + if ap == "am": + hour = 0 if hour == 12 else hour + elif ap == "pm": + hour = hour if hour == 12 else hour + 12 + elif k == "M": + minute = int(v) + elif k == "S": + second = int(v) + elif k == "f": + fraction = int(v + "0" * (6 - len(v))) + elif k == "A" or k == "a": + weekday = _DAY.index(v[:3].lower()) + elif k == "w": + weekday = (int(v) - 1) % 7 + elif k == "u": + weekday = int(v) - 1 + elif k == "j": + julian = int(v) + elif k == "U" or k == "W": + week = int(v) + week_start = 6 if k == "U" else 0 + elif k == "V": + iso_week = int(v) + elif k == "z": + if v == "Z" or v == "z": + gmtoff = 0 + else: + sign = -1 if v[0] == "-" else 1 + s = v[1:].replace(":", "") + hh, mm = int(s[0:2]), int(s[2:4]) + ss = int(s[4:6]) if len(s) >= 6 else 0 + frac = s[7:] if len(s) > 7 else "" + gmtoff = sign * (hh * 3600 + mm * 60 + ss) + gmtoff_fraction = sign * int(frac + "0" * (6 - len(frac))) if frac else 0 + elif k == "Z": + low = v.lower() + if low in ("utc", "gmt"): + tz = 0 + if year is None: + if month == 2 and day == 29: + year = 1904 + else: + year = 1900 + if iso_year is not None and iso_week is not None and weekday is not None: + import datetime + dd = datetime.date.fromisocalendar(iso_year, iso_week, weekday + 1) + year, month, day = dd.year, dd.month, dd.day + elif julian is None and week is not None and weekday is not None: + first = (_t._days_from_civil(year, 1, 1) + 3) % 7 + if week_start == 0: + off = (7 - first) % 7 + julian = off + 1 + (week - 1) * 7 + weekday if week else 1 + weekday - first + else: + first = (first + 1) % 7 + wd = (weekday + 1) % 7 + off = (7 - first) % 7 + julian = off + 1 + (week - 1) * 7 + wd if week else 1 + wd - first + if julian is not None: + base = _t._days_from_civil(year, 1, 1) + julian - 1 + year, month, day = _t._civil_from_days(base) + else: + if month == 2 and day == 29 and not _t._leap(year): + raise ValueError("day is out of range for month") + mdays = [31, 29 if _t._leap(year) else 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] + if day > mdays[month - 1]: + raise ValueError("day is out of range for month") + julian = _t._yday(year, month, day) + weekday = (_t._days_from_civil(year, month, day) + 3) % 7 + return ((year, month, day, hour, minute, second, weekday, julian, tz, None, gmtoff), + fraction, gmtoff_fraction) + + +def _strptime_time(data, fmt="%a %b %d %H:%M:%S %Y"): + tt = _strptime(data, fmt)[0] + return _t.struct_time(tt[:9], tt[9], tt[10]) + + +def _strptime_datetime(cls, data, fmt="%a %b %d %H:%M:%S %Y"): + tt, fraction, gmtoff_fraction = _strptime(data, fmt) + args = tt[:6] + (fraction,) + gmtoff = tt[10] + if gmtoff is not None: + import datetime + delta = datetime.timedelta(seconds=gmtoff, microseconds=gmtoff_fraction) + args += (datetime.timezone(delta),) + return cls(*args) + + +def _strptime_datetime_date(cls, data, fmt): + tt = _strptime(data, fmt)[0] + return cls(tt[0], tt[1], tt[2]) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/abc.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/abc.py new file mode 100644 index 00000000000..0556e9cdc1c --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/abc.py @@ -0,0 +1,75 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Abstract base classes.""" + + +def abstractmethod(funcobj): + funcobj.__isabstractmethod__ = True + return funcobj + + +def _is_abstract(value): + if isinstance(value, (classmethod, staticmethod)): + value = value.__func__ + elif isinstance(value, property): + return any(getattr(f, "__isabstractmethod__", False) for f in (value.fget, value.fset, value.fdel)) + return getattr(value, "__isabstractmethod__", False) + + +class ABCMeta(type): + def __new__(mcls, name, bases, namespace, **kwargs): + cls = super().__new__(mcls, name, bases, namespace, **kwargs) + abstracts = {n for n, v in namespace.items() if _is_abstract(v)} + for base in bases: + for n in getattr(base, "__abstractmethods__", ()): + if _is_abstract(getattr(cls, n, None)): + abstracts.add(n) + cls.__abstractmethods__ = frozenset(abstracts) + cls._abc_registry = [] + return cls + + def __call__(cls, *args, **kwargs): + missing = cls.__abstractmethods__ + if missing: + names = sorted(missing) + raise TypeError("Can't instantiate abstract class %s without an implementation for abstract method%s %s" + % (cls.__name__, "s" if len(names) > 1 else "", ", ".join("'" + n + "'" for n in names))) + return super().__call__(*args, **kwargs) + + def register(cls, subclass): + if not cls.__subclasscheck__(subclass): + cls._abc_registry.append(subclass) + return subclass + + def __instancecheck__(cls, instance): + return cls.__subclasscheck__(type(instance)) + + def __subclasscheck__(cls, subclass): + if cls in subclass.__mro__: + return True + hook = getattr(cls, "__subclasshook__", None) + if hook is not None: + ok = hook(subclass) + if ok is not NotImplemented: + return bool(ok) + for k in subclass.__mro__: + for r in cls._abc_registry: + if k is r or issubclass(k, r): + return True + for sub in cls.__subclasses__() if hasattr(cls, "__subclasses__") else (): + if issubclass(subclass, sub): + return True + return False + + +class ABC(metaclass=ABCMeta): + __slots__ = () + + +def get_cache_token(): + return 0 + + +def update_abstractmethods(cls): + return cls diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/asyncio.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/asyncio.py new file mode 100644 index 00000000000..dad66f7236c --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/asyncio.py @@ -0,0 +1,920 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Coroutines on a single-threaded event loop. + +The loop runs callbacks, timers and tasks in the thread that called run(). +There is no selector: a loop with nothing ready sleeps until its next +timer. Networking, subprocesses and executors are not part of this module. +""" + +import time as _time +from heapq import heappush as _heappush, heappop as _heappop + +__all__ = [ + "AbstractEventLoop", "BaseEventLoop", "CancelledError", "Condition", "Event", "Future", + "InvalidStateError", "Lock", "Queue", "QueueEmpty", "QueueFull", "Semaphore", "BoundedSemaphore", + "Task", "TimeoutError", "all_tasks", "as_completed", "create_task", "current_task", "ensure_future", + "gather", "get_event_loop", "get_running_loop", "iscoroutine", "iscoroutinefunction", "new_event_loop", + "run", "set_event_loop", "shield", "sleep", "wait", "wait_for", "FIRST_COMPLETED", "FIRST_EXCEPTION", + "ALL_COMPLETED", "timeout", +] + +FIRST_COMPLETED = "FIRST_COMPLETED" +FIRST_EXCEPTION = "FIRST_EXCEPTION" +ALL_COMPLETED = "ALL_COMPLETED" + + +class CancelledError(BaseException): + pass + + +class InvalidStateError(Exception): + pass + + +class QueueEmpty(Exception): + pass + + +class QueueFull(Exception): + pass + + +_PENDING = "PENDING" +_CANCELLED = "CANCELLED" +_FINISHED = "FINISHED" + + +def iscoroutine(obj): + return type(obj).__name__ == "coroutine" + + +def iscoroutinefunction(func): + return False + + +# ---- futures -------------------------------------------------------------- + +class Future: + _asyncio_future_blocking = False + + def __init__(self, *, loop=None): + self._loop = loop if loop is not None else get_event_loop() + self._state = _PENDING + self._result = None + self._exception = None + self._callbacks = [] + self._cancel_message = None + + def get_loop(self): + return self._loop + + def done(self): + return self._state != _PENDING + + def cancelled(self): + return self._state == _CANCELLED + + def cancel(self, msg=None): + if self._state != _PENDING: + return False + self._state = _CANCELLED + self._cancel_message = msg + self._schedule_callbacks() + return True + + def result(self): + if self._state == _CANCELLED: + raise self._make_cancelled_error() + if self._state != _FINISHED: + raise InvalidStateError("Result is not ready.") + if self._exception is not None: + raise self._exception + return self._result + + def exception(self): + if self._state == _CANCELLED: + raise self._make_cancelled_error() + if self._state != _FINISHED: + raise InvalidStateError("Exception is not set.") + return self._exception + + def set_result(self, result): + if self._state != _PENDING: + raise InvalidStateError("invalid state") + self._result = result + self._state = _FINISHED + self._schedule_callbacks() + + def set_exception(self, exception): + if self._state != _PENDING: + raise InvalidStateError("invalid state") + if isinstance(exception, type): + exception = exception() + self._exception = exception + self._state = _FINISHED + self._schedule_callbacks() + + def add_done_callback(self, fn, *, context=None): + if self._state != _PENDING: + self._loop.call_soon(fn, self) + else: + self._callbacks.append(fn) + + def remove_done_callback(self, fn): + kept = [f for f in self._callbacks if f != fn] + removed = len(self._callbacks) - len(kept) + self._callbacks = kept + return removed + + def _schedule_callbacks(self): + callbacks = self._callbacks + self._callbacks = [] + for fn in callbacks: + self._loop.call_soon(fn, self) + + def _make_cancelled_error(self): + if self._cancel_message is None: + return CancelledError() + return CancelledError(self._cancel_message) + + def __await__(self): + if not self.done(): + self._asyncio_future_blocking = True + yield self + if not self.done(): + raise RuntimeError("await wasn't used with future") + return self.result() + + __iter__ = __await__ + + def __repr__(self): + return "" + + +class _Yield: + """`await _Yield()` gives every other ready callback one turn.""" + + def __await__(self): + yield + + +_current_tasks = {} +_all_tasks = [] + + +class Task(Future): + _counter = 0 + + def __init__(self, coro, *, loop=None, name=None, context=None): + Future.__init__(self, loop=loop) + if not iscoroutine(coro): + raise TypeError("a coroutine was expected, got " + repr(coro)) + self._coro = coro + self._fut_waiter = None + self._must_cancel = False + Task._counter += 1 + self._name = name if name is not None else "Task-" + str(Task._counter) + self._loop.call_soon(self._step) + _all_tasks.append(self) + + def get_name(self): + return self._name + + def set_name(self, value): + self._name = str(value) + + def get_coro(self): + return self._coro + + def cancel(self, msg=None): + if self.done(): + return False + if self._fut_waiter is not None: + if self._fut_waiter.cancel(msg): + return True + self._must_cancel = True + self._cancel_message = msg + return True + + def _step(self, exc=None): + if self.done(): + return + if self._must_cancel: + if not isinstance(exc, CancelledError): + exc = self._make_cancelled_error() + self._must_cancel = False + self._fut_waiter = None + coro = self._coro + _current_tasks[id(self._loop)] = self + try: + if exc is None: + result = coro.send(None) + else: + result = coro.throw(exc) + except StopIteration as e: + if self._must_cancel: + self._must_cancel = False + Future.cancel(self, self._cancel_message) + else: + Future.set_result(self, e.value) + except CancelledError as e: + self._cancel_message = e.args[0] if e.args else None + Future.cancel(self, self._cancel_message) + except (KeyboardInterrupt, SystemExit) as e: + Future.set_exception(self, e) + raise + except BaseException as e: + Future.set_exception(self, e) + else: + if getattr(result, "_asyncio_future_blocking", False): + result._asyncio_future_blocking = False + result.add_done_callback(self._wakeup) + self._fut_waiter = result + if self._must_cancel: + if self._fut_waiter.cancel(self._cancel_message): + self._must_cancel = False + elif result is None: + self._loop.call_soon(self._step) + else: + self._loop.call_soon(self._step, RuntimeError("Task got bad yield: " + repr(result))) + finally: + _current_tasks.pop(id(self._loop), None) + if self.done() and self in _all_tasks: + _all_tasks.remove(self) + + def _wakeup(self, future): + try: + future.result() + except BaseException as e: + self._step(e) + else: + self._step() + + def __repr__(self): + return "" + + +# ---- the loop ------------------------------------------------------------- + +class Handle: + def __init__(self, callback, args, loop): + self._callback = callback + self._args = args + self._loop = loop + self._cancelled = False + + def cancel(self): + self._cancelled = True + + def cancelled(self): + return self._cancelled + + def _run(self): + self._callback(*self._args) + + +class TimerHandle(Handle): + def __init__(self, when, callback, args, loop): + Handle.__init__(self, callback, args, loop) + self._when = when + + def when(self): + return self._when + + +class AbstractEventLoop: + pass + + +class BaseEventLoop(AbstractEventLoop): + def __init__(self): + self._ready = [] + self._scheduled = [] + self._seq = 0 + self._stopping = False + self._running = False + self._closed = False + + def time(self): + return _time.monotonic() + + def call_soon(self, callback, *args, context=None): + h = Handle(callback, args, self) + self._ready.append(h) + return h + + call_soon_threadsafe = call_soon + + def call_later(self, delay, callback, *args, context=None): + return self.call_at(self.time() + delay, callback, *args) + + def call_at(self, when, callback, *args, context=None): + h = TimerHandle(when, callback, args, self) + self._seq += 1 + _heappush(self._scheduled, (when, self._seq, h)) + return h + + def create_future(self): + return Future(loop=self) + + def create_task(self, coro, *, name=None, context=None): + return Task(coro, loop=self, name=name) + + def is_running(self): + return self._running + + def is_closed(self): + return self._closed + + def stop(self): + self._stopping = True + + def close(self): + if self._running: + raise RuntimeError("Cannot close a running event loop") + self._closed = True + self._ready = [] + self._scheduled = [] + + def run_forever(self): + if self._running: + raise RuntimeError("This event loop is already running") + if self._closed: + raise RuntimeError("Event loop is closed") + global _running_loop + previous = _running_loop + _running_loop = self + self._running = True + try: + while True: + self._run_once() + if self._stopping: + break + finally: + self._stopping = False + self._running = False + _running_loop = previous + + def run_until_complete(self, future): + future = ensure_future(future, loop=self) + + def done(f): + self.stop() + + future.add_done_callback(done) + self.run_forever() + if not future.done(): + raise RuntimeError("Event loop stopped before Future completed.") + return future.result() + + def _run_once(self): + scheduled = self._scheduled + while scheduled and scheduled[0][2]._cancelled: + _heappop(scheduled) + if not self._ready and scheduled and not self._stopping: + wait = scheduled[0][0] - self.time() + if wait > 0: + _time.sleep(wait) + now = self.time() + while scheduled and scheduled[0][0] <= now: + h = _heappop(scheduled)[2] + if not h._cancelled: + self._ready.append(h) + ready = self._ready + self._ready = [] + for h in ready: + if not h._cancelled: + h._run() + + +_running_loop = None +_event_loop = None + + +def get_running_loop(): + if _running_loop is None: + raise RuntimeError("no running event loop") + return _running_loop + + +def get_event_loop(): + global _event_loop + if _running_loop is not None: + return _running_loop + if _event_loop is None or _event_loop.is_closed(): + _event_loop = BaseEventLoop() + return _event_loop + + +def new_event_loop(): + return BaseEventLoop() + + +def set_event_loop(loop): + global _event_loop + _event_loop = loop + + +def current_task(loop=None): + if loop is None: + loop = get_running_loop() + return _current_tasks.get(id(loop)) + + +def all_tasks(loop=None): + if loop is None: + loop = get_running_loop() + return set(t for t in _all_tasks if t._loop is loop and not t.done()) + + +def ensure_future(coro_or_future, *, loop=None): + if isinstance(coro_or_future, Future): + return coro_or_future + if loop is None: + loop = get_event_loop() + if iscoroutine(coro_or_future): + return loop.create_task(coro_or_future) + if hasattr(coro_or_future, "__await__"): + return loop.create_task(_wrap_awaitable(coro_or_future)) + raise TypeError("An asyncio.Future, a coroutine or an awaitable is required") + + +async def _wrap_awaitable(awaitable): + return await awaitable + + +def create_task(coro, *, name=None, context=None): + return get_running_loop().create_task(coro, name=name) + + +def run(main, *, debug=None): + if _running_loop is not None: + raise RuntimeError("asyncio.run() cannot be called from a running event loop") + if not iscoroutine(main): + raise ValueError("a coroutine was expected, got " + repr(main)) + loop = new_event_loop() + set_event_loop(loop) + try: + return loop.run_until_complete(main) + finally: + try: + pending = [t for t in _all_tasks if t._loop is loop and not t.done()] + for t in pending: + t.cancel() + if pending: + loop.run_until_complete(gather(*pending, return_exceptions=True)) + finally: + set_event_loop(None) + loop.close() + + +async def sleep(delay, result=None): + if delay <= 0: + await _Yield() + return result + loop = get_running_loop() + future = loop.create_future() + h = loop.call_later(delay, _set_result_unless_cancelled, future, result) + try: + return await future + finally: + h.cancel() + + +def _set_result_unless_cancelled(fut, result): + if not fut.cancelled(): + fut.set_result(result) + + +def gather(*coros_or_futures, return_exceptions=False): + loop = get_event_loop() if _running_loop is None else _running_loop + children = [ensure_future(c, loop=loop) for c in coros_or_futures] + outer = loop.create_future() + if not children: + outer.set_result([]) + return outer + state = [0] + + def child_done(fut): + state[0] += 1 + if outer.done(): + return + if not return_exceptions: + if fut.cancelled(): + outer.set_exception(fut._make_cancelled_error()) + return + if fut.exception() is not None: + outer.set_exception(fut.exception()) + return + if state[0] == len(children): + results = [] + for c in children: + if c.cancelled(): + results.append(c._make_cancelled_error()) + elif c.exception() is not None: + results.append(c.exception()) + else: + results.append(c.result()) + outer.set_result(results) + + for c in children: + c.add_done_callback(child_done) + return outer + + +def shield(arg): + inner = ensure_future(arg) + if inner.done(): + return inner + outer = inner._loop.create_future() + + def done(f): + if outer.cancelled(): + return + if f.cancelled(): + outer.cancel() + elif f.exception() is not None: + outer.set_exception(f.exception()) + else: + outer.set_result(f.result()) + + inner.add_done_callback(done) + return outer + + +async def wait_for(fut, timeout): + fut = ensure_future(fut) + if timeout is None: + return await fut + if fut.done(): + return fut.result() + loop = get_running_loop() + timed_out = [False] + + def expire(): + timed_out[0] = True + fut.cancel() + + h = loop.call_later(timeout, expire) + try: + return await fut + except CancelledError: + if timed_out[0]: + raise TimeoutError() + raise + finally: + h.cancel() + + +async def wait(fs, *, timeout=None, return_when=ALL_COMPLETED): + fs = set(ensure_future(f) for f in fs) + if not fs: + raise ValueError("Set of Tasks/Futures is empty.") + loop = get_running_loop() + waiter = loop.create_future() + counter = [len(fs)] + + def check(f): + counter[0] -= 1 + if waiter.done(): + return + if (counter[0] <= 0 or return_when == FIRST_COMPLETED + or return_when == FIRST_EXCEPTION and not f.cancelled() and f.exception() is not None): + waiter.set_result(None) + + h = None + if timeout is not None: + h = loop.call_later(timeout, _set_result_unless_cancelled, waiter, None) + for f in fs: + f.add_done_callback(check) + try: + await waiter + finally: + if h is not None: + h.cancel() + for f in fs: + f.remove_done_callback(check) + done = set(f for f in fs if f.done()) + return done, fs - done + + +def as_completed(fs, *, timeout=None): + todo = [ensure_future(f) for f in fs] + loop = get_event_loop() if _running_loop is None else _running_loop + queue = Queue() + + def on_done(f): + queue.put_nowait(f) + + for f in todo: + f.add_done_callback(on_done) + + async def next_done(): + f = await queue.get() + return f.result() + + return [next_done() for _ in todo] + + +class timeout: + """`async with asyncio.timeout(delay):` cancels the block after `delay`.""" + + def __init__(self, delay): + self._delay = delay + self._handle = None + self._task = None + self._expired = False + + def reschedule(self, when): + if self._handle is not None: + self._handle.cancel() + self._handle = None + if when is not None: + loop = get_running_loop() + self._handle = loop.call_at(when, self._on_timeout) + + def expired(self): + return self._expired + + def _on_timeout(self): + self._expired = True + self._task.cancel() + + async def __aenter__(self): + self._task = current_task() + if self._delay is not None: + self._handle = get_running_loop().call_later(self._delay, self._on_timeout) + return self + + async def __aexit__(self, exc_type, exc, tb): + if self._handle is not None: + self._handle.cancel() + self._handle = None + if self._expired and exc_type is CancelledError: + raise TimeoutError() + return False + + +# ---- synchronization primitives -------------------------------------------- + +class Event: + def __init__(self): + self._waiters = [] + self._value = False + + def is_set(self): + return self._value + + def set(self): + if not self._value: + self._value = True + for fut in self._waiters: + if not fut.done(): + fut.set_result(True) + + def clear(self): + self._value = False + + async def wait(self): + if self._value: + return True + fut = get_running_loop().create_future() + self._waiters.append(fut) + try: + await fut + return True + finally: + self._waiters.remove(fut) + + +class Lock: + def __init__(self): + self._waiters = [] + self._locked = False + + def locked(self): + return self._locked + + async def acquire(self): + if not self._locked and not self._waiters: + self._locked = True + return True + fut = get_running_loop().create_future() + self._waiters.append(fut) + try: + await fut + finally: + self._waiters.remove(fut) + self._locked = True + return True + + def release(self): + if not self._locked: + raise RuntimeError("Lock is not acquired.") + self._locked = False + for fut in self._waiters: + if not fut.done(): + fut.set_result(True) + break + + async def __aenter__(self): + await self.acquire() + return None + + async def __aexit__(self, exc_type, exc, tb): + self.release() + return False + + +class Condition: + def __init__(self, lock=None): + self._lock = lock if lock is not None else Lock() + self._waiters = [] + + def locked(self): + return self._lock.locked() + + async def acquire(self): + return await self._lock.acquire() + + def release(self): + self._lock.release() + + async def __aenter__(self): + await self._lock.acquire() + return None + + async def __aexit__(self, exc_type, exc, tb): + self._lock.release() + return False + + async def wait(self): + fut = get_running_loop().create_future() + self._waiters.append(fut) + self._lock.release() + try: + await fut + return True + finally: + self._waiters.remove(fut) + await self._lock.acquire() + + async def wait_for(self, predicate): + result = predicate() + while not result: + await self.wait() + result = predicate() + return result + + def notify(self, n=1): + count = 0 + for fut in self._waiters: + if count >= n: + break + if not fut.done(): + count += 1 + fut.set_result(False) + + def notify_all(self): + self.notify(len(self._waiters)) + + +class Semaphore: + def __init__(self, value=1): + if value < 0: + raise ValueError("Semaphore initial value must be >= 0") + self._value = value + self._waiters = [] + + def locked(self): + return self._value == 0 + + async def acquire(self): + while self._value <= 0: + fut = get_running_loop().create_future() + self._waiters.append(fut) + try: + await fut + finally: + self._waiters.remove(fut) + self._value -= 1 + return True + + def release(self): + self._value += 1 + for fut in self._waiters: + if not fut.done(): + fut.set_result(True) + break + + async def __aenter__(self): + await self.acquire() + return None + + async def __aexit__(self, exc_type, exc, tb): + self.release() + return False + + +class BoundedSemaphore(Semaphore): + def __init__(self, value=1): + Semaphore.__init__(self, value) + self._bound_value = value + + def release(self): + if self._value >= self._bound_value: + raise ValueError("BoundedSemaphore released too many times") + Semaphore.release(self) + + +class Queue: + def __init__(self, maxsize=0): + self._maxsize = maxsize + self._items = [] + self._head = 0 + self._getters = [] + self._putters = [] + self._unfinished = 0 + self._finished = Event() + self._finished.set() + + @property + def maxsize(self): + return self._maxsize + + def qsize(self): + return len(self._items) - self._head + + def empty(self): + return self.qsize() == 0 + + def full(self): + if self._maxsize <= 0: + return False + return self.qsize() >= self._maxsize + + def _wake(self, waiters): + while waiters: + fut = waiters.pop(0) + if not fut.done(): + fut.set_result(None) + break + + def put_nowait(self, item): + if self.full(): + raise QueueFull + self._items.append(item) + self._unfinished += 1 + self._finished.clear() + self._wake(self._getters) + + def get_nowait(self): + if self.empty(): + raise QueueEmpty + item = self._items[self._head] + self._items[self._head] = None + self._head += 1 + if self._head > 32 and self._head * 2 > len(self._items): + self._items = self._items[self._head:] + self._head = 0 + self._wake(self._putters) + return item + + async def put(self, item): + while self.full(): + fut = get_running_loop().create_future() + self._putters.append(fut) + try: + await fut + except BaseException: + fut.cancel() + if fut in self._putters: + self._putters.remove(fut) + raise + self.put_nowait(item) + + async def get(self): + while self.empty(): + fut = get_running_loop().create_future() + self._getters.append(fut) + try: + await fut + except BaseException: + fut.cancel() + if fut in self._getters: + self._getters.remove(fut) + raise + return self.get_nowait() + + def task_done(self): + if self._unfinished <= 0: + raise ValueError("task_done() called too many times") + self._unfinished -= 1 + if self._unfinished == 0: + self._finished.set() + + async def join(self): + if self._unfinished > 0: + await self._finished.wait() diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/base64.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/base64.py new file mode 100644 index 00000000000..890234aedea --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/base64.py @@ -0,0 +1,308 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Base16, Base32, Base64, Base85 and Ascii85 data encodings.""" + +import binascii + +__all__ = ["encode", "decode", "encodebytes", "decodebytes", "b64encode", "b64decode", "b32encode", + "b32decode", "b32hexencode", "b32hexdecode", "b16encode", "b16decode", "b85encode", + "b85decode", "a85encode", "a85decode", "z85encode", "z85decode", "standard_b64encode", + "standard_b64decode", "urlsafe_b64encode", "urlsafe_b64decode"] + +bytes_types = (bytes, bytearray) + + +def _bytes_from_decode_data(s): + if isinstance(s, str): + try: + return s.encode("ascii") + except UnicodeEncodeError: + raise ValueError("string argument should contain only ASCII characters") + if isinstance(s, (bytes, bytearray, memoryview)): + return bytes(s) + raise TypeError("argument should be a bytes-like object or ASCII string, not %r" % s.__class__.__name__) + + +def _input(s): + if isinstance(s, (bytes, bytearray, memoryview)): + return bytes(s) + raise TypeError("a bytes-like object is required, not '%s'" % type(s).__name__) + + +def b64encode(s, altchars=None): + encoded = binascii.b2a_base64(_input(s), newline=False) + if altchars is not None: + altchars = _input(altchars) + if len(altchars) != 2: + raise ValueError("invalid altchars: %r" % (altchars,)) + return encoded.translate(bytes.maketrans(b"+/", altchars)) + return encoded + + +def b64decode(s, altchars=None, validate=False): + s = _bytes_from_decode_data(s) + if altchars is not None: + altchars = _bytes_from_decode_data(altchars) + if len(altchars) != 2: + raise ValueError("invalid altchars: %r" % (altchars,)) + s = s.translate(bytes.maketrans(altchars, b"+/")) + return binascii.a2b_base64(s, strict_mode=validate) + + +def standard_b64encode(s): + return b64encode(s) + + +def standard_b64decode(s): + return b64decode(s) + + +def urlsafe_b64encode(s): + return b64encode(s).translate(bytes.maketrans(b"+/", b"-_")) + + +def urlsafe_b64decode(s): + s = _bytes_from_decode_data(s) + s = s.translate(bytes.maketrans(b"-_", b"+/")) + return b64decode(s) + + +_B32 = b"ABCDEFGHIJKLMNOPQRSTUVWXYZ234567" +_B32HEX = b"0123456789ABCDEFGHIJKLMNOPQRSTUV" + + +def _b32encode(alphabet, s): + s = _input(s) + leftover = len(s) % 5 + if leftover: + s = s + b"\0" * (5 - leftover) + out = bytearray() + for i in range(0, len(s), 5): + c = int.from_bytes(s[i:i + 5], "big") + for shift in range(35, -1, -5): + out.append(alphabet[(c >> shift) & 31]) + if leftover == 1: + out[-6:] = b"======" + elif leftover == 2: + out[-4:] = b"====" + elif leftover == 3: + out[-3:] = b"===" + elif leftover == 4: + out[-1:] = b"=" + return bytes(out) + + +def _b32decode(alphabet, s, casefold=False, map01=None): + s = _bytes_from_decode_data(s) + if len(s) % 8: + raise binascii.Error("Incorrect padding") + if map01 is not None: + map01 = _bytes_from_decode_data(map01) + s = s.translate(bytes.maketrans(b"01", b"O" + map01)) + if casefold: + s = s.upper() + n = len(s) + s = s.rstrip(b"=") + padchars = n - len(s) + rev = {c: i for i, c in enumerate(alphabet)} + out = bytearray() + for i in range(0, len(s), 8): + quanta = s[i:i + 8] + acc = 0 + for c in quanta: + v = rev.get(c) + if v is None: + raise binascii.Error("Non-base32 digit found") + acc = (acc << 5) | v + acc <<= 5 * (8 - len(quanta)) + out += acc.to_bytes(5, "big") + if n % 8 or padchars not in (0, 1, 3, 4, 6): + raise binascii.Error("Incorrect padding") + if padchars and out: + leftover = (43 - 5 * padchars) // 8 + del out[leftover - 5:] + return bytes(out) + + +def b32encode(s): + return _b32encode(_B32, s) + + +def b32decode(s, casefold=False, map01=None): + return _b32decode(_B32, s, casefold, map01) + + +def b32hexencode(s): + return _b32encode(_B32HEX, s) + + +def b32hexdecode(s, casefold=False): + return _b32decode(_B32HEX, s, casefold) + + +def b16encode(s): + return _input(s).hex().upper().encode("ascii") + + +def b16decode(s, casefold=False): + s = _bytes_from_decode_data(s) + if casefold: + s = s.upper() + for c in s: + if c not in b"0123456789ABCDEF": + raise binascii.Error("Non-base16 digit found") + return binascii.unhexlify(s) + + +_B85 = (b"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" + b"abcdefghijklmnopqrstuvwxyz!#$%&()*+-;<=>?@^_`{|}~") +_Z85 = (b"0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" + b".-:+=^!/*?&<>()[]{}@%$#") + + +def _85encode(b, chars, pad=False, foldnuls=False, foldspaces=False): + b = _input(b) + padding = (-len(b)) % 4 + if padding: + b = b + b"\0" * padding + out = bytearray() + for i in range(0, len(b), 4): + word = int.from_bytes(b[i:i + 4], "big") + if foldnuls and not word: + out += b"z" + elif foldspaces and word == 0x20202020: + out += b"y" + else: + digits = bytearray(5) + for k in range(4, -1, -1): + word, digits[k] = divmod(word, 85) + for d in digits: + out.append(chars[d]) + if padding and not pad: + if out[-1:] == b"z": + out[-1:] = chars[0:1] * 5 + del out[len(out) - padding:] + return bytes(out) + + +def b85encode(b, pad=False): + return _85encode(b, _B85, pad) + + +def _85decode(b, chars, name): + b = _bytes_from_decode_data(b) + rev = {c: i for i, c in enumerate(chars)} + padding = (-len(b)) % 5 + b = b + chars[84:85] * padding + out = bytearray() + for i in range(0, len(b), 5): + acc = 0 + for j, c in enumerate(b[i:i + 5]): + v = rev.get(c) + if v is None: + raise ValueError("bad %s character at position %d" % (name, i + j)) + acc = acc * 85 + v + if acc > 0xFFFFFFFF: + raise ValueError("%s overflow in hunk starting at byte %d" % (name, i)) + out += acc.to_bytes(4, "big") + if padding: + del out[len(out) - padding:] + return bytes(out) + + +def b85decode(b): + return _85decode(b, _B85, "base85") + + +def z85encode(s): + return _85encode(s, _Z85, True) + + +def z85decode(s): + return _85decode(s, _Z85, "z85") + + +_A85START = b"<~" +_A85END = b"~>" + + +def a85encode(b, *, foldspaces=False, wrapcol=0, pad=False, adobe=False): + chars = bytes(range(33, 118)) + result = _85encode(b, chars, pad, True, foldspaces) + if adobe: + result = _A85START + result + if wrapcol: + wrapcol = max(2 if adobe else 1, wrapcol) + chunks = [result[i:i + wrapcol] for i in range(0, len(result), wrapcol)] + if adobe and len(chunks[-1]) + 2 > wrapcol: + chunks.append(b"") + result = b"\n".join(chunks) + if adobe: + result += _A85END + return result + + +def a85decode(b, *, foldspaces=False, adobe=False, ignorechars=b" \t\n\r\x0b"): + b = _bytes_from_decode_data(b) + if adobe: + if not b.endswith(_A85END): + raise ValueError("Ascii85 encoded byte sequences must end with %r" % (_A85END,)) + b = b[2:-2] if b.startswith(_A85START) else b[:-2] + out = bytearray() + curr = [] + for x in b + b"u" * 4: + if 33 <= x <= 117: + curr.append(x) + if len(curr) == 5: + acc = 0 + for c in curr: + acc = 85 * acc + (c - 33) + if acc > 0xFFFFFFFF: + raise ValueError("Ascii85 overflow") + out += acc.to_bytes(4, "big") + curr = [] + elif x == 122: + if curr: + raise ValueError("z inside Ascii85 5-tuple") + out += b"\0\0\0\0" + elif foldspaces and x == 121: + if curr: + raise ValueError("y inside Ascii85 5-tuple") + out += b"\x20\x20\x20\x20" + elif x in ignorechars: + continue + else: + raise ValueError("Non-Ascii85 digit found: %c" % x) + padding = 4 - len(curr) + if padding: + del out[len(out) - padding:] + return bytes(out) + + +MAXLINESIZE = 76 +MAXBINSIZE = (MAXLINESIZE // 4) * 3 + + +def encodebytes(s): + s = _input(s) + pieces = [] + for i in range(0, len(s), MAXBINSIZE): + pieces.append(binascii.b2a_base64(s[i:i + MAXBINSIZE])) + return b"".join(pieces) + + +def decodebytes(s): + return binascii.a2b_base64(_input(s)) + + +def encode(input, output): + while True: + s = input.read(MAXBINSIZE) + if not s: + break + output.write(binascii.b2a_base64(s)) + + +def decode(input, output): + output.write(binascii.a2b_base64(input.read())) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/binascii.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/binascii.py new file mode 100644 index 00000000000..6c2cb225fbe --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/binascii.py @@ -0,0 +1,164 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Conversions between binary data and ASCII encodings.""" + +__all__ = ["Error", "Incomplete", "hexlify", "unhexlify", "b2a_hex", "a2b_hex", "b2a_base64", + "a2b_base64", "crc32", "crc_hqx"] + + +class Error(ValueError): + pass + + +class Incomplete(Exception): + pass + + +Error.__module__ = "binascii" +Incomplete.__module__ = "binascii" + +_B64 = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" +_B64_REV = [-1] * 256 +for _i in range(64): + _B64_REV[_B64[_i]] = _i +_HEX = b"0123456789abcdef" + + +def _bytes(data, name): + if isinstance(data, str): + try: + return data.encode("ascii") + except UnicodeEncodeError: + raise ValueError("string argument should contain only ASCII characters") + if isinstance(data, (bytes, bytearray, memoryview)): + return bytes(data) + raise TypeError("a bytes-like object is required, not '%s'" % type(data).__name__) + + +def hexlify(data, sep=None, bytes_per_sep=1): + if isinstance(data, str): + raise TypeError("a bytes-like object is required, not 'str'") + data = _bytes(data, "data") + if sep is None: + return data.hex().encode("ascii") + if isinstance(sep, bytes): + sep = sep.decode("ascii") + return data.hex(sep, bytes_per_sep).encode("ascii") + + +b2a_hex = hexlify + + +def unhexlify(hexstr): + hexstr = _bytes(hexstr, "hexstr") + if len(hexstr) % 2: + raise Error("Odd-length string") + try: + return bytes.fromhex(hexstr.decode("ascii").replace(" ", "!")) + except ValueError: + raise Error("Non-hexadecimal digit found") + + +a2b_hex = unhexlify + + +def b2a_base64(data, *, newline=True): + if isinstance(data, str): + raise TypeError("a bytes-like object is required, not 'str'") + data = _bytes(data, "data") + out = bytearray() + n = len(data) + i = 0 + t = _B64 + while i + 2 < n: + v = (data[i] << 16) | (data[i + 1] << 8) | data[i + 2] + out.append(t[v >> 18]) + out.append(t[(v >> 12) & 63]) + out.append(t[(v >> 6) & 63]) + out.append(t[v & 63]) + i += 3 + rest = n - i + if rest == 1: + v = data[i] << 16 + out += bytes((t[v >> 18], t[(v >> 12) & 63], 61, 61)) + elif rest == 2: + v = (data[i] << 16) | (data[i + 1] << 8) + out += bytes((t[v >> 18], t[(v >> 12) & 63], t[(v >> 6) & 63], 61)) + if newline: + out.append(10) + return bytes(out) + + +def a2b_base64(data, *, strict_mode=False): + data = _bytes(data, "data") + out = bytearray() + rev = _B64_REV + acc = 0 + bits = 0 + quad = 0 + pads = 0 + for i, c in enumerate(data): + if c == 61: + if strict_mode and quad == 0: + raise Error("Excess padding not allowed" if i else "Leading padding not allowed") + if quad >= 2: + pads += 1 + if quad + pads >= 4: + if strict_mode and i != len(data) - 1: + raise Error("Excess data after padding") + acc = bits = quad = pads = 0 + if not strict_mode: + continue + break + elif strict_mode: + raise Error("Discontinuous padding not allowed") + continue + v = rev[c] + if v < 0: + if strict_mode: + raise Error("Only base64 data is allowed") + continue + if strict_mode and pads: + raise Error("Discontinuous padding not allowed") + pads = 0 + acc = (acc << 6) | v + bits += 6 + quad = (quad + 1) % 4 + if bits >= 8: + bits -= 8 + out.append((acc >> bits) & 255) + acc &= (1 << bits) - 1 + if quad != 0: + if quad == 1: + raise Error("Invalid base64-encoded string: number of data characters (%d) cannot be " + "1 more than a multiple of 4" % (len([c for c in data if rev[c] >= 0]))) + raise Error("Incorrect padding") + return bytes(out) + + +_CRC_TABLE = [] +for _n in range(256): + _c = _n + for _k in range(8): + _c = (0xEDB88320 ^ (_c >> 1)) if _c & 1 else _c >> 1 + _CRC_TABLE.append(_c) + + +def crc32(data, crc=0): + data = _bytes(data, "data") + crc = ~crc & 0xFFFFFFFF + t = _CRC_TABLE + for b in data: + crc = t[(crc ^ b) & 0xFF] ^ (crc >> 8) + return ~crc & 0xFFFFFFFF + + +def crc_hqx(data, value): + data = _bytes(data, "data") + crc = value & 0xFFFF + for b in data: + crc ^= b << 8 + for _ in range(8): + crc = ((crc << 1) ^ 0x1021) & 0xFFFF if crc & 0x8000 else (crc << 1) & 0xFFFF + return crc diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/bisect.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/bisect.py new file mode 100644 index 00000000000..163d15fe62c --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/bisect.py @@ -0,0 +1,46 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Bisection algorithms over sorted lists.""" + + +def bisect_left(a, x, lo=0, hi=None, *, key=None): + if lo < 0: + raise ValueError("lo must be non-negative") + if hi is None: + hi = len(a) + while lo < hi: + mid = (lo + hi) // 2 + v = a[mid] if key is None else key(a[mid]) + if v < x: + lo = mid + 1 + else: + hi = mid + return lo + + +def bisect_right(a, x, lo=0, hi=None, *, key=None): + if lo < 0: + raise ValueError("lo must be non-negative") + if hi is None: + hi = len(a) + while lo < hi: + mid = (lo + hi) // 2 + v = a[mid] if key is None else key(a[mid]) + if x < v: + hi = mid + else: + lo = mid + 1 + return lo + + +def insort_left(a, x, lo=0, hi=None, *, key=None): + a.insert(bisect_left(a, x if key is None else key(x), lo, hi, key=key), x) + + +def insort_right(a, x, lo=0, hi=None, *, key=None): + a.insert(bisect_right(a, x if key is None else key(x), lo, hi, key=key), x) + + +bisect = bisect_right +insort = insort_right diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/cmath.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/cmath.py new file mode 100644 index 00000000000..e7cf0aceda4 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/cmath.py @@ -0,0 +1,219 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Mathematical functions over complex numbers.""" + +import math + +__all__ = ["acos", "acosh", "asin", "asinh", "atan", "atanh", "cos", "cosh", "e", "exp", "inf", "infj", + "isclose", "isfinite", "isinf", "isnan", "log", "log10", "nan", "nanj", "phase", "pi", "polar", + "rect", "sin", "sinh", "sqrt", "tan", "tanh", "tau"] + +pi = math.pi +e = math.e +tau = math.tau +inf = math.inf +nan = math.nan +infj = complex(0.0, inf) +nanj = complex(0.0, nan) + + +def _c(z): + if isinstance(z, complex): + return z + if hasattr(z, "__complex__"): + return z.__complex__() + try: + return complex(float(z), 0.0) + except TypeError: + raise TypeError("must be real number, not %s" % type(z).__name__) + + +def phase(z): + z = _c(z) + return math.atan2(z.imag, z.real) + + +def polar(z): + z = _c(z) + return (abs(z), math.atan2(z.imag, z.real)) + + +def rect(r, phi): + if math.isinf(r) and math.isfinite(phi) and phi != 0.0: + re = math.copysign(math.inf, math.cos(phi)) if r > 0 else -math.copysign(math.inf, math.cos(phi)) + im = math.copysign(math.inf, math.sin(phi)) if r > 0 else -math.copysign(math.inf, math.sin(phi)) + return complex(re, im) + if phi == 0.0: + return complex(r, phi * r if math.isfinite(r) else phi) + return complex(r * math.cos(phi), r * math.sin(phi)) + + +def isfinite(z): + z = _c(z) + return math.isfinite(z.real) and math.isfinite(z.imag) + + +def isinf(z): + z = _c(z) + return math.isinf(z.real) or math.isinf(z.imag) + + +def isnan(z): + z = _c(z) + return math.isnan(z.real) or math.isnan(z.imag) + + +def isclose(a, b, *, rel_tol=1e-09, abs_tol=0.0): + a = _c(a) + b = _c(b) + if rel_tol < 0.0 or abs_tol < 0.0: + raise ValueError("tolerances must be non-negative") + if a == b: + return True + if isinf(a) or isinf(b): + return False + diff = abs(b - a) + return diff <= abs(rel_tol * b) or diff <= abs(rel_tol * a) or diff <= abs_tol + + +def exp(z): + z = _c(z) + if z.imag == 0.0: + return complex(math.exp(z.real), z.imag) + l = math.exp(z.real) + return complex(l * math.cos(z.imag), l * math.sin(z.imag)) + + +def sqrt(z): + z = _c(z) + x, y = z.real, z.imag + if x == 0.0 and y == 0.0: + return complex(0.0, y) + if math.isinf(y): + return complex(math.inf, y) + if math.isnan(x) or math.isnan(y): + return complex(math.nan, math.nan) + if math.isinf(x): + if x > 0: + return complex(x, math.copysign(0.0, y)) + return complex(0.0, math.copysign(math.inf, y)) + ax = math.fabs(x) + ay = math.fabs(y) + s = 2.0 * math.sqrt(ax / 8.0 + math.hypot(ax / 8.0, ay / 8.0)) + d = ay / (2.0 * s) + if x >= 0.0: + return complex(s, math.copysign(d, y)) + return complex(d, math.copysign(s, y)) + + +def log(z, base=None): + z = _c(z) + if z.real == 0.0 and z.imag == 0.0: + raise ValueError("math domain error") + r = complex(math.log(abs(z)), math.atan2(z.imag, z.real)) + if base is None: + return r + return r / log(base) + + +def log10(z): + r = log(z) + return complex(r.real / math.log(10.0), r.imag / math.log(10.0)) + + +def sinh(z): + z = _c(z) + return complex(math.sinh(z.real) * math.cos(z.imag), math.cosh(z.real) * math.sin(z.imag)) + + +def cosh(z): + z = _c(z) + return complex(math.cosh(z.real) * math.cos(z.imag), math.sinh(z.real) * math.sin(z.imag)) + + +def tanh(z): + z = _c(z) + x, y = z.real, z.imag + if math.fabs(x) > 354.0: + return complex(math.copysign(1.0, x), 4.0 * math.sin(y) * math.cos(y) * math.exp(-2.0 * math.fabs(x))) + tx = math.tanh(x) + ty = math.tan(y) + cx = 1.0 / math.cosh(x) + txty = tx * ty + denom = 1.0 + txty * txty + return complex(tx * (1.0 + ty * ty) / denom, ((ty / denom) * cx) * cx) + + +def sin(z): + z = _c(z) + r = sinh(complex(-z.imag, z.real)) + return complex(r.imag, -r.real) + + +def cos(z): + z = _c(z) + return cosh(complex(-z.imag, z.real)) + + +def tan(z): + z = _c(z) + r = tanh(complex(-z.imag, z.real)) + return complex(r.imag, -r.real) + + +def asinh(z): + z = _c(z) + s1 = sqrt(complex(1.0 + z.imag, -z.real)) + s2 = sqrt(complex(1.0 - z.imag, z.real)) + re = math.asinh(s1.real * s2.imag - s2.real * s1.imag) + im = math.atan2(z.imag, s1.real * s2.real - s1.imag * s2.imag) + return complex(re, im) + + +def asin(z): + z = _c(z) + r = asinh(complex(-z.imag, z.real)) + return complex(r.imag, -r.real) + + +def acos(z): + z = _c(z) + s1 = sqrt(complex(1.0 - z.real, -z.imag)) + s2 = sqrt(complex(1.0 + z.real, z.imag)) + re = 2.0 * math.atan2(s1.real, s2.real) + im = math.asinh(s2.real * s1.imag - s2.imag * s1.real) + return complex(re, im) + + +def acosh(z): + z = _c(z) + s1 = sqrt(complex(z.real - 1.0, z.imag)) + s2 = sqrt(complex(z.real + 1.0, z.imag)) + re = math.asinh(s1.real * s2.real + s1.imag * s2.imag) + im = 2.0 * math.atan2(s1.imag, s2.real) + return complex(re, im) + + +def atanh(z): + z = _c(z) + if z.real < 0.0: + r = atanh(complex(-z.real, -z.imag)) + return complex(-r.real, -r.imag) + x, y = z.real, z.imag + ay = math.fabs(y) + if x == 1.0 and ay < 1e-154: + if ay == 0.0: + raise ValueError("math domain error") + re = -math.log(math.sqrt(ay) / math.sqrt(math.hypot(ay, 2.0))) + im = math.copysign(math.atan2(2.0, -ay) / 2.0, y) + return complex(re, im) + re = math.log1p(4.0 * x / ((1.0 - x) * (1.0 - x) + ay * ay)) / 4.0 + im = -math.atan2(-2.0 * y, (1.0 - x) * (1.0 + x) - ay * ay) / 2.0 + return complex(re, im) + + +def atan(z): + z = _c(z) + r = atanh(complex(-z.imag, z.real)) + return complex(r.imag, -r.real) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/collections/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/collections/__init__.py new file mode 100644 index 00000000000..d26033aa413 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/collections/__init__.py @@ -0,0 +1,633 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Container datatypes.""" + +from operator import itemgetter as _itemgetter +from itertools import chain as _chain, repeat as _repeat, starmap as _starmap +import heapq as _heapq + +__all__ = ["ChainMap", "Counter", "OrderedDict", "UserDict", "UserList", "UserString", "defaultdict", "deque", + "namedtuple"] + + +class deque: + """A double-ended queue over a circular buffer.""" + + def __init__(self, iterable=(), maxlen=None): + if maxlen is not None and maxlen < 0: + raise ValueError("maxlen must be non-negative") + self._buf = [None] * 8 + self._head = 0 + self._size = 0 + self._maxlen = maxlen + for x in iterable: + self.append(x) + + @property + def maxlen(self): + return self._maxlen + + def _grow(self): + old = self._buf + n = len(old) + self._buf = [old[(self._head + i) % n] for i in range(self._size)] + [None] * n + self._head = 0 + + def _index(self, i): + n = self._size + if i < 0: + i += n + if i < 0 or i >= n: + raise IndexError("deque index out of range") + return (self._head + i) % len(self._buf) + + def append(self, x): + if self._maxlen == 0: + return + if self._size == self._maxlen: + self.popleft() + if self._size == len(self._buf): + self._grow() + self._buf[(self._head + self._size) % len(self._buf)] = x + self._size += 1 + + def appendleft(self, x): + if self._maxlen == 0: + return + if self._size == self._maxlen: + self.pop() + if self._size == len(self._buf): + self._grow() + self._head = (self._head - 1) % len(self._buf) + self._buf[self._head] = x + self._size += 1 + + def pop(self): + if not self._size: + raise IndexError("pop from an empty deque") + self._size -= 1 + i = (self._head + self._size) % len(self._buf) + x = self._buf[i] + self._buf[i] = None + return x + + def popleft(self): + if not self._size: + raise IndexError("pop from an empty deque") + x = self._buf[self._head] + self._buf[self._head] = None + self._head = (self._head + 1) % len(self._buf) + self._size -= 1 + return x + + def extend(self, iterable): + if iterable is self: + iterable = list(iterable) + for x in iterable: + self.append(x) + + def extendleft(self, iterable): + if iterable is self: + iterable = list(iterable) + for x in iterable: + self.appendleft(x) + + def clear(self): + self._buf = [None] * 8 + self._head = 0 + self._size = 0 + + def copy(self): + return deque(self, self._maxlen) + + __copy__ = copy + + def count(self, x): + return sum(1 for v in self if v is x or v == x) + + def index(self, x, start=0, stop=None): + n = self._size + stop = n if stop is None else stop + for i in range(start, min(stop, n)): + v = self[i] + if v is x or v == x: + return i + raise ValueError(repr(x) + " is not in deque") + + def insert(self, i, x): + if self._maxlen is not None and self._size == self._maxlen: + raise IndexError("deque already at its maximum size") + items = list(self) + items.insert(i, x) + self.clear() + self.extend(items) + + def remove(self, value): + for i in range(self._size): + v = self[i] + if v is value or v == value: + del self[i] + return + raise ValueError(repr(value) + " is not in deque") + + def reverse(self): + items = list(self) + items.reverse() + self.clear() + self.extend(items) + + def rotate(self, n=1): + size = self._size + if not size: + return + n %= size + for _ in range(n): + self.appendleft(self.pop()) + + def __len__(self): + return self._size + + def __bool__(self): + return self._size > 0 + + def __iter__(self): + buf = self._buf + head = self._head + for i in range(self._size): + yield buf[(head + i) % len(buf)] + + def __reversed__(self): + for i in range(self._size - 1, -1, -1): + yield self[i] + + def __getitem__(self, i): + return self._buf[self._index(i)] + + def __setitem__(self, i, x): + self._buf[self._index(i)] = x + + def __delitem__(self, i): + items = list(self) + del items[i] + self.clear() + self.extend(items) + + def __contains__(self, x): + for v in self: + if v is x or v == x: + return True + return False + + def __eq__(self, other): + if not isinstance(other, deque): + return NotImplemented + return list(self) == list(other) + + def __lt__(self, other): + return list(self) < list(other) + + def __le__(self, other): + return list(self) <= list(other) + + def __gt__(self, other): + return list(self) > list(other) + + def __ge__(self, other): + return list(self) >= list(other) + + __hash__ = None + + def __add__(self, other): + r = deque(self, self._maxlen) + r.extend(other) + return r + + def __iadd__(self, other): + self.extend(other) + return self + + def __mul__(self, n): + r = deque((), self._maxlen) + for _ in range(n): + r.extend(list(self)) + return r + + def __repr__(self): + body = "[" + ", ".join(repr(x) for x in self) + "]" + if self._maxlen is None: + return "deque(" + body + ")" + return "deque(" + body + ", maxlen=" + str(self._maxlen) + ")" + + +class defaultdict(dict): + def __init__(self, default_factory=None, *args, **kwargs): + if default_factory is not None and not callable(default_factory): + raise TypeError("first argument must be callable or None") + super().__init__(*args, **kwargs) + self.default_factory = default_factory + + def __missing__(self, key): + if self.default_factory is None: + raise KeyError(key) + value = self.default_factory() + self[key] = value + return value + + def copy(self): + return defaultdict(self.default_factory, self) + + __copy__ = copy + + def __repr__(self): + f = self.default_factory + name = "None" if f is None else (getattr(f, "__qualname__", None) or repr(f)) + if isinstance(f, type): + name = repr(f) + return "defaultdict(" + name + ", " + dict.__repr__(self) + ")" + + def __or__(self, other): + if not isinstance(other, dict): + return NotImplemented + r = self.copy() + r.update(other) + return r + + +class OrderedDict(dict): + def move_to_end(self, key, last=True): + value = self.pop(key) + if last: + self[key] = value + else: + rest = list(self.items()) + self.clear() + self[key] = value + for k, v in rest: + self[k] = v + + def popitem(self, last=True): + if not self: + raise KeyError("dictionary is empty") + key = next(reversed(self)) if last else next(iter(self)) + return key, self.pop(key) + + def copy(self): + return OrderedDict(self) + + __copy__ = copy + + def __eq__(self, other): + if isinstance(other, OrderedDict): + return list(self.items()) == list(other.items()) + return dict.__eq__(self, other) + + def __ne__(self, other): + r = self.__eq__(other) + return r if r is NotImplemented else not r + + __hash__ = None + + def __repr__(self): + if not self: + return "OrderedDict()" + return "OrderedDict(" + dict.__repr__(self) + ")" + + @classmethod + def fromkeys(cls, iterable, value=None): + r = cls() + for k in iterable: + r[k] = value + return r + + +class Counter(dict): + def __init__(self, iterable=None, /, **kwds): + super().__init__() + self.update(iterable, **kwds) + + def __missing__(self, key): + return 0 + + def total(self): + return sum(self.values()) + + def most_common(self, n=None): + if n is None: + return sorted(self.items(), key=_itemgetter(1), reverse=True) + return _heapq.nlargest(n, self.items(), key=_itemgetter(1)) + + def elements(self): + return _chain.from_iterable(_starmap(_repeat, self.items())) + + @classmethod + def fromkeys(cls, iterable, v=None): + raise NotImplementedError("Counter.fromkeys() is undefined. Use Counter(iterable) instead.") + + def update(self, iterable=None, /, **kwds): + if iterable is not None: + if isinstance(iterable, dict): + for elem, count in iterable.items(): + self[elem] = self[elem] + count + else: + for elem in iterable: + self[elem] = self[elem] + 1 + if kwds: + self.update(kwds) + + def subtract(self, iterable=None, /, **kwds): + if iterable is not None: + if isinstance(iterable, dict): + for elem, count in iterable.items(): + self[elem] = self[elem] - count + else: + for elem in iterable: + self[elem] = self[elem] - 1 + if kwds: + self.subtract(kwds) + + def copy(self): + return Counter(self) + + __copy__ = copy + + def __delitem__(self, elem): + if elem in self: + dict.__delitem__(self, elem) + + def __repr__(self): + if not self: + return type(self).__name__ + "()" + items = ", ".join(repr(k) + ": " + repr(v) for k, v in self.most_common()) + return type(self).__name__ + "({" + items + "})" + + def __eq__(self, other): + if not isinstance(other, Counter): + return NotImplemented + return all(self[e] == other[e] for c in (self, other) for e in c) + + def __ne__(self, other): + r = self.__eq__(other) + return r if r is NotImplemented else not r + + __hash__ = None + + def __add__(self, other): + if not isinstance(other, Counter): + return NotImplemented + result = Counter() + for elem, count in self.items(): + newcount = count + other[elem] + if newcount > 0: + result[elem] = newcount + for elem, count in other.items(): + if elem not in self and count > 0: + result[elem] = count + return result + + def __sub__(self, other): + if not isinstance(other, Counter): + return NotImplemented + result = Counter() + for elem, count in self.items(): + newcount = count - other[elem] + if newcount > 0: + result[elem] = newcount + for elem, count in other.items(): + if elem not in self and count < 0: + result[elem] = 0 - count + return result + + def __or__(self, other): + if not isinstance(other, Counter): + return NotImplemented + result = Counter() + for elem, count in self.items(): + other_count = other[elem] + newcount = other_count if count < other_count else count + if newcount > 0: + result[elem] = newcount + for elem, count in other.items(): + if elem not in self and count > 0: + result[elem] = count + return result + + def __and__(self, other): + if not isinstance(other, Counter): + return NotImplemented + result = Counter() + for elem, count in self.items(): + other_count = other[elem] + newcount = count if count < other_count else other_count + if newcount > 0: + result[elem] = newcount + return result + + def __pos__(self): + result = Counter() + for elem, count in self.items(): + if count > 0: + result[elem] = count + return result + + def __neg__(self): + result = Counter() + for elem, count in self.items(): + if count < 0: + result[elem] = 0 - count + return result + + +def namedtuple(typename, field_names, *, rename=False, defaults=None, module=None): + if isinstance(field_names, str): + field_names = field_names.replace(",", " ").split() + field_names = tuple(str(f) for f in field_names) + if rename: + seen = set() + names = [] + for i, name in enumerate(field_names): + if not name.isidentifier() or name.startswith("_") or name in seen: + name = "_" + str(i) + seen.add(name) + names.append(name) + field_names = tuple(names) + for name in (typename,) + field_names: + if not name.isidentifier(): + raise ValueError("Type names and field names must be valid identifiers: " + repr(name)) + seen = set() + for name in field_names: + if name.startswith("_") and not rename: + raise ValueError("Field names cannot start with an underscore: " + repr(name)) + if name in seen: + raise ValueError("Encountered duplicate field name: " + repr(name)) + seen.add(name) + field_defaults = {} + if defaults is not None: + defaults = tuple(defaults) + if len(defaults) > len(field_names): + raise TypeError("Got more default values than field names") + field_defaults = dict(zip(field_names[len(field_names) - len(defaults):], defaults)) + n = len(field_names) + + def __new__(cls, *args, **kwargs): + if len(args) > n: + raise TypeError("%s.__new__() takes %d positional arguments but %d were given" % (typename, n + 1, len(args) + 1)) + values = list(args) + for name in field_names[len(args):]: + if name in kwargs: + values.append(kwargs.pop(name)) + elif name in field_defaults: + values.append(field_defaults[name]) + else: + raise TypeError("%s.__new__() missing 1 required positional argument: %r" % (typename, name)) + if kwargs: + name = next(iter(kwargs)) + if name in field_names: + raise TypeError("%s.__new__() got multiple values for argument %r" % (typename, name)) + raise TypeError("%s.__new__() got an unexpected keyword argument %r" % (typename, name)) + return tuple.__new__(cls, values) + + def _make(cls, iterable): + result = tuple.__new__(cls, iterable) + if len(result) != n: + raise TypeError("Expected %d arguments, got %d" % (n, len(result))) + return result + + def _replace(self, **kwds): + result = self._make([kwds.pop(name) if name in kwds else v for name, v in zip(field_names, self)]) + if kwds: + raise ValueError("Got unexpected field names: %r" % list(kwds)) + return result + + def __repr__(self): + return typename + "(" + ", ".join(name + "=" + repr(v) for name, v in zip(field_names, self)) + ")" + + def _asdict(self): + return dict(zip(field_names, self)) + + def __getnewargs__(self): + return tuple(self) + + namespace = { + "__doc__": typename + "(" + ", ".join(field_names) + ")", + "__slots__": (), + "_fields": field_names, + "_field_defaults": field_defaults, + "__new__": __new__, + "_make": classmethod(_make), + "_replace": _replace, + "__repr__": __repr__, + "_asdict": _asdict, + "__getnewargs__": __getnewargs__, + "__match_args__": field_names, + } + for index, name in enumerate(field_names): + namespace[name] = property(_itemgetter(index), doc="Alias for field number " + str(index)) + result = type(typename, (tuple,), namespace) + if module is not None: + result.__module__ = module + return result + + +class ChainMap(dict): + """A view of several mappings searched in order; writes go to the first.""" + + def __init__(self, *maps): + self.maps = list(maps) or [{}] + + def __missing__(self, key): + raise KeyError(key) + + def __getitem__(self, key): + for mapping in self.maps: + if key in mapping: + return mapping[key] + return self.__missing__(key) + + def get(self, key, default=None): + return self[key] if key in self else default + + def __len__(self): + return len(set().union(*self.maps)) + + def __iter__(self): + d = {} + for mapping in reversed(self.maps): + for k in mapping: + d[k] = None + return iter(d) + + def __contains__(self, key): + return any(key in m for m in self.maps) + + def __bool__(self): + return any(self.maps) + + def keys(self): + return list(iter(self)) + + def items(self): + return [(k, self[k]) for k in self] + + def values(self): + return [self[k] for k in self] + + def __repr__(self): + return "ChainMap(" + ", ".join(repr(m) for m in self.maps) + ")" + + def new_child(self, m=None, **kwargs): + if m is None: + m = kwargs + elif kwargs: + m.update(kwargs) + return ChainMap(m, *self.maps) + + @property + def parents(self): + return ChainMap(*self.maps[1:]) + + def __setitem__(self, key, value): + self.maps[0][key] = value + + def __delitem__(self, key): + try: + del self.maps[0][key] + except KeyError: + raise KeyError("Key not found in the first mapping: " + repr(key)) from None + + def pop(self, key, *args): + try: + return self.maps[0].pop(key, *args) + except KeyError: + raise KeyError("Key not found in the first mapping: " + repr(key)) from None + + def clear(self): + self.maps[0].clear() + + def copy(self): + return ChainMap(self.maps[0].copy(), *self.maps[1:]) + + def __eq__(self, other): + return dict(self.items()) == dict(other.items()) + + +class UserDict(dict): + def __init__(self, dict=None, /, **kwargs): + super().__init__() + self.data = self + if dict is not None: + self.update(dict) + if kwargs: + self.update(kwargs) + + +class UserList(list): + def __init__(self, initlist=None): + super().__init__() + self.data = self + if initlist is not None: + self.extend(initlist) + + +class UserString(str): + pass diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/collections/abc.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/collections/abc.py new file mode 100644 index 00000000000..f04c1de4ad8 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/collections/abc.py @@ -0,0 +1,481 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Abstract base classes for containers.""" + +from abc import ABCMeta, abstractmethod + + +def _has(c, *methods): + for m in methods: + for k in c.__mro__: + if m in k.__dict__: + if k.__dict__[m] is None: + return NotImplemented + break + else: + return NotImplemented + return True + + +class Hashable(metaclass=ABCMeta): + @abstractmethod + def __hash__(self): + return 0 + + @classmethod + def __subclasshook__(cls, c): + return _has(c, "__hash__") if cls is Hashable else NotImplemented + + +class Awaitable(metaclass=ABCMeta): + @abstractmethod + def __await__(self): + yield + + @classmethod + def __subclasshook__(cls, c): + return _has(c, "__await__") if cls is Awaitable else NotImplemented + + +class Coroutine(Awaitable): + pass + + +class AsyncIterable(metaclass=ABCMeta): + @abstractmethod + def __aiter__(self): + return None + + @classmethod + def __subclasshook__(cls, c): + return _has(c, "__aiter__") if cls is AsyncIterable else NotImplemented + + +class AsyncIterator(AsyncIterable): + @abstractmethod + async def __anext__(self): + raise StopAsyncIteration + + def __aiter__(self): + return self + + @classmethod + def __subclasshook__(cls, c): + return _has(c, "__anext__", "__aiter__") if cls is AsyncIterator else NotImplemented + + +class AsyncGenerator(AsyncIterator): + pass + + +class Iterable(metaclass=ABCMeta): + @abstractmethod + def __iter__(self): + while False: + yield None + + @classmethod + def __subclasshook__(cls, c): + return _has(c, "__iter__") if cls is Iterable else NotImplemented + + +class Iterator(Iterable): + @abstractmethod + def __next__(self): + raise StopIteration + + def __iter__(self): + return self + + @classmethod + def __subclasshook__(cls, c): + return _has(c, "__iter__", "__next__") if cls is Iterator else NotImplemented + + +class Reversible(Iterable): + @abstractmethod + def __reversed__(self): + while False: + yield None + + @classmethod + def __subclasshook__(cls, c): + return _has(c, "__reversed__", "__iter__") if cls is Reversible else NotImplemented + + +class Generator(Iterator): + @classmethod + def __subclasshook__(cls, c): + return _has(c, "__iter__", "__next__", "send", "throw", "close") if cls is Generator else NotImplemented + + +class Sized(metaclass=ABCMeta): + @abstractmethod + def __len__(self): + return 0 + + @classmethod + def __subclasshook__(cls, c): + return _has(c, "__len__") if cls is Sized else NotImplemented + + +class Container(metaclass=ABCMeta): + @abstractmethod + def __contains__(self, x): + return False + + @classmethod + def __subclasshook__(cls, c): + return _has(c, "__contains__") if cls is Container else NotImplemented + + +class Collection(Sized, Iterable, Container): + @classmethod + def __subclasshook__(cls, c): + return _has(c, "__len__", "__iter__", "__contains__") if cls is Collection else NotImplemented + + +class Callable(metaclass=ABCMeta): + @abstractmethod + def __call__(self, *args, **kwds): + return False + + @classmethod + def __subclasshook__(cls, c): + return _has(c, "__call__") if cls is Callable else NotImplemented + + +class Set(Collection): + def __le__(self, other): + if len(self) > len(other): + return False + return all(e in other for e in self) + + def __lt__(self, other): + return len(self) < len(other) and self.__le__(other) + + def __ge__(self, other): + return len(self) >= len(other) and all(e in self for e in other) + + def __gt__(self, other): + return len(self) > len(other) and self.__ge__(other) + + def __eq__(self, other): + if not isinstance(other, Set): + return NotImplemented + return len(self) == len(other) and self.__le__(other) + + @classmethod + def _from_iterable(cls, it): + return cls(it) + + def __and__(self, other): + return self._from_iterable(v for v in other if v in self) + + def __or__(self, other): + return self._from_iterable(e for s in (self, other) for e in s) + + def __sub__(self, other): + return self._from_iterable(v for v in self if v not in other) + + def __xor__(self, other): + return (self - other) | (other - self) + + def isdisjoint(self, other): + return all(v not in self for v in other) + + +class MutableSet(Set): + @abstractmethod + def add(self, value): + raise NotImplementedError + + @abstractmethod + def discard(self, value): + raise NotImplementedError + + def remove(self, value): + if value not in self: + raise KeyError(value) + self.discard(value) + + def pop(self): + it = iter(self) + try: + value = next(it) + except StopIteration: + raise KeyError from None + self.discard(value) + return value + + def clear(self): + try: + while True: + self.pop() + except KeyError: + pass + + def __ior__(self, it): + for value in it: + self.add(value) + return self + + def __iand__(self, it): + for value in self - it: + self.discard(value) + return self + + def __isub__(self, it): + for value in it: + self.discard(value) + return self + + +class Mapping(Collection): + @abstractmethod + def __getitem__(self, key): + raise KeyError + + def get(self, key, default=None): + try: + return self[key] + except KeyError: + return default + + def __contains__(self, key): + try: + self[key] + except KeyError: + return False + return True + + def keys(self): + return KeysView(self) + + def items(self): + return ItemsView(self) + + def values(self): + return ValuesView(self) + + def __eq__(self, other): + if not isinstance(other, Mapping): + return NotImplemented + return dict(self.items()) == dict(other.items()) + + __reversed__ = None + + +class MappingView(Sized): + def __init__(self, mapping): + self._mapping = mapping + + def __len__(self): + return len(self._mapping) + + def __repr__(self): + return "%s(%r)" % (type(self).__name__, self._mapping) + + +class KeysView(MappingView, Set): + @classmethod + def _from_iterable(cls, it): + return set(it) + + def __contains__(self, key): + return key in self._mapping + + def __iter__(self): + yield from self._mapping + + +class ItemsView(MappingView, Set): + @classmethod + def _from_iterable(cls, it): + return set(it) + + def __contains__(self, item): + key, value = item + try: + v = self._mapping[key] + except KeyError: + return False + return v is value or v == value + + def __iter__(self): + for key in self._mapping: + yield (key, self._mapping[key]) + + +class ValuesView(MappingView, Collection): + def __contains__(self, value): + for key in self._mapping: + v = self._mapping[key] + if v is value or v == value: + return True + return False + + def __iter__(self): + for key in self._mapping: + yield self._mapping[key] + + +_MARKER = object() + + +class MutableMapping(Mapping): + @abstractmethod + def __setitem__(self, key, value): + raise KeyError + + @abstractmethod + def __delitem__(self, key): + raise KeyError + + def pop(self, key, default=_MARKER): + try: + value = self[key] + except KeyError: + if default is _MARKER: + raise + return default + del self[key] + return value + + def popitem(self): + try: + key = next(iter(self)) + except StopIteration: + raise KeyError from None + value = self[key] + del self[key] + return key, value + + def clear(self): + try: + while True: + self.popitem() + except KeyError: + pass + + def update(self, other=(), **kwds): + if isinstance(other, Mapping): + for key in other: + self[key] = other[key] + elif hasattr(other, "keys"): + for key in other.keys(): + self[key] = other[key] + else: + for key, value in other: + self[key] = value + for key, value in kwds.items(): + self[key] = value + + def setdefault(self, key, default=None): + try: + return self[key] + except KeyError: + self[key] = default + return default + + +class Sequence(Reversible, Collection): + @abstractmethod + def __getitem__(self, index): + raise IndexError + + def __iter__(self): + i = 0 + try: + while True: + yield self[i] + i += 1 + except IndexError: + return + + def __contains__(self, value): + for v in self: + if v is value or v == value: + return True + return False + + def __reversed__(self): + for i in reversed(range(len(self))): + yield self[i] + + def index(self, value, start=0, stop=None): + i = start + while stop is None or i < stop: + try: + v = self[i] + except IndexError: + break + if v is value or v == value: + return i + i += 1 + raise ValueError + + def count(self, value): + return sum(1 for v in self if v is value or v == value) + + +class MutableSequence(Sequence): + @abstractmethod + def __setitem__(self, index, value): + raise IndexError + + @abstractmethod + def __delitem__(self, index): + raise IndexError + + @abstractmethod + def insert(self, index, value): + raise IndexError + + def append(self, value): + self.insert(len(self), value) + + def clear(self): + try: + while True: + self.pop() + except IndexError: + pass + + def reverse(self): + n = len(self) + for i in range(n // 2): + self[i], self[n - i - 1] = self[n - i - 1], self[i] + + def extend(self, values): + if values is self: + values = list(values) + for v in values: + self.append(v) + + def pop(self, index=-1): + v = self[index] + del self[index] + return v + + def remove(self, value): + del self[self.index(value)] + + def __iadd__(self, values): + self.extend(values) + return self + + +class ByteString(Sequence): + pass + + +Sequence.register(tuple) +Sequence.register(str) +Sequence.register(range) +MutableSequence.register(list) +Set.register(frozenset) +MutableSet.register(set) +Mapping.register(dict) +MutableMapping.register(dict) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/contextlib.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/contextlib.py new file mode 100644 index 00000000000..ce3386c86a0 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/contextlib.py @@ -0,0 +1,330 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Utilities for with-statement contexts.""" + +from functools import wraps +from collections import deque + + +class AbstractContextManager: + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + return None + + +class AbstractAsyncContextManager: + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc_value, traceback): + return None + + +class ContextDecorator: + def _recreate_cm(self): + return self + + def __call__(self, func): + @wraps(func) + def inner(*args, **kwds): + with self._recreate_cm(): + return func(*args, **kwds) + return inner + + +class _GeneratorContextManager(ContextDecorator, AbstractContextManager): + def __init__(self, func, args, kwds): + self.gen = func(*args, **kwds) + self.func, self.args, self.kwds = func, args, kwds + + def _recreate_cm(self): + return self.__class__(self.func, self.args, self.kwds) + + def __enter__(self): + try: + return next(self.gen) + except StopIteration: + raise RuntimeError("generator didn't yield") from None + + def __exit__(self, typ, value, traceback): + if typ is None: + try: + next(self.gen) + except StopIteration: + return False + raise RuntimeError("generator didn't stop") + if value is None: + value = typ() + try: + self.gen.throw(value) + except StopIteration as exc: + return exc is not value + except RuntimeError as exc: + if exc is value: + return False + if isinstance(value, StopIteration) and exc.__cause__ is value: + return False + raise + except BaseException as exc: + if exc is not value: + raise + return False + raise RuntimeError("generator didn't stop after throw()") + + +class _AsyncGeneratorContextManager(AbstractAsyncContextManager): + def __init__(self, func, args, kwds): + self.gen = func(*args, **kwds) + + async def __aenter__(self): + try: + return await self.gen.__anext__() + except StopAsyncIteration: + raise RuntimeError("generator didn't yield") from None + + async def __aexit__(self, typ, value, traceback): + if typ is None: + try: + await self.gen.__anext__() + except StopAsyncIteration: + return False + raise RuntimeError("generator didn't stop") + if value is None: + value = typ() + try: + await self.gen.athrow(value) + except StopAsyncIteration as exc: + return exc is not value + except BaseException as exc: + if exc is not value: + raise + return False + raise RuntimeError("generator didn't stop after athrow()") + + +def contextmanager(func): + @wraps(func) + def helper(*args, **kwds): + return _GeneratorContextManager(func, args, kwds) + return helper + + +def asynccontextmanager(func): + @wraps(func) + def helper(*args, **kwds): + return _AsyncGeneratorContextManager(func, args, kwds) + return helper + + +class closing(AbstractContextManager): + def __init__(self, thing): + self.thing = thing + + def __enter__(self): + return self.thing + + def __exit__(self, *exc_info): + self.thing.close() + + +class aclosing(AbstractAsyncContextManager): + def __init__(self, thing): + self.thing = thing + + async def __aenter__(self): + return self.thing + + async def __aexit__(self, *exc_info): + await self.thing.aclose() + + +class suppress(AbstractContextManager): + def __init__(self, *exceptions): + self._exceptions = exceptions + + def __enter__(self): + return None + + def __exit__(self, exctype, excinst, exctb): + if exctype is None: + return False + if issubclass(exctype, self._exceptions): + return True + if isinstance(excinst, BaseExceptionGroup): + match, rest = excinst.split(self._exceptions) + if rest is None: + return True + raise rest + return False + + +class nullcontext(AbstractContextManager, AbstractAsyncContextManager): + def __init__(self, enter_result=None): + self.enter_result = enter_result + + def __enter__(self): + return self.enter_result + + def __exit__(self, *excinfo): + return None + + async def __aenter__(self): + return self.enter_result + + async def __aexit__(self, *excinfo): + return None + + +class _RedirectStream(AbstractContextManager): + _stream = None + + def __init__(self, new_target): + self._new_target = new_target + self._old_targets = [] + + def __enter__(self): + import sys + self._old_targets.append(getattr(sys, self._stream)) + setattr(sys, self._stream, self._new_target) + return self._new_target + + def __exit__(self, exctype, excinst, exctb): + import sys + setattr(sys, self._stream, self._old_targets.pop()) + + +class redirect_stdout(_RedirectStream): + _stream = "stdout" + + +class redirect_stderr(_RedirectStream): + _stream = "stderr" + + +class ExitStack(AbstractContextManager): + def __init__(self): + self._exit_callbacks = deque() + + def pop_all(self): + new_stack = type(self)() + new_stack._exit_callbacks = self._exit_callbacks + self._exit_callbacks = deque() + return new_stack + + def push(self, exit): + if hasattr(exit, "__exit__"): + cm = exit + self._exit_callbacks.append(lambda t, v, tb: cm.__exit__(t, v, tb)) + else: + self._exit_callbacks.append(exit) + return exit + + def enter_context(self, cm): + result = cm.__enter__() + self._exit_callbacks.append(lambda t, v, tb: cm.__exit__(t, v, tb)) + return result + + def callback(self, callback, /, *args, **kwds): + def _exit_wrapper(exc_type, exc, tb): + callback(*args, **kwds) + self._exit_callbacks.append(_exit_wrapper) + return callback + + def close(self): + self.__exit__(None, None, None) + + def __exit__(self, *exc_details): + received_exc = exc_details[0] is not None + pending_raise = False + suppressed = False + exc = exc_details[1] + while self._exit_callbacks: + cb = self._exit_callbacks.pop() + try: + if exc is None: + details = (None, None, None) + else: + details = (type(exc), exc, None) + if cb(*details): + suppressed = True + pending_raise = False + exc = None + except BaseException as new_exc: + if exc is not None and new_exc is not exc: + new_exc.__context__ = exc + pending_raise = True + exc = new_exc + if pending_raise: + raise exc + return received_exc and suppressed + + +class AsyncExitStack(AbstractAsyncContextManager): + def __init__(self): + self._exit_callbacks = deque() + + async def enter_async_context(self, cm): + result = await cm.__aenter__() + self._exit_callbacks.append((True, cm)) + return result + + def enter_context(self, cm): + result = cm.__enter__() + self._exit_callbacks.append((False, cm)) + return result + + def push_async_callback(self, callback, /, *args, **kwds): + self._exit_callbacks.append((True, (callback, args, kwds))) + return callback + + def callback(self, callback, /, *args, **kwds): + self._exit_callbacks.append((False, (callback, args, kwds))) + return callback + + async def aclose(self): + await self.__aexit__(None, None, None) + + async def __aexit__(self, *exc_details): + received_exc = exc_details[0] is not None + suppressed = False + pending_raise = False + exc = exc_details[1] + while self._exit_callbacks: + is_async, cb = self._exit_callbacks.pop() + try: + details = (None, None, None) if exc is None else (type(exc), exc, None) + if isinstance(cb, tuple): + r = cb[0](*cb[1], **cb[2]) + if is_async: + await r + r = False + elif is_async: + r = await cb.__aexit__(*details) + else: + r = cb.__exit__(*details) + if r: + suppressed = True + pending_raise = False + exc = None + except BaseException as new_exc: + if exc is not None and new_exc is not exc: + new_exc.__context__ = exc + pending_raise = True + exc = new_exc + if pending_raise: + raise exc + return received_exc and suppressed + + +class chdir(AbstractContextManager): + def __init__(self, path): + self.path = path + + def __enter__(self): + return None + + def __exit__(self, *excinfo): + return None diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/copy.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/copy.py new file mode 100644 index 00000000000..0e181ab97c9 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/copy.py @@ -0,0 +1,144 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Shallow and deep copy operations.""" + + +class Error(Exception): + pass + + +error = Error + +_ATOMIC = (type(None), int, float, bool, str, bytes, type, range, type(Ellipsis), + type(NotImplemented), property) + + +_FUNCTIONS = ("function", "builtin_function_or_method", "method") + + +def copy(x): + cls = type(x) + if cls in _ATOMIC or cls is tuple or cls is frozenset or isinstance(x, type) or cls.__name__ in _FUNCTIONS: + return x + if cls is list: + return x[:] + if cls is dict: + return dict(x) + if cls is set: + return set(x) + copier = getattr(cls, "__copy__", None) + if copier is not None: + return copier(x) + return _reconstruct(x, None) + + +def deepcopy(x, memo=None, _nil=[]): + if memo is None: + memo = {} + d = id(x) + y = memo.get(d, _nil) + if y is not _nil: + return y + cls = type(x) + if cls in _ATOMIC or isinstance(x, type): + return x + if cls is list: + y = [] + memo[d] = y + for a in x: + y.append(deepcopy(a, memo)) + elif cls is tuple: + y = tuple(deepcopy(a, memo) for a in x) + if all(a is b for a, b in zip(x, y)): + y = x + elif cls is dict: + y = {} + memo[d] = y + for k, v in x.items(): + y[deepcopy(k, memo)] = deepcopy(v, memo) + elif cls is set: + y = {deepcopy(a, memo) for a in x} + elif cls is frozenset: + y = frozenset(deepcopy(a, memo) for a in x) + elif hasattr(cls, "__deepcopy__"): + y = x.__deepcopy__(memo) + elif cls.__name__ in _FUNCTIONS: + return x + else: + y = _reconstruct(x, memo) + if y is not x: + memo[d] = y + _keep_alive(x, memo) + return y + + +def _keep_alive(x, memo): + try: + memo[id(memo)].append(x) + except KeyError: + memo[id(memo)] = [x] + + +def _reconstruct(x, memo): + cls = type(x) + y = cls.__new__(cls) if not _builtin_base(cls) else _new_like(x, cls, memo) + if memo is not None: + memo[id(x)] = y + state = getattr(x, "__getstate__", None) + state = state() if state is not None and getattr(cls, "__getstate__", None) is not getattr(object, "__getstate__", None) else _state_of(x) + if state: + if memo is not None: + state = deepcopy(state, memo) + if hasattr(y, "__setstate__"): + y.__setstate__(state) + else: + for k, v in state.items(): + setattr(y, k, v) + return y + + +def _state_of(x): + try: + return dict(x.__dict__) + except AttributeError: + state = {} + for k in type(x).__mro__: + for name in k.__dict__.get("__slots__", ()): + if hasattr(x, name): + state[name] = getattr(x, name) + return state + + +def _builtin_base(cls): + for k in cls.__mro__: + if k in (list, dict, set, tuple, frozenset, int, float, str, bytes): + return k + return None + + +def _new_like(x, cls, memo): + base = _builtin_base(cls) + if base in (list, dict, set): + y = cls.__new__(cls) + if base is list: + y.extend(x if memo is None else [deepcopy(a, memo) for a in x]) + elif base is dict: + for k, v in x.items(): + y[k if memo is None else deepcopy(k, memo)] = v if memo is None else deepcopy(v, memo) + else: + for a in x: + y.add(a if memo is None else deepcopy(a, memo)) + return y + args = x.__getnewargs__() if hasattr(x, "__getnewargs__") else (base(x),) + if memo is not None: + args = deepcopy(args, memo) + return cls.__new__(cls, *args) + + +def replace(obj, /, **changes): + cls = type(obj) + func = getattr(cls, "__replace__", None) + if func is None: + raise TypeError("replace() does not support " + cls.__name__ + " objects") + return func(obj, **changes) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/csv.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/csv.py new file mode 100644 index 00000000000..315c537c2d3 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/csv.py @@ -0,0 +1,692 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""CSV reading and writing, with the parsing rules of CPython's _csv.""" + +from io import StringIO + +__all__ = ["QUOTE_MINIMAL", "QUOTE_ALL", "QUOTE_NONNUMERIC", "QUOTE_NONE", "QUOTE_STRINGS", "QUOTE_NOTNULL", + "Error", "Dialect", "excel", "excel_tab", "unix_dialect", "field_size_limit", "reader", "writer", + "register_dialect", "get_dialect", "list_dialects", "unregister_dialect", "DictReader", "DictWriter", + "Sniffer"] + +QUOTE_MINIMAL, QUOTE_ALL, QUOTE_NONNUMERIC, QUOTE_NONE, QUOTE_STRINGS, QUOTE_NOTNULL = range(6) +_QUOTINGS = (QUOTE_MINIMAL, QUOTE_ALL, QUOTE_NONNUMERIC, QUOTE_NONE, QUOTE_STRINGS, QUOTE_NOTNULL) + +__version__ = "1.0" + +_field_limit = 128 * 1024 + + +class Error(Exception): + pass + + +def field_size_limit(*args): + global _field_limit + if len(args) > 1: + raise TypeError("field_size_limit expected at most 1 argument, got %d" % len(args)) + old = _field_limit + if args: + if not isinstance(args[0], int): + raise TypeError("limit must be an integer") + _field_limit = args[0] + return old + + +class Dialect: + _name = "" + _valid = False + delimiter = None + quotechar = None + escapechar = None + doublequote = None + skipinitialspace = None + lineterminator = None + quoting = None + strict = None + + def __init__(self): + if self.__class__ != Dialect: + self._valid = True + self._validate() + + def _validate(self): + try: + _Dialect(self) + except TypeError as e: + raise Error(str(e)) from None + + +class excel(Dialect): + delimiter = "," + quotechar = '"' + doublequote = True + skipinitialspace = False + lineterminator = "\r\n" + quoting = QUOTE_MINIMAL + + +class excel_tab(excel): + delimiter = "\t" + + +class unix_dialect(Dialect): + delimiter = "," + quotechar = '"' + doublequote = True + skipinitialspace = False + lineterminator = "\n" + quoting = QUOTE_ALL + + +_dialects = {} + + +def _char(name, value, allow_none): + if value is None: + if allow_none: + return None + raise TypeError('"%s" must be a unicode character, not NoneType' % name) + if not isinstance(value, str): + raise TypeError('"%s" must be a unicode character or None, not %s' % (name, type(value).__name__) + if allow_none else '"%s" must be a unicode character, not %s' % (name, type(value).__name__)) + if len(value) != 1: + raise TypeError('"%s" must be a unicode character or None, not a string of length %d' % (name, len(value)) + if allow_none else '"%s" must be a unicode character, not a string of length %d' + % (name, len(value))) + return value + + +class _Dialect: + """The validated parameters a reader or writer works with.""" + + def __init__(self, dialect=None, **kw): + if isinstance(dialect, str): + dialect = get_dialect(dialect) + + def pick(name, default): + if name in kw: + return kw[name] + if dialect is not None and getattr(dialect, name, None) is not None: + return getattr(dialect, name) + return default + + self.delimiter = _char("delimiter", pick("delimiter", ","), False) + self.doublequote = bool(pick("doublequote", True)) + self.escapechar = _char("escapechar", pick("escapechar", None), True) + self.lineterminator = pick("lineterminator", "\r\n") + if not isinstance(self.lineterminator, str): + raise TypeError('"lineterminator" must be a string, not %s' % type(self.lineterminator).__name__) + quoting = pick("quoting", QUOTE_MINIMAL) + if quoting not in _QUOTINGS: + raise TypeError("bad \"quoting\" value") + self.quoting = quoting + qc = pick("quotechar", '"') + if qc is None and "quotechar" not in kw and (dialect is None or getattr(dialect, "quotechar", 0) is not None): + qc = '"' + self.quotechar = _char("quotechar", qc, True) + self.skipinitialspace = bool(pick("skipinitialspace", False)) + self.strict = bool(pick("strict", False)) + if self.quotechar is None and self.quoting != QUOTE_NONE: + if "quoting" in kw or (dialect is not None and getattr(dialect, "quoting", None) is not None): + raise TypeError("quotechar must be set if quoting enabled") + self.quoting = QUOTE_NONE + if self.delimiter == " " and self.skipinitialspace: + raise ValueError("bad delimiter or skipinitialspace value") + if self.delimiter in ("\r", "\n"): + raise ValueError("bad delimiter value") + if self.quotechar in ("\r", "\n"): + raise ValueError("bad quotechar value") + if self.escapechar in ("\r", "\n"): + raise ValueError("bad escapechar value") + if "\r" in self.lineterminator or "\n" in self.lineterminator: + pass + if self.escapechar is not None and self.escapechar == self.delimiter: + raise ValueError("bad delimiter or escapechar value") + if self.quotechar is not None and self.quotechar == self.delimiter: + raise ValueError("bad delimiter or quotechar value") + if self.escapechar is not None and self.escapechar == self.quotechar: + raise ValueError("bad escapechar or quotechar value") + + +def register_dialect(name, dialect=None, **fmtparams): + if not isinstance(name, str): + raise TypeError("dialect name must be a string") + d = _Dialect(dialect, **fmtparams) + _dialects[name] = _Frozen(d) + + +def unregister_dialect(name): + if name not in _dialects: + raise Error("unknown dialect") + del _dialects[name] + + +def get_dialect(name): + try: + return _dialects[name] + except (KeyError, TypeError): + raise Error("unknown dialect") + + +def list_dialects(): + return list(_dialects) + + +class _Frozen: + __slots__ = ("delimiter", "quotechar", "escapechar", "doublequote", "skipinitialspace", "lineterminator", + "quoting", "strict") + + def __init__(self, d): + for n in self.__slots__: + object.__setattr__(self, n, getattr(d, n)) + + def __setattr__(self, name, value): + raise AttributeError("attribute '%s' of '_csv.Dialect' objects is not writable" % name) + + +register_dialect("excel", excel) +register_dialect("excel-tab", excel_tab) +register_dialect("unix", unix_dialect) + +(_START_RECORD, _START_FIELD, _ESCAPED_CHAR, _IN_FIELD, _IN_QUOTED_FIELD, _ESCAPE_IN_QUOTED_FIELD, + _QUOTE_IN_QUOTED_FIELD, _EAT_CRNL, _AFTER_ESCAPED_CRNL) = range(9) + + +class _Reader: + def __init__(self, iterable, dialect="excel", **fmtparams): + self._it = iter(iterable) + self.dialect = _Frozen(_Dialect(dialect, **fmtparams)) + self._d = self.dialect + self.line_num = 0 + + def __iter__(self): + return self + + def _parse_reset(self): + self._fields = [] + self._field = [] + self._state = _START_RECORD + self._unquoted = True + self._quoted_once = False + + def _save_field(self): + field = "".join(self._field) + self._field = [] + d = self._d + if self._unquoted and d.quoting != QUOTE_NONE: + if d.quoting == QUOTE_NONNUMERIC and field: + field = float(field) + elif d.quoting in (QUOTE_NOTNULL, QUOTE_STRINGS) and not field and not self._quoted_once: + field = None + elif d.quoting == QUOTE_STRINGS and field: + field = float(field) + self._fields.append(field) + self._unquoted = True + self._quoted_once = False + + def _add_char(self, c): + if len(self._field) >= _field_limit: + raise Error("field larger than field limit (%d)" % _field_limit) + self._field.append(c) + + def _process_char(self, c): + d = self._d + s = self._state + if s == _START_RECORD: + if c is None: + return + elif c == "\n" or c == "\r": + self._state = _EAT_CRNL + return + s = self._state = _START_FIELD + if s == _START_FIELD: + if c is None or c in ("\n", "\r"): + self._save_field() + self._state = _START_RECORD if c is None else _EAT_CRNL + elif c == d.quotechar and d.quoting != QUOTE_NONE: + self._unquoted = False + self._quoted_once = True + self._state = _IN_QUOTED_FIELD + elif c == d.escapechar: + self._state = _ESCAPED_CHAR + elif c == " " and d.skipinitialspace: + pass + elif c == d.delimiter: + self._save_field() + else: + if d.quoting == QUOTE_NONNUMERIC: + self._numeric = True + self._add_char(c) + self._state = _IN_FIELD + elif s == _ESCAPED_CHAR: + if c in ("\n", "\r"): + self._add_char(c) + self._state = _AFTER_ESCAPED_CRNL + return + if c is None: + c = "\n" + self._add_char(c) + self._state = _IN_FIELD + elif s == _AFTER_ESCAPED_CRNL: + if c is None: + return + self._state = _IN_FIELD + self._process_char(c) + elif s == _IN_FIELD: + if c is None or c in ("\n", "\r"): + self._save_field() + self._state = _START_RECORD if c is None else _EAT_CRNL + elif c == d.escapechar: + self._state = _ESCAPED_CHAR + elif c == d.delimiter: + self._save_field() + self._state = _START_FIELD + else: + self._add_char(c) + elif s == _IN_QUOTED_FIELD: + if c is None: + pass + elif c == d.escapechar: + self._state = _ESCAPE_IN_QUOTED_FIELD + elif c == d.quotechar and d.quoting != QUOTE_NONE: + if d.doublequote: + self._state = _QUOTE_IN_QUOTED_FIELD + else: + self._state = _IN_FIELD + else: + self._add_char(c) + elif s == _ESCAPE_IN_QUOTED_FIELD: + if c is None: + c = "\n" + self._add_char(c) + self._state = _IN_QUOTED_FIELD + elif s == _QUOTE_IN_QUOTED_FIELD: + if d.quoting != QUOTE_NONE and c == d.quotechar: + self._add_char(c) + self._state = _IN_QUOTED_FIELD + elif c == d.delimiter: + self._save_field() + self._state = _START_FIELD + elif c is None or c in ("\n", "\r"): + self._save_field() + self._state = _START_RECORD if c is None else _EAT_CRNL + elif not d.strict: + self._add_char(c) + self._state = _IN_FIELD + else: + raise Error("'%c' expected after '%c'" % (d.delimiter, d.quotechar)) + elif s == _EAT_CRNL: + if c in ("\n", "\r"): + pass + elif c is None: + self._state = _START_RECORD + else: + raise Error("new-line character seen in unquoted field - do you need to open the file with " + "newline=''?") + + def __next__(self): + self._parse_reset() + while True: + try: + line = next(self._it) + except StopIteration: + if len(self._field) != 0 or self._state == _IN_QUOTED_FIELD: + if self._d.strict: + raise Error("unexpected end of data") + self._save_field() + break + if self._fields or self._state == _START_FIELD and False: + break + raise + if not isinstance(line, str): + raise Error("iterator should return strings, not %.200s (the file should be opened in text mode)" + % type(line).__name__) + self.line_num += 1 + for c in line: + self._process_char(c) + self._process_char(None) + if self._state == _START_RECORD: + break + fields = self._fields + self._fields = [] + return fields + + +def reader(csvfile, dialect="excel", **fmtparams): + return _Reader(csvfile, dialect, **fmtparams) + + +class _Writer: + def __init__(self, fileobj, dialect="excel", **fmtparams): + if not hasattr(fileobj, "write"): + raise TypeError("argument 1 must have a \"write\" method") + self._write = fileobj.write + self.dialect = _Frozen(_Dialect(dialect, **fmtparams)) + self._d = self.dialect + + def _join_append(self, out, field, quoted): + d = self._d + need_escape = [] + specials = (d.delimiter, d.escapechar, d.quotechar, "\n", "\r") + if field and (field[0] == " " or field[-1] == " ") and False: + quoted = True + chars = [] + for c in field: + want_escape = False + if c in specials and c is not None: + if d.quoting == QUOTE_NONE: + want_escape = True + else: + if c == d.quotechar: + if d.doublequote: + chars.append(d.quotechar) + else: + want_escape = True + elif c == d.escapechar: + want_escape = True + if not want_escape: + quoted = True + if want_escape: + if d.escapechar is None: + raise Error("need to escape, but no escapechar set") + chars.append(d.escapechar) + chars.append(c) + body = "".join(chars) + if not field and d.quoting == QUOTE_NONE and len(self._row_fields) == 1: + pass + if quoted: + body = d.quotechar + body + d.quotechar + out.append(body) + + def writerow(self, row): + d = self._d + try: + it = iter(row) + except TypeError: + raise Error("iterable expected, not %s" % type(row).__name__) from None + self._row_fields = list(it) + out = [] + n = len(self._row_fields) + for i, field in enumerate(self._row_fields): + if i: + out.append(d.delimiter) + if d.quoting == QUOTE_NONNUMERIC: + quoted = not isinstance(field, (int, float)) or isinstance(field, bool) and False + if field is None: + quoted = True + elif d.quoting == QUOTE_ALL: + quoted = True + elif d.quoting == QUOTE_STRINGS: + quoted = isinstance(field, str) + elif d.quoting == QUOTE_NOTNULL: + quoted = field is not None + else: + quoted = False + if field is None: + text = "" + elif isinstance(field, str): + text = field + else: + text = str(field) + if (d.quoting == QUOTE_MINIMAL and not quoted and not text and n == 1): + quoted = True + if d.quoting == QUOTE_NONE and not text and n == 1: + raise Error("single empty field record must be quoted") + self._join_append(out, text, quoted) + out.append(d.lineterminator) + return self._write("".join(out)) + + def writerows(self, rows): + for row in rows: + self.writerow(row) + + +def writer(csvfile, dialect="excel", **fmtparams): + return _Writer(csvfile, dialect, **fmtparams) + + +class DictReader: + def __init__(self, f, fieldnames=None, restkey=None, restval=None, dialect="excel", *args, **kwds): + if fieldnames is not None and iter(fieldnames) is fieldnames: + fieldnames = list(fieldnames) + self._fieldnames = fieldnames + self.restkey = restkey + self.restval = restval + self.reader = reader(f, dialect, *args, **kwds) + self.dialect = dialect + self.line_num = 0 + + def __iter__(self): + return self + + @property + def fieldnames(self): + if self._fieldnames is None: + try: + self._fieldnames = next(self.reader) + except StopIteration: + pass + self.line_num = self.reader.line_num + return self._fieldnames + + @fieldnames.setter + def fieldnames(self, value): + self._fieldnames = value + + def __next__(self): + if self.line_num == 0: + self.fieldnames + row = next(self.reader) + self.line_num = self.reader.line_num + while row == []: + row = next(self.reader) + d = dict(zip(self.fieldnames, row)) + lf = len(self.fieldnames) + lr = len(row) + if lf < lr: + d[self.restkey] = row[lf:] + elif lf > lr: + for key in self.fieldnames[lr:]: + d[key] = self.restval + return d + + +class DictWriter: + def __init__(self, f, fieldnames, restval="", extrasaction="raise", dialect="excel", *args, **kwds): + if fieldnames is not None and iter(fieldnames) is fieldnames: + fieldnames = list(fieldnames) + self.fieldnames = fieldnames + self.restval = restval + extrasaction = extrasaction.lower() + if extrasaction not in ("raise", "ignore"): + raise ValueError("extrasaction (%s) must be 'raise' or 'ignore'" % extrasaction) + self.extrasaction = extrasaction + self.writer = writer(f, dialect, *args, **kwds) + + def writeheader(self): + header = dict(zip(self.fieldnames, self.fieldnames)) + return self.writerow(header) + + def _dict_to_list(self, rowdict): + if self.extrasaction == "raise": + wrong_fields = rowdict.keys() - self.fieldnames + if wrong_fields: + raise ValueError("dict contains fields not in fieldnames: " + ", ".join([repr(x) for x in wrong_fields])) + return (rowdict.get(key, self.restval) for key in self.fieldnames) + + def writerow(self, rowdict): + return self.writer.writerow(self._dict_to_list(rowdict)) + + def writerows(self, rowdicts): + return self.writer.writerows(map(self._dict_to_list, rowdicts)) + + +class Sniffer: + def __init__(self): + self.preferred = [",", "\t", ";", " ", ":"] + + def sniff(self, sample, delimiters=None): + quotechar, doublequote, delimiter, skipinitialspace = self._guess_quote_and_delimiter(sample, delimiters) + if not delimiter: + delimiter, skipinitialspace = self._guess_delimiter(sample, delimiters) + if not delimiter: + raise Error("Could not determine delimiter") + + class dialect(Dialect): + _name = "sniffed" + lineterminator = "\r\n" + quoting = QUOTE_MINIMAL + + dialect.doublequote = doublequote + dialect.delimiter = delimiter + dialect.quotechar = quotechar or '"' + dialect.skipinitialspace = skipinitialspace + return dialect + + def _guess_quote_and_delimiter(self, data, delimiters): + import re + matches = [] + for restr in (r'(?P[^\w\n"\'])(?P ?)(?P["\']).*?(?P=quote)(?P=delim)', + r'(?:^|\n)(?P["\']).*?(?P=quote)(?P[^\w\n"\'])(?P ?)', + r'(?P[^\w\n"\'])(?P ?)(?P["\']).*?(?P=quote)(?:$|\n)', + r'(?:^|\n)(?P["\']).*?(?P=quote)(?:$|\n)'): + regexp = re.compile(restr, re.DOTALL | re.MULTILINE) + matches = regexp.findall(data) + if matches: + break + if not matches: + return ("", False, None, 0) + quotes = {} + delims = {} + spaces = 0 + groupindex = regexp.groupindex + for m in matches: + n = groupindex["quote"] - 1 + key = m[n] + if key: + quotes[key] = quotes.get(key, 0) + 1 + try: + n = groupindex["delim"] - 1 + key = m[n] + except KeyError: + continue + if key and (delimiters is None or key in delimiters): + delims[key] = delims.get(key, 0) + 1 + try: + n = groupindex["space"] - 1 + except KeyError: + continue + if m[n]: + spaces += 1 + quotechar = max(quotes, key=quotes.get) + if delims: + delim = max(delims, key=delims.get) + skipinitialspace = delims[delim] == spaces + if delim == "\n": + delim = "" + else: + delim = "" + skipinitialspace = 0 + dq_regexp = re.compile(r"((%(delim)s)|^)\W*%(quote)s[^%(delim)s\n]*%(quote)s[^%(delim)s\n]*%(quote)s\W*((%(delim)s)|$)" + % {"delim": re.escape(delim), "quote": quotechar}, re.MULTILINE) + doublequote = bool(dq_regexp.search(data)) + return (quotechar, doublequote, delim, skipinitialspace) + + def _guess_delimiter(self, data, delimiters): + data = list(filter(None, data.split("\n"))) + ascii = [chr(c) for c in range(127)] + chunkLength = min(10, len(data)) + iteration = 0 + charFrequency = {} + modes = {} + delims = {} + start, end = 0, chunkLength + while start < len(data): + iteration += 1 + for line in data[start:end]: + for char in ascii: + metaFrequency = charFrequency.get(char, {}) + freq = line.count(char) + metaFrequency[freq] = metaFrequency.get(freq, 0) + 1 + charFrequency[char] = metaFrequency + for char in charFrequency.keys(): + items = list(charFrequency[char].items()) + if len(items) == 1 and items[0][0] == 0: + continue + if len(items) > 1: + modes[char] = max(items, key=lambda x: x[1]) + items.remove(modes[char]) + modes[char] = (modes[char][0], modes[char][1] - sum(item[1] for item in items)) + else: + modes[char] = items[0] + modeList = modes.items() + total = float(min(chunkLength * iteration, len(data))) + consistency = 1.0 + threshold = 0.9 + while len(delims) == 0 and consistency >= threshold: + for k, v in modeList: + if v[0] > 0 and v[1] > 0: + if ((v[1] / total) >= consistency and (delimiters is None or k in delimiters)): + delims[k] = v + consistency -= 0.01 + if len(delims) == 1: + delim = list(delims.keys())[0] + skipinitialspace = (data[0].count(delim) == data[0].count("%c " % delim)) + return (delim, skipinitialspace) + start = end + end += chunkLength + if not delims: + return ("", 0) + if len(delims) > 1: + for d in self.preferred: + if d in delims.keys(): + skipinitialspace = (data[0].count(d) == data[0].count("%c " % d)) + return (d, skipinitialspace) + items = [(v, k) for (k, v) in delims.items()] + items.sort() + delim = items[-1][1] + skipinitialspace = (data[0].count(delim) == data[0].count("%c " % delim)) + return (delim, skipinitialspace) + + def has_header(self, sample): + rdr = reader(StringIO(sample), self.sniff(sample)) + header = next(rdr) + columns = len(header) + columnTypes = {} + for i in range(columns): + columnTypes[i] = None + checked = 0 + for row in rdr: + if checked > 20: + break + checked += 1 + if len(row) != columns: + continue + for col in list(columnTypes.keys()): + thisType = complex + try: + thisType(row[col]) + except (ValueError, OverflowError): + thisType = len(row[col]) + if thisType != columnTypes[col]: + if columnTypes[col] is None: + columnTypes[col] = thisType + else: + del columnTypes[col] + hasHeader = 0 + for col, colType in columnTypes.items(): + if isinstance(colType, int): + if len(header[col]) != colType: + hasHeader += 1 + else: + hasHeader -= 1 + else: + try: + colType(header[col]) + except (ValueError, TypeError): + hasHeader += 1 + else: + hasHeader -= 1 + return hasHeader > 0 diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/dataclasses.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/dataclasses.py new file mode 100644 index 00000000000..7887a25510e --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/dataclasses.py @@ -0,0 +1,511 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Generated __init__, __repr__, __eq__ and friends for classes of fields. + +CPython builds these methods by exec'ing source text; the methods here are +closures over the field list instead, since no source is compiled at run time. +""" + +import copy as _copy + +__all__ = ["dataclass", "field", "Field", "FrozenInstanceError", "InitVar", "KW_ONLY", "MISSING", "fields", + "asdict", "astuple", "make_dataclass", "replace", "is_dataclass"] + + +class _MissingType: + def __repr__(self): + return "MISSING" + + +MISSING = _MissingType() + + +class _KwOnly: + pass + + +KW_ONLY = _KwOnly + + +class FrozenInstanceError(AttributeError): + pass + + +class InitVar: + def __init__(self, type): + self.type = type + + def __class_getitem__(cls, type): + return InitVar(type) + + def __repr__(self): + t = self.type + return "dataclasses.InitVar[" + (t.__name__ if isinstance(t, type) else repr(t)) + "]" + + +_FIELD = "_FIELD" +_FIELD_CLASSVAR = "_FIELD_CLASSVAR" +_FIELD_INITVAR = "_FIELD_INITVAR" + + +class Field: + __slots__ = ("name", "type", "default", "default_factory", "repr", "hash", "init", "compare", "metadata", + "kw_only", "_field_type") + + def __init__(self, default, default_factory, init, repr, hash, compare, metadata, kw_only): + self.name = None + self.type = None + self.default = default + self.default_factory = default_factory + self.init = init + self.repr = repr + self.hash = hash + self.compare = compare + self.metadata = {} if metadata is None else metadata + self.kw_only = kw_only + self._field_type = None + + def __repr__(self): + return ("Field(name=%r,type=%r,default=%r,default_factory=%r,init=%r,repr=%r,hash=%r,compare=%r," + "metadata=%r,kw_only=%r,_field_type=%s)" % (self.name, self.type, self.default, + self.default_factory, self.init, self.repr, self.hash, + self.compare, self.metadata, self.kw_only, + self._field_type)) + + def __set_name__(self, owner, name): + func = getattr(type(self.default), "__set_name__", None) + if func: + func(self.default, owner, name) + + +def field(*, default=MISSING, default_factory=MISSING, init=True, repr=True, hash=None, compare=True, + metadata=None, kw_only=MISSING): + if default is not MISSING and default_factory is not MISSING: + raise ValueError("cannot specify both default and default_factory") + return Field(default, default_factory, init, repr, hash, compare, metadata, kw_only) + + +_PARAMS = "__dataclass_params__" +_FIELDS = "__dataclass_fields__" + + +class _DataclassParams: + def __init__(self, init, repr, eq, order, unsafe_hash, frozen, match_args, kw_only, slots, weakref_slot): + self.init = init + self.repr = repr + self.eq = eq + self.order = order + self.unsafe_hash = unsafe_hash + self.frozen = frozen + self.match_args = match_args + self.kw_only = kw_only + self.slots = slots + self.weakref_slot = weakref_slot + + +def _is_classvar(a_type): + if a_type is _ClassVarMarker: + return True + name = getattr(a_type, "_name", None) or getattr(a_type, "__name__", None) + if name == "ClassVar": + return True + if isinstance(a_type, str): + return a_type.startswith("ClassVar") or a_type.startswith("typing.ClassVar") + origin = getattr(a_type, "__origin__", None) + return origin is not None and getattr(origin, "_name", None) == "ClassVar" + + +_ClassVarMarker = object() + + +def _is_initvar(a_type): + return a_type is InitVar or isinstance(a_type, InitVar) or isinstance(a_type, str) and \ + a_type.startswith("InitVar") + + +def _is_kw_only(a_type): + return a_type is KW_ONLY or isinstance(a_type, str) and a_type in ("KW_ONLY", "dataclasses.KW_ONLY") + + +def _get_field(cls, a_name, a_type, default_kw_only): + default = getattr(cls, a_name, MISSING) + if isinstance(default, Field): + f = default + else: + f = field(default=default) + f.name = a_name + f.type = a_type + f._field_type = _FIELD + if _is_classvar(a_type): + f._field_type = _FIELD_CLASSVAR + elif _is_initvar(a_type): + f._field_type = _FIELD_INITVAR + if f._field_type in (_FIELD_CLASSVAR, _FIELD_INITVAR): + if f.default_factory is not MISSING: + raise TypeError("field " + f.name + " cannot have a default factory") + if f._field_type in (_FIELD, _FIELD_INITVAR): + if f.kw_only is MISSING: + f.kw_only = default_kw_only + else: + f.kw_only = False + if f._field_type is _FIELD and f.default_factory is MISSING and isinstance(f.default, (list, dict, set)): + raise ValueError("mutable default " + str(type(f.default)) + " for field " + f.name + + " is not allowed: use default_factory") + return f + + +def _set_new_attribute(cls, name, value): + if name in cls.__dict__: + return True + setattr(cls, name, value) + return False + + +def _fields_in_init_order(fields): + std = [f for f in fields if f.init and not f.kw_only] + kw = [f for f in fields if f.init and f.kw_only] + return std, kw + + +def _make_init(cls, fields, std_fields, kw_only_fields, frozen, has_post_init): + seen_default = None + for f in std_fields: + if f.default is not MISSING or f.default_factory is not MISSING: + seen_default = f + elif seen_default is not None: + raise TypeError("non-default argument " + repr(f.name) + " follows default argument " + + repr(seen_default.name)) + qualname = cls.__qualname__ + ".__init__" + nstd = len(std_fields) + nreq = len([f for f in std_fields if f.default is MISSING and f.default_factory is MISSING]) + names = [f.name for f in std_fields + kw_only_fields] + initvars = [f.name for f in fields if f._field_type is _FIELD_INITVAR] + + def __init__(self, *args, **kwargs): + if len(args) > nstd: + if nreq == nstd: + takes = "%d positional argument%s" % (nstd + 1, "" if nstd == 0 else "s") + else: + takes = "from %d to %d positional arguments" % (nreq + 1, nstd + 1) + raise TypeError("%s() takes %s but %d were given" % (qualname, takes, len(args) + 1)) + values = {} + for i in range(len(args)): + values[std_fields[i].name] = args[i] + for k, v in kwargs.items(): + if k not in names: + raise TypeError(qualname + "() got an unexpected keyword argument " + repr(k)) + if k in values: + raise TypeError(qualname + "() got multiple values for argument " + repr(k)) + values[k] = v + missing = [f.name for f in std_fields if f.name not in values and f.default is MISSING and + f.default_factory is MISSING] + if missing: + raise TypeError("%s() missing %d required positional argument%s: %s" + % (qualname, len(missing), "" if len(missing) == 1 else "s", + _and_join(["'" + m + "'" for m in missing]))) + missing = [f.name for f in kw_only_fields if f.name not in values and f.default is MISSING and + f.default_factory is MISSING] + if missing: + raise TypeError("%s() missing %d required keyword-only argument%s: %s" + % (qualname, len(missing), "" if len(missing) == 1 else "s", + _and_join(["'" + m + "'" for m in missing]))) + for f in fields: + if f._field_type is not _FIELD: + continue + if f.init and f.name in values: + v = values[f.name] + elif f.default_factory is not MISSING: + v = f.default_factory() + elif f.default is not MISSING: + if not f.init: + continue + v = f.default + else: + continue + if frozen: + object.__setattr__(self, f.name, v) + else: + setattr(self, f.name, v) + if has_post_init: + extra = [] + for name in initvars: + if name in values: + extra.append(values[name]) + else: + extra.append(next(f.default for f in fields if f.name == name)) + self.__post_init__(*extra) + __init__.__qualname__ = qualname + return __init__ + + +def _and_join(items): + if len(items) == 1: + return items[0] + if len(items) == 2: + return items[0] + " and " + items[1] + return ", ".join(items[:-1]) + ", and " + items[-1] + + +def _make_repr(fields): + shown = [f for f in fields if f._field_type is _FIELD and f.repr] + active = set() + + def __repr__(self): + key = id(self) + if key in active: + return "..." + active.add(key) + try: + return type(self).__qualname__ + "(" + ", ".join(f.name + "=" + repr(getattr(self, f.name)) + for f in shown) + ")" + finally: + active.discard(key) + # pprint lays out a dataclass field by field only when its repr is ours. + __repr__._dataclass_repr = True + return __repr__ + + +def _tuple_of(fields): + compared = [f.name for f in fields if f._field_type is _FIELD and f.compare] + + def as_tuple(self): + return tuple(getattr(self, n) for n in compared) + return as_tuple + + +def _make_eq(as_tuple): + def __eq__(self, other): + if self is other: + return True + if other.__class__ is self.__class__: + return as_tuple(self) == as_tuple(other) + return NotImplemented + return __eq__ + + +def _make_order(as_tuple, op): + def compare(self, other): + if other.__class__ is not self.__class__: + return NotImplemented + a = as_tuple(self) + b = as_tuple(other) + if op == "__lt__": + return a < b + if op == "__le__": + return a <= b + if op == "__gt__": + return a > b + return a >= b + compare.__name__ = op + return compare + + +def _make_hash(fields): + hashed = [f.name for f in fields if f._field_type is _FIELD and (f.compare if f.hash is None else f.hash)] + + def __hash__(self): + return hash(tuple(getattr(self, n) for n in hashed)) + return __hash__ + + +def _make_frozen(cls, fields): + names = {f.name for f in fields} + + def __setattr__(self, name, value): + if type(self) is cls or name in names: + raise FrozenInstanceError("cannot assign to field " + repr(name)) + object.__setattr__(self, name, value) + + def __delattr__(self, name): + if type(self) is cls or name in names: + raise FrozenInstanceError("cannot delete field " + repr(name)) + object.__delattr__(self, name) + return __setattr__, __delattr__ + + +def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, match_args, kw_only, slots, weakref_slot): + fields = {} + setattr(cls, _PARAMS, _DataclassParams(init, repr, eq, order, unsafe_hash, frozen, match_args, kw_only, + slots, weakref_slot)) + any_frozen_base = False + has_dataclass_bases = False + for b in cls.__mro__[-1:0:-1]: + base_fields = getattr(b, _FIELDS, None) + if base_fields is not None: + has_dataclass_bases = True + for f in base_fields.values(): + fields[f.name] = f + if getattr(b, _PARAMS).frozen: + any_frozen_base = True + cls_annotations = cls.__dict__.get("__annotations__", {}) + cls_fields = [] + kw_only_seen = False + for name, type_ in cls_annotations.items(): + if _is_kw_only(type_): + if kw_only_seen: + raise TypeError(cls.__name__ + " is KW_ONLY, but KW_ONLY has already been specified") + kw_only_seen = True + kw_only = True + else: + cls_fields.append(_get_field(cls, name, type_, kw_only)) + for f in cls_fields: + fields[f.name] = f + if isinstance(getattr(cls, f.name, None), Field): + if f.default is MISSING: + delattr(cls, f.name) + else: + setattr(cls, f.name, f.default) + for name, value in cls.__dict__.items(): + if isinstance(value, Field) and name not in cls_annotations: + raise TypeError(repr(name) + " is a field but has no type annotation") + if has_dataclass_bases: + if any_frozen_base and not frozen: + raise TypeError("cannot inherit non-frozen dataclass from a frozen one") + if not any_frozen_base and frozen: + raise TypeError("cannot inherit frozen dataclass from a non-frozen one") + setattr(cls, _FIELDS, fields) + class_hash = cls.__dict__.get("__hash__", MISSING) + has_explicit_hash = not (class_hash is MISSING or (class_hash is None and "__eq__" in cls.__dict__)) + all_fields = list(fields.values()) + field_list = [f for f in all_fields if f._field_type is _FIELD] + std, kw = _fields_in_init_order([f for f in all_fields if f._field_type in (_FIELD, _FIELD_INITVAR)]) + if init: + has_post_init = hasattr(cls, "__post_init__") + _set_new_attribute(cls, "__init__", _make_init(cls, all_fields, std, kw, frozen, has_post_init)) + if repr: + _set_new_attribute(cls, "__repr__", _make_repr(field_list)) + as_tuple = _tuple_of(field_list) + if eq: + _set_new_attribute(cls, "__eq__", _make_eq(as_tuple)) + if order: + for op in ("__lt__", "__le__", "__gt__", "__ge__"): + if _set_new_attribute(cls, op, _make_order(as_tuple, op)): + raise TypeError("Cannot overwrite attribute " + op + " in class " + cls.__name__ + + ". Consider using functools.total_ordering") + if frozen: + setter, deleter = _make_frozen(cls, field_list) + for name, fn in (("__setattr__", setter), ("__delattr__", deleter)): + if _set_new_attribute(cls, name, fn): + raise TypeError("Cannot overwrite attribute " + name + " in class " + cls.__name__) + if unsafe_hash: + if has_explicit_hash: + raise TypeError("Cannot overwrite attribute __hash__ in class " + cls.__name__) + cls.__hash__ = _make_hash(field_list) + elif not has_explicit_hash: + if eq and frozen: + cls.__hash__ = _make_hash(field_list) + elif eq: + cls.__hash__ = None + if match_args: + _set_new_attribute(cls, "__match_args__", tuple(f.name for f in std)) + if "__replace__" not in cls.__dict__: + cls.__replace__ = _replace + return cls + + +def dataclass(cls=None, /, *, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False, + match_args=True, kw_only=False, slots=False, weakref_slot=False): + def wrap(cls): + if order and not eq: + raise ValueError("eq must be true if order is true") + return _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, match_args, kw_only, slots, + weakref_slot) + if cls is None: + return wrap + return wrap(cls) + + +def fields(class_or_instance): + try: + fs = getattr(class_or_instance, _FIELDS) + except AttributeError: + raise TypeError("must be called with a dataclass type or instance") from None + return tuple(f for f in fs.values() if f._field_type is _FIELD) + + +def is_dataclass(obj): + cls = obj if isinstance(obj, type) else type(obj) + return hasattr(cls, _FIELDS) + + +def _is_dataclass_instance(obj): + return hasattr(type(obj), _FIELDS) + + +def asdict(obj, *, dict_factory=dict): + if not _is_dataclass_instance(obj): + raise TypeError("asdict() should be called on dataclass instances") + return _asdict_inner(obj, dict_factory) + + +def _asdict_inner(obj, dict_factory): + if _is_dataclass_instance(obj): + return dict_factory([(f.name, _asdict_inner(getattr(obj, f.name), dict_factory)) for f in fields(obj)]) + if isinstance(obj, tuple) and hasattr(obj, "_fields"): + return type(obj)(*[_asdict_inner(v, dict_factory) for v in obj]) + if isinstance(obj, (list, tuple)): + return type(obj)(_asdict_inner(v, dict_factory) for v in obj) + if isinstance(obj, dict): + return type(obj)((_asdict_inner(k, dict_factory), _asdict_inner(v, dict_factory)) for k, v in obj.items()) + return _copy.deepcopy(obj) + + +def astuple(obj, *, tuple_factory=tuple): + if not _is_dataclass_instance(obj): + raise TypeError("astuple() should be called on dataclass instances") + return _astuple_inner(obj, tuple_factory) + + +def _astuple_inner(obj, tuple_factory): + if _is_dataclass_instance(obj): + return tuple_factory([_astuple_inner(getattr(obj, f.name), tuple_factory) for f in fields(obj)]) + if isinstance(obj, tuple) and hasattr(obj, "_fields"): + return type(obj)(*[_astuple_inner(v, tuple_factory) for v in obj]) + if isinstance(obj, (list, tuple)): + return type(obj)(_astuple_inner(v, tuple_factory) for v in obj) + if isinstance(obj, dict): + return type(obj)((_astuple_inner(k, tuple_factory), _astuple_inner(v, tuple_factory)) + for k, v in obj.items()) + return _copy.deepcopy(obj) + + +def make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, repr=True, eq=True, order=False, + unsafe_hash=False, frozen=False, match_args=True, kw_only=False, slots=False, + weakref_slot=False, module=None): + ns = {} if namespace is None else dict(namespace) + annotations = {} + for item in fields: + if isinstance(item, str): + annotations[item] = "typing.Any" + elif len(item) == 2: + annotations[item[0]] = item[1] + else: + annotations[item[0]] = item[1] + ns[item[0]] = item[2] + ns["__annotations__"] = annotations + cls = type(cls_name, bases, ns) + return dataclass(cls, init=init, repr=repr, eq=eq, order=order, unsafe_hash=unsafe_hash, frozen=frozen, + match_args=match_args, kw_only=kw_only, slots=slots, weakref_slot=weakref_slot) + + +def _replace(self, /, **changes): + for f in getattr(self, _FIELDS).values(): + if f._field_type is _FIELD_CLASSVAR: + continue + if not f.init: + if f.name in changes: + raise TypeError("field " + f.name + " is declared with init=False, it cannot be specified with " + "replace()") + continue + if f.name not in changes: + if f._field_type is _FIELD_INITVAR and f.default is MISSING: + raise TypeError("InitVar " + repr(f.name) + " must be specified with replace()") + changes[f.name] = getattr(self, f.name) + return self.__class__(**changes) + + +def replace(obj, /, **changes): + if not _is_dataclass_instance(obj): + raise TypeError("replace() should be called on dataclass instances") + return _replace(obj, **changes) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/datetime.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/datetime.py new file mode 100644 index 00000000000..59f676e05f6 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/datetime.py @@ -0,0 +1,1258 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Dates, times, durations and fixed-offset time zones. + +Dates are proleptic Gregorian. Local time is the device's default zone, +asked for its UTC offset at each instant. +""" + +import time as _time +import _time as _clock + +__all__ = ["date", "datetime", "time", "timedelta", "timezone", "tzinfo", "MINYEAR", "MAXYEAR", "UTC"] + +MINYEAR = 1 +MAXYEAR = 9999 +_MAXORDINAL = 3652059 +_EPOCH_ORDINAL = 719163 +_DIM = [0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] + + +def _is_leap(y): + return y % 4 == 0 and (y % 100 != 0 or y % 400 == 0) + + +def _days_in_month(y, m): + return 29 if m == 2 and _is_leap(y) else _DIM[m] + + +def _ymd2ord(y, m, d): + return _time._days_from_civil(y, m, d) + _EPOCH_ORDINAL + + +def _ord2ymd(n): + return _time._civil_from_days(n - _EPOCH_ORDINAL) + + +def _cmp(x, y): + return 0 if x == y else (1 if x > y else -1) + + +def _name(obj): + t = type(obj) + return "%s.%s" % (t.__module__, t.__qualname__) + + +def _index(v, what): + if isinstance(v, bool) or not isinstance(v, int): + if isinstance(v, float): + raise TypeError("'float' object cannot be interpreted as an integer") + try: + return v.__index__() + except AttributeError: + raise TypeError("'%s' object cannot be interpreted as an integer" % type(v).__name__) + return v + + +def _check_date(year, month, day): + year = _index(year, "year") + month = _index(month, "month") + day = _index(day, "day") + if not MINYEAR <= year <= MAXYEAR: + raise ValueError("year must be in %d..%d, not %d" % (MINYEAR, MAXYEAR, year)) + if not 1 <= month <= 12: + raise ValueError("month must be in 1..12, not %d" % month) + dim = _days_in_month(year, month) + if not 1 <= day <= dim: + raise ValueError("day %d must be in range 1..%d for month %d in year %d" % (day, dim, month, year)) + return year, month, day + + +def _check_time(hour, minute, second, microsecond, fold): + hour = _index(hour, "hour") + minute = _index(minute, "minute") + second = _index(second, "second") + microsecond = _index(microsecond, "microsecond") + if not 0 <= hour <= 23: + raise ValueError("hour must be in 0..23") + if not 0 <= minute <= 59: + raise ValueError("minute must be in 0..59") + if not 0 <= second <= 59: + raise ValueError("second must be in 0..59") + if not 0 <= microsecond <= 999999: + raise ValueError("microsecond must be in 0..999999") + if fold not in (0, 1): + raise ValueError("fold must be either 0 or 1") + return hour, minute, second, microsecond, fold + + +def _check_tzinfo(tz): + if tz is not None and not isinstance(tz, tzinfo): + raise TypeError("tzinfo argument must be None or of a tzinfo subclass, not type '%s'" + % type(tz).__name__) + + +def _check_offset(name, offset): + if offset is None: + return + if not isinstance(offset, timedelta): + raise TypeError("tzinfo.%s() must return None or timedelta, not '%s'" % (name, type(offset).__name__)) + if not -timedelta(1) < offset < timedelta(1): + raise ValueError("offset must be a timedelta strictly between -timedelta(hours=24) and " + "timedelta(hours=24), not %r." % (offset,)) + + +def _format_time(hh, mm, ss, us, timespec="auto"): + if timespec == "auto": + timespec = "microseconds" if us else "seconds" + elif timespec == "milliseconds": + us //= 1000 + if timespec == "hours": + return "%02d" % hh + if timespec == "minutes": + return "%02d:%02d" % (hh, mm) + if timespec == "seconds": + return "%02d:%02d:%02d" % (hh, mm, ss) + if timespec == "milliseconds": + return "%02d:%02d:%02d.%03d" % (hh, mm, ss, us) + if timespec == "microseconds": + return "%02d:%02d:%02d.%06d" % (hh, mm, ss, us) + raise ValueError("Unknown timespec value") + + +def _format_offset(off, sep=":"): + if off is None: + return "" + if off.days < 0: + sign = "-" + off = -off + else: + sign = "+" + hh, mm = divmod(off, timedelta(hours=1)) + mm, ss = divmod(mm, timedelta(minutes=1)) + s = "%s%02d%s%02d" % (sign, hh, sep, mm) + if ss or ss.microseconds: + s += "%s%02d" % (sep, ss.seconds) + if ss.microseconds: + s += ".%06d" % ss.microseconds + return s + + +def _round_half_even(x): + return int(round(x)) + + +class timedelta: + __slots__ = ("_days", "_seconds", "_microseconds", "_hashcode") + + def __new__(cls, days=0, seconds=0, microseconds=0, milliseconds=0, minutes=0, hours=0, weeks=0): + exact = 0 + inexact = 0.0 + for v, scale in ((days, 86400000000), (seconds, 1000000), (microseconds, 1), + (milliseconds, 1000), (minutes, 60000000), (hours, 3600000000), + (weeks, 604800000000)): + if isinstance(v, bool) or isinstance(v, int): + exact += v * scale + elif isinstance(v, float): + ip = int(v) + exact += ip * scale + inexact += (v - ip) * scale + else: + raise TypeError("unsupported type for timedelta %s component: %s" + % (_component(scale), type(v).__name__)) + whole = int(inexact) + total = exact + whole + frac = inexact - whole + if frac > 0.5 or (frac == 0.5 and total % 2 == 1): + total += 1 + elif frac < -0.5 or (frac == -0.5 and total % 2 == 1): + total -= 1 + return cls._from_us(total) + + @classmethod + def _from_us(cls, total): + d, rem = divmod(total, 86400000000) + s, us = divmod(rem, 1000000) + if abs(d) > 999999999: + raise OverflowError("days=%d; must have magnitude <= 999999999" % d) + self = object.__new__(cls) + self._days = d + self._seconds = s + self._microseconds = us + self._hashcode = -1 + return self + + def _us(self): + return (self._days * 86400 + self._seconds) * 1000000 + self._microseconds + + @property + def days(self): + return self._days + + @property + def seconds(self): + return self._seconds + + @property + def microseconds(self): + return self._microseconds + + def total_seconds(self): + return self._us() / 10 ** 6 + + def __repr__(self): + args = [] + if self._days: + args.append("days=%d" % self._days) + if self._seconds: + args.append("seconds=%d" % self._seconds) + if self._microseconds: + args.append("microseconds=%d" % self._microseconds) + if not args: + args.append("0") + return "%s(%s)" % (_name(self), ", ".join(args)) + + def __str__(self): + mm, ss = divmod(self._seconds, 60) + hh, mm = divmod(mm, 60) + s = "%d:%02d:%02d" % (hh, mm, ss) + if self._days: + s = "%d day%s, " % (self._days, "" if abs(self._days) == 1 else "s") + s + if self._microseconds: + s += ".%06d" % self._microseconds + return s + + def __add__(self, other): + if isinstance(other, timedelta): + return timedelta._from_us(self._us() + other._us()) + return NotImplemented + + __radd__ = __add__ + + def __sub__(self, other): + if isinstance(other, timedelta): + return timedelta._from_us(self._us() - other._us()) + return NotImplemented + + def __rsub__(self, other): + if isinstance(other, timedelta): + return -self + other + return NotImplemented + + def __neg__(self): + return timedelta._from_us(-self._us()) + + def __pos__(self): + return self + + def __abs__(self): + return -self if self._days < 0 else self + + def __mul__(self, other): + if isinstance(other, int): + return timedelta._from_us(self._us() * other) + if isinstance(other, float): + a, b = other.as_integer_ratio() + return timedelta._from_us(_div_even(self._us() * a, b)) + return NotImplemented + + __rmul__ = __mul__ + + def __floordiv__(self, other): + if isinstance(other, timedelta): + return self._us() // other._us() + if isinstance(other, int): + return timedelta._from_us(self._us() // other) + return NotImplemented + + def __truediv__(self, other): + if isinstance(other, timedelta): + return self._us() / other._us() + if isinstance(other, int): + return timedelta._from_us(_div_even(self._us(), other)) + if isinstance(other, float): + a, b = other.as_integer_ratio() + return timedelta._from_us(_div_even(b * self._us(), a)) + return NotImplemented + + def __mod__(self, other): + if isinstance(other, timedelta): + return timedelta._from_us(self._us() % other._us()) + return NotImplemented + + def __divmod__(self, other): + if isinstance(other, timedelta): + q, r = divmod(self._us(), other._us()) + return q, timedelta._from_us(r) + return NotImplemented + + def __eq__(self, other): + if isinstance(other, timedelta): + return self._us() == other._us() + return NotImplemented + + def __lt__(self, other): + if isinstance(other, timedelta): + return self._us() < other._us() + return NotImplemented + + def __le__(self, other): + if isinstance(other, timedelta): + return self._us() <= other._us() + return NotImplemented + + def __gt__(self, other): + if isinstance(other, timedelta): + return self._us() > other._us() + return NotImplemented + + def __ge__(self, other): + if isinstance(other, timedelta): + return self._us() >= other._us() + return NotImplemented + + def __hash__(self): + if self._hashcode == -1: + self._hashcode = hash((self._days, self._seconds, self._microseconds)) + return self._hashcode + + def __bool__(self): + return self._days != 0 or self._seconds != 0 or self._microseconds != 0 + + def __reduce__(self): + return (type(self), (self._days, self._seconds, self._microseconds)) + + +def _component(scale): + return {86400000000: "days", 1000000: "seconds", 1: "microseconds", 1000: "milliseconds", + 60000000: "minutes", 3600000000: "hours", 604800000000: "weeks"}[scale] + + +def _div_even(a, b): + q, r = divmod(a, b) + r2 = r * 2 + if b > 0: + if r2 > b or (r2 == b and q % 2 == 1): + q += 1 + else: + if r2 < b or (r2 == b and q % 2 == 1): + q += 1 + return q + + +timedelta.min = timedelta(-999999999) +timedelta.max = timedelta(days=999999999, hours=23, minutes=59, seconds=59, microseconds=999999) +timedelta.resolution = timedelta(microseconds=1) + + +class date: + __slots__ = ("_year", "_month", "_day", "_hashcode") + + def __new__(cls, year, month=None, day=None): + year, month, day = _check_date(year, month, day) + self = object.__new__(cls) + self._year = year + self._month = month + self._day = day + self._hashcode = -1 + return self + + @classmethod + def fromtimestamp(cls, t): + y, m, d, hh, mm, ss, us = _local_fields(t) + return cls(y, m, d) + + @classmethod + def today(cls): + return cls.fromtimestamp(_time.time()) + + @classmethod + def fromordinal(cls, n): + if not 1 <= n <= _MAXORDINAL: + raise ValueError("ordinal must be in 1..%d" % _MAXORDINAL) + y, m, d = _ord2ymd(n) + return cls(y, m, d) + + @classmethod + def fromisoformat(cls, s): + if not isinstance(s, str): + raise TypeError("fromisoformat: argument must be str") + try: + ymd, n = _parse_date(s) + if n != len(s): + raise ValueError + except (ValueError, IndexError): + raise ValueError("Invalid isoformat string: %r" % (s,)) + return cls(*ymd) + + @classmethod + def fromisocalendar(cls, year, week, day): + return cls(*_isocal_to_ymd(year, week, day)) + + @classmethod + def strptime(cls, date_string, format): + import _strptime + return _strptime._strptime_datetime_date(cls, date_string, format) + + @property + def year(self): + return self._year + + @property + def month(self): + return self._month + + @property + def day(self): + return self._day + + def __repr__(self): + return "%s(%d, %d, %d)" % (_name(self), self._year, self._month, self._day) + + def isoformat(self): + return "%04d-%02d-%02d" % (self._year, self._month, self._day) + + __str__ = isoformat + + def ctime(self): + wd = self.toordinal() % 7 or 7 + return "%s %s %2d 00:00:00 %04d" % (_time._DAYS[wd - 1][:3], _time._MONTHS[self._month - 1][:3], + self._day, self._year) + + def strftime(self, format): + return _time._format(format, self._tuple(0, 0, 0), None, None, 0) + + def __format__(self, fmt): + if not isinstance(fmt, str): + raise TypeError("must be str, not %s" % type(fmt).__name__) + return self.strftime(fmt) if fmt else str(self) + + def _tuple(self, hh, mm, ss): + return (self._year, self._month, self._day, hh, mm, ss, self.weekday(), + _time._yday(self._year, self._month, self._day), -1) + + def timetuple(self): + return _time.struct_time(self._tuple(0, 0, 0)) + + def toordinal(self): + return _ymd2ord(self._year, self._month, self._day) + + def replace(self, year=None, month=None, day=None): + return type(self)(self._year if year is None else year, self._month if month is None else month, + self._day if day is None else day) + + __replace__ = replace + + def weekday(self): + return (self.toordinal() + 6) % 7 + + def isoweekday(self): + return self.toordinal() % 7 or 7 + + def isocalendar(self): + y, w, d = _time._iso(self._year, self._month, self._day) + return IsoCalendarDate(y, w, d) + + def _cmpkey(self): + return (self._year, self._month, self._day) + + def _plain(self, other): + return isinstance(other, date) and not isinstance(other, datetime) and not isinstance(self, datetime) + + def __eq__(self, other): + if self._plain(other): + return self._cmpkey() == other._cmpkey() + return NotImplemented + + def __lt__(self, other): + if self._plain(other): + return self._cmpkey() < other._cmpkey() + return NotImplemented + + def __le__(self, other): + if self._plain(other): + return self._cmpkey() <= other._cmpkey() + return NotImplemented + + def __gt__(self, other): + if self._plain(other): + return self._cmpkey() > other._cmpkey() + return NotImplemented + + def __ge__(self, other): + if self._plain(other): + return self._cmpkey() >= other._cmpkey() + return NotImplemented + + def __hash__(self): + if self._hashcode == -1: + self._hashcode = hash(self._cmpkey()) + return self._hashcode + + def __add__(self, other): + if isinstance(other, timedelta): + o = self.toordinal() + other.days + if 0 < o <= _MAXORDINAL: + return type(self).fromordinal(o) + raise OverflowError("result out of range") + return NotImplemented + + __radd__ = __add__ + + def __sub__(self, other): + if isinstance(other, timedelta): + return self + timedelta(-other.days) + if isinstance(other, date) and not isinstance(other, datetime): + return timedelta(self.toordinal() - other.toordinal()) + return NotImplemented + + def __reduce__(self): + return (type(self), (self._year, self._month, self._day)) + + +date.min = date(1, 1, 1) +date.max = date(9999, 12, 31) +date.resolution = timedelta(days=1) + + +class IsoCalendarDate(tuple): + def __new__(cls, year, week, weekday): + return tuple.__new__(cls, (year, week, weekday)) + + year = property(lambda self: self[0]) + week = property(lambda self: self[1]) + weekday = property(lambda self: self[2]) + + def __repr__(self): + return "datetime.IsoCalendarDate(year=%d, week=%d, weekday=%d)" % tuple(self) + + def __reduce__(self): + return (tuple, (tuple(self),)) + + +def _isocal_to_ymd(year, week, day): + if not MINYEAR <= year <= MAXYEAR: + raise ValueError("year %d is out of range" % year) + if not 0 < week < 53: + out = True + if week == 53: + first = _ymd2ord(year, 1, 1) % 7 + if first == 4 or (first == 3 and _is_leap(year)): + out = False + if out: + raise ValueError("Invalid week: %d" % week) + if not 0 < day < 8: + raise ValueError("Invalid weekday: %d (range is [1, 7])" % day) + first = _ymd2ord(year, 1, 1) + wd = (first + 6) % 7 + week1 = first - wd + if wd > 3: + week1 += 7 + return _ord2ymd(week1 + (week - 1) * 7 + day - 1) + + +class tzinfo: + __slots__ = () + + def tzname(self, dt): + raise NotImplementedError("tzinfo subclass must override tzname()") + + def utcoffset(self, dt): + raise NotImplementedError("tzinfo subclass must override utcoffset()") + + def dst(self, dt): + raise NotImplementedError("tzinfo subclass must override dst()") + + def fromutc(self, dt): + if not isinstance(dt, datetime): + raise TypeError("fromutc() requires a datetime argument") + if dt.tzinfo is not self: + raise ValueError("dt.tzinfo is not self") + dtoff = dt.utcoffset() + if dtoff is None: + raise ValueError("fromutc() requires a non-None utcoffset() result") + dtdst = dt.dst() + if dtdst is None: + raise ValueError("fromutc() requires a non-None dst() result") + delta = dtoff - dtdst + if delta: + dt += delta + dtdst = dt.dst() + if dtdst is None: + raise ValueError("fromutc(): dt.dst gave inconsistent results; cannot convert") + return dt + dtdst + + +class timezone(tzinfo): + __slots__ = ("_offset", "_name") + + _Omitted = object() + + def __new__(cls, offset, name=_Omitted): + if not isinstance(offset, timedelta): + raise TypeError("timezone() argument 1 must be datetime.timedelta, not %s" % type(offset).__name__) + if name is cls._Omitted: + if not offset: + return cls.utc + name = None + elif not isinstance(name, str): + raise TypeError("timezone() argument 2 must be str, not %s" % type(name).__name__) + if not cls._minoffset <= offset <= cls._maxoffset: + raise ValueError("offset must be a timedelta strictly between -timedelta(hours=24) and " + "timedelta(hours=24), not %r" % (offset,)) + return cls._create(offset, name) + + @classmethod + def _create(cls, offset, name=None): + self = tzinfo.__new__(cls) + self._offset = offset + self._name = name + return self + + def __eq__(self, other): + if isinstance(other, timezone): + return self._offset == other._offset + return NotImplemented + + def __hash__(self): + return hash(self._offset) + + def __repr__(self): + if self is self.utc: + return "datetime.timezone.utc" + if self._name is None: + return "%s(%r)" % (_name(self), self._offset) + return "%s(%r, %r)" % (_name(self), self._offset, self._name) + + def __str__(self): + return self.tzname(None) + + def utcoffset(self, dt): + if isinstance(dt, datetime) or dt is None: + return self._offset + raise TypeError("utcoffset() argument must be a datetime instance or None") + + def tzname(self, dt): + if isinstance(dt, datetime) or dt is None: + if self._name is None: + return self._name_from_offset(self._offset) + return self._name + raise TypeError("tzname() argument must be a datetime instance or None") + + def dst(self, dt): + if isinstance(dt, datetime) or dt is None: + return None + raise TypeError("dst() argument must be a datetime instance or None") + + def fromutc(self, dt): + if isinstance(dt, datetime): + if dt.tzinfo is not self: + raise ValueError("fromutc: dt.tzinfo is not self") + return dt + self._offset + raise TypeError("fromutc() argument must be a datetime instance or None") + + @staticmethod + def _name_from_offset(delta): + if not delta: + return "UTC" + return "UTC" + _format_offset(delta) + + def __getinitargs__(self): + if self._name is None: + return (self._offset,) + return (self._offset, self._name) + + +timezone._maxoffset = timedelta(hours=24, microseconds=-1) +timezone._minoffset = -timezone._maxoffset +timezone.utc = timezone._create(timedelta(0)) +timezone.min = timezone._create(-timedelta(hours=23, minutes=59)) +timezone.max = timezone._create(timedelta(hours=23, minutes=59)) +UTC = timezone.utc + + +class time: + __slots__ = ("_hour", "_minute", "_second", "_microsecond", "_tzinfo", "_fold", "_hashcode") + + def __new__(cls, hour=0, minute=0, second=0, microsecond=0, tzinfo=None, *, fold=0): + hour, minute, second, microsecond, fold = _check_time(hour, minute, second, microsecond, fold) + _check_tzinfo(tzinfo) + self = object.__new__(cls) + self._hour = hour + self._minute = minute + self._second = second + self._microsecond = microsecond + self._tzinfo = tzinfo + self._fold = fold + self._hashcode = -1 + return self + + hour = property(lambda self: self._hour) + minute = property(lambda self: self._minute) + second = property(lambda self: self._second) + microsecond = property(lambda self: self._microsecond) + tzinfo = property(lambda self: self._tzinfo) + fold = property(lambda self: self._fold) + + @classmethod + def fromisoformat(cls, s): + if not isinstance(s, str): + raise TypeError("fromisoformat: argument must be str") + try: + t = s[1:] if s[:1] == "T" else s + fields, n = _parse_time(t, 0) + if n != len(t): + raise ValueError + except (ValueError, IndexError): + raise ValueError("Invalid isoformat string: %r" % (s,)) + return cls(*fields) + + @classmethod + def strptime(cls, date_string, format): + import _strptime + d = _strptime._strptime_datetime(datetime, date_string, format) + return d.timetz() + + def _off(self): + if self._tzinfo is None: + return None + off = self._tzinfo.utcoffset(None) + _check_offset("utcoffset", off) + return off + + def utcoffset(self): + return self._off() + + def tzname(self): + if self._tzinfo is None: + return None + return self._tzinfo.tzname(None) + + def dst(self): + if self._tzinfo is None: + return None + off = self._tzinfo.dst(None) + _check_offset("dst", off) + return off + + def _cmp(self, other): + a = self._off() + b = other._off() + if (a is None) != (b is None): + return None + x = (self._hour, self._minute, self._second, self._microsecond) + y = (other._hour, other._minute, other._second, other._microsecond) + if a is not None and a != b: + ma = self._hour * 60 + self._minute - a // timedelta(minutes=1) + mb = other._hour * 60 + other._minute - b // timedelta(minutes=1) + x = (ma, self._second, self._microsecond) + y = (mb, other._second, other._microsecond) + return _cmp(x, y) + + def __eq__(self, other): + if isinstance(other, time): + c = self._cmp(other) + return c == 0 if c is not None else False + return NotImplemented + + def _order(self, other): + c = self._cmp(other) + if c is None: + raise TypeError("can't compare offset-naive and offset-aware times") + return c + + def __lt__(self, other): + if isinstance(other, time): + return self._order(other) < 0 + return NotImplemented + + def __le__(self, other): + if isinstance(other, time): + return self._order(other) <= 0 + return NotImplemented + + def __gt__(self, other): + if isinstance(other, time): + return self._order(other) > 0 + return NotImplemented + + def __ge__(self, other): + if isinstance(other, time): + return self._order(other) >= 0 + return NotImplemented + + def __hash__(self): + if self._hashcode == -1: + off = self._off() + if not off: + self._hashcode = hash((self._hour, self._minute, self._second, self._microsecond)) + else: + h, m = divmod(timedelta(hours=self._hour, minutes=self._minute) - off, timedelta(hours=1)) + self._hashcode = hash((h, m // timedelta(minutes=1), self._second, self._microsecond)) + return self._hashcode + + def isoformat(self, timespec="auto"): + s = _format_time(self._hour, self._minute, self._second, self._microsecond, timespec) + return s + _format_offset(self._off()) + + __str__ = isoformat + + def __repr__(self): + if self._microsecond: + s = ", %d, %d" % (self._second, self._microsecond) + elif self._second: + s = ", %d" % self._second + else: + s = "" + s = "%s(%d, %d%s)" % (_name(self), self._hour, self._minute, s) + if self._tzinfo is not None: + s = s[:-1] + ", tzinfo=%r)" % self._tzinfo + if self._fold: + s = s[:-1] + ", fold=1)" + return s + + def strftime(self, format): + tt = (1900, 1, 1, self._hour, self._minute, self._second, 0, 1, -1) + off = self._off() + return _time._format(format, tt, self.tzname() if self._tzinfo is not None else None, + None if off is None else off.total_seconds(), self._microsecond) + + def __format__(self, fmt): + if not isinstance(fmt, str): + raise TypeError("must be str, not %s" % type(fmt).__name__) + return self.strftime(fmt) if fmt else str(self) + + def replace(self, hour=None, minute=None, second=None, microsecond=None, tzinfo=True, *, fold=None): + return type(self)(self._hour if hour is None else hour, self._minute if minute is None else minute, + self._second if second is None else second, + self._microsecond if microsecond is None else microsecond, + self._tzinfo if tzinfo is True else tzinfo, fold=self._fold if fold is None else fold) + + __replace__ = replace + + def __bool__(self): + return True + + def __reduce__(self): + return (type(self), (self._hour, self._minute, self._second, self._microsecond, self._tzinfo)) + + +time.min = time(0, 0, 0) +time.max = time(23, 59, 59, 999999) +time.resolution = timedelta(microseconds=1) + + +def _local_fields(t): + if isinstance(t, bool) or not isinstance(t, (int, float)): + raise TypeError("'%s' object cannot be interpreted as an integer" % type(t).__name__) + return _fields(t, _clock.utcoffset(t)) + + +def _fields(t, off): + if isinstance(t, int): + secs = t + us = 0 + else: + secs = int(t // 1) + us = _round_half_even((t - secs) * 1000000) + if us >= 1000000: + secs += 1 + us -= 1000000 + secs += off + days, rem = divmod(secs, 86400) + y, m, d = _time._civil_from_days(days) + if not MINYEAR <= y <= MAXYEAR: + raise ValueError("year %d is out of range" % y) + return y, m, d, rem // 3600, rem // 60 % 60, rem % 60, us + + +class datetime(date): + __slots__ = ("_hour", "_minute", "_second", "_microsecond", "_tzinfo", "_fold") + + def __new__(cls, year, month=None, day=None, hour=0, minute=0, second=0, microsecond=0, tzinfo=None, + *, fold=0): + year, month, day = _check_date(year, month, day) + hour, minute, second, microsecond, fold = _check_time(hour, minute, second, microsecond, fold) + _check_tzinfo(tzinfo) + self = object.__new__(cls) + self._year = year + self._month = month + self._day = day + self._hour = hour + self._minute = minute + self._second = second + self._microsecond = microsecond + self._tzinfo = tzinfo + self._fold = fold + self._hashcode = -1 + return self + + hour = property(lambda self: self._hour) + minute = property(lambda self: self._minute) + second = property(lambda self: self._second) + microsecond = property(lambda self: self._microsecond) + tzinfo = property(lambda self: self._tzinfo) + fold = property(lambda self: self._fold) + + @classmethod + def fromtimestamp(cls, t, tz=None): + _check_tzinfo(tz) + if tz is None: + return cls(*_local_fields(t)) + y, m, d, hh, mm, ss, us = _fields(t, 0) + return tz.fromutc(cls(y, m, d, hh, mm, ss, us, tz)) + + @classmethod + def utcfromtimestamp(cls, t): + return cls(*_fields(t, 0)) + + @classmethod + def now(cls, tz=None): + return cls.fromtimestamp(_time.time(), tz) + + @classmethod + def utcnow(cls): + return cls.utcfromtimestamp(_time.time()) + + @classmethod + def combine(cls, date_, time_, tzinfo=True): + if not isinstance(date_, date): + raise TypeError("combine() argument 1 must be datetime.date, not %s" % type(date_).__name__) + if not isinstance(time_, time): + raise TypeError("combine() argument 2 must be datetime.time, not %s" % type(time_).__name__) + if tzinfo is True: + tzinfo = time_.tzinfo + return cls(date_.year, date_.month, date_.day, time_.hour, time_.minute, time_.second, + time_.microsecond, tzinfo, fold=time_.fold) + + @classmethod + def fromisoformat(cls, s): + if not isinstance(s, str): + raise TypeError("fromisoformat: argument must be str") + try: + ymd, n = _parse_date(s) + tfields = (0, 0, 0, 0, None) + if n < len(s): + n += 1 + tfields, m = _parse_time(s, n) + if m != len(s): + raise ValueError + except (ValueError, IndexError): + raise ValueError("Invalid isoformat string: %r" % (s,)) + hh, mi, ss, us, tz = tfields + extra = 0 + if hh == 24: + hh = 0 + extra = 1 + r = cls(ymd[0], ymd[1], ymd[2], hh, mi, ss, us, tz) + return r + timedelta(days=extra) if extra else r + + @classmethod + def strptime(cls, date_string, format): + import _strptime + return _strptime._strptime_datetime(cls, date_string, format) + + def date(self): + return date(self._year, self._month, self._day) + + def time(self): + return time(self._hour, self._minute, self._second, self._microsecond, fold=self._fold) + + def timetz(self): + return time(self._hour, self._minute, self._second, self._microsecond, self._tzinfo, fold=self._fold) + + def _off(self): + if self._tzinfo is None: + return None + off = self._tzinfo.utcoffset(self) + _check_offset("utcoffset", off) + return off + + def utcoffset(self): + return self._off() + + def tzname(self): + if self._tzinfo is None: + return None + name = self._tzinfo.tzname(self) + if name is not None and not isinstance(name, str): + raise TypeError("tzinfo.tzname() must return None or a string, not '%s'" % type(name).__name__) + return name + + def dst(self): + if self._tzinfo is None: + return None + off = self._tzinfo.dst(self) + _check_offset("dst", off) + return off + + def _local_us(self): + return ((self.toordinal() - _EPOCH_ORDINAL) * 86400 + self._hour * 3600 + self._minute * 60 + + self._second) * 1000000 + self._microsecond + + def _naive_epoch(self): + local = (self.toordinal() - _EPOCH_ORDINAL) * 86400 + self._hour * 3600 + self._minute * 60 + self._second + off = _clock.utcoffset(local) + off = _clock.utcoffset(local - off) + return local - off + + def timestamp(self): + off = self._off() + if off is None: + return self._naive_epoch() + self._microsecond / 1e6 + return (self._local_us() - off._us()) / 10 ** 6 + + def timetuple(self): + dst = self.dst() + isdst = -1 if dst is None else (1 if dst else 0) + return _time.struct_time(self._tuple(self._hour, self._minute, self._second)[:8] + (isdst,)) + + def utctimetuple(self): + off = self._off() + d = self - off if off else self + return _time.struct_time(d._tuple(d._hour, d._minute, d._second)[:8] + (0,)) + + def astimezone(self, tz=None): + if tz is None: + ts = self.timestamp() + off = _clock.utcoffset(ts) + tz = timezone(timedelta(seconds=off), _time._abbrev(off)) + elif not isinstance(tz, tzinfo): + raise TypeError("tzinfo argument must be None or of a tzinfo subclass, not type '%s'" + % type(tz).__name__) + mytz = self._tzinfo + if mytz is None: + off = timedelta(seconds=(self._local_us() // 1000000) - self._naive_epoch()) + mytz = timezone(off) + myoff = off + else: + if tz is mytz: + return self + myoff = self._off() + if myoff is None: + return self.replace(tzinfo=None).astimezone(tz) + utc = (self - myoff).replace(tzinfo=tz) + return tz.fromutc(utc) + + def ctime(self): + wd = self.toordinal() % 7 or 7 + return "%s %s %2d %02d:%02d:%02d %04d" % (_time._DAYS[wd - 1][:3], _time._MONTHS[self._month - 1][:3], + self._day, self._hour, self._minute, self._second, self._year) + + def isoformat(self, sep="T", timespec="auto"): + return ("%04d-%02d-%02d%s" % (self._year, self._month, self._day, sep) + + _format_time(self._hour, self._minute, self._second, self._microsecond, timespec) + + _format_offset(self._off())) + + def __str__(self): + return self.isoformat(sep=" ") + + def __repr__(self): + L = [self._year, self._month, self._day, self._hour, self._minute, self._second, self._microsecond] + if L[-1] == 0: + del L[-1] + if L[-1] == 0: + del L[-1] + s = "%s(%s)" % (_name(self), ", ".join(map(str, L))) + if self._tzinfo is not None: + s = s[:-1] + ", tzinfo=%r)" % self._tzinfo + if self._fold: + s = s[:-1] + ", fold=1)" + return s + + def strftime(self, format): + off = self._off() + name = None + if self._tzinfo is not None: + name = self.tzname() + if name is None: + name = "" + tt = self._tuple(self._hour, self._minute, self._second) + return _time._format(format, tt, name, None if off is None else off.days * 86400 + off.seconds + + (off.microseconds / 1e6 if off.microseconds else 0), self._microsecond) + + def replace(self, year=None, month=None, day=None, hour=None, minute=None, second=None, microsecond=None, + tzinfo=True, *, fold=None): + return type(self)(self._year if year is None else year, self._month if month is None else month, + self._day if day is None else day, self._hour if hour is None else hour, + self._minute if minute is None else minute, + self._second if second is None else second, + self._microsecond if microsecond is None else microsecond, + self._tzinfo if tzinfo is True else tzinfo, fold=self._fold if fold is None else fold) + + __replace__ = replace + + def _cmp_dt(self, other, allow_mixed): + mytz = self._tzinfo + ottz = other._tzinfo + myoff = otoff = None + if mytz is ottz: + base = True + else: + myoff = self._off() + otoff = other._off() + base = myoff == otoff + if base: + return _cmp(self._local_us(), other._local_us()) + if myoff is None or otoff is None: + if allow_mixed: + return 2 + raise TypeError("can't compare offset-naive and offset-aware datetimes") + return _cmp(self._local_us() - myoff._us(), other._local_us() - otoff._us()) + + def __eq__(self, other): + if isinstance(other, datetime): + return self._cmp_dt(other, True) == 0 + return NotImplemented + + def __lt__(self, other): + if isinstance(other, datetime): + return self._cmp_dt(other, False) < 0 + return NotImplemented + + def __le__(self, other): + if isinstance(other, datetime): + return self._cmp_dt(other, False) <= 0 + return NotImplemented + + def __gt__(self, other): + if isinstance(other, datetime): + return self._cmp_dt(other, False) > 0 + return NotImplemented + + def __ge__(self, other): + if isinstance(other, datetime): + return self._cmp_dt(other, False) >= 0 + return NotImplemented + + def __hash__(self): + if self._hashcode == -1: + off = self._off() + us = self._local_us() - (off._us() if off else 0) + self._hashcode = hash(us) + return self._hashcode + + def __add__(self, other): + if not isinstance(other, timedelta): + return NotImplemented + total = self._local_us() + other._us() + days, rem = divmod(total, 86400000000) + o = days + _EPOCH_ORDINAL + if not 0 < o <= _MAXORDINAL: + raise OverflowError("date value out of range") + y, m, d = _ord2ymd(o) + s, us = divmod(rem, 1000000) + return type(self).combine(date(y, m, d), time(s // 3600, s // 60 % 60, s % 60, us, self._tzinfo)) + + __radd__ = __add__ + + def __sub__(self, other): + if not isinstance(other, datetime): + if isinstance(other, timedelta): + return self + -other + return NotImplemented + a = self._local_us() + b = other._local_us() + if self._tzinfo is other._tzinfo: + return timedelta._from_us(a - b) + myoff = self._off() + otoff = other._off() + if myoff == otoff: + return timedelta._from_us(a - b) + if myoff is None or otoff is None: + raise TypeError("can't subtract offset-naive and offset-aware datetimes") + return timedelta._from_us((a - myoff._us()) - (b - otoff._us())) + + def __reduce__(self): + return (type(self), (self._year, self._month, self._day, self._hour, self._minute, self._second, + self._microsecond, self._tzinfo)) + + +datetime.min = datetime(1, 1, 1) +datetime.max = datetime(9999, 12, 31, 23, 59, 59, 999999) +datetime.resolution = timedelta(microseconds=1) + + +def _digits(s, i, n): + part = s[i:i + n] + if len(part) != n or not part.isdigit() or not part.isascii(): + raise ValueError + return int(part) + + +def _parse_date(s): + """(year, month, day) and the index after the date part of `s`.""" + year = _digits(s, 0, 4) + if s[4:5] == "-": + if s[5:6] == "W": + week = _digits(s, 6, 2) + n = 8 + day = 1 + if s[8:9] == "-": + day = _digits(s, 9, 1) + n = 10 + return _isocal_to_ymd(year, week, day), n + month = _digits(s, 5, 2) + if s[7:8] != "-": + raise ValueError + return (year, month, _digits(s, 8, 2)), 10 + if s[4:5] == "W": + week = _digits(s, 5, 2) + n = 7 + day = 1 + if s[7:8].isdigit(): + day = _digits(s, 7, 1) + n = 8 + return _isocal_to_ymd(year, week, day), n + return (year, _digits(s, 4, 2), _digits(s, 6, 2)), 8 + + +def _parse_time(s, i): + """(hour, minute, second, microsecond, tzinfo) parsed at s[i:].""" + hh = _digits(s, i, 2) + i += 2 + mi = ss = us = 0 + colon = s[i:i + 1] == ":" + if i < len(s) and s[i] not in "+-Z": + if colon: + i += 1 + mi = _digits(s, i, 2) + i += 2 + if i < len(s) and s[i] not in "+-Z": + if colon: + if s[i] != ":": + raise ValueError + i += 1 + ss = _digits(s, i, 2) + i += 2 + if i < len(s) and s[i] in ".,": + i += 1 + j = i + while j < len(s) and s[j].isdigit(): + j += 1 + frac = s[i:j] + if not frac: + raise ValueError + us = int((frac + "00000")[:6]) + i = j + tz = None + if i < len(s): + c = s[i] + if c == "Z": + tz = timezone.utc + i += 1 + elif c in "+-": + i += 1 + (oh, om, os_, ous, _), _n = _parse_time(s, i) + i = len(s) + off = timedelta(hours=oh, minutes=om, seconds=os_, microseconds=ous) + if c == "-": + off = -off + tz = timezone(off) + else: + raise ValueError + if hh == 24: + if mi or ss or us: + raise ValueError + return (hh, mi, ss, us, tz), i diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/decimal.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/decimal.py new file mode 100644 index 00000000000..ace6c854d02 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/decimal.py @@ -0,0 +1,2163 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Decimal fixed and floating point arithmetic (the General Decimal +Arithmetic Specification), with the results and signals of CPython's +decimal module. + +A coefficient is kept as a string of digits, the way the specification +describes it. Every operation the specification defines exactly is exact +here; exp, ln, log10 and non-integral powers are computed with twenty +guard digits and then rounded once, which agrees with CPython's correctly +rounded results except for an input whose true result lies within +1e-20 ulp of a rounding boundary. There is one context for the whole +program: an application is single threaded. +""" + +import math +import numbers +from collections import namedtuple as _namedtuple + +__all__ = ["Decimal", "Context", "DecimalTuple", "DefaultContext", "BasicContext", "ExtendedContext", + "DecimalException", "Clamped", "InvalidOperation", "DivisionByZero", "Inexact", "Rounded", "Subnormal", + "Overflow", "Underflow", "FloatOperation", "DivisionImpossible", "InvalidContext", "ConversionSyntax", + "DivisionUndefined", "ROUND_DOWN", "ROUND_HALF_UP", "ROUND_HALF_EVEN", "ROUND_CEILING", "ROUND_FLOOR", + "ROUND_UP", "ROUND_HALF_DOWN", "ROUND_05UP", "setcontext", "getcontext", "localcontext", "MAX_PREC", + "MAX_EMAX", "MIN_EMIN", "MIN_ETINY", "HAVE_THREADS", "HAVE_CONTEXTVAR"] + +__version__ = "1.70" +__libmpdec_version__ = "2.5.1" + +ROUND_DOWN = "ROUND_DOWN" +ROUND_HALF_UP = "ROUND_HALF_UP" +ROUND_HALF_EVEN = "ROUND_HALF_EVEN" +ROUND_CEILING = "ROUND_CEILING" +ROUND_FLOOR = "ROUND_FLOOR" +ROUND_UP = "ROUND_UP" +ROUND_HALF_DOWN = "ROUND_HALF_DOWN" +ROUND_05UP = "ROUND_05UP" +_ROUNDINGS = (ROUND_DOWN, ROUND_HALF_UP, ROUND_HALF_EVEN, ROUND_CEILING, ROUND_FLOOR, ROUND_UP, ROUND_HALF_DOWN, + ROUND_05UP) + +HAVE_THREADS = True +HAVE_CONTEXTVAR = True +MAX_PREC = 999999999999999999 +MAX_EMAX = 999999999999999999 +MIN_EMIN = -999999999999999999 +MIN_ETINY = MIN_EMIN - (MAX_PREC - 1) + +DecimalTuple = _namedtuple("DecimalTuple", "sign digits exponent", module="decimal") + + +class DecimalException(ArithmeticError): + def handle(self, context, *args): + pass + + +class Clamped(DecimalException): + pass + + +class InvalidOperation(DecimalException): + def handle(self, context, *args): + return _NaN + + +class ConversionSyntax(InvalidOperation): + pass + + +class DivisionByZero(DecimalException, ZeroDivisionError): + def handle(self, context, sign, *args): + return _SignedInfinity[sign] + + +class DivisionImpossible(InvalidOperation): + pass + + +class DivisionUndefined(InvalidOperation, ZeroDivisionError): + pass + + +class Inexact(DecimalException): + pass + + +class InvalidContext(InvalidOperation): + pass + + +class Rounded(DecimalException): + pass + + +class Subnormal(DecimalException): + pass + + +class Overflow(Inexact, Rounded): + def handle(self, context, sign, *args): + if context.rounding in (ROUND_HALF_UP, ROUND_HALF_EVEN, ROUND_HALF_DOWN, ROUND_UP): + return _SignedInfinity[sign] + if sign == 0: + if context.rounding == ROUND_CEILING: + return _SignedInfinity[sign] + return _dec(sign, "9" * context.prec, context.Emax - context.prec + 1) + if context.rounding == ROUND_FLOOR: + return _SignedInfinity[sign] + return _dec(sign, "9" * context.prec, context.Emax - context.prec + 1) + + +class Underflow(Inexact, Rounded, Subnormal): + pass + + +class FloatOperation(DecimalException, TypeError): + pass + + +# The order CPython lists them in a context's repr. +_signals = [Clamped, InvalidOperation, DivisionByZero, Inexact, FloatOperation, Overflow, Rounded, Subnormal, + Underflow] + +_condition_map = {ConversionSyntax: InvalidOperation, DivisionImpossible: InvalidOperation, + DivisionUndefined: InvalidOperation, InvalidContext: InvalidOperation} + +_rounding_names = {r: r for r in _ROUNDINGS} + + +class Context: + def __init__(self, prec=None, rounding=None, Emin=None, Emax=None, capitals=None, clamp=None, flags=None, + traps=None, _ignored_flags=None): + dc = DefaultContext if "DefaultContext" in globals() else None + self.prec = prec if prec is not None else (dc.prec if dc else 28) + self.rounding = rounding if rounding is not None else (dc.rounding if dc else ROUND_HALF_EVEN) + self.Emin = Emin if Emin is not None else (dc.Emin if dc else -999999) + self.Emax = Emax if Emax is not None else (dc.Emax if dc else 999999) + self.capitals = capitals if capitals is not None else (dc.capitals if dc else 1) + self.clamp = clamp if clamp is not None else (dc.clamp if dc else 0) + self._ignored_flags = [] if _ignored_flags is None else _ignored_flags + if traps is None: + self.traps = dict(dc.traps) if dc else {s: False for s in _signals} + elif not isinstance(traps, dict): + self.traps = {s: s in traps for s in _signals} + else: + self.traps = {s: bool(traps.get(s, False)) for s in _signals} + if flags is None: + self.flags = {s: False for s in _signals} + elif not isinstance(flags, dict): + self.flags = {s: s in flags for s in _signals} + else: + self.flags = {s: bool(flags.get(s, False)) for s in _signals} + + def __setattr__(self, name, value): + if name == "prec": + if not isinstance(value, int): + raise TypeError("an integer is required") + if not 1 <= value <= MAX_PREC: + raise ValueError("valid range for prec is [1, MAX_PREC]") + elif name == "Emin": + if not isinstance(value, int): + raise TypeError("an integer is required") + if not MIN_EMIN <= value <= 0: + raise ValueError("valid range for Emin is [MIN_EMIN, 0]") + elif name == "Emax": + if not isinstance(value, int): + raise TypeError("an integer is required") + if not 0 <= value <= MAX_EMAX: + raise ValueError("valid range for Emax is [0, MAX_EMAX]") + elif name == "capitals" or name == "clamp": + if value not in (0, 1): + raise ValueError("valid values for %s are [0, 1]" % name) + elif name == "rounding": + if value not in _ROUNDINGS: + raise TypeError("valid values for rounding are:\n [ROUND_CEILING, ROUND_FLOOR, ROUND_UP, ROUND_DOWN," + "\n ROUND_HALF_UP, ROUND_HALF_DOWN, ROUND_HALF_EVEN,\n ROUND_05UP]") + object.__setattr__(self, name, value) + + def __repr__(self): + return ("Context(prec=%d, rounding=%s, Emin=%d, Emax=%d, capitals=%d, clamp=%d, flags=[%s], traps=[%s])" + % (self.prec, self.rounding, self.Emin, self.Emax, self.capitals, self.clamp, + ", ".join(s.__name__ for s in _signals if self.flags[s]), + ", ".join(s.__name__ for s in _signals if self.traps[s]))) + + def clear_flags(self): + for flag in self.flags: + self.flags[flag] = False + + def clear_traps(self): + for flag in self.traps: + self.traps[flag] = False + + def _shallow_copy(self): + return Context(self.prec, self.rounding, self.Emin, self.Emax, self.capitals, self.clamp, self.flags, + self.traps, self._ignored_flags) + + def copy(self): + return Context(self.prec, self.rounding, self.Emin, self.Emax, self.capitals, self.clamp, self.flags.copy(), + self.traps.copy(), self._ignored_flags) + + __copy__ = copy + + def _raise_error(self, condition, explanation=None, *args): + error = _condition_map.get(condition, condition) + if error in self._ignored_flags: + return error().handle(self, *args) + self.flags[error] = True + if not self.traps[error]: + return condition().handle(self, *args) + raise error([condition]) + + def _quiet(self): + """A copy that records flags but never raises, for internal steps.""" + c = self.copy() + c.clear_traps() + return c + + def Etiny(self): + return int(self.Emin - self.prec + 1) + + def Etop(self): + return int(self.Emax - self.prec + 1) + + def _set_rounding(self, type): + rounding = self.rounding + self.rounding = type + return rounding + + def create_decimal(self, num="0"): + if isinstance(num, str) and (num != num.strip() or "_" in num): + return self._raise_error(ConversionSyntax, "trailing or leading whitespace and underscores are not " + "permitted.") + d = Decimal(num, context=self) + if d._isnan() and len(d._int) > self.prec - self.clamp: + return self._raise_error(ConversionSyntax, "diagnostic info too long in NaN") + return d._fix(self) + + def create_decimal_from_float(self, f): + d = Decimal.from_float(f) + return d._fix(self) + + def abs(self, a): + return _convert(a, True).__abs__(context=self) + + def add(self, a, b): + return _convert(a, True).__add__(b, context=self) + + def subtract(self, a, b): + return _convert(a, True).__sub__(b, context=self) + + def multiply(self, a, b): + return _convert(a, True).__mul__(b, context=self) + + def divide(self, a, b): + return _convert(a, True).__truediv__(b, context=self) + + def divide_int(self, a, b): + return _convert(a, True).__floordiv__(b, context=self) + + def remainder(self, a, b): + return _convert(a, True).__mod__(b, context=self) + + def divmod(self, a, b): + return _convert(a, True).__divmod__(b, context=self) + + def power(self, a, b, modulo=None): + return _convert(a, True).__pow__(b, modulo, context=self) + + def plus(self, a): + return _convert(a, True).__pos__(context=self) + + def minus(self, a): + return _convert(a, True).__neg__(context=self) + + def sqrt(self, a): + return _convert(a, True).sqrt(context=self) + + def exp(self, a): + return _convert(a, True).exp(context=self) + + def ln(self, a): + return _convert(a, True).ln(context=self) + + def log10(self, a): + return _convert(a, True).log10(context=self) + + def quantize(self, a, b): + return _convert(a, True).quantize(b, context=self) + + def compare(self, a, b): + return _convert(a, True).compare(b, context=self) + + def max(self, a, b): + return _convert(a, True).max(b, context=self) + + def min(self, a, b): + return _convert(a, True).min(b, context=self) + + def normalize(self, a): + return _convert(a, True).normalize(context=self) + + def fma(self, a, b, c): + return _convert(a, True).fma(b, c, context=self) + + def to_sci_string(self, a): + return _convert(a, True).__str__(context=self) + + def to_eng_string(self, a): + return _convert(a, True).to_eng_string(context=self) + + def to_integral_value(self, a): + return _convert(a, True).to_integral_value(context=self) + + to_integral = to_integral_value + + def to_integral_exact(self, a): + return _convert(a, True).to_integral_exact(context=self) + + def is_zero(self, a): + return _convert(a, True).is_zero() + + def is_signed(self, a): + return _convert(a, True).is_signed() + + def copy_abs(self, a): + return _convert(a, True).copy_abs() + + def copy_negate(self, a): + return _convert(a, True).copy_negate() + + def number_class(self, a): + return _convert(a, True).number_class(context=self) + + def radix(self): + return Decimal(10) + + +DefaultContext = Context(prec=28, rounding=ROUND_HALF_EVEN, traps=[DivisionByZero, Overflow, InvalidOperation], + flags=[], Emax=999999, Emin=-999999, capitals=1, clamp=0) +BasicContext = Context(prec=9, rounding=ROUND_HALF_UP, traps=[DivisionByZero, Overflow, InvalidOperation, Clamped, + Underflow], flags=[]) +ExtendedContext = Context(prec=9, rounding=ROUND_HALF_EVEN, traps=[], flags=[]) + +_current = None + + +def getcontext(): + global _current + if _current is None: + _current = DefaultContext.copy() + return _current + + +def setcontext(context): + global _current + if context in (DefaultContext, BasicContext, ExtendedContext): + context = context.copy() + context.clear_flags() + _current = context + + +class _ContextManager: + def __init__(self, new_context): + self.new_context = new_context.copy() + + def __enter__(self): + self.saved_context = getcontext() + setcontext(self.new_context) + return self.new_context + + def __exit__(self, t, v, tb): + setcontext(self.saved_context) + + +def localcontext(ctx=None, **kwargs): + if ctx is None: + ctx = getcontext() + manager = _ContextManager(ctx) + for key, value in kwargs.items(): + if key == "flags" or key == "traps": + setattr(manager.new_context, key, {s: s in value for s in _signals}) + elif key in ("prec", "rounding", "Emin", "Emax", "capitals", "clamp"): + setattr(manager.new_context, key, value) + else: + raise TypeError("'%s' is an invalid keyword argument for this function" % key) + return manager + + +def _all_zeros(s, i): + return s[i:].strip("0") == "" + + +def _exact_half(s, i): + return i < len(s) and s[i] == "5" and _all_zeros(s, i + 1) + + +def _dec(sign, coefficient, exponent, special=False): + self = object.__new__(Decimal) + self._sign = sign + self._int = coefficient + self._exp = exponent + self._is_special = special + return self + + +def _convert(other, raiseit=False, allow_float=False): + if isinstance(other, Decimal): + return other + if isinstance(other, int): + return Decimal(other) + if allow_float and isinstance(other, float): + return Decimal.from_float(other) + if raiseit: + raise TypeError("conversion from %s to Decimal is not supported" % type(other).__name__) + return NotImplemented + + +def _parse(s): + """Answers (sign, digits, exponent, special) for a numeric string, or None.""" + s = s.strip() + if "_" in s: + for i, ch in enumerate(s): + if ch == "_" and (i == 0 or i == len(s) - 1 or not s[i - 1].isdigit() or not s[i + 1].isdigit()): + return None + s = s.replace("_", "") + sign = 0 + if s[:1] in ("+", "-"): + sign = 1 if s[0] == "-" else 0 + s = s[1:] + low = s.lower() + if low in ("inf", "infinity"): + return (sign, "0", "F", True) + for prefix, kind in (("snan", "N"), ("nan", "n")): + if low.startswith(prefix): + payload = low[len(prefix):] + if payload and not (payload.isdigit() and payload.isascii()): + return None + payload = payload.lstrip("0") + return (sign, payload, kind, True) + exp = 0 + epos = low.find("e") + if epos >= 0: + e = low[epos + 1:] + low = low[:epos] + if e[:1] in ("+", "-"): + digits = e[1:] + else: + digits = e + if not digits or not (digits.isdigit() and digits.isascii()): + return None + exp = int(e) + if "." in low: + intpart, _, frac = low.partition(".") + else: + intpart, frac = low, "" + if not intpart and not frac: + return None + for part in (intpart, frac): + if part and not (part.isdigit() and part.isascii()): + return None + digits = (intpart + frac).lstrip("0") or "0" + return (sign, digits, exp - len(frac), False) + + +class Decimal: + __slots__ = ("_exp", "_int", "_sign", "_is_special") + + def __new__(cls, value="0", context=None): + self = object.__new__(cls) + if isinstance(value, str): + p = _parse(value) + if p is None: + if context is None: + context = getcontext() + return context._raise_error(ConversionSyntax, "Invalid literal for Decimal: %r" % value) + self._sign, self._int, self._exp, self._is_special = p + return self + if isinstance(value, int): + self._sign = 1 if value < 0 else 0 + self._int = str(abs(value)) + self._exp = 0 + self._is_special = False + return self + if isinstance(value, Decimal): + self._sign = value._sign + self._int = value._int + self._exp = value._exp + self._is_special = value._is_special + return self + if isinstance(value, (list, tuple)): + if len(value) != 3: + raise ValueError("argument must be a sequence of length 3") + if not (isinstance(value[0], int) and value[0] in (0, 1)): + raise ValueError("sign must be an integer with the value 0 or 1") + if value[2] == "F": + self._int = "0" + self._exp = value[2] + self._is_special = True + else: + digits = [] + for digit in value[1]: + if isinstance(digit, int) and 0 <= digit <= 9: + if digits or digit != 0: + digits.append(digit) + else: + raise ValueError("The second value in the tuple must be composed of integers in the range " + "0 through 9.") + if value[2] in ("n", "N"): + self._int = "".join(map(str, digits)) + self._exp = value[2] + self._is_special = True + elif isinstance(value[2], int): + self._int = "".join(map(str, digits or [0])) + self._exp = value[2] + self._is_special = False + else: + raise ValueError("The third value in the tuple must be an integer, or one of the strings " + "'F', 'n', 'N'.") + self._sign = value[0] + return self + if isinstance(value, float): + if context is None: + context = getcontext() + context._raise_error(FloatOperation, "strict semantics for mixing floats and Decimals are enabled") + value = Decimal.from_float(value) + self._sign = value._sign + self._int = value._int + self._exp = value._exp + self._is_special = value._is_special + return self + raise TypeError("conversion from %s to Decimal is not supported" % type(value).__name__) + + @classmethod + def from_float(cls, f): + if isinstance(f, int): + sign = 0 if f >= 0 else 1 + k = 0 + coeff = str(abs(f)) + elif isinstance(f, float): + if math.isinf(f) or math.isnan(f): + return cls(repr(f)) + sign = 0 if math.copysign(1.0, f) == 1.0 else 1 + n, d = abs(f).as_integer_ratio() + k = d.bit_length() - 1 + coeff = str(n * 5 ** k) + else: + raise TypeError("argument must be int or float.") + result = _dec(sign, coeff, -k) + if cls is Decimal: + return result + return cls(result) + + @classmethod + def from_number(cls, number): + if isinstance(number, (int, Decimal, float)): + return cls(number) if not isinstance(number, float) else cls.from_float(number) + raise TypeError("conversion from %s to Decimal is not supported" % type(number).__name__) + + # -- classification ------------------------------------------------- + + def _isnan(self): + if self._is_special: + exp = self._exp + if exp == "n": + return 1 + elif exp == "N": + return 2 + return 0 + + def _isinfinity(self): + if self._exp == "F": + return -1 if self._sign else 1 + return 0 + + def _check_nans(self, other=None, context=None): + self_is_nan = self._isnan() + other_is_nan = other._isnan() if other is not None else False + if self_is_nan or other_is_nan: + if context is None: + context = getcontext() + if self_is_nan == 2: + return context._raise_error(InvalidOperation, "sNaN", self) + if other_is_nan == 2: + return context._raise_error(InvalidOperation, "sNaN", other) + if self_is_nan: + return self._fix_nan(context) + return other._fix_nan(context) + return 0 + + def _compare_check_nans(self, other, context): + if context is None: + context = getcontext() + if self._is_special or other._is_special: + if self._isnan() == 2 or other._isnan() == 2: + return context._raise_error(InvalidOperation, "comparison involving sNaN") + if self._isnan() or other._isnan(): + return context._raise_error(InvalidOperation, "comparison involving NaN") + return 0 + + def __bool__(self): + return self._is_special or self._int != "0" + + def _isinteger(self): + if self._is_special: + return False + if self._exp >= 0: + return True + rest = self._int[self._exp:] + return rest == "0" * len(rest) + + def _iseven(self): + if not self or self._exp > 0: + return True + return self._int[-1 + self._exp] in "02468" + + def adjusted(self): + try: + return self._exp + len(self._int) - 1 + except TypeError: + return 0 + + def is_canonical(self): + return True + + def is_finite(self): + return not self._is_special + + def is_infinite(self): + return self._exp == "F" + + def is_nan(self): + return self._exp in ("n", "N") + + def is_qnan(self): + return self._exp == "n" + + def is_snan(self): + return self._exp == "N" + + def is_signed(self): + return self._sign == 1 + + def is_zero(self): + return not self._is_special and self._int == "0" + + def is_normal(self, context=None): + if self._is_special or not self: + return False + if context is None: + context = getcontext() + return context.Emin <= self.adjusted() + + def is_subnormal(self, context=None): + if self._is_special or not self: + return False + if context is None: + context = getcontext() + return self.adjusted() < context.Emin + + def number_class(self, context=None): + if self.is_snan(): + return "sNaN" + if self.is_qnan(): + return "NaN" + inf = self._isinfinity() + if inf == 1: + return "+Infinity" + if inf == -1: + return "-Infinity" + if self.is_zero(): + return "-Zero" if self._sign else "+Zero" + if self.is_subnormal(context): + return "-Subnormal" if self._sign else "+Subnormal" + return "-Normal" if self._sign else "+Normal" + + def canonical(self): + return self + + def radix(self): + return Decimal(10) + + def same_quantum(self, other, context=None): + other = _convert(other, raiseit=True) + if self._is_special or other._is_special: + return (self.is_nan() and other.is_nan() or self.is_infinite() and other.is_infinite()) + return self._exp == other._exp + + # -- text -------------------------------------------------------------- + + def __str__(self, eng=False, context=None): + sign = ["", "-"][self._sign] + if self._is_special: + if self._exp == "F": + return sign + "Infinity" + elif self._exp == "n": + return sign + "NaN" + self._int + else: + return sign + "sNaN" + self._int + leftdigits = self._exp + len(self._int) + if self._exp <= 0 and leftdigits > -6: + dotplace = leftdigits + elif not eng: + dotplace = 1 + elif self._int == "0": + dotplace = (leftdigits + 1) % 3 - 1 + else: + dotplace = (leftdigits - 1) % 3 + 1 + if dotplace <= 0: + intpart = "0" + fracpart = "." + "0" * (-dotplace) + self._int + elif dotplace >= len(self._int): + intpart = self._int + "0" * (dotplace - len(self._int)) + fracpart = "" + else: + intpart = self._int[:dotplace] + fracpart = "." + self._int[dotplace:] + if leftdigits == dotplace: + exp = "" + else: + if context is None: + context = getcontext() + exp = ["e", "E"][context.capitals] + "%+d" % (leftdigits - dotplace) + return sign + intpart + fracpart + exp + + def to_eng_string(self, context=None): + return self.__str__(eng=True, context=context) + + def __repr__(self): + return "Decimal('%s')" % str(self) + + def as_tuple(self): + if self._is_special: + return DecimalTuple(self._sign, tuple(map(int, self._int)), self._exp) + return DecimalTuple(self._sign, tuple(map(int, self._int)), self._exp) + + def as_integer_ratio(self): + if self._is_special: + if self._isnan(): + raise ValueError("cannot convert NaN to integer ratio") + raise OverflowError("cannot convert Infinity to integer ratio") + if not self: + return 0, 1 + n = int(self._int) + if self._exp >= 0: + n, d = n * 10 ** self._exp, 1 + else: + d5 = -self._exp + while d5 > 0 and n % 5 == 0: + n //= 5 + d5 -= 1 + d2 = -self._exp + shift2 = min((n & -n).bit_length() - 1, d2) + if shift2: + n >>= shift2 + d2 -= shift2 + d = 5 ** d5 << d2 + if self._sign: + n = -n + return n, d + + # -- conversion -------------------------------------------------------- + + def __int__(self): + if self._is_special: + if self._isnan(): + raise ValueError("cannot convert NaN to integer") + raise OverflowError("cannot convert Infinity to integer") + s = (-1) ** self._sign + if self._exp >= 0: + return s * int(self._int) * 10 ** self._exp + return s * int(self._int[:self._exp] or "0") + + __trunc__ = __int__ + + def __index__(self): + raise TypeError("'decimal.Decimal' object cannot be interpreted as an integer") + + def __float__(self): + if self._isnan(): + if self.is_snan(): + raise ValueError("cannot convert signaling NaN to float") + s = "-nan" if self._sign else "nan" + else: + s = str(self) + return float(s) + + def __complex__(self): + return complex(float(self)) + + @property + def real(self): + return self + + @property + def imag(self): + return Decimal(0) + + def conjugate(self): + return self + + def __floor__(self): + if self._is_special: + if self.is_nan(): + raise ValueError("cannot round a NaN") + raise OverflowError("cannot round an infinity") + return int(self._rescale(0, ROUND_FLOOR)) + + def __ceil__(self): + if self._is_special: + if self.is_nan(): + raise ValueError("cannot round a NaN") + raise OverflowError("cannot round an infinity") + return int(self._rescale(0, ROUND_CEILING)) + + def __round__(self, n=None): + if n is not None: + if not isinstance(n, int): + raise TypeError("Second argument to round should be integral") + exp = _dec(0, "1", -n) + return self.quantize(exp) + if self._is_special: + if self.is_nan(): + raise ValueError("cannot round a NaN") + raise OverflowError("cannot round an infinity") + return int(self._rescale(0, ROUND_HALF_EVEN)) + + # -- hashing and comparison -------------------------------------------- + + def __hash__(self): + if self._is_special: + if self.is_snan(): + raise TypeError("Cannot hash a signaling NaN value.") + elif self.is_nan(): + return object.__hash__(self) + return -314159 if self._sign else 314159 + modulus = (1 << 61) - 1 + if self._exp >= 0: + exp_hash = pow(10, self._exp, modulus) + else: + exp_hash = pow(_PyHASH_10INV, -self._exp, modulus) + hash_ = int(self._int) * exp_hash % modulus + ans = hash_ if self >= 0 else -hash_ + return -2 if ans == -1 else ans + + def _cmp(self, other): + if self._is_special or other._is_special: + self_inf = self._isinfinity() + other_inf = other._isinfinity() + if self_inf == other_inf: + return 0 + elif self_inf < other_inf: + return -1 + else: + return 1 + if not self: + if not other: + return 0 + return -((-1) ** other._sign) + if not other: + return (-1) ** self._sign + if other._sign < self._sign: + return -1 + if self._sign < other._sign: + return 1 + self_adjusted = self.adjusted() + other_adjusted = other.adjusted() + if self_adjusted == other_adjusted: + self_padded = self._int + "0" * (self._exp - other._exp) + other_padded = other._int + "0" * (other._exp - self._exp) + if self_padded == other_padded: + return 0 + elif self_padded < other_padded: + return -((-1) ** self._sign) + return (-1) ** self._sign + elif self_adjusted > other_adjusted: + return (-1) ** self._sign + return -((-1) ** self._sign) + + def _rational_other(self, other, equality_op=False): + """Converts the other operand of a comparison, or answers NotImplemented.""" + if isinstance(other, Decimal): + return other + if isinstance(other, int): + return Decimal(other) + if isinstance(other, float): + context = getcontext() + if equality_op: + context.flags[FloatOperation] = True + else: + context._raise_error(FloatOperation, "strict semantics for mixing floats and Decimals are enabled") + return Decimal.from_float(other) + if isinstance(other, numbers.Rational): + return ("rational", other.numerator, other.denominator) + return NotImplemented + + def _cmp_rational(self, r): + if self._is_special: + return 1 if self._isinfinity() == 1 else -1 + n, d = self.as_integer_ratio() + a = n * r[2] + b = r[1] * d + return (a > b) - (a < b) + + def __eq__(self, other, context=None): + other = self._rational_other(other, equality_op=True) + if other is NotImplemented: + return other + if isinstance(other, tuple): + return not self._isnan() and self._cmp_rational(other) == 0 + if self._isnan() or other._isnan(): + return False + return self._cmp(other) == 0 + + def _order(self, other, context): + other = self._rational_other(other) + if other is NotImplemented: + return other + if isinstance(other, tuple): + if self._isnan(): + ans = self._compare_check_nans(self, context) + return None + return self._cmp_rational(other) + if self._compare_check_nans(other, context): + return None + return self._cmp(other) + + def __lt__(self, other, context=None): + c = self._order(other, context) + if c is NotImplemented: + return c + return c is not None and c < 0 + + def __le__(self, other, context=None): + c = self._order(other, context) + if c is NotImplemented: + return c + return c is not None and c <= 0 + + def __gt__(self, other, context=None): + c = self._order(other, context) + if c is NotImplemented: + return c + return c is not None and c > 0 + + def __ge__(self, other, context=None): + c = self._order(other, context) + if c is NotImplemented: + return c + return c is not None and c >= 0 + + def compare(self, other, context=None): + other = _convert(other, raiseit=True) + if self._is_special or other._is_special: + ans = self._check_nans(other, context) + if ans: + return ans + return Decimal(self._cmp(other)) + + def compare_total(self, other, context=None): + other = _convert(other, raiseit=True) + if self._sign and not other._sign: + return _NegativeOne + if not self._sign and other._sign: + return _One + sign = self._sign + self_nan = self._isnan() + other_nan = other._isnan() + if self_nan or other_nan: + if self_nan == other_nan: + self_key = len(self._int), self._int + other_key = len(other._int), other._int + if self_key < other_key: + return _NegativeOne if not sign else _One + if self_key > other_key: + return _One if not sign else _NegativeOne + return _Zero + if sign: + if self_nan == 1: + return _NegativeOne + if other_nan == 1: + return _One + if self_nan == 2: + return _NegativeOne + if other_nan == 2: + return _One + else: + if self_nan == 1: + return _One + if other_nan == 1: + return _NegativeOne + if self_nan == 2: + return _One + if other_nan == 2: + return _NegativeOne + if self < other: + return _NegativeOne + if self > other: + return _One + if self._exp < other._exp: + return _One if sign else _NegativeOne + if self._exp > other._exp: + return _NegativeOne if sign else _One + return _Zero + + def max(self, other, context=None): + other = _convert(other, raiseit=True) + if context is None: + context = getcontext() + if self._is_special or other._is_special: + sn = self._isnan() + on = other._isnan() + if sn or on: + if on == 1 and sn == 0: + return self._fix(context) + if sn == 1 and on == 0: + return other._fix(context) + return self._check_nans(other, context) + c = self._cmp(other) + if c == 0: + c = self.compare_total(other)._cmp(_Zero) + ans = other if c == -1 else self + return ans._fix(context) + + def min(self, other, context=None): + other = _convert(other, raiseit=True) + if context is None: + context = getcontext() + if self._is_special or other._is_special: + sn = self._isnan() + on = other._isnan() + if sn or on: + if on == 1 and sn == 0: + return self._fix(context) + if sn == 1 and on == 0: + return other._fix(context) + return self._check_nans(other, context) + c = self._cmp(other) + if c == 0: + c = self.compare_total(other)._cmp(_Zero) + ans = self if c == -1 else other + return ans._fix(context) + + # -- rounding ------------------------------------------------------------ + + def _fix_nan(self, context): + payload = self._int + max_payload_len = context.prec - context.clamp + if len(payload) > max_payload_len: + payload = payload[len(payload) - max_payload_len:].lstrip("0") + return _dec(self._sign, payload, self._exp, True) + return Decimal(self) + + def _fix(self, context): + if self._is_special: + if self._isnan(): + return self._fix_nan(context) + return Decimal(self) + Etiny = context.Etiny() + Etop = context.Etop() + if not self: + exp_max = [context.Emax, Etop][context.clamp] + new_exp = min(max(self._exp, Etiny), exp_max) + if new_exp != self._exp: + context._raise_error(Clamped) + return _dec(self._sign, "0", new_exp) + return Decimal(self) + exp_min = len(self._int) + self._exp - context.prec + if exp_min > Etop: + ans = context._raise_error(Overflow, "above Emax", self._sign) + context._raise_error(Inexact) + context._raise_error(Rounded) + return ans + self_is_subnormal = exp_min < Etiny + if self_is_subnormal: + exp_min = Etiny + if self._exp < exp_min: + digits = len(self._int) + self._exp - exp_min + if digits < 0: + self = _dec(self._sign, "1", exp_min - 1) + digits = 0 + changed = self._round_by(context.rounding, digits) + coeff = self._int[:digits] or "0" + if changed > 0: + coeff = str(int(coeff) + 1) + if len(coeff) > context.prec: + coeff = coeff[:-1] + exp_min += 1 + if exp_min > Etop: + ans = context._raise_error(Overflow, "above Emax", self._sign) + else: + ans = _dec(self._sign, coeff, exp_min) + if changed and self_is_subnormal: + context._raise_error(Underflow) + if self_is_subnormal: + context._raise_error(Subnormal) + if changed: + context._raise_error(Inexact) + context._raise_error(Rounded) + if not ans: + context._raise_error(Clamped) + return ans + if self_is_subnormal: + context._raise_error(Subnormal) + if context.clamp == 1 and self._exp > Etop: + context._raise_error(Clamped) + self_padded = self._int + "0" * (self._exp - Etop) + return _dec(self._sign, self_padded, Etop) + return Decimal(self) + + def _round_down(self, prec): + return 0 if _all_zeros(self._int, prec) else -1 + + def _round_by(self, rounding, prec): + """1 if dropping the digits from prec on rounds up, 0 if exact, -1 if it truncates.""" + s = self._int + if rounding == ROUND_DOWN: + return self._round_down(prec) + if rounding == ROUND_UP: + return -self._round_down(prec) + if rounding == ROUND_HALF_UP: + if s[prec] in "56789": + return 1 + return 0 if _all_zeros(s, prec) else -1 + if rounding == ROUND_HALF_DOWN: + if _exact_half(s, prec): + return -1 + return self._round_by(ROUND_HALF_UP, prec) + if rounding == ROUND_HALF_EVEN: + if _exact_half(s, prec) and (prec == 0 or s[prec - 1] in "02468"): + return -1 + return self._round_by(ROUND_HALF_UP, prec) + if rounding == ROUND_CEILING: + return self._round_down(prec) if self._sign else -self._round_down(prec) + if rounding == ROUND_FLOOR: + return -self._round_down(prec) if self._sign else self._round_down(prec) + if prec and s[prec - 1] not in "05": + return self._round_down(prec) + return -self._round_down(prec) + + def _rescale(self, exp, rounding): + if self._is_special: + return Decimal(self) + if not self: + return _dec(self._sign, "0", exp) + if self._exp >= exp: + return _dec(self._sign, self._int + "0" * (self._exp - exp), exp) + digits = len(self._int) + self._exp - exp + if digits < 0: + self = _dec(self._sign, "1", exp - 1) + digits = 0 + changed = self._round_by(rounding, digits) + coeff = self._int[:digits] or "0" + if changed == 1: + coeff = str(int(coeff) + 1) + return _dec(self._sign, coeff, exp) + + def _round(self, places, rounding): + if places <= 0: + raise ValueError("argument should be at least 1 in _round") + if self._is_special or not self: + return Decimal(self) + ans = self._rescale(self.adjusted() + 1 - places, rounding) + if ans.adjusted() != self.adjusted(): + ans = ans._rescale(ans.adjusted() + 1 - places, rounding) + return ans + + def quantize(self, exp, rounding=None, context=None): + exp = _convert(exp, raiseit=True) + if context is None: + context = getcontext() + if rounding is None: + rounding = context.rounding + if self._is_special or exp._is_special: + ans = self._check_nans(exp, context) + if ans: + return ans + if exp._isinfinity() or self._isinfinity(): + if exp._isinfinity() and self._isinfinity(): + return Decimal(self) + return context._raise_error(InvalidOperation, "quantize with one INF") + if not (context.Etiny() <= exp._exp <= context.Emax): + return context._raise_error(InvalidOperation, "target exponent out of bounds in quantize") + if not self: + ans = _dec(self._sign, "0", exp._exp) + return ans._fix(context) + self_adjusted = self.adjusted() + if self_adjusted > context.Emax: + return context._raise_error(InvalidOperation, "exponent of quantize result too large for current " + "context") + if self_adjusted - exp._exp + 1 > context.prec: + return context._raise_error(InvalidOperation, "quantize result has too many digits for current context") + ans = self._rescale(exp._exp, rounding) + if ans.adjusted() > context.Emax: + return context._raise_error(InvalidOperation, "exponent of quantize result too large for current " + "context") + if len(ans._int) > context.prec: + return context._raise_error(InvalidOperation, "quantize result has too many digits for current context") + if ans and ans.adjusted() < context.Emin: + context._raise_error(Subnormal) + if ans._exp > self._exp: + if ans != self: + context._raise_error(Inexact) + context._raise_error(Rounded) + return ans._fix(context) + + def to_integral_exact(self, rounding=None, context=None): + if self._is_special: + ans = self._check_nans(context=context) + if ans: + return ans + return Decimal(self) + if self._exp >= 0: + return Decimal(self) + if not self: + return _dec(self._sign, "0", 0) + if context is None: + context = getcontext() + if rounding is None: + rounding = context.rounding + ans = self._rescale(0, rounding) + if ans != self: + context._raise_error(Inexact) + context._raise_error(Rounded) + return ans + + def to_integral_value(self, rounding=None, context=None): + if context is None: + context = getcontext() + if rounding is None: + rounding = context.rounding + if self._is_special: + ans = self._check_nans(context=context) + if ans: + return ans + return Decimal(self) + if self._exp >= 0: + return Decimal(self) + return self._rescale(0, rounding) + + to_integral = to_integral_value + + def normalize(self, context=None): + if context is None: + context = getcontext() + if self._is_special: + ans = self._check_nans(context=context) + if ans: + return ans + dup = self._fix(context) + if dup._isinfinity(): + return dup + if not dup: + return _dec(dup._sign, "0", 0) + exp_max = [context.Emax, context.Etop()][context.clamp] + end = len(dup._int) + exp = dup._exp + while dup._int[end - 1] == "0" and exp < exp_max: + exp += 1 + end -= 1 + return _dec(dup._sign, dup._int[:end], exp) + + def scaleb(self, other, context=None): + if context is None: + context = getcontext() + other = _convert(other, raiseit=True) + ans = self._check_nans(other, context) + if ans: + return ans + if other._exp != 0 or other._is_special: + return context._raise_error(InvalidOperation) + if self._isinfinity(): + return Decimal(self) + d = _dec(self._sign, self._int, self._exp + int(other)) + return d._fix(context) + + def logb(self, context=None): + ans = self._check_nans(context=context) + if ans: + return ans + if context is None: + context = getcontext() + if self._isinfinity(): + return _Infinity + if not self: + return context._raise_error(DivisionByZero, "logb(0)", 1) + return Decimal(self.adjusted())._fix(context) + + # -- sign operations --------------------------------------------------- + + def copy_abs(self): + return _dec(0, self._int, self._exp, self._is_special) + + def copy_negate(self): + return _dec(1 - self._sign, self._int, self._exp, self._is_special) + + def copy_sign(self, other, context=None): + other = _convert(other, raiseit=True) + return _dec(other._sign, self._int, self._exp, self._is_special) + + def __neg__(self, context=None): + if self._is_special: + ans = self._check_nans(context=context) + if ans: + return ans + if context is None: + context = getcontext() + if not self and context.rounding != ROUND_FLOOR: + ans = self.copy_abs() + else: + ans = self.copy_negate() + return ans._fix(context) + + def __pos__(self, context=None): + if self._is_special: + ans = self._check_nans(context=context) + if ans: + return ans + if context is None: + context = getcontext() + if not self and context.rounding != ROUND_FLOOR: + ans = self.copy_abs() + else: + ans = Decimal(self) + return ans._fix(context) + + def __abs__(self, round=True, context=None): + if not round: + return self.copy_abs() + if self._is_special: + ans = self._check_nans(context=context) + if ans: + return ans + if self._sign: + return self.__neg__(context=context) + return self.__pos__(context=context) + + # -- arithmetic -------------------------------------------------------- + + def __add__(self, other, context=None): + other = _convert(other) + if other is NotImplemented: + return other + if context is None: + context = getcontext() + if self._is_special or other._is_special: + ans = self._check_nans(other, context) + if ans: + return ans + if self._isinfinity(): + if self._sign != other._sign and other._isinfinity(): + return context._raise_error(InvalidOperation, "-INF + INF") + return Decimal(self) + if other._isinfinity(): + return Decimal(other) + exp = min(self._exp, other._exp) + negativezero = 0 + if context.rounding == ROUND_FLOOR and self._sign != other._sign: + negativezero = 1 + if not self and not other: + sign = min(self._sign, other._sign) + if negativezero: + sign = 1 + return _dec(sign, "0", exp)._fix(context) + if not self: + exp = max(exp, other._exp - context.prec - 1) + return other._rescale(exp, context.rounding)._fix(context) + if not other: + exp = max(exp, self._exp - context.prec - 1) + return self._rescale(exp, context.rounding)._fix(context) + a = int(self._int) * 10 ** (self._exp - exp) + b = int(other._int) * 10 ** (other._exp - exp) + if self._sign: + a = -a + if other._sign: + b = -b + total = a + b + if total == 0: + return _dec(negativezero, "0", exp)._fix(context) + return _dec(1 if total < 0 else 0, str(abs(total)), exp)._fix(context) + + __radd__ = __add__ + + def __sub__(self, other, context=None): + other = _convert(other) + if other is NotImplemented: + return other + if self._is_special or other._is_special: + ans = self._check_nans(other, context=context) + if ans: + return ans + return self.__add__(other.copy_negate(), context=context) + + def __rsub__(self, other, context=None): + other = _convert(other) + if other is NotImplemented: + return other + return other.__sub__(self, context=context) + + def __mul__(self, other, context=None): + other = _convert(other) + if other is NotImplemented: + return other + if context is None: + context = getcontext() + resultsign = self._sign ^ other._sign + if self._is_special or other._is_special: + ans = self._check_nans(other, context) + if ans: + return ans + if self._isinfinity(): + if not other: + return context._raise_error(InvalidOperation, "(+-)INF * 0") + return _SignedInfinity[resultsign] + if other._isinfinity(): + if not self: + return context._raise_error(InvalidOperation, "0 * (+-)INF") + return _SignedInfinity[resultsign] + resultexp = self._exp + other._exp + if not self or not other: + return _dec(resultsign, "0", resultexp)._fix(context) + return _dec(resultsign, str(int(self._int) * int(other._int)), resultexp)._fix(context) + + __rmul__ = __mul__ + + def __truediv__(self, other, context=None): + other = _convert(other) + if other is NotImplemented: + return other + if context is None: + context = getcontext() + sign = self._sign ^ other._sign + if self._is_special or other._is_special: + ans = self._check_nans(other, context) + if ans: + return ans + if self._isinfinity() and other._isinfinity(): + return context._raise_error(InvalidOperation, "(+-)INF/(+-)INF") + if self._isinfinity(): + return _SignedInfinity[sign] + if other._isinfinity(): + context._raise_error(Clamped, "Division by infinity") + return _dec(sign, "0", context.Etiny()) + if not other: + if not self: + return context._raise_error(DivisionUndefined, "0 / 0") + return context._raise_error(DivisionByZero, "x / 0", sign) + if not self: + exp = self._exp - other._exp + coeff = 0 + else: + shift = len(other._int) - len(self._int) + context.prec + 1 + exp = self._exp - other._exp - shift + if shift >= 0: + coeff, remainder = divmod(int(self._int) * 10 ** shift, int(other._int)) + else: + coeff, remainder = divmod(int(self._int), int(other._int) * 10 ** -shift) + if remainder: + if coeff % 5 == 0: + coeff += 1 + else: + ideal_exp = self._exp - other._exp + while exp < ideal_exp and coeff % 10 == 0: + coeff //= 10 + exp += 1 + return _dec(sign, str(coeff), exp)._fix(context) + + def __rtruediv__(self, other, context=None): + other = _convert(other) + if other is NotImplemented: + return other + return other.__truediv__(self, context=context) + + def _divide(self, other, context): + sign = self._sign ^ other._sign + if other._isinfinity(): + ideal_exp = self._exp + else: + ideal_exp = min(self._exp, other._exp) + expdiff = self.adjusted() - other.adjusted() + if not self or other._isinfinity() or expdiff <= -2: + return (_dec(sign, "0", 0), self._rescale(ideal_exp, context.rounding)) + if expdiff <= context.prec: + a = int(self._int) + b = int(other._int) + if self._exp >= other._exp: + a *= 10 ** (self._exp - other._exp) + else: + b *= 10 ** (other._exp - self._exp) + q, r = divmod(a, b) + if q < 10 ** context.prec: + return (_dec(sign, str(q), 0), _dec(self._sign, str(r), ideal_exp)) + ans = context._raise_error(DivisionImpossible, "quotient too large in //, % or divmod") + return ans, ans + + def __divmod__(self, other, context=None): + other = _convert(other) + if other is NotImplemented: + return other + if context is None: + context = getcontext() + ans = self._check_nans(other, context) + if ans: + return (ans, ans) + sign = self._sign ^ other._sign + if self._isinfinity(): + if other._isinfinity(): + ans = context._raise_error(InvalidOperation, "divmod(INF, INF)") + return ans, ans + return (_SignedInfinity[sign], context._raise_error(InvalidOperation, "INF % x")) + if not other: + if not self: + ans = context._raise_error(DivisionUndefined, "divmod(0, 0)") + return ans, ans + return (context._raise_error(DivisionByZero, "x // 0", sign), + context._raise_error(InvalidOperation, "x % 0")) + quotient, remainder = self._divide(other, context) + remainder = remainder._fix(context) + return quotient, remainder + + def __rdivmod__(self, other, context=None): + other = _convert(other) + if other is NotImplemented: + return other + return other.__divmod__(self, context=context) + + def __mod__(self, other, context=None): + other = _convert(other) + if other is NotImplemented: + return other + if context is None: + context = getcontext() + ans = self._check_nans(other, context) + if ans: + return ans + if self._isinfinity(): + return context._raise_error(InvalidOperation, "INF % x") + elif not other: + if self: + return context._raise_error(InvalidOperation, "x % 0") + return context._raise_error(DivisionUndefined, "0 % 0") + remainder = self._divide(other, context)[1] + return remainder._fix(context) + + def __rmod__(self, other, context=None): + other = _convert(other) + if other is NotImplemented: + return other + return other.__mod__(self, context=context) + + def remainder_near(self, other, context=None): + if context is None: + context = getcontext() + other = _convert(other, raiseit=True) + ans = self._check_nans(other, context) + if ans: + return ans + if self._isinfinity(): + return context._raise_error(InvalidOperation, "remainder_near(infinity, x)") + if not other: + if self: + return context._raise_error(InvalidOperation, "remainder_near(x, 0)") + return context._raise_error(DivisionUndefined, "remainder_near(0, 0)") + if other._isinfinity(): + return Decimal(self)._fix(context) + ideal_exponent = min(self._exp, other._exp) + if not self: + return _dec(self._sign, "0", ideal_exponent)._fix(context) + expdiff = self.adjusted() - other.adjusted() + if expdiff >= context.prec + 1: + return context._raise_error(DivisionImpossible) + if expdiff <= -2: + return self._rescale(ideal_exponent, context.rounding)._fix(context) + a = int(self._int) + b = int(other._int) + if self._exp >= other._exp: + a *= 10 ** (self._exp - other._exp) + else: + b *= 10 ** (other._exp - self._exp) + q, r = divmod(a, b) + if 2 * r + (q & 1) > b: + r -= b + q += 1 + if q >= 10 ** context.prec: + return context._raise_error(DivisionImpossible) + sign = self._sign + if r < 0: + sign = 1 - sign + r = -r + return _dec(sign, str(r), ideal_exponent)._fix(context) + + def __floordiv__(self, other, context=None): + other = _convert(other) + if other is NotImplemented: + return other + if context is None: + context = getcontext() + ans = self._check_nans(other, context) + if ans: + return ans + if self._isinfinity(): + if other._isinfinity(): + return context._raise_error(InvalidOperation, "INF // INF") + return _SignedInfinity[self._sign ^ other._sign] + if not other: + if self: + return context._raise_error(DivisionByZero, "x // 0", self._sign ^ other._sign) + return context._raise_error(DivisionUndefined, "0 // 0") + return self._divide(other, context)[0] + + def __rfloordiv__(self, other, context=None): + other = _convert(other) + if other is NotImplemented: + return other + return other.__floordiv__(self, context=context) + + def fma(self, other, third, context=None): + other = _convert(other, raiseit=True) + third = _convert(third, raiseit=True) + if context is None: + context = getcontext() + if self._is_special or other._is_special: + if self.is_snan() or other.is_snan() or self.is_nan() or other.is_nan(): + ans = self._check_nans(other, context) + return ans + product = self.__mul__(other, context) + else: + product = _dec(self._sign ^ other._sign, str(int(self._int) * int(other._int)), self._exp + other._exp) + return product.__add__(third, context) + + # -- powers, roots and logarithms -------------------------------------- + + def sqrt(self, context=None): + if context is None: + context = getcontext() + if self._is_special: + ans = self._check_nans(context=context) + if ans: + return ans + if self._isinfinity() and self._sign == 0: + return Decimal(self) + if not self: + return _dec(self._sign, "0", self._exp // 2)._fix(context) + if self._sign == 1: + return context._raise_error(InvalidOperation, "sqrt(-x), x > 0") + prec = context.prec + 1 + e = self._exp >> 1 + if self._exp & 1: + c = int(self._int) * 10 + l = (len(self._int) >> 1) + 1 + else: + c = int(self._int) + l = len(self._int) + 1 >> 1 + shift = prec - l + if shift >= 0: + c *= 100 ** shift + exact = True + else: + c, remainder = divmod(c, 100 ** -shift) + exact = not remainder + e -= shift + n = 10 ** prec + while True: + q = c // n + if n <= q: + break + n = n + q >> 1 + exact = exact and n * n == c + if exact: + if shift >= 0: + n //= 10 ** shift + else: + n *= 10 ** -shift + e += shift + else: + if n % 5 == 0: + n += 1 + ans = _dec(0, str(n), e) + context = context._shallow_copy() + rounding = context._set_rounding(ROUND_HALF_EVEN) + ans = ans._fix(context) + context.rounding = rounding + return ans + + def _guarded(self, context): + """A trap-free context with twenty guard digits, for the series below.""" + c = Context(prec=context.prec + 20, rounding=ROUND_HALF_EVEN, Emin=MIN_EMIN, Emax=MAX_EMAX, traps=[]) + return c + + def _round_result(self, value, context): + """Rounds a guarded result to context, signalling Inexact and Rounded.""" + ans = value._fix(context) + if ans != value or len(value._int) > context.prec: + context._raise_error(Inexact) + context._raise_error(Rounded) + return ans + + def exp(self, context=None): + if context is None: + context = getcontext() + ans = self._check_nans(context=context) + if ans: + return ans + if self._isinfinity() == -1: + return _Zero + if not self: + return _One + if self._isinfinity() == 1: + return Decimal(self) + adj = self.adjusted() + if self._sign == 0 and adj > len(str((context.Emax + 1) * 3)): + ans = _dec(0, "1", context.Emax + 1) + return ans._fix(context) + if self._sign == 1 and adj > len(str((-context.Etiny() + 1) * 3)): + ans = _dec(0, "1", context.Etiny() - 1) + return ans._fix(context) + g = self._guarded(context) + return self._round_result(_exp_series(self, g), context) + + def ln(self, context=None): + if context is None: + context = getcontext() + ans = self._check_nans(context=context) + if ans: + return ans + if not self: + return _NegativeInfinity + if self._isinfinity() == 1: + return _Infinity + if self == _One: + return _Zero + if self._sign == 1: + return context._raise_error(InvalidOperation, "ln of a negative value") + g = self._guarded(context) + return self._round_result(_ln(self, g), context) + + def log10(self, context=None): + if context is None: + context = getcontext() + ans = self._check_nans(context=context) + if ans: + return ans + if not self: + return _NegativeInfinity + if self._isinfinity() == 1: + return _Infinity + if self._sign == 1: + return context._raise_error(InvalidOperation, "log10 of a negative value") + if self._int[0] == "1" and self._int[1:] == "0" * (len(self._int) - 1): + return Decimal(self._exp + len(self._int) - 1) + g = self._guarded(context) + return self._round_result(_ln(self, g).__truediv__(_ln10(g), g), context) + + def __pow__(self, other, modulo=None, context=None): + if modulo is not None: + return self._power_modulo(other, modulo, context) + other = _convert(other) + if other is NotImplemented: + return other + if context is None: + context = getcontext() + ans = self._check_nans(other, context) + if ans: + return ans + if not other: + if not self: + return context._raise_error(InvalidOperation, "0 ** 0") + return _One + result_sign = 0 + if self._sign == 1: + if other._isinteger(): + if not other._iseven(): + result_sign = 1 + elif self: + return context._raise_error(InvalidOperation, "x ** y with x negative and y not an integer") + self = self.copy_negate() + if not self: + if other._sign == 0: + return _dec(result_sign, "0", 0) + return _SignedInfinity[result_sign] + if self._isinfinity(): + if other._sign == 0: + return _SignedInfinity[result_sign] + return _dec(result_sign, "0", 0) + if other._isinfinity(): + c = self._cmp(_One) + if c == 0: + context._raise_error(Inexact) + context._raise_error(Rounded) + return _dec(0, "1" + "0" * (context.prec - 1), 1 - context.prec) + if (c > 0) == (other._sign == 0): + return _Infinity + return _Zero + if other._isinteger(): + n = int(other) + # An exact power when it is small enough to compute exactly. + if 0 < n and len(self._int) * n <= 4 * context.prec + 400: + coeff = int(self._int) ** n + ans = _dec(result_sign, str(coeff), self._exp * n) + return ans._fix(context) + if n < 0 and len(self._int) * -n <= 4 * context.prec + 400: + denom = _dec(0, str(int(self._int) ** -n), self._exp * -n) + ans = _One.__truediv__(denom, context) + if result_sign: + ans = ans.copy_negate() + return ans + g = self._guarded(context) + g.prec += len(str(abs(other.adjusted()))) + 5 + val = _exp_series(_ln(self, g).__mul__(other, g), g) + if other._isinteger(): + pass + else: + exact = _exact_root(self, other, context) + if exact is not None: + val = exact + ans = self._round_result(val, context) + if result_sign: + ans = ans.copy_negate() + return ans + + def __rpow__(self, other, modulo=None, context=None): + other = _convert(other) + if other is NotImplemented: + return other + return other.__pow__(self, modulo, context=context) + + def _power_modulo(self, other, modulo, context=None): + if context is None: + context = getcontext() + other = _convert(other, raiseit=True) + modulo = _convert(modulo, raiseit=True) + for v in (self, other, modulo): + if v._is_special: + return context._raise_error(InvalidOperation, "pow() 3rd argument not allowed unless all " + "arguments are integers") + if not other._isinteger() or not self._isinteger() or not modulo._isinteger(): + return context._raise_error(InvalidOperation, "pow() 3rd argument not allowed unless all arguments " + "are integers") + if other < 0: + return context._raise_error(InvalidOperation, "pow() 2nd argument cannot be negative when 3rd " + "argument specified") + if not modulo: + return context._raise_error(InvalidOperation, "pow() 3rd argument cannot be 0") + if modulo.adjusted() >= context.prec: + return context._raise_error(InvalidOperation, "insufficient precision: pow() 3rd argument must not " + "have more than precision digits") + if not other and not self: + return context._raise_error(InvalidOperation, "at least one of pow() 1st argument and 2nd argument " + "must be nonzero; 0**0 is not defined") + sign = 1 if other._iseven() is False and self._sign else 0 + m = abs(int(modulo)) + r = pow(abs(int(self)), int(other), m) + return _dec(sign, str(r), 0) + + # -- formatting -------------------------------------------------------- + + def __format__(self, specifier, context=None, _localeconv=None): + if context is None: + context = getcontext() + spec = _parse_format_specifier(specifier) + if self._is_special: + sign = _format_sign(self._sign, spec) + body = str(self.copy_abs()) + if spec["type"] == "%": + body += "%" + return _format_align(sign, body, spec) + if spec["type"] is None: + spec["type"] = ["g", "G"][context.capitals] + if spec["type"] == "%": + self = _dec(self._sign, self._int, self._exp + 2) + rounding = context.rounding + precision = spec["precision"] + if precision is not None: + if spec["type"] in "eE": + self = self._round(precision + 1, rounding) + elif spec["type"] in "fF%": + self = self._rescale(-precision, rounding) + elif spec["type"] in "gG" and len(self._int) > precision: + self = self._round(precision, rounding) + if not self and self._exp > 0 and spec["type"] in "fF%": + self = self._rescale(0, rounding) + if not self and spec["no_neg_0"] and self._sign: + adjusted_sign = 0 + else: + adjusted_sign = self._sign + leftdigits = self._exp + len(self._int) + if spec["type"] in "eE": + if not self and precision is not None: + dotplace = 1 - precision + else: + dotplace = 1 + elif spec["type"] in "fF%": + dotplace = leftdigits + else: + if self._exp <= 0 and leftdigits > -6: + dotplace = leftdigits + else: + dotplace = 1 + if dotplace < 0: + intpart = "0" + fracpart = "0" * (-dotplace) + self._int + elif dotplace > len(self._int): + intpart = self._int + "0" * (dotplace - len(self._int)) + fracpart = "" + else: + intpart = self._int[:dotplace] or "0" + fracpart = self._int[dotplace:] + exp = leftdigits - dotplace + return _format_number(adjusted_sign, intpart, fracpart, exp, spec) + + def __reduce__(self): + return (self.__class__, (str(self),)) + + def __copy__(self): + return self + + def __deepcopy__(self, memo): + return self + + +def _exp_series(x, g): + """e**x in the guarded context g.""" + k = 0 + # Halve until |x| < 0.1, then square the result back up. + ax = x.copy_abs() + while ax._cmp(_dec(0, "1", -1)) > 0: + ax = ax.__truediv__(_Two, g) + k += 1 + y = x.copy_abs() + for _ in range(k): + y = y.__truediv__(_Two, g) + total = _One + term = _One + n = 1 + eps = _dec(0, "1", -(g.prec + 2)) + while True: + term = term.__mul__(y, g).__truediv__(Decimal(n), g) + if term.copy_abs()._cmp(eps) < 0: + break + total = total.__add__(term, g) + n += 1 + for _ in range(k): + total = total.__mul__(total, g) + if x._sign: + total = _One.__truediv__(total, g) + return total + + +_ln10_cache = {} + + +def _ln10(g): + v = _ln10_cache.get(g.prec) + if v is None: + v = _ln_reduced(Decimal(10), g) + _ln10_cache[g.prec] = v + return v + + +def _ln(x, g): + """ln(x) for a finite positive x, in the guarded context g.""" + k = x.adjusted() + m = _dec(0, x._int, -(len(x._int) - 1)) + r = _ln_reduced(m, g) + if k: + r = r.__add__(_ln10(g).__mul__(Decimal(k), g), g) + return r + + +def _ln_reduced(m, g): + """ln(m) for m near [1, 10), by square roots and the atanh series.""" + j = 0 + lo = _dec(0, "9", -1) + hi = _dec(0, "11", -1) + while m._cmp(hi) > 0 or m._cmp(lo) < 0: + m = m.sqrt(g) + j += 1 + z = m.__sub__(_One, g).__truediv__(m.__add__(_One, g), g) + z2 = z.__mul__(z, g) + total = z + term = z + n = 3 + eps = _dec(0, "1", -(g.prec + 2)) + while True: + term = term.__mul__(z2, g) + t = term.__truediv__(Decimal(n), g) + if t.copy_abs()._cmp(eps) < 0: + break + total = total.__add__(t, g) + n += 2 + return total.__mul__(Decimal(2 ** (j + 1)), g) + + +def _exact_root(x, y, context): + """x ** y when y = p/q and x has an exact q-th root, padded as CPython pads it; else None.""" + n, d = y.as_integer_ratio() + if d > 64 or n <= 0: + return None + coeff = int(x._int) + exp = x._exp + while exp % d: + coeff *= 10 + exp -= 1 + lo, hi = 0, 1 + while hi ** d <= coeff: + hi *= 2 + while lo < hi: + mid = (lo + hi) // 2 + if mid ** d < coeff: + lo = mid + 1 + else: + hi = mid + if lo ** d != coeff: + return None + root = lo ** n + rexp = exp // d * n + digits = str(root) + pad = context.prec - len(digits) + if pad > 0: + digits += "0" * pad + rexp -= pad + return _dec(0, digits, rexp) + + +def _parse_format_specifier(spec): + fill = " " + align = None + i = 0 + if len(spec) >= 2 and spec[1] in "<>=^": + fill = spec[0] + align = spec[1] + i = 2 + elif spec[:1] in ("<", ">", "=", "^"): + align = spec[0] + i = 1 + sign = "-" + if spec[i:i + 1] in ("+", "-", " "): + sign = spec[i] + i += 1 + no_neg_0 = False + if spec[i:i + 1] == "z": + no_neg_0 = True + i += 1 + alt = False + if spec[i:i + 1] == "#": + alt = True + i += 1 + zeropad = False + if spec[i:i + 1] == "0": + zeropad = True + if align is None: + fill = "0" + align = "=" + i += 1 + j = i + while j < len(spec) and spec[j].isdigit(): + j += 1 + minimumwidth = int(spec[i:j]) if j > i else 0 + i = j + thousands_sep = "" + if spec[i:i + 1] in (",", "_"): + thousands_sep = spec[i] + i += 1 + precision = None + if spec[i:i + 1] == ".": + j = i + 1 + while j < len(spec) and spec[j].isdigit(): + j += 1 + if j == i + 1: + raise ValueError("Invalid format specifier: " + spec) + precision = int(spec[i + 1:j]) + i = j + type = None + if i < len(spec): + type = spec[i] + i += 1 + if type not in "eEfFgGn%": + raise ValueError("Invalid format specifier: " + spec) + if i != len(spec): + raise ValueError("Invalid format specifier: " + spec) + if type in ("g", "G", "n") and precision == 0: + precision = 1 + if type == "n": + type = "g" + if align is None: + align = ">" + return {"fill": fill, "align": align, "sign": sign, "no_neg_0": no_neg_0, "alt": alt, "zeropad": zeropad, + "minimumwidth": minimumwidth, "thousands_sep": thousands_sep, "precision": precision, "type": type, + "decimal_point": ".", "grouping": [3, 0]} + + +def _format_sign(is_negative, spec): + if is_negative: + return "-" + elif spec["sign"] in " +": + return spec["sign"] + return "" + + +def _format_align(sign, body, spec): + minimumwidth = spec["minimumwidth"] + fill = spec["fill"] + padding = fill * (minimumwidth - len(sign) - len(body)) + align = spec["align"] + if align == "<": + result = sign + body + padding + elif align == ">": + result = padding + sign + body + elif align == "=": + result = sign + padding + body + else: + half = len(padding) // 2 + result = padding[:half] + sign + body + padding[half:] + return result + + +def _insert_thousands_sep(digits, spec, min_width=1): + sep = spec["thousands_sep"] + if not sep: + if len(digits) < min_width: + digits = "0" * (min_width - len(digits)) + digits + return digits + groups = [] + min_width = max(min_width, 1) + while True: + l = min(max(len(digits), min_width, 1), 3) + groups.append("0" * (l - len(digits)) + digits[-l:]) + digits = digits[:-l] + min_width -= l + if not digits and min_width <= 0: + break + min_width -= len(sep) + return sep.join(reversed(groups)) + + +def _format_number(is_negative, intpart, fracpart, exp, spec): + sign = _format_sign(is_negative, spec) + if fracpart or spec["alt"]: + fracpart = spec["decimal_point"] + fracpart + if exp != 0 or spec["type"] in "eE": + echar = {"E": "E", "e": "e", "G": "E", "g": "e"}[spec["type"]] + fracpart += "{0}{1:+}".format(echar, exp) + if spec["type"] == "%": + fracpart += "%" + if spec["zeropad"]: + min_width = spec["minimumwidth"] - len(fracpart) - len(sign) + else: + min_width = 0 + intpart = _insert_thousands_sep(intpart, spec, min_width) + return _format_align(sign, intpart + fracpart, spec) + + +_PyHASH_10INV = pow(10, (1 << 61) - 3, (1 << 61) - 1) + +_Infinity = Decimal("Inf") +_NegativeInfinity = Decimal("-Inf") +_NaN = Decimal("NaN") +_Zero = Decimal(0) +_One = Decimal(1) +_Two = Decimal(2) +_NegativeOne = Decimal(-1) +_SignedInfinity = (_Infinity, _NegativeInfinity) + +numbers.Number.register(Decimal) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/enum.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/enum.py new file mode 100644 index 00000000000..cb33f065871 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/enum.py @@ -0,0 +1,521 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Enumerations: classes whose attributes are a fixed set of named members.""" + +__all__ = ["EnumType", "EnumMeta", "Enum", "IntEnum", "StrEnum", "Flag", "IntFlag", "ReprEnum", "auto", "unique", + "member", "nonmember", "global_enum", "verify", "UNIQUE", "CONTINUOUS", "NAMED_FLAGS", "EnumCheck", + "FlagBoundary", "STRICT", "CONFORM", "EJECT", "KEEP", "property"] + +_MIXINS = (int, str, float, bytes) +builtins_property = property +# The library's own bases, whose __new__ never builds a member. +_BASES = [] + + +class auto: + """A placeholder replaced by an automatically chosen value.""" + _counter = 0 + + def __init__(self, value=None): + self.value = value + auto._counter += 1 + self._order = auto._counter + + def __repr__(self): + return "auto(" + repr(self.value) + ")" + + +class member: + def __init__(self, value): + self.value = value + + +class nonmember: + def __init__(self, value): + self.value = value + + + + +def _is_dunder(name): + return len(name) > 4 and name[:2] == name[-2:] == "__" and name[2] != "_" and name[-3] != "_" + + +def _is_sunder(name): + return len(name) > 2 and name[0] == name[-1] == "_" and name[1] != "_" and name[-2] != "_" + + +def _is_private(cls_name, name): + return name.startswith("_" + cls_name.lstrip("_") + "__") + + +def _is_descriptor(obj): + t = type(obj) + if t.__name__ in ("function", "builtin_function_or_method", "method"): + return True + return isinstance(obj, (classmethod, staticmethod, type)) or hasattr(t, "__get__") or \ + hasattr(t, "__set__") or hasattr(t, "__delete__") + + +def _mixin_of(bases): + for base in bases: + for k in base.__mro__: + if k in _MIXINS: + return k + return None + + +class EnumType(type): + @classmethod + def __prepare__(metacls, cls, bases, **kwds): + return {} + + def __new__(metacls, cls, bases, classdict, *, boundary=None, _simple=False, **kwds): + enum_class = super().__new__(metacls, cls, bases, classdict, **kwds) + type.__setattr__(enum_class, "_member_names_", []) + type.__setattr__(enum_class, "_member_map_", {}) + type.__setattr__(enum_class, "_value2member_map_", {}) + type.__setattr__(enum_class, "_unhashable_values_", []) + mixin = _mixin_of(bases) + type.__setattr__(enum_class, "_member_type_", mixin if mixin is not None else object) + flag = Flag is not None and any(isinstance(b, EnumType) and issubclass(b, Flag) for b in bases) + type.__setattr__(enum_class, "_is_flag_", flag) + type.__setattr__(enum_class, "_boundary_", boundary) + if any(isinstance(b, EnumType) and b._member_names_ for b in bases): + raise TypeError( + " cannot extend %s" % (cls, ", ".join("" % b.__name__ for b in bases + if isinstance(b, EnumType) and b._member_names_))) + last_values = [] + all_bits = 0 + for name, value in list(classdict.items()): + if isinstance(value, nonmember): + type.__setattr__(enum_class, name, value.value) + continue + if isinstance(value, member): + value = value.value + elif _is_dunder(name) or _is_sunder(name) or _is_private(cls, name) or _is_descriptor(value): + continue + if name in enum_class._member_map_: + raise TypeError("%r already defined as %r" % (name, enum_class._member_map_[name])) + if isinstance(value, auto): + if value.value is None: + value = enum_class._generate_next_value_(name, 1, len(enum_class._member_names_), + last_values[:]) + else: + value = value.value + elif isinstance(value, tuple) and any(isinstance(v, auto) for v in value): + value = tuple(enum_class._generate_next_value_(name, 1, len(enum_class._member_names_), + last_values[:]) if isinstance(v, auto) else v + for v in value) + last_values.append(value) + m = metacls._make_member(enum_class, name, value, mixin) + canonical = None + for other in enum_class._member_names_: + o = enum_class._member_map_[other] + if o._value_ == m._value_: + canonical = o + break + if canonical is not None: + m = canonical + else: + enum_class._member_names_.append(name) + if flag and isinstance(m._value_, int): + all_bits |= m._value_ + enum_class._member_map_[name] = m + try: + enum_class._value2member_map_.setdefault(m._value_, m) + except TypeError: + enum_class._unhashable_values_.append(m._value_) + type.__setattr__(enum_class, name, m) + if mixin is not None: + # The mixin type comes first in the MRO, so its __repr__ and + # __str__ would shadow the enum's: take the nearest enum's own. + for dunder in ("__repr__", "__str__", "__format__"): + if dunder in classdict: + continue + for k in enum_class.__mro__: + if isinstance(k, EnumType) and dunder in k.__dict__: + type.__setattr__(enum_class, dunder, k.__dict__[dunder]) + break + type.__setattr__(enum_class, "_flag_mask_", all_bits) + type.__setattr__(enum_class, "_all_bits_", all_bits) + return enum_class + + @staticmethod + def _make_member(enum_class, name, value, mixin): + args = value if isinstance(value, tuple) else (value,) + custom_new = enum_class.__dict__.get("_new_member_", None) + new = None + for k in enum_class.__mro__: + if "__new__" in k.__dict__ and k not in _BASES and k is not object: + if k not in _MIXINS: + new = k.__dict__["__new__"] + break + if new is not None: + m = new(enum_class, *args) + elif mixin is None: + m = object.__new__(enum_class) + else: + m = mixin.__new__(enum_class, *args) + if not hasattr(m, "_value_"): + if mixin is None: + m._value_ = value + else: + m._value_ = mixin(*args) + m._name_ = name + m._sort_order_ = len(enum_class._member_names_) + init = getattr(enum_class, "__init__", None) + if init is not None and init is not Enum.__init__: + m.__init__(*args) + return m + + def __call__(cls, value, names=None, *values, module=None, qualname=None, type=None, start=1, boundary=None): + if names is None: + return cls.__new_member_lookup__(value) + return cls._create_(value, names, module=module, qualname=qualname, type=type, start=start, + boundary=boundary) + + def __new_member_lookup__(cls, value): + if isinstance(value, cls): + return value + try: + return cls._value2member_map_[value] + except (KeyError, TypeError): + pass + for m in cls._member_map_.values(): + if m._value_ == value: + return m + result = cls._missing_(value) + if isinstance(result, cls): + return result + if result is None: + raise ValueError("%r is not a valid %s" % (value, cls.__qualname__)) + raise TypeError("error in %s._missing_: returned %r instead of None or a valid member" + % (cls.__name__, result)) + + def _create_(cls, class_name, names, *, module=None, qualname=None, type=None, start=1, boundary=None): + bases = (cls,) if type is None else (type, cls) + classdict = {} + if isinstance(names, str): + names = names.replace(",", " ").split() + if isinstance(names, (tuple, list)) and names and isinstance(names[0], str): + original, names, last = names, [], [] + for count, name in enumerate(original): + value = cls._generate_next_value_(name, start, count, last[:]) + last.append(value) + names.append((name, value)) + if isinstance(names, dict): + names = list(names.items()) + for name, value in names: + classdict[name] = value + return EnumType(class_name, bases, classdict, boundary=boundary) + + def __contains__(cls, value): + if isinstance(value, cls): + return True + try: + if value in cls._value2member_map_: + return True + except TypeError: + pass + return value in cls._unhashable_values_ + + def __delattr__(cls, attr): + if attr in cls._member_map_: + raise AttributeError("%r cannot delete member %r." % (cls.__name__, attr)) + super().__delattr__(attr) + + def __getitem__(cls, name): + return cls._member_map_[name] + + def __iter__(cls): + return (cls._member_map_[name] for name in cls._member_names_) + + def __reversed__(cls): + return (cls._member_map_[name] for name in reversed(cls._member_names_)) + + def __len__(cls): + return len(cls._member_names_) + + def __bool__(cls): + return True + + @builtins_property + def __members__(cls): + return dict(cls._member_map_) + + def __repr__(cls): + if Flag is not None and issubclass(cls, Flag): + return "" % cls.__name__ + return "" % cls.__name__ + + def __setattr__(cls, name, value): + member_map = cls.__dict__.get("_member_map_", {}) + if name in member_map: + raise AttributeError("cannot reassign member %r" % (name,)) + super().__setattr__(name, value) + + +EnumMeta = EnumType +Flag = None + + +class Enum(metaclass=EnumType): + def __init__(self, *args, **kwds): + pass + + @staticmethod + def _generate_next_value_(name, start, count, last_values): + if not last_values: + return start + try: + last = last_values[-1] + for v in reversed(last_values): + if isinstance(v, int) and not isinstance(v, bool): + last = v + break + return last + 1 + except TypeError: + raise TypeError("unable to sort non-numeric values") from None + + @classmethod + def _missing_(cls, value): + return None + + def __repr__(self): + v_repr = type(self)._member_type_.__repr__ if type(self)._member_type_ is not object else repr + return "<%s.%s: %s>" % (type(self).__name__, self._name_, v_repr(self._value_)) + + def __str__(self): + return "%s.%s" % (type(self).__name__, self._name_) + + def __format__(self, format_spec): + return str.__format__(str(self), format_spec) + + def __hash__(self): + return hash(self._name_) + + def __reduce_ex__(self, proto): + return self.__class__, (self._value_,) + + def __deepcopy__(self, memo): + return self + + def __copy__(self): + return self + + def __dir__(self): + return ["__class__", "__doc__", "__eq__", "__hash__", "__module__", "name", "value"] + + @builtins_property + def name(self): + return self._name_ + + @builtins_property + def value(self): + return self._value_ + + +_BASES.append(Enum) + + +class ReprEnum(Enum): + def __str__(self): + mixin = type(self)._member_type_ + return str.__str__(self) if issubclass(mixin, str) else mixin.__repr__(self) + + def __format__(self, format_spec): + return type(self)._member_type_.__format__(self, format_spec) + + +class IntEnum(int, ReprEnum): + pass + + +_BASES.extend((ReprEnum, IntEnum)) + + +class StrEnum(str, ReprEnum): + @staticmethod + def _generate_next_value_(name, start, count, last_values): + return name.lower() + + +_BASES.append(StrEnum) + + +class FlagBoundary(StrEnum): + STRICT = auto() + CONFORM = auto() + EJECT = auto() + KEEP = auto() + + +STRICT, CONFORM, EJECT, KEEP = FlagBoundary.STRICT, FlagBoundary.CONFORM, FlagBoundary.EJECT, FlagBoundary.KEEP + + +def _bits(n): + out = [] + while n: + low = n & (0 - n) + out.append(low) + n ^= low + return out + + +class Flag(Enum, boundary=STRICT): + @staticmethod + def _generate_next_value_(name, start, count, last_values): + if not count: + return start if start is not None else 1 + last = max(v for v in last_values if isinstance(v, int)) if last_values else 0 + return 1 << last.bit_length() if last else 1 + + @classmethod + def _missing_(cls, value): + if not isinstance(value, int): + raise ValueError("%r is not a valid %s" % (value, cls.__qualname__)) + mask = cls._flag_mask_ + boundary = cls._boundary_ + neg = value < 0 + if neg: + value = value & mask if mask else value + if value & ~mask: + if boundary is STRICT: + raise ValueError("%s invalid value %r\n given %s\n allowed %s" + % (cls.__name__, value, bin(value), bin(mask))) + if boundary is CONFORM: + value = value & mask + elif boundary is EJECT: + return value + mixin = cls._member_type_ + if mixin is object: + pseudo = object.__new__(cls) + else: + pseudo = mixin.__new__(cls, value) + pseudo._value_ = value + names = [m._name_ for m in cls._member_map_.values() if m._value_ and m._value_ & value == m._value_ + and len(_bits(m._value_)) == 1] + unnamed = value + for m in cls: + if m._value_ & value == m._value_: + unnamed &= ~m._value_ + if unnamed and boundary is KEEP: + names.append(hex(unnamed)) + pseudo._name_ = "|".join(names) if names or value else None + if pseudo._name_ is not None and not names: + pseudo._name_ = None + pseudo._sort_order_ = 0 + cls._value2member_map_.setdefault(value, pseudo) + return cls._value2member_map_[value] + + def __contains__(self, other): + if not isinstance(other, self.__class__): + raise TypeError("unsupported operand type(s) for 'in': %r and %r" + % (type(other).__qualname__, self.__class__.__qualname__)) + return other._value_ & self._value_ == other._value_ + + def __iter__(self): + for m in type(self): + if m._value_ and m._value_ & self._value_ == m._value_: + yield m + + def __len__(self): + return len(_bits(self._value_)) + + def __repr__(self): + cls_name = self.__class__.__name__ + v_repr = type(self)._member_type_.__repr__ if type(self)._member_type_ is not object else repr + if self._name_ is None: + return "<%s: %s>" % (cls_name, v_repr(self._value_)) + return "<%s.%s: %s>" % (cls_name, self._name_, v_repr(self._value_)) + + def __str__(self): + cls_name = self.__class__.__name__ + if self._name_ is None: + return "%s(%r)" % (cls_name, self._value_) + return "%s.%s" % (cls_name, self._name_) + + def __bool__(self): + return bool(self._value_) + + def _get_value(self, flag): + if isinstance(flag, self.__class__): + return flag._value_ + if self._member_type_ is not object and isinstance(flag, self._member_type_): + return flag + return NotImplemented + + def __or__(self, other): + v = self._get_value(other) + if v is NotImplemented: + return NotImplemented + return self.__class__(self._value_ | v) + + def __and__(self, other): + v = self._get_value(other) + if v is NotImplemented: + return NotImplemented + return self.__class__(self._value_ & v) + + def __xor__(self, other): + v = self._get_value(other) + if v is NotImplemented: + return NotImplemented + return self.__class__(self._value_ ^ v) + + def __invert__(self): + return self.__class__(self._flag_mask_ ^ self._value_) + + __ror__ = __or__ + __rand__ = __and__ + __rxor__ = __xor__ + + +class IntFlag(int, ReprEnum, Flag, boundary=KEEP): + __str__ = ReprEnum.__str__ + __format__ = ReprEnum.__format__ + __repr__ = Flag.__repr__ + __or__ = __ror__ = Flag.__or__ + __and__ = __rand__ = Flag.__and__ + __xor__ = __rxor__ = Flag.__xor__ + __invert__ = Flag.__invert__ + + +_BASES.extend((Flag, IntFlag)) +property = builtins_property + + +def unique(enumeration): + duplicates = [] + for name, m in enumeration.__members__.items(): + if name != m._name_: + duplicates.append((name, m._name_)) + if duplicates: + alias_details = ", ".join("%s -> %s" % (alias, name) for (alias, name) in duplicates) + raise ValueError("duplicate values found in %r: %s" % (enumeration, alias_details)) + return enumeration + + +class EnumCheck(StrEnum): + CONTINUOUS = "no skipped integer values" + NAMED_FLAGS = "multi-flag aliases may not contain unnamed flags" + UNIQUE = "one name per value" + + +CONTINUOUS, NAMED_FLAGS, UNIQUE = EnumCheck.CONTINUOUS, EnumCheck.NAMED_FLAGS, EnumCheck.UNIQUE + + +class verify: + def __init__(self, *checks): + self.checks = checks + + def __call__(self, enumeration): + if UNIQUE in self.checks: + unique(enumeration) + return enumeration + + +def global_enum(cls): + return cls diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/fnmatch.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/fnmatch.py new file mode 100644 index 00000000000..bdb0da47b11 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/fnmatch.py @@ -0,0 +1,111 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Shell-style wildcard matching: *, ?, [seq] and [!seq].""" + +import re +import os + +__all__ = ["filter", "filterfalse", "fnmatch", "fnmatchcase", "translate"] + +_cache = {} + + +def _compile(pat): + r = _cache.get(pat) + if r is None: + if isinstance(pat, bytes): + r = re.compile(translate(pat.decode("latin-1")).encode("latin-1")).match + else: + r = re.compile(translate(pat)).match + if len(_cache) >= 32768: + _cache.clear() + _cache[pat] = r + return r + + +def fnmatch(name, pat): + name = os.path.normcase(name) + pat = os.path.normcase(pat) + return fnmatchcase(name, pat) + + +def fnmatchcase(name, pat): + return _compile(pat)(name) is not None + + +def filter(names, pat): + m = _compile(os.path.normcase(pat)) + return [n for n in names if m(os.path.normcase(n))] + + +def filterfalse(names, pat): + m = _compile(os.path.normcase(pat)) + return [n for n in names if not m(os.path.normcase(n))] + + +def translate(pat): + return "(?s:" + _translate(pat) + ")\\z" + + +def _translate(pat): + res = [] + i = 0 + n = len(pat) + while i < n: + c = pat[i] + i += 1 + if c == "*": + if not res or res[-1] != ".*": + res.append(".*") + elif c == "?": + res.append(".") + elif c == "[": + j = i + if j < n and pat[j] == "!": + j += 1 + if j < n and pat[j] == "]": + j += 1 + while j < n and pat[j] != "]": + j += 1 + if j >= n: + res.append("\\[") + else: + stuff = pat[i:j] + if "-" not in stuff: + stuff = stuff.replace("\\", "\\\\") + else: + chunks = [] + k = i + 2 if pat[i] == "!" else i + 1 + while True: + k = pat.find("-", k, j) + if k < 0: + break + chunks.append(pat[i:k]) + i = k + 1 + k = k + 3 + chunk = pat[i:j] + if chunk: + chunks.append(chunk) + else: + chunks[-1] += "-" + for k in range(len(chunks) - 1, 0, -1): + if chunks[k - 1][-1] > chunks[k][0]: + chunks[k - 1] = chunks[k - 1][:-1] + chunks[k][1:] + del chunks[k] + stuff = "-".join(s.replace("\\", "\\\\").replace("-", "\\-") for s in chunks) + stuff = re.sub(r"([&~|])", r"\\\1", stuff) + i = j + 1 + if not stuff: + res.append("(?!)") + elif stuff == "!": + res.append(".") + else: + if stuff[0] == "!": + stuff = "^" + stuff[1:] + elif stuff[0] in ("^", "["): + stuff = "\\" + stuff + res.append("[" + stuff + "]") + else: + res.append(re.escape(c)) + return "".join(res) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/fractions.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/fractions.py new file mode 100644 index 00000000000..adbc7d95a5d --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/fractions.py @@ -0,0 +1,460 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Rational numbers.""" + +import math +import numbers +import operator +import re + +__all__ = ["Fraction"] + +_HASH_MODULUS = (1 << 61) - 1 +_HASH_INF = 314159 + +_RATIONAL_FORMAT = re.compile(r""" + \A\s* + (?P[-+]?) + (?=\d|\.\d) + (?P\d*|\d+(_\d+)*) + (?: + (?:\s*/\s*(?P\d+(_\d+)*))? + | + (?:\.(?P\d*|\d+(_\d+)*))? + (?:E(?P[-+]?\d+(_\d+)*))? + ) + \s*\Z +""", re.VERBOSE | re.IGNORECASE) + + +class Fraction(numbers.Rational): + __slots__ = ("_numerator", "_denominator") + + def __new__(cls, numerator=0, denominator=None): + self = object.__new__(cls) + if denominator is None: + if type(numerator) is int: + self._numerator = numerator + self._denominator = 1 + return self + if isinstance(numerator, Fraction): + self._numerator = numerator._numerator + self._denominator = numerator._denominator + return self + if isinstance(numerator, numbers.Rational) and not isinstance(numerator, float): + self._numerator = int(numerator.numerator) + self._denominator = int(numerator.denominator) + return self + if isinstance(numerator, (float, int)) or hasattr(numerator, "as_integer_ratio"): + if isinstance(numerator, int): + self._numerator = int(numerator) + self._denominator = 1 + return self + self._numerator, self._denominator = numerator.as_integer_ratio() + return self + if isinstance(numerator, str): + m = _RATIONAL_FORMAT.match(numerator) + if m is None: + raise ValueError("Invalid literal for Fraction: %r" % numerator) + numerator = int(m.group("num") or "0") + denom = m.group("denom") + if denom: + denominator = int(denom) + else: + denominator = 1 + decimal = m.group("decimal") + if decimal: + decimal = decimal.replace("_", "") + scale = 10 ** len(decimal) + numerator = numerator * scale + int(decimal) + denominator *= scale + exp = m.group("exp") + if exp: + exp = int(exp) + if exp >= 0: + numerator *= 10 ** exp + else: + denominator *= 10 ** -exp + if m.group("sign") == "-": + numerator = -numerator + else: + raise TypeError("argument should be a string or a Rational instance or have the " + "as_integer_ratio() method") + elif type(numerator) is int and type(denominator) is int: + pass + elif isinstance(numerator, numbers.Rational) and isinstance(denominator, numbers.Rational) \ + and not isinstance(numerator, float) and not isinstance(denominator, float): + numerator, denominator = (numerator.numerator * denominator.denominator, + denominator.numerator * numerator.denominator) + else: + raise TypeError("both arguments should be Rational instances") + if denominator == 0: + raise ZeroDivisionError("Fraction(%s, 0)" % numerator) + g = math.gcd(numerator, denominator) + if denominator < 0: + g = -g + self._numerator = numerator // g + self._denominator = denominator // g + return self + + @classmethod + def _from_coprime_ints(cls, numerator, denominator): + obj = object.__new__(cls) + obj._numerator = numerator + obj._denominator = denominator + return obj + + @classmethod + def from_float(cls, f): + if isinstance(f, numbers.Integral): + return cls(f) + if not isinstance(f, float): + raise TypeError("%s.from_float() only takes floats, not %r (%s)" % (cls.__name__, f, type(f).__name__)) + return cls._from_coprime_ints(*f.as_integer_ratio()) + + @classmethod + def from_decimal(cls, dec): + from decimal import Decimal + if isinstance(dec, numbers.Integral): + dec = Decimal(int(dec)) + elif not isinstance(dec, Decimal): + raise TypeError("%s.from_decimal() only takes Decimals, not %r (%s)" + % (cls.__name__, dec, type(dec).__name__)) + return cls._from_coprime_ints(*dec.as_integer_ratio()) + + @classmethod + def from_number(cls, number): + if isinstance(number, numbers.Rational) and not isinstance(number, float): + return cls(number) + if not isinstance(number, type) and hasattr(number, "as_integer_ratio"): + return cls._from_coprime_ints(*number.as_integer_ratio()) + raise TypeError("%s.from_number() only takes numbers, not %r (%s)" + % (cls.__name__, number, type(number).__name__)) + + def is_integer(self): + return self._denominator == 1 + + def as_integer_ratio(self): + return (self._numerator, self._denominator) + + def limit_denominator(self, max_denominator=1000000): + if max_denominator < 1: + raise ValueError("max_denominator should be at least 1") + if self._denominator <= max_denominator: + return Fraction(self) + p0, q0, p1, q1 = 0, 1, 1, 0 + n, d = self._numerator, self._denominator + while True: + a = n // d + q2 = q0 + a * q1 + if q2 > max_denominator: + break + p0, q0, p1, q1 = p1, q1, p0 + a * p1, q2 + n, d = d, n - a * d + k = (max_denominator - q0) // q1 + if 2 * d * (q0 + k * q1) <= self._denominator: + return Fraction._from_coprime_ints(p1, q1) + return Fraction._from_coprime_ints(p0 + k * p1, q0 + k * q1) + + @property + def numerator(a): + return a._numerator + + @property + def denominator(a): + return a._denominator + + def __repr__(self): + return "%s(%s, %s)" % (self.__class__.__name__, self._numerator, self._denominator) + + def __str__(self): + if self._denominator == 1: + return str(self._numerator) + return "%s/%s" % (self._numerator, self._denominator) + + def __format__(self, format_spec): + if not format_spec: + return str(self) + m = re.fullmatch(r"(?:(?P.)?(?P[<>=^]))?(?P[-+ ]?)(?P\#)?(?P0)?" + r"(?P\d+)?(?P[,_])?(?:\.(?P\d+))?(?P[fF%])?", format_spec) + if m is None or (m.group("type") is None and m.group("prec") is None): + if m is not None and m.group("type") is None: + return format(str(self), format_spec.replace(m.group("sign") or "\0", "") if False else format_spec) + return format(float(self), format_spec) + prec = int(m.group("prec") or 6) + kind = m.group("type") or "f" + value = self * 100 if kind == "%" else self + neg = value < 0 + n = abs(value) + scaled = n._numerator * 10 ** prec + q, r = divmod(scaled, n._denominator) + if 2 * r > n._denominator or (2 * r == n._denominator and q % 2 == 1): + q += 1 + digits = str(q).rjust(prec + 1, "0") + head = digits[:len(digits) - prec] if prec else digits + if m.group("thousands"): + sep = m.group("thousands") + parts = [] + while len(head) > 3: + parts.insert(0, head[-3:]) + head = head[:-3] + parts.insert(0, head) + head = sep.join(parts) + body = head + ("." + digits[len(digits) - prec:] if prec else ("." if m.group("alt") else "")) + if kind == "%": + body += "%" + sign = "-" if neg and q else ({"+": "+", " ": " "}.get(m.group("sign"), "")) + width = int(m.group("width") or 0) + fill = m.group("fill") or (" " if not m.group("zero") or m.group("align") else "0") + align = m.group("align") or ("=" if m.group("zero") else ">") + pad = max(0, width - len(sign) - len(body)) + if align == "<": + return sign + body + fill * pad + if align == "^": + return fill * (pad // 2) + sign + body + fill * (pad - pad // 2) + if align == "=": + return sign + fill * pad + body + return fill * pad + sign + body + + def _operator_fallbacks(monomorphic_operator, fallback_operator): + def forward(a, b): + if isinstance(b, Fraction): + return monomorphic_operator(a, b) + elif isinstance(b, int): + return monomorphic_operator(a, Fraction._from_coprime_ints(b, 1)) + elif isinstance(b, float): + return fallback_operator(float(a), b) + elif isinstance(b, complex): + return fallback_operator(complex(a), b) + else: + return NotImplemented + + def reverse(b, a): + if isinstance(a, numbers.Rational) and not isinstance(a, float): + return monomorphic_operator(Fraction(a), b) + elif isinstance(a, numbers.Real): + return fallback_operator(float(a), float(b)) + elif isinstance(a, numbers.Complex): + return fallback_operator(complex(a), complex(b)) + else: + return NotImplemented + + return forward, reverse + + def _add(a, b): + na, da = a._numerator, a._denominator + nb, db = b._numerator, b._denominator + g = math.gcd(da, db) + if g == 1: + return Fraction._from_coprime_ints(na * db + da * nb, da * db) + s = da // g + t = na * (db // g) + nb * s + g2 = math.gcd(t, g) + if g2 == 1: + return Fraction._from_coprime_ints(t, s * db) + return Fraction._from_coprime_ints(t // g2, s * (db // g2)) + + __add__, __radd__ = _operator_fallbacks(_add, operator.add) + + def _sub(a, b): + na, da = a._numerator, a._denominator + nb, db = b._numerator, b._denominator + g = math.gcd(da, db) + if g == 1: + return Fraction._from_coprime_ints(na * db - da * nb, da * db) + s = da // g + t = na * (db // g) - nb * s + g2 = math.gcd(t, g) + if g2 == 1: + return Fraction._from_coprime_ints(t, s * db) + return Fraction._from_coprime_ints(t // g2, s * (db // g2)) + + __sub__, __rsub__ = _operator_fallbacks(_sub, operator.sub) + + def _mul(a, b): + na, da = a._numerator, a._denominator + nb, db = b._numerator, b._denominator + g1 = math.gcd(na, db) + if g1 > 1: + na //= g1 + db //= g1 + g2 = math.gcd(nb, da) + if g2 > 1: + nb //= g2 + da //= g2 + return Fraction._from_coprime_ints(na * nb, db * da) + + __mul__, __rmul__ = _operator_fallbacks(_mul, operator.mul) + + def _div(a, b): + nb, db = b._numerator, b._denominator + if nb == 0: + raise ZeroDivisionError("Fraction(%s, 0)" % (db,)) + na, da = a._numerator, a._denominator + g1 = math.gcd(na, nb) + if g1 > 1: + na //= g1 + nb //= g1 + g2 = math.gcd(db, da) + if g2 > 1: + da //= g2 + db //= g2 + n, d = na * db, nb * da + if d < 0: + n, d = -n, -d + return Fraction._from_coprime_ints(n, d) + + __truediv__, __rtruediv__ = _operator_fallbacks(_div, operator.truediv) + + def _floordiv(a, b): + return (a.numerator * b.denominator) // (a.denominator * b.numerator) + + __floordiv__, __rfloordiv__ = _operator_fallbacks(_floordiv, operator.floordiv) + + def _divmod(a, b): + da, db = a.denominator, b.denominator + div, n_mod = divmod(a.numerator * db, da * b.numerator) + return div, Fraction(n_mod, da * db) + + __divmod__, __rdivmod__ = _operator_fallbacks(_divmod, divmod) + + def _mod(a, b): + da, db = a.denominator, b.denominator + return Fraction((a.numerator * db) % (b.numerator * da), da * db) + + __mod__, __rmod__ = _operator_fallbacks(_mod, operator.mod) + + def __pow__(a, b, modulo=None): + if modulo is not None: + return NotImplemented + if isinstance(b, numbers.Rational) and not isinstance(b, float): + if b.denominator == 1: + power = b.numerator + if power >= 0: + return Fraction._from_coprime_ints(a._numerator ** power, a._denominator ** power) + elif a._numerator > 0: + return Fraction._from_coprime_ints(a._denominator ** -power, a._numerator ** -power) + elif a._numerator == 0: + raise ZeroDivisionError("Fraction(%s, 0)" % (a._denominator ** -power,)) + else: + return Fraction._from_coprime_ints((-a._denominator) ** -power, (-a._numerator) ** -power) + else: + return float(a) ** float(b) + elif isinstance(b, (float, complex)): + return float(a) ** b + else: + return NotImplemented + + def __rpow__(b, a, modulo=None): + if modulo is not None: + return NotImplemented + if b._denominator == 1 and b._numerator >= 0: + return a ** b._numerator + if isinstance(a, numbers.Rational) and not isinstance(a, float): + return Fraction(a.numerator, a.denominator) ** b + if b._denominator == 1: + return a ** b._numerator + return a ** float(b) + + def __pos__(a): + return Fraction._from_coprime_ints(a._numerator, a._denominator) + + def __neg__(a): + return Fraction._from_coprime_ints(-a._numerator, a._denominator) + + def __abs__(a): + return Fraction._from_coprime_ints(abs(a._numerator), a._denominator) + + def __int__(a): + if a._numerator < 0: + return -(-a._numerator // a._denominator) + return a._numerator // a._denominator + + def __trunc__(a): + return a.__int__() + + def __floor__(a): + return a._numerator // a._denominator + + def __ceil__(a): + return -(-a._numerator // a._denominator) + + def __round__(self, ndigits=None): + if ndigits is None: + d = self._denominator + floor, remainder = divmod(self._numerator, d) + if remainder * 2 < d: + return floor + elif remainder * 2 > d: + return floor + 1 + elif floor % 2 == 0: + return floor + return floor + 1 + shift = 10 ** abs(ndigits) + if ndigits > 0: + return Fraction(round(self * shift), shift) + return Fraction(round(self / shift) * shift) + + def __float__(a): + return a._numerator / a._denominator + + def __hash__(self): + try: + dinv = pow(self._denominator, -1, _HASH_MODULUS) + except ValueError: + hash_ = _HASH_INF + else: + hash_ = hash(hash(abs(self._numerator)) * dinv) + result = hash_ if self._numerator >= 0 else -hash_ + return -2 if result == -1 else result + + def __eq__(a, b): + if type(b) is int: + return a._numerator == b and a._denominator == 1 + if isinstance(b, numbers.Rational) and not isinstance(b, float): + return a._numerator == b.numerator and a._denominator == b.denominator + if isinstance(b, numbers.Complex) and b.imag == 0: + b = b.real + if isinstance(b, float): + if math.isnan(b) or math.isinf(b): + return 0.0 == b + return a == a.from_float(b) + return NotImplemented + + def _richcmp(self, other, op): + if isinstance(other, numbers.Rational) and not isinstance(other, float): + return op(self._numerator * other.denominator, self._denominator * other.numerator) + if isinstance(other, float): + if math.isnan(other) or math.isinf(other): + return op(0.0, other) + return op(self, self.from_float(other)) + return NotImplemented + + def __lt__(a, b): + return a._richcmp(b, operator.lt) + + def __gt__(a, b): + return a._richcmp(b, operator.gt) + + def __le__(a, b): + return a._richcmp(b, operator.le) + + def __ge__(a, b): + return a._richcmp(b, operator.ge) + + def __bool__(a): + return bool(a._numerator) + + def __reduce__(self): + return (self.__class__, (self._numerator, self._denominator)) + + def __copy__(self): + if type(self) == Fraction: + return self + return self.__class__(self._numerator, self._denominator) + + def __deepcopy__(self, memo): + if type(self) == Fraction: + return self + return self.__class__(self._numerator, self._denominator) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/functools.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/functools.py new file mode 100644 index 00000000000..7399af601f3 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/functools.py @@ -0,0 +1,250 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Higher-order functions and operations on callable objects.""" + +WRAPPER_ASSIGNMENTS = ("__module__", "__name__", "__qualname__", "__doc__") +WRAPPER_UPDATES = ("__dict__",) +_MISSING = object() + + +def reduce(function, iterable, initial=_MISSING): + it = iter(iterable) + if initial is _MISSING: + try: + value = next(it) + except StopIteration: + raise TypeError("reduce() of empty iterable with no initial value") from None + else: + value = initial + for x in it: + value = function(value, x) + return value + + +class partial: + def __init__(self, func, *args, **keywords): + if not callable(func): + raise TypeError("the first argument must be callable") + if isinstance(func, partial): + args = func.args + args + keywords = {**func.keywords, **keywords} + func = func.func + self.func = func + self.args = args + self.keywords = keywords + + def __call__(self, *args, **keywords): + return self.func(*self.args, *args, **{**self.keywords, **keywords}) + + def __repr__(self): + parts = [repr(self.func)] + parts.extend(repr(a) for a in self.args) + parts.extend(k + "=" + repr(v) for k, v in self.keywords.items()) + return "functools.partial(" + ", ".join(parts) + ")" + + +def update_wrapper(wrapper, wrapped, assigned=WRAPPER_ASSIGNMENTS, updated=WRAPPER_UPDATES): + for attr in assigned: + try: + value = getattr(wrapped, attr) + except AttributeError: + continue + try: + setattr(wrapper, attr, value) + except (AttributeError, TypeError): + pass + wrapper.__wrapped__ = wrapped + return wrapper + + +def wraps(wrapped, assigned=WRAPPER_ASSIGNMENTS, updated=WRAPPER_UPDATES): + def deco(wrapper): + return update_wrapper(wrapper, wrapped, assigned, updated) + return deco + + +def cmp_to_key(mycmp): + class K: + __slots__ = ["obj"] + + def __init__(self, obj): + self.obj = obj + + def __lt__(self, other): + return mycmp(self.obj, other.obj) < 0 + + def __gt__(self, other): + return mycmp(self.obj, other.obj) > 0 + + def __eq__(self, other): + return mycmp(self.obj, other.obj) == 0 + + def __le__(self, other): + return mycmp(self.obj, other.obj) <= 0 + + def __ge__(self, other): + return mycmp(self.obj, other.obj) >= 0 + + __hash__ = None + return K + + +def total_ordering(cls): + def has(name): + own = cls.__dict__.get(name) + return own is not None and own is not getattr(object, name, None) + roots = [op for op in ("__lt__", "__le__", "__gt__", "__ge__") if has(op)] + if not roots: + raise ValueError("must define at least one ordering operation: < > <= >=") + root = roots[0] + + def lt_from(self, other): + r = getattr(self, root)(other) + if r is NotImplemented: + return r + if root == "__lt__": + return r + if root == "__le__": + return r and self != other + if root == "__gt__": + return not r and self != other + return not r + + def le(self, other): + r = lt_from(self, other) + return r if r is NotImplemented else (r or self == other) + + def gt(self, other): + r = lt_from(self, other) + return r if r is NotImplemented else (not r and self != other) + + def ge(self, other): + r = lt_from(self, other) + return r if r is NotImplemented else not r + + for name, fn in (("__lt__", lt_from), ("__le__", le), ("__gt__", gt), ("__ge__", ge)): + if name not in roots: + setattr(cls, name, fn) + return cls + + +class _CacheInfo(tuple): + def __new__(cls, hits, misses, maxsize, currsize): + return tuple.__new__(cls, (hits, misses, maxsize, currsize)) + + hits = property(lambda self: self[0]) + misses = property(lambda self: self[1]) + maxsize = property(lambda self: self[2]) + currsize = property(lambda self: self[3]) + + def __repr__(self): + return "CacheInfo(hits=%r, misses=%r, maxsize=%r, currsize=%r)" % tuple(self) + + +def _make_key(args, kwds, typed): + key = args + if kwds: + key = key + (_MISSING,) + for item in kwds.items(): + key = key + item + if typed: + key = key + tuple(type(v) for v in args) + if kwds: + key = key + tuple(type(v) for v in kwds.values()) + elif len(key) == 1 and type(key[0]) in (int, str): + return key[0] + return key + + +def lru_cache(maxsize=128, typed=False): + if callable(maxsize) and not isinstance(maxsize, int): + return lru_cache(128, False)(maxsize) + if maxsize is not None and maxsize < 0: + maxsize = 0 + + def decorating(user_function): + cache = {} + stats = [0, 0] + + def wrapper(*args, **kwds): + key = _make_key(args, kwds, typed) + if key in cache: + stats[0] += 1 + result = cache.pop(key) + cache[key] = result + return result + stats[1] += 1 + result = user_function(*args, **kwds) + if maxsize != 0: + cache[key] = result + if maxsize is not None and len(cache) > maxsize: + del cache[next(iter(cache))] + return result + + def cache_info(): + return _CacheInfo(stats[0], stats[1], maxsize, len(cache)) + + def cache_clear(): + cache.clear() + stats[0] = 0 + stats[1] = 0 + + wrapper.cache_info = cache_info + wrapper.cache_clear = cache_clear + return update_wrapper(wrapper, user_function) + return decorating + + +def cache(user_function): + return lru_cache(None)(user_function) + + +class cached_property: + def __init__(self, func): + self.func = func + self.attrname = None + self.__doc__ = getattr(func, "__doc__", None) + + def __set_name__(self, owner, name): + self.attrname = name + + def __get__(self, instance, owner=None): + if instance is None: + return self + d = instance.__dict__ + if self.attrname in d: + return d[self.attrname] + val = self.func(instance) + d[self.attrname] = val + return val + + +def singledispatch(func): + registry = {object: func} + + def dispatch(cls): + for k in cls.__mro__: + if k in registry: + return registry[k] + return func + + def register(cls, f=None): + if f is None: + if isinstance(cls, type): + return lambda g: register(cls, g) + f = cls + ann = getattr(f, "__annotations__", {}) + cls = next(iter(v for k, v in ann.items() if k != "return")) + registry[cls] = f + return f + + def wrapper(*args, **kw): + if not args: + raise TypeError(getattr(func, "__name__", "singledispatch function") + " requires at least 1 positional argument") + return dispatch(type(args[0]))(*args, **kw) + + wrapper.register = register + wrapper.dispatch = dispatch + wrapper.registry = registry + return update_wrapper(wrapper, func) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/glob.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/glob.py new file mode 100644 index 00000000000..47fb3ebe1aa --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/glob.py @@ -0,0 +1,90 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Path names matching a shell pattern.""" + +import os +import fnmatch + +__all__ = ["glob", "iglob", "escape", "has_magic"] + + +def has_magic(s): + return any(c in s for c in "*?[") + + +def escape(pathname): + drive, rest = "", pathname + out = [] + for c in rest: + out.append("[" + c + "]" if c in "*?[" else c) + return drive + "".join(out) + + +def glob(pathname, *, root_dir=None, dir_fd=None, recursive=False, include_hidden=False): + return list(iglob(pathname, root_dir=root_dir, recursive=recursive, include_hidden=include_hidden)) + + +def iglob(pathname, *, root_dir=None, dir_fd=None, recursive=False, include_hidden=False): + absolute = pathname.startswith("/") + parts = [p for p in pathname.split("/") if p] + base = "/" if absolute else "" + yield from _walk(root_dir, base, parts, recursive, include_hidden, pathname.endswith("/")) + + +def _full(root_dir, p): + if root_dir is None or p.startswith("/"): + return p or "." + return os.path.join(root_dir, p) if p else root_dir + + +def _walk(root_dir, prefix, parts, recursive, hidden, want_dir): + if not parts: + if os.path.exists(_full(root_dir, prefix)): + if not want_dir or os.path.isdir(_full(root_dir, prefix)): + yield prefix + ("/" if want_dir and not prefix.endswith("/") else "") + return + head = parts[0] + rest = parts[1:] + if recursive and head == "**": + for sub in _rdirs(root_dir, prefix, hidden): + if rest: + yield from _walk(root_dir, sub, rest, recursive, hidden, want_dir) + elif sub != prefix or prefix: + yield (sub.rstrip("/") if sub not in ("/",) else sub) + ("/" if want_dir and sub else "") + return + if not has_magic(head): + yield from _walk(root_dir, _join(prefix, head), rest, recursive, hidden, want_dir) + return + d = _full(root_dir, prefix) + if not os.path.isdir(d): + return + for name in os.listdir(d): + if name.startswith(".") and not hidden and not head.startswith("."): + continue + if fnmatch.fnmatchcase(name, head): + p = _join(prefix, name) + if rest and not os.path.isdir(_full(root_dir, p)): + continue + yield from _walk(root_dir, p, rest, recursive, hidden, want_dir) + + +def _join(prefix, name): + if not prefix: + return name + return prefix + name if prefix.endswith("/") else prefix + "/" + name + + +def _rdirs(root_dir, prefix, hidden): + yield prefix + d = _full(root_dir, prefix) + if not os.path.isdir(d): + return + for name in os.listdir(d): + if name.startswith(".") and not hidden: + continue + p = _join(prefix, name) + if os.path.isdir(_full(root_dir, p)): + yield from _rdirs(root_dir, p, hidden) + else: + yield p diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/hashlib.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/hashlib.py new file mode 100644 index 00000000000..c9bcaeb0713 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/hashlib.py @@ -0,0 +1,358 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Message digests: MD5 (RFC 1321), SHA-1 and the SHA-2 family (FIPS 180-4).""" + +__all__ = ["md5", "sha1", "sha224", "sha256", "sha384", "sha512", "new", "algorithms_available", + "algorithms_guaranteed", "pbkdf2_hmac", "file_digest"] + +algorithms_guaranteed = frozenset(["md5", "sha1", "sha224", "sha256", "sha384", "sha512"]) +algorithms_available = algorithms_guaranteed + +_M32 = 0xFFFFFFFF +_M64 = 0xFFFFFFFFFFFFFFFF + + +def _rotl32(x, n): + return ((x << n) | (x >> (32 - n))) & _M32 + + +def _rotr32(x, n): + return ((x >> n) | (x << (32 - n))) & _M32 + + +def _rotr64(x, n): + return ((x >> n) | (x << (64 - n))) & _M64 + + +def _as_bytes(data): + if isinstance(data, str): + raise TypeError("Strings must be encoded before hashing") + return bytes(data) + + +class _Hash: + """Buffers input into blocks and hands each full block to `_compress`.""" + + def __init__(self, data=b""): + self._buf = b"" + self._count = 0 + self._h = list(self._IV) + if data: + self.update(data) + + def update(self, data): + data = _as_bytes(data) + self._count += len(data) + buf = self._buf + data + bs = self.block_size + n = len(buf) - len(buf) % bs + for i in range(0, n, bs): + self._compress(buf[i:i + bs]) + self._buf = buf[n:] + + def copy(self): + c = type(self).__new__(type(self)) + c._buf = self._buf + c._count = self._count + c._h = list(self._h) + return c + + def _padded(self, length_bytes, order): + bs = self.block_size + bits = self._count * 8 + tail = self._buf + b"\x80" + pad = (bs - length_bytes - len(tail)) % bs + return tail + b"\x00" * pad + bits.to_bytes(length_bytes, order) + + def digest(self): + c = self.copy() + tail = c._padded(self._LEN_BYTES, self._ORDER) + for i in range(0, len(tail), self.block_size): + c._compress(tail[i:i + self.block_size]) + w = self._WORD + out = b"".join(v.to_bytes(w, self._ORDER) for v in c._h) + return out[:self.digest_size] + + def hexdigest(self): + return self.digest().hex() + + +class _MD5(_Hash): + name = "md5" + digest_size = 16 + block_size = 64 + _IV = (0x67452301, 0xEFCDAB89, 0x98BADCFE, 0x10325476) + _LEN_BYTES = 8 + _ORDER = "little" + _WORD = 4 + _S = (7, 12, 17, 22) * 4 + (5, 9, 14, 20) * 4 + (4, 11, 16, 23) * 4 + (6, 10, 15, 21) * 4 + _K = (0xd76aa478, 0xe8c7b756, 0x242070db, 0xc1bdceee, 0xf57c0faf, 0x4787c62a, 0xa8304613, + 0xfd469501, 0x698098d8, 0x8b44f7af, 0xffff5bb1, 0x895cd7be, 0x6b901122, 0xfd987193, + 0xa679438e, 0x49b40821, 0xf61e2562, 0xc040b340, 0x265e5a51, 0xe9b6c7aa, 0xd62f105d, + 0x02441453, 0xd8a1e681, 0xe7d3fbc8, 0x21e1cde6, 0xc33707d6, 0xf4d50d87, 0x455a14ed, + 0xa9e3e905, 0xfcefa3f8, 0x676f02d9, 0x8d2a4c8a, 0xfffa3942, 0x8771f681, 0x6d9d6122, + 0xfde5380c, 0xa4beea44, 0x4bdecfa9, 0xf6bb4b60, 0xbebfbc70, 0x289b7ec6, 0xeaa127fa, + 0xd4ef3085, 0x04881d05, 0xd9d4d039, 0xe6db99e5, 0x1fa27cf8, 0xc4ac5665, 0xf4292244, + 0x432aff97, 0xab9423a7, 0xfc93a039, 0x655b59c3, 0x8f0ccc92, 0xffeff47d, 0x85845dd1, + 0x6fa87e4f, 0xfe2ce6e0, 0xa3014314, 0x4e0811a1, 0xf7537e82, 0xbd3af235, 0x2ad7d2bb, + 0xeb86d391) + + def _compress(self, block): + m = [int.from_bytes(block[i:i + 4], "little") for i in range(0, 64, 4)] + a, b, c, d = self._h + for i in range(64): + if i < 16: + f = (b & c) | (~b & d) + g = i + elif i < 32: + f = (d & b) | (~d & c) + g = (5 * i + 1) % 16 + elif i < 48: + f = b ^ c ^ d + g = (3 * i + 5) % 16 + else: + f = c ^ (b | (~d & _M32)) + g = (7 * i) % 16 + f = (f + a + self._K[i] + m[g]) & _M32 + a = d + d = c + c = b + b = (b + _rotl32(f, self._S[i])) & _M32 + h = self._h + self._h = [(h[0] + a) & _M32, (h[1] + b) & _M32, (h[2] + c) & _M32, (h[3] + d) & _M32] + + +class _SHA1(_Hash): + name = "sha1" + digest_size = 20 + block_size = 64 + _IV = (0x67452301, 0xEFCDAB89, 0x98BADCFE, 0x10325476, 0xC3D2E1F0) + _LEN_BYTES = 8 + _ORDER = "big" + _WORD = 4 + + def _compress(self, block): + w = [int.from_bytes(block[i:i + 4], "big") for i in range(0, 64, 4)] + for i in range(16, 80): + w.append(_rotl32(w[i - 3] ^ w[i - 8] ^ w[i - 14] ^ w[i - 16], 1)) + a, b, c, d, e = self._h + for i in range(80): + if i < 20: + f = (b & c) | (~b & d) + k = 0x5A827999 + elif i < 40: + f = b ^ c ^ d + k = 0x6ED9EBA1 + elif i < 60: + f = (b & c) | (b & d) | (c & d) + k = 0x8F1BBCDC + else: + f = b ^ c ^ d + k = 0xCA62C1D6 + t = (_rotl32(a, 5) + (f & _M32) + e + k + w[i]) & _M32 + e = d + d = c + c = _rotl32(b, 30) + b = a + a = t + self._h = [(x + y) & _M32 for x, y in zip(self._h, (a, b, c, d, e))] + + +_K256 = (0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, + 0xab1c5ed5, 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, + 0x9bdc06a7, 0xc19bf174, 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, + 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, + 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, + 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, 0xa2bfe8a1, 0xa81a664b, + 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, 0x19a4c116, + 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, + 0xc67178f2) + + +class _SHA256(_Hash): + name = "sha256" + digest_size = 32 + block_size = 64 + _IV = (0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab, + 0x5be0cd19) + _LEN_BYTES = 8 + _ORDER = "big" + _WORD = 4 + + def _compress(self, block): + w = [int.from_bytes(block[i:i + 4], "big") for i in range(0, 64, 4)] + for i in range(16, 64): + x = w[i - 15] + y = w[i - 2] + s0 = _rotr32(x, 7) ^ _rotr32(x, 18) ^ (x >> 3) + s1 = _rotr32(y, 17) ^ _rotr32(y, 19) ^ (y >> 10) + w.append((w[i - 16] + s0 + w[i - 7] + s1) & _M32) + a, b, c, d, e, f, g, h = self._h + for i in range(64): + s1 = _rotr32(e, 6) ^ _rotr32(e, 11) ^ _rotr32(e, 25) + ch = (e & f) ^ (~e & g) + t1 = (h + s1 + ch + _K256[i] + w[i]) & _M32 + s0 = _rotr32(a, 2) ^ _rotr32(a, 13) ^ _rotr32(a, 22) + maj = (a & b) ^ (a & c) ^ (b & c) + t2 = (s0 + maj) & _M32 + h = g + g = f + f = e + e = (d + t1) & _M32 + d = c + c = b + b = a + a = (t1 + t2) & _M32 + self._h = [(x + y) & _M32 for x, y in zip(self._h, (a, b, c, d, e, f, g, h))] + + +class _SHA224(_SHA256): + name = "sha224" + digest_size = 28 + _IV = (0xc1059ed8, 0x367cd507, 0x3070dd17, 0xf70e5939, 0xffc00b31, 0x68581511, 0x64f98fa7, + 0xbefa4fa4) + + +_K512 = (0x428a2f98d728ae22, 0x7137449123ef65cd, 0xb5c0fbcfec4d3b2f, 0xe9b5dba58189dbbc, + 0x3956c25bf348b538, 0x59f111f1b605d019, 0x923f82a4af194f9b, 0xab1c5ed5da6d8118, + 0xd807aa98a3030242, 0x12835b0145706fbe, 0x243185be4ee4b28c, 0x550c7dc3d5ffb4e2, + 0x72be5d74f27b896f, 0x80deb1fe3b1696b1, 0x9bdc06a725c71235, 0xc19bf174cf692694, + 0xe49b69c19ef14ad2, 0xefbe4786384f25e3, 0x0fc19dc68b8cd5b5, 0x240ca1cc77ac9c65, + 0x2de92c6f592b0275, 0x4a7484aa6ea6e483, 0x5cb0a9dcbd41fbd4, 0x76f988da831153b5, + 0x983e5152ee66dfab, 0xa831c66d2db43210, 0xb00327c898fb213f, 0xbf597fc7beef0ee4, + 0xc6e00bf33da88fc2, 0xd5a79147930aa725, 0x06ca6351e003826f, 0x142929670a0e6e70, + 0x27b70a8546d22ffc, 0x2e1b21385c26c926, 0x4d2c6dfc5ac42aed, 0x53380d139d95b3df, + 0x650a73548baf63de, 0x766a0abb3c77b2a8, 0x81c2c92e47edaee6, 0x92722c851482353b, + 0xa2bfe8a14cf10364, 0xa81a664bbc423001, 0xc24b8b70d0f89791, 0xc76c51a30654be30, + 0xd192e819d6ef5218, 0xd69906245565a910, 0xf40e35855771202a, 0x106aa07032bbd1b8, + 0x19a4c116b8d2d0c8, 0x1e376c085141ab53, 0x2748774cdf8eeb99, 0x34b0bcb5e19b48a8, + 0x391c0cb3c5c95a63, 0x4ed8aa4ae3418acb, 0x5b9cca4f7763e373, 0x682e6ff3d6b2b8a3, + 0x748f82ee5defb2fc, 0x78a5636f43172f60, 0x84c87814a1f0ab72, 0x8cc702081a6439ec, + 0x90befffa23631e28, 0xa4506cebde82bde9, 0xbef9a3f7b2c67915, 0xc67178f2e372532b, + 0xca273eceea26619c, 0xd186b8c721c0c207, 0xeada7dd6cde0eb1e, 0xf57d4f7fee6ed178, + 0x06f067aa72176fba, 0x0a637dc5a2c898a6, 0x113f9804bef90dae, 0x1b710b35131c471b, + 0x28db77f523047d84, 0x32caab7b40c72493, 0x3c9ebe0a15c9bebc, 0x431d67c49c100d4c, + 0x4cc5d4becb3e42b6, 0x597f299cfc657e2a, 0x5fcb6fab3ad6faec, 0x6c44198c4a475817) + + +class _SHA512(_Hash): + name = "sha512" + digest_size = 64 + block_size = 128 + _IV = (0x6a09e667f3bcc908, 0xbb67ae8584caa73b, 0x3c6ef372fe94f82b, 0xa54ff53a5f1d36f1, + 0x510e527fade682d1, 0x9b05688c2b3e6c1f, 0x1f83d9abfb41bd6b, 0x5be0cd19137e2179) + _LEN_BYTES = 16 + _ORDER = "big" + _WORD = 8 + + def _compress(self, block): + w = [int.from_bytes(block[i:i + 8], "big") for i in range(0, 128, 8)] + for i in range(16, 80): + x = w[i - 15] + y = w[i - 2] + s0 = _rotr64(x, 1) ^ _rotr64(x, 8) ^ (x >> 7) + s1 = _rotr64(y, 19) ^ _rotr64(y, 61) ^ (y >> 6) + w.append((w[i - 16] + s0 + w[i - 7] + s1) & _M64) + a, b, c, d, e, f, g, h = self._h + for i in range(80): + s1 = _rotr64(e, 14) ^ _rotr64(e, 18) ^ _rotr64(e, 41) + ch = (e & f) ^ (~e & g) + t1 = (h + s1 + ch + _K512[i] + w[i]) & _M64 + s0 = _rotr64(a, 28) ^ _rotr64(a, 34) ^ _rotr64(a, 39) + maj = (a & b) ^ (a & c) ^ (b & c) + t2 = (s0 + maj) & _M64 + h = g + g = f + f = e + e = (d + t1) & _M64 + d = c + c = b + b = a + a = (t1 + t2) & _M64 + self._h = [(x + y) & _M64 for x, y in zip(self._h, (a, b, c, d, e, f, g, h))] + + +class _SHA384(_SHA512): + name = "sha384" + digest_size = 48 + _IV = (0xcbbb9d5dc1059ed8, 0x629a292a367cd507, 0x9159015a3070dd17, 0x152fecd8f70e5939, + 0x67332667ffc00b31, 0x8eb44a8768581511, 0xdb0c2e0d64f98fa7, 0x47b5481dbefa4fa4) + + +_BY_NAME = {"md5": _MD5, "sha1": _SHA1, "sha224": _SHA224, "sha256": _SHA256, + "sha384": _SHA384, "sha512": _SHA512} + + +def md5(data=b"", *, usedforsecurity=True): + return _MD5(data) + + +def sha1(data=b"", *, usedforsecurity=True): + return _SHA1(data) + + +def sha224(data=b"", *, usedforsecurity=True): + return _SHA224(data) + + +def sha256(data=b"", *, usedforsecurity=True): + return _SHA256(data) + + +def sha384(data=b"", *, usedforsecurity=True): + return _SHA384(data) + + +def sha512(data=b"", *, usedforsecurity=True): + return _SHA512(data) + + +def new(name, data=b"", **kwargs): + cls = _BY_NAME.get(name.lower().replace("-", "")) if isinstance(name, str) else None + if cls is None: + raise ValueError("unsupported hash type " + str(name)) + return cls(data) + + +def file_digest(fileobj, digest, /): + h = new(digest) if isinstance(digest, str) else digest() + while True: + chunk = fileobj.read(65536) + if not chunk: + return h + h.update(chunk) + + +def _hmac(cls, key, msg): + bs = cls.block_size + if len(key) > bs: + key = cls(key).digest() + key = key + b"\x00" * (bs - len(key)) + inner = cls(bytes(b ^ 0x36 for b in key)) + inner.update(msg) + outer = cls(bytes(b ^ 0x5C for b in key)) + outer.update(inner.digest()) + return outer.digest() + + +def pbkdf2_hmac(hash_name, password, salt, iterations, dklen=None): + cls = new(hash_name).__class__ + password = _as_bytes(password) + salt = _as_bytes(salt) + if iterations < 1: + raise ValueError("iteration value must be greater than 0.") + if dklen is None: + dklen = cls.digest_size + out = b"" + block = 1 + while len(out) < dklen: + u = _hmac(cls, password, salt + block.to_bytes(4, "big")) + acc = int.from_bytes(u, "big") + for _ in range(iterations - 1): + u = _hmac(cls, password, u) + acc ^= int.from_bytes(u, "big") + out += acc.to_bytes(cls.digest_size, "big") + block += 1 + return out[:dklen] diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/heapq.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/heapq.py new file mode 100644 index 00000000000..195625f6e7d --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/heapq.py @@ -0,0 +1,107 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Heap queue algorithm: a list kept as a binary min-heap.""" + +__all__ = ["heappush", "heappop", "heapify", "heapreplace", "heappushpop", "nlargest", "nsmallest", + "merge"] + + +def _up(heap, start, pos): + item = heap[pos] + while pos > start: + parent = (pos - 1) >> 1 + p = heap[parent] + if item < p: + heap[pos] = p + pos = parent + else: + break + heap[pos] = item + + +def _down(heap, pos): + end = len(heap) + start = pos + item = heap[pos] + child = 2 * pos + 1 + while child < end: + right = child + 1 + if right < end and not heap[child] < heap[right]: + child = right + heap[pos] = heap[child] + pos = child + child = 2 * pos + 1 + heap[pos] = item + _up(heap, start, pos) + + +def heappush(heap, item): + heap.append(item) + _up(heap, 0, len(heap) - 1) + + +def heappop(heap): + last = heap.pop() + if heap: + item = heap[0] + heap[0] = last + _down(heap, 0) + return item + return last + + +def heapreplace(heap, item): + top = heap[0] + heap[0] = item + _down(heap, 0) + return top + + +def heappushpop(heap, item): + if heap and heap[0] < item: + item, heap[0] = heap[0], item + _down(heap, 0) + return item + + +def heapify(x): + for i in reversed(range(len(x) // 2)): + _down(x, i) + + +def nsmallest(n, iterable, key=None): + if key is None: + return sorted(iterable)[:n] + return sorted(iterable, key=key)[:n] + + +def nlargest(n, iterable, key=None): + if key is None: + return sorted(iterable, reverse=True)[:n] + return sorted(iterable, key=key, reverse=True)[:n] + + +def merge(*iterables, key=None, reverse=False): + """Lazily merges sorted inputs; equal items come in input order.""" + its = [iter(i) for i in iterables] + heads = [] + for idx in range(len(its)): + for v in its[idx]: + heads.append([v if key is None else key(v), idx, v]) + break + while heads: + best = 0 + for i in range(1, len(heads)): + if reverse: + if heads[best][0] < heads[i][0]: + best = i + elif heads[i][0] < heads[best][0]: + best = i + idx = heads[best][1] + yield heads[best][2] + for v in its[idx]: + heads[best] = [v if key is None else key(v), idx, v] + break + else: + del heads[best] diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/html/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/html/__init__.py new file mode 100644 index 00000000000..8b8bc529492 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/html/__init__.py @@ -0,0 +1,73 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""General functions for HTML manipulation.""" + +import re as _re +from html.entities import html5 as _html5 + +__all__ = ["escape", "unescape"] + + +def escape(s, quote=True): + s = s.replace("&", "&") + s = s.replace("<", "<") + s = s.replace(">", ">") + if quote: + s = s.replace('"', """) + s = s.replace("'", "'") + return s + + +_invalid_charrefs = { + 0x00: "�", 0x0d: "\r", 0x80: "€", 0x81: "\x81", 0x82: "‚", 0x83: "ƒ", + 0x84: "„", 0x85: "…", 0x86: "†", 0x87: "‡", 0x88: "ˆ", 0x89: "‰", + 0x8a: "Š", 0x8b: "‹", 0x8c: "Œ", 0x8d: "\x8d", 0x8e: "Ž", 0x8f: "\x8f", + 0x90: "\x90", 0x91: "‘", 0x92: "’", 0x93: "“", 0x94: "”", 0x95: "•", + 0x96: "–", 0x97: "—", 0x98: "˜", 0x99: "™", 0x9a: "š", 0x9b: "›", + 0x9c: "œ", 0x9d: "\x9d", 0x9e: "ž", 0x9f: "Ÿ", +} + +_invalid_codepoints = { + 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0xe, 0xf, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x7f, 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, + 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, + 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, 0xfdd0, 0xfdd1, 0xfdd2, 0xfdd3, 0xfdd4, + 0xfdd5, 0xfdd6, 0xfdd7, 0xfdd8, 0xfdd9, 0xfdda, 0xfddb, 0xfddc, 0xfddd, 0xfdde, 0xfddf, 0xfde0, + 0xfde1, 0xfde2, 0xfde3, 0xfde4, 0xfde5, 0xfde6, 0xfde7, 0xfde8, 0xfde9, 0xfdea, 0xfdeb, 0xfdec, + 0xfded, 0xfdee, 0xfdef, 0xb, 0xfffe, 0xffff, 0x1fffe, 0x1ffff, 0x2fffe, 0x2ffff, 0x3fffe, 0x3ffff, + 0x4fffe, 0x4ffff, 0x5fffe, 0x5ffff, 0x6fffe, 0x6ffff, 0x7fffe, 0x7ffff, 0x8fffe, 0x8ffff, 0x9fffe, + 0x9ffff, 0xafffe, 0xaffff, 0xbfffe, 0xbffff, 0xcfffe, 0xcffff, 0xdfffe, 0xdffff, 0xefffe, 0xeffff, + 0xffffe, 0xfffff, 0x10fffe, 0x10ffff, +} + + +def _replace_charref(s): + s = s.group(1) + if s[0] == "#": + if s[1] in "xX": + num = int(s[2:].rstrip(";"), 16) + else: + num = int(s[1:].rstrip(";")) + if num in _invalid_charrefs: + return _invalid_charrefs[num] + if 0xD800 <= num <= 0xDFFF or num > 0x10FFFF: + return "�" + if num in _invalid_codepoints: + return "" + return chr(num) + if s in _html5: + return _html5[s] + for x in range(len(s) - 1, 1, -1): + if s[:x] in _html5: + return _html5[s[:x]] + s[x:] + return "&" + s + + +_charref = _re.compile(r"&(#[0-9]+;?|#[xX][0-9a-fA-F]+;?|[^\t\n\f <&#;]{1,32};?)") + + +def unescape(s): + if "&" not in s: + return s + return _charref.sub(_replace_charref, s) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/html/entities.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/html/entities.py new file mode 100644 index 00000000000..b8df1e6deaa --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/html/entities.py @@ -0,0 +1,2503 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""HTML character entity references, as the HTML5 specification lists them.""" + +__all__ = ["html5", "name2codepoint", "codepoint2name", "entitydefs"] + +name2codepoint = { + "AElig": 0x00c6, + "Aacute": 0x00c1, + "Acirc": 0x00c2, + "Agrave": 0x00c0, + "Alpha": 0x0391, + "Aring": 0x00c5, + "Atilde": 0x00c3, + "Auml": 0x00c4, + "Beta": 0x0392, + "Ccedil": 0x00c7, + "Chi": 0x03a7, + "Dagger": 0x2021, + "Delta": 0x0394, + "ETH": 0x00d0, + "Eacute": 0x00c9, + "Ecirc": 0x00ca, + "Egrave": 0x00c8, + "Epsilon": 0x0395, + "Eta": 0x0397, + "Euml": 0x00cb, + "Gamma": 0x0393, + "Iacute": 0x00cd, + "Icirc": 0x00ce, + "Igrave": 0x00cc, + "Iota": 0x0399, + "Iuml": 0x00cf, + "Kappa": 0x039a, + "Lambda": 0x039b, + "Mu": 0x039c, + "Ntilde": 0x00d1, + "Nu": 0x039d, + "OElig": 0x0152, + "Oacute": 0x00d3, + "Ocirc": 0x00d4, + "Ograve": 0x00d2, + "Omega": 0x03a9, + "Omicron": 0x039f, + "Oslash": 0x00d8, + "Otilde": 0x00d5, + "Ouml": 0x00d6, + "Phi": 0x03a6, + "Pi": 0x03a0, + "Prime": 0x2033, + "Psi": 0x03a8, + "Rho": 0x03a1, + "Scaron": 0x0160, + "Sigma": 0x03a3, + "THORN": 0x00de, + "Tau": 0x03a4, + "Theta": 0x0398, + "Uacute": 0x00da, + "Ucirc": 0x00db, + "Ugrave": 0x00d9, + "Upsilon": 0x03a5, + "Uuml": 0x00dc, + "Xi": 0x039e, + "Yacute": 0x00dd, + "Yuml": 0x0178, + "Zeta": 0x0396, + "aacute": 0x00e1, + "acirc": 0x00e2, + "acute": 0x00b4, + "aelig": 0x00e6, + "agrave": 0x00e0, + "alefsym": 0x2135, + "alpha": 0x03b1, + "amp": 0x0026, + "and": 0x2227, + "ang": 0x2220, + "aring": 0x00e5, + "asymp": 0x2248, + "atilde": 0x00e3, + "auml": 0x00e4, + "bdquo": 0x201e, + "beta": 0x03b2, + "brvbar": 0x00a6, + "bull": 0x2022, + "cap": 0x2229, + "ccedil": 0x00e7, + "cedil": 0x00b8, + "cent": 0x00a2, + "chi": 0x03c7, + "circ": 0x02c6, + "clubs": 0x2663, + "cong": 0x2245, + "copy": 0x00a9, + "crarr": 0x21b5, + "cup": 0x222a, + "curren": 0x00a4, + "dArr": 0x21d3, + "dagger": 0x2020, + "darr": 0x2193, + "deg": 0x00b0, + "delta": 0x03b4, + "diams": 0x2666, + "divide": 0x00f7, + "eacute": 0x00e9, + "ecirc": 0x00ea, + "egrave": 0x00e8, + "empty": 0x2205, + "emsp": 0x2003, + "ensp": 0x2002, + "epsilon": 0x03b5, + "equiv": 0x2261, + "eta": 0x03b7, + "eth": 0x00f0, + "euml": 0x00eb, + "euro": 0x20ac, + "exist": 0x2203, + "fnof": 0x0192, + "forall": 0x2200, + "frac12": 0x00bd, + "frac14": 0x00bc, + "frac34": 0x00be, + "frasl": 0x2044, + "gamma": 0x03b3, + "ge": 0x2265, + "gt": 0x003e, + "hArr": 0x21d4, + "harr": 0x2194, + "hearts": 0x2665, + "hellip": 0x2026, + "iacute": 0x00ed, + "icirc": 0x00ee, + "iexcl": 0x00a1, + "igrave": 0x00ec, + "image": 0x2111, + "infin": 0x221e, + "int": 0x222b, + "iota": 0x03b9, + "iquest": 0x00bf, + "isin": 0x2208, + "iuml": 0x00ef, + "kappa": 0x03ba, + "lArr": 0x21d0, + "lambda": 0x03bb, + "lang": 0x2329, + "laquo": 0x00ab, + "larr": 0x2190, + "lceil": 0x2308, + "ldquo": 0x201c, + "le": 0x2264, + "lfloor": 0x230a, + "lowast": 0x2217, + "loz": 0x25ca, + "lrm": 0x200e, + "lsaquo": 0x2039, + "lsquo": 0x2018, + "lt": 0x003c, + "macr": 0x00af, + "mdash": 0x2014, + "micro": 0x00b5, + "middot": 0x00b7, + "minus": 0x2212, + "mu": 0x03bc, + "nabla": 0x2207, + "nbsp": 0x00a0, + "ndash": 0x2013, + "ne": 0x2260, + "ni": 0x220b, + "not": 0x00ac, + "notin": 0x2209, + "nsub": 0x2284, + "ntilde": 0x00f1, + "nu": 0x03bd, + "oacute": 0x00f3, + "ocirc": 0x00f4, + "oelig": 0x0153, + "ograve": 0x00f2, + "oline": 0x203e, + "omega": 0x03c9, + "omicron": 0x03bf, + "oplus": 0x2295, + "or": 0x2228, + "ordf": 0x00aa, + "ordm": 0x00ba, + "oslash": 0x00f8, + "otilde": 0x00f5, + "otimes": 0x2297, + "ouml": 0x00f6, + "para": 0x00b6, + "part": 0x2202, + "permil": 0x2030, + "perp": 0x22a5, + "phi": 0x03c6, + "pi": 0x03c0, + "piv": 0x03d6, + "plusmn": 0x00b1, + "pound": 0x00a3, + "prime": 0x2032, + "prod": 0x220f, + "prop": 0x221d, + "psi": 0x03c8, + "quot": 0x0022, + "rArr": 0x21d2, + "radic": 0x221a, + "rang": 0x232a, + "raquo": 0x00bb, + "rarr": 0x2192, + "rceil": 0x2309, + "rdquo": 0x201d, + "real": 0x211c, + "reg": 0x00ae, + "rfloor": 0x230b, + "rho": 0x03c1, + "rlm": 0x200f, + "rsaquo": 0x203a, + "rsquo": 0x2019, + "sbquo": 0x201a, + "scaron": 0x0161, + "sdot": 0x22c5, + "sect": 0x00a7, + "shy": 0x00ad, + "sigma": 0x03c3, + "sigmaf": 0x03c2, + "sim": 0x223c, + "spades": 0x2660, + "sub": 0x2282, + "sube": 0x2286, + "sum": 0x2211, + "sup": 0x2283, + "sup1": 0x00b9, + "sup2": 0x00b2, + "sup3": 0x00b3, + "supe": 0x2287, + "szlig": 0x00df, + "tau": 0x03c4, + "there4": 0x2234, + "theta": 0x03b8, + "thetasym": 0x03d1, + "thinsp": 0x2009, + "thorn": 0x00fe, + "tilde": 0x02dc, + "times": 0x00d7, + "trade": 0x2122, + "uArr": 0x21d1, + "uacute": 0x00fa, + "uarr": 0x2191, + "ucirc": 0x00fb, + "ugrave": 0x00f9, + "uml": 0x00a8, + "upsih": 0x03d2, + "upsilon": 0x03c5, + "uuml": 0x00fc, + "weierp": 0x2118, + "xi": 0x03be, + "yacute": 0x00fd, + "yen": 0x00a5, + "yuml": 0x00ff, + "zeta": 0x03b6, + "zwj": 0x200d, + "zwnj": 0x200c, +} + +html5 = { + "Aacute": "\u00c1", + "aacute": "\u00e1", + "Aacute;": "\u00c1", + "aacute;": "\u00e1", + "Abreve;": "\u0102", + "abreve;": "\u0103", + "ac;": "\u223e", + "acd;": "\u223f", + "acE;": "\u223e\u0333", + "Acirc": "\u00c2", + "acirc": "\u00e2", + "Acirc;": "\u00c2", + "acirc;": "\u00e2", + "acute": "\u00b4", + "acute;": "\u00b4", + "Acy;": "\u0410", + "acy;": "\u0430", + "AElig": "\u00c6", + "aelig": "\u00e6", + "AElig;": "\u00c6", + "aelig;": "\u00e6", + "af;": "\u2061", + "Afr;": "\U0001d504", + "afr;": "\U0001d51e", + "Agrave": "\u00c0", + "agrave": "\u00e0", + "Agrave;": "\u00c0", + "agrave;": "\u00e0", + "alefsym;": "\u2135", + "aleph;": "\u2135", + "Alpha;": "\u0391", + "alpha;": "\u03b1", + "Amacr;": "\u0100", + "amacr;": "\u0101", + "amalg;": "\u2a3f", + "AMP": "&", + "amp": "&", + "AMP;": "&", + "amp;": "&", + "And;": "\u2a53", + "and;": "\u2227", + "andand;": "\u2a55", + "andd;": "\u2a5c", + "andslope;": "\u2a58", + "andv;": "\u2a5a", + "ang;": "\u2220", + "ange;": "\u29a4", + "angle;": "\u2220", + "angmsd;": "\u2221", + "angmsdaa;": "\u29a8", + "angmsdab;": "\u29a9", + "angmsdac;": "\u29aa", + "angmsdad;": "\u29ab", + "angmsdae;": "\u29ac", + "angmsdaf;": "\u29ad", + "angmsdag;": "\u29ae", + "angmsdah;": "\u29af", + "angrt;": "\u221f", + "angrtvb;": "\u22be", + "angrtvbd;": "\u299d", + "angsph;": "\u2222", + "angst;": "\u00c5", + "angzarr;": "\u237c", + "Aogon;": "\u0104", + "aogon;": "\u0105", + "Aopf;": "\U0001d538", + "aopf;": "\U0001d552", + "ap;": "\u2248", + "apacir;": "\u2a6f", + "apE;": "\u2a70", + "ape;": "\u224a", + "apid;": "\u224b", + "apos;": "'", + "ApplyFunction;": "\u2061", + "approx;": "\u2248", + "approxeq;": "\u224a", + "Aring": "\u00c5", + "aring": "\u00e5", + "Aring;": "\u00c5", + "aring;": "\u00e5", + "Ascr;": "\U0001d49c", + "ascr;": "\U0001d4b6", + "Assign;": "\u2254", + "ast;": "*", + "asymp;": "\u2248", + "asympeq;": "\u224d", + "Atilde": "\u00c3", + "atilde": "\u00e3", + "Atilde;": "\u00c3", + "atilde;": "\u00e3", + "Auml": "\u00c4", + "auml": "\u00e4", + "Auml;": "\u00c4", + "auml;": "\u00e4", + "awconint;": "\u2233", + "awint;": "\u2a11", + "backcong;": "\u224c", + "backepsilon;": "\u03f6", + "backprime;": "\u2035", + "backsim;": "\u223d", + "backsimeq;": "\u22cd", + "Backslash;": "\u2216", + "Barv;": "\u2ae7", + "barvee;": "\u22bd", + "Barwed;": "\u2306", + "barwed;": "\u2305", + "barwedge;": "\u2305", + "bbrk;": "\u23b5", + "bbrktbrk;": "\u23b6", + "bcong;": "\u224c", + "Bcy;": "\u0411", + "bcy;": "\u0431", + "bdquo;": "\u201e", + "becaus;": "\u2235", + "Because;": "\u2235", + "because;": "\u2235", + "bemptyv;": "\u29b0", + "bepsi;": "\u03f6", + "bernou;": "\u212c", + "Bernoullis;": "\u212c", + "Beta;": "\u0392", + "beta;": "\u03b2", + "beth;": "\u2136", + "between;": "\u226c", + "Bfr;": "\U0001d505", + "bfr;": "\U0001d51f", + "bigcap;": "\u22c2", + "bigcirc;": "\u25ef", + "bigcup;": "\u22c3", + "bigodot;": "\u2a00", + "bigoplus;": "\u2a01", + "bigotimes;": "\u2a02", + "bigsqcup;": "\u2a06", + "bigstar;": "\u2605", + "bigtriangledown;": "\u25bd", + "bigtriangleup;": "\u25b3", + "biguplus;": "\u2a04", + "bigvee;": "\u22c1", + "bigwedge;": "\u22c0", + "bkarow;": "\u290d", + "blacklozenge;": "\u29eb", + "blacksquare;": "\u25aa", + "blacktriangle;": "\u25b4", + "blacktriangledown;": "\u25be", + "blacktriangleleft;": "\u25c2", + "blacktriangleright;": "\u25b8", + "blank;": "\u2423", + "blk12;": "\u2592", + "blk14;": "\u2591", + "blk34;": "\u2593", + "block;": "\u2588", + "bne;": "=\u20e5", + "bnequiv;": "\u2261\u20e5", + "bNot;": "\u2aed", + "bnot;": "\u2310", + "Bopf;": "\U0001d539", + "bopf;": "\U0001d553", + "bot;": "\u22a5", + "bottom;": "\u22a5", + "bowtie;": "\u22c8", + "boxbox;": "\u29c9", + "boxDL;": "\u2557", + "boxDl;": "\u2556", + "boxdL;": "\u2555", + "boxdl;": "\u2510", + "boxDR;": "\u2554", + "boxDr;": "\u2553", + "boxdR;": "\u2552", + "boxdr;": "\u250c", + "boxH;": "\u2550", + "boxh;": "\u2500", + "boxHD;": "\u2566", + "boxHd;": "\u2564", + "boxhD;": "\u2565", + "boxhd;": "\u252c", + "boxHU;": "\u2569", + "boxHu;": "\u2567", + "boxhU;": "\u2568", + "boxhu;": "\u2534", + "boxminus;": "\u229f", + "boxplus;": "\u229e", + "boxtimes;": "\u22a0", + "boxUL;": "\u255d", + "boxUl;": "\u255c", + "boxuL;": "\u255b", + "boxul;": "\u2518", + "boxUR;": "\u255a", + "boxUr;": "\u2559", + "boxuR;": "\u2558", + "boxur;": "\u2514", + "boxV;": "\u2551", + "boxv;": "\u2502", + "boxVH;": "\u256c", + "boxVh;": "\u256b", + "boxvH;": "\u256a", + "boxvh;": "\u253c", + "boxVL;": "\u2563", + "boxVl;": "\u2562", + "boxvL;": "\u2561", + "boxvl;": "\u2524", + "boxVR;": "\u2560", + "boxVr;": "\u255f", + "boxvR;": "\u255e", + "boxvr;": "\u251c", + "bprime;": "\u2035", + "Breve;": "\u02d8", + "breve;": "\u02d8", + "brvbar": "\u00a6", + "brvbar;": "\u00a6", + "Bscr;": "\u212c", + "bscr;": "\U0001d4b7", + "bsemi;": "\u204f", + "bsim;": "\u223d", + "bsime;": "\u22cd", + "bsol;": "\\", + "bsolb;": "\u29c5", + "bsolhsub;": "\u27c8", + "bull;": "\u2022", + "bullet;": "\u2022", + "bump;": "\u224e", + "bumpE;": "\u2aae", + "bumpe;": "\u224f", + "Bumpeq;": "\u224e", + "bumpeq;": "\u224f", + "Cacute;": "\u0106", + "cacute;": "\u0107", + "Cap;": "\u22d2", + "cap;": "\u2229", + "capand;": "\u2a44", + "capbrcup;": "\u2a49", + "capcap;": "\u2a4b", + "capcup;": "\u2a47", + "capdot;": "\u2a40", + "CapitalDifferentialD;": "\u2145", + "caps;": "\u2229\ufe00", + "caret;": "\u2041", + "caron;": "\u02c7", + "Cayleys;": "\u212d", + "ccaps;": "\u2a4d", + "Ccaron;": "\u010c", + "ccaron;": "\u010d", + "Ccedil": "\u00c7", + "ccedil": "\u00e7", + "Ccedil;": "\u00c7", + "ccedil;": "\u00e7", + "Ccirc;": "\u0108", + "ccirc;": "\u0109", + "Cconint;": "\u2230", + "ccups;": "\u2a4c", + "ccupssm;": "\u2a50", + "Cdot;": "\u010a", + "cdot;": "\u010b", + "cedil": "\u00b8", + "cedil;": "\u00b8", + "Cedilla;": "\u00b8", + "cemptyv;": "\u29b2", + "cent": "\u00a2", + "cent;": "\u00a2", + "CenterDot;": "\u00b7", + "centerdot;": "\u00b7", + "Cfr;": "\u212d", + "cfr;": "\U0001d520", + "CHcy;": "\u0427", + "chcy;": "\u0447", + "check;": "\u2713", + "checkmark;": "\u2713", + "Chi;": "\u03a7", + "chi;": "\u03c7", + "cir;": "\u25cb", + "circ;": "\u02c6", + "circeq;": "\u2257", + "circlearrowleft;": "\u21ba", + "circlearrowright;": "\u21bb", + "circledast;": "\u229b", + "circledcirc;": "\u229a", + "circleddash;": "\u229d", + "CircleDot;": "\u2299", + "circledR;": "\u00ae", + "circledS;": "\u24c8", + "CircleMinus;": "\u2296", + "CirclePlus;": "\u2295", + "CircleTimes;": "\u2297", + "cirE;": "\u29c3", + "cire;": "\u2257", + "cirfnint;": "\u2a10", + "cirmid;": "\u2aef", + "cirscir;": "\u29c2", + "ClockwiseContourIntegral;": "\u2232", + "CloseCurlyDoubleQuote;": "\u201d", + "CloseCurlyQuote;": "\u2019", + "clubs;": "\u2663", + "clubsuit;": "\u2663", + "Colon;": "\u2237", + "colon;": ":", + "Colone;": "\u2a74", + "colone;": "\u2254", + "coloneq;": "\u2254", + "comma;": ",", + "commat;": "@", + "comp;": "\u2201", + "compfn;": "\u2218", + "complement;": "\u2201", + "complexes;": "\u2102", + "cong;": "\u2245", + "congdot;": "\u2a6d", + "Congruent;": "\u2261", + "Conint;": "\u222f", + "conint;": "\u222e", + "ContourIntegral;": "\u222e", + "Copf;": "\u2102", + "copf;": "\U0001d554", + "coprod;": "\u2210", + "Coproduct;": "\u2210", + "COPY": "\u00a9", + "copy": "\u00a9", + "COPY;": "\u00a9", + "copy;": "\u00a9", + "copysr;": "\u2117", + "CounterClockwiseContourIntegral;": "\u2233", + "crarr;": "\u21b5", + "Cross;": "\u2a2f", + "cross;": "\u2717", + "Cscr;": "\U0001d49e", + "cscr;": "\U0001d4b8", + "csub;": "\u2acf", + "csube;": "\u2ad1", + "csup;": "\u2ad0", + "csupe;": "\u2ad2", + "ctdot;": "\u22ef", + "cudarrl;": "\u2938", + "cudarrr;": "\u2935", + "cuepr;": "\u22de", + "cuesc;": "\u22df", + "cularr;": "\u21b6", + "cularrp;": "\u293d", + "Cup;": "\u22d3", + "cup;": "\u222a", + "cupbrcap;": "\u2a48", + "CupCap;": "\u224d", + "cupcap;": "\u2a46", + "cupcup;": "\u2a4a", + "cupdot;": "\u228d", + "cupor;": "\u2a45", + "cups;": "\u222a\ufe00", + "curarr;": "\u21b7", + "curarrm;": "\u293c", + "curlyeqprec;": "\u22de", + "curlyeqsucc;": "\u22df", + "curlyvee;": "\u22ce", + "curlywedge;": "\u22cf", + "curren": "\u00a4", + "curren;": "\u00a4", + "curvearrowleft;": "\u21b6", + "curvearrowright;": "\u21b7", + "cuvee;": "\u22ce", + "cuwed;": "\u22cf", + "cwconint;": "\u2232", + "cwint;": "\u2231", + "cylcty;": "\u232d", + "Dagger;": "\u2021", + "dagger;": "\u2020", + "daleth;": "\u2138", + "Darr;": "\u21a1", + "dArr;": "\u21d3", + "darr;": "\u2193", + "dash;": "\u2010", + "Dashv;": "\u2ae4", + "dashv;": "\u22a3", + "dbkarow;": "\u290f", + "dblac;": "\u02dd", + "Dcaron;": "\u010e", + "dcaron;": "\u010f", + "Dcy;": "\u0414", + "dcy;": "\u0434", + "DD;": "\u2145", + "dd;": "\u2146", + "ddagger;": "\u2021", + "ddarr;": "\u21ca", + "DDotrahd;": "\u2911", + "ddotseq;": "\u2a77", + "deg": "\u00b0", + "deg;": "\u00b0", + "Del;": "\u2207", + "Delta;": "\u0394", + "delta;": "\u03b4", + "demptyv;": "\u29b1", + "dfisht;": "\u297f", + "Dfr;": "\U0001d507", + "dfr;": "\U0001d521", + "dHar;": "\u2965", + "dharl;": "\u21c3", + "dharr;": "\u21c2", + "DiacriticalAcute;": "\u00b4", + "DiacriticalDot;": "\u02d9", + "DiacriticalDoubleAcute;": "\u02dd", + "DiacriticalGrave;": "`", + "DiacriticalTilde;": "\u02dc", + "diam;": "\u22c4", + "Diamond;": "\u22c4", + "diamond;": "\u22c4", + "diamondsuit;": "\u2666", + "diams;": "\u2666", + "die;": "\u00a8", + "DifferentialD;": "\u2146", + "digamma;": "\u03dd", + "disin;": "\u22f2", + "div;": "\u00f7", + "divide": "\u00f7", + "divide;": "\u00f7", + "divideontimes;": "\u22c7", + "divonx;": "\u22c7", + "DJcy;": "\u0402", + "djcy;": "\u0452", + "dlcorn;": "\u231e", + "dlcrop;": "\u230d", + "dollar;": "$", + "Dopf;": "\U0001d53b", + "dopf;": "\U0001d555", + "Dot;": "\u00a8", + "dot;": "\u02d9", + "DotDot;": "\u20dc", + "doteq;": "\u2250", + "doteqdot;": "\u2251", + "DotEqual;": "\u2250", + "dotminus;": "\u2238", + "dotplus;": "\u2214", + "dotsquare;": "\u22a1", + "doublebarwedge;": "\u2306", + "DoubleContourIntegral;": "\u222f", + "DoubleDot;": "\u00a8", + "DoubleDownArrow;": "\u21d3", + "DoubleLeftArrow;": "\u21d0", + "DoubleLeftRightArrow;": "\u21d4", + "DoubleLeftTee;": "\u2ae4", + "DoubleLongLeftArrow;": "\u27f8", + "DoubleLongLeftRightArrow;": "\u27fa", + "DoubleLongRightArrow;": "\u27f9", + "DoubleRightArrow;": "\u21d2", + "DoubleRightTee;": "\u22a8", + "DoubleUpArrow;": "\u21d1", + "DoubleUpDownArrow;": "\u21d5", + "DoubleVerticalBar;": "\u2225", + "DownArrow;": "\u2193", + "Downarrow;": "\u21d3", + "downarrow;": "\u2193", + "DownArrowBar;": "\u2913", + "DownArrowUpArrow;": "\u21f5", + "DownBreve;": "\u0311", + "downdownarrows;": "\u21ca", + "downharpoonleft;": "\u21c3", + "downharpoonright;": "\u21c2", + "DownLeftRightVector;": "\u2950", + "DownLeftTeeVector;": "\u295e", + "DownLeftVector;": "\u21bd", + "DownLeftVectorBar;": "\u2956", + "DownRightTeeVector;": "\u295f", + "DownRightVector;": "\u21c1", + "DownRightVectorBar;": "\u2957", + "DownTee;": "\u22a4", + "DownTeeArrow;": "\u21a7", + "drbkarow;": "\u2910", + "drcorn;": "\u231f", + "drcrop;": "\u230c", + "Dscr;": "\U0001d49f", + "dscr;": "\U0001d4b9", + "DScy;": "\u0405", + "dscy;": "\u0455", + "dsol;": "\u29f6", + "Dstrok;": "\u0110", + "dstrok;": "\u0111", + "dtdot;": "\u22f1", + "dtri;": "\u25bf", + "dtrif;": "\u25be", + "duarr;": "\u21f5", + "duhar;": "\u296f", + "dwangle;": "\u29a6", + "DZcy;": "\u040f", + "dzcy;": "\u045f", + "dzigrarr;": "\u27ff", + "Eacute": "\u00c9", + "eacute": "\u00e9", + "Eacute;": "\u00c9", + "eacute;": "\u00e9", + "easter;": "\u2a6e", + "Ecaron;": "\u011a", + "ecaron;": "\u011b", + "ecir;": "\u2256", + "Ecirc": "\u00ca", + "ecirc": "\u00ea", + "Ecirc;": "\u00ca", + "ecirc;": "\u00ea", + "ecolon;": "\u2255", + "Ecy;": "\u042d", + "ecy;": "\u044d", + "eDDot;": "\u2a77", + "Edot;": "\u0116", + "eDot;": "\u2251", + "edot;": "\u0117", + "ee;": "\u2147", + "efDot;": "\u2252", + "Efr;": "\U0001d508", + "efr;": "\U0001d522", + "eg;": "\u2a9a", + "Egrave": "\u00c8", + "egrave": "\u00e8", + "Egrave;": "\u00c8", + "egrave;": "\u00e8", + "egs;": "\u2a96", + "egsdot;": "\u2a98", + "el;": "\u2a99", + "Element;": "\u2208", + "elinters;": "\u23e7", + "ell;": "\u2113", + "els;": "\u2a95", + "elsdot;": "\u2a97", + "Emacr;": "\u0112", + "emacr;": "\u0113", + "empty;": "\u2205", + "emptyset;": "\u2205", + "EmptySmallSquare;": "\u25fb", + "emptyv;": "\u2205", + "EmptyVerySmallSquare;": "\u25ab", + "emsp13;": "\u2004", + "emsp14;": "\u2005", + "emsp;": "\u2003", + "ENG;": "\u014a", + "eng;": "\u014b", + "ensp;": "\u2002", + "Eogon;": "\u0118", + "eogon;": "\u0119", + "Eopf;": "\U0001d53c", + "eopf;": "\U0001d556", + "epar;": "\u22d5", + "eparsl;": "\u29e3", + "eplus;": "\u2a71", + "epsi;": "\u03b5", + "Epsilon;": "\u0395", + "epsilon;": "\u03b5", + "epsiv;": "\u03f5", + "eqcirc;": "\u2256", + "eqcolon;": "\u2255", + "eqsim;": "\u2242", + "eqslantgtr;": "\u2a96", + "eqslantless;": "\u2a95", + "Equal;": "\u2a75", + "equals;": "=", + "EqualTilde;": "\u2242", + "equest;": "\u225f", + "Equilibrium;": "\u21cc", + "equiv;": "\u2261", + "equivDD;": "\u2a78", + "eqvparsl;": "\u29e5", + "erarr;": "\u2971", + "erDot;": "\u2253", + "Escr;": "\u2130", + "escr;": "\u212f", + "esdot;": "\u2250", + "Esim;": "\u2a73", + "esim;": "\u2242", + "Eta;": "\u0397", + "eta;": "\u03b7", + "ETH": "\u00d0", + "eth": "\u00f0", + "ETH;": "\u00d0", + "eth;": "\u00f0", + "Euml": "\u00cb", + "euml": "\u00eb", + "Euml;": "\u00cb", + "euml;": "\u00eb", + "euro;": "\u20ac", + "excl;": "!", + "exist;": "\u2203", + "Exists;": "\u2203", + "expectation;": "\u2130", + "ExponentialE;": "\u2147", + "exponentiale;": "\u2147", + "fallingdotseq;": "\u2252", + "Fcy;": "\u0424", + "fcy;": "\u0444", + "female;": "\u2640", + "ffilig;": "\ufb03", + "fflig;": "\ufb00", + "ffllig;": "\ufb04", + "Ffr;": "\U0001d509", + "ffr;": "\U0001d523", + "filig;": "\ufb01", + "FilledSmallSquare;": "\u25fc", + "FilledVerySmallSquare;": "\u25aa", + "fjlig;": "fj", + "flat;": "\u266d", + "fllig;": "\ufb02", + "fltns;": "\u25b1", + "fnof;": "\u0192", + "Fopf;": "\U0001d53d", + "fopf;": "\U0001d557", + "ForAll;": "\u2200", + "forall;": "\u2200", + "fork;": "\u22d4", + "forkv;": "\u2ad9", + "Fouriertrf;": "\u2131", + "fpartint;": "\u2a0d", + "frac12": "\u00bd", + "frac12;": "\u00bd", + "frac13;": "\u2153", + "frac14": "\u00bc", + "frac14;": "\u00bc", + "frac15;": "\u2155", + "frac16;": "\u2159", + "frac18;": "\u215b", + "frac23;": "\u2154", + "frac25;": "\u2156", + "frac34": "\u00be", + "frac34;": "\u00be", + "frac35;": "\u2157", + "frac38;": "\u215c", + "frac45;": "\u2158", + "frac56;": "\u215a", + "frac58;": "\u215d", + "frac78;": "\u215e", + "frasl;": "\u2044", + "frown;": "\u2322", + "Fscr;": "\u2131", + "fscr;": "\U0001d4bb", + "gacute;": "\u01f5", + "Gamma;": "\u0393", + "gamma;": "\u03b3", + "Gammad;": "\u03dc", + "gammad;": "\u03dd", + "gap;": "\u2a86", + "Gbreve;": "\u011e", + "gbreve;": "\u011f", + "Gcedil;": "\u0122", + "Gcirc;": "\u011c", + "gcirc;": "\u011d", + "Gcy;": "\u0413", + "gcy;": "\u0433", + "Gdot;": "\u0120", + "gdot;": "\u0121", + "gE;": "\u2267", + "ge;": "\u2265", + "gEl;": "\u2a8c", + "gel;": "\u22db", + "geq;": "\u2265", + "geqq;": "\u2267", + "geqslant;": "\u2a7e", + "ges;": "\u2a7e", + "gescc;": "\u2aa9", + "gesdot;": "\u2a80", + "gesdoto;": "\u2a82", + "gesdotol;": "\u2a84", + "gesl;": "\u22db\ufe00", + "gesles;": "\u2a94", + "Gfr;": "\U0001d50a", + "gfr;": "\U0001d524", + "Gg;": "\u22d9", + "gg;": "\u226b", + "ggg;": "\u22d9", + "gimel;": "\u2137", + "GJcy;": "\u0403", + "gjcy;": "\u0453", + "gl;": "\u2277", + "gla;": "\u2aa5", + "glE;": "\u2a92", + "glj;": "\u2aa4", + "gnap;": "\u2a8a", + "gnapprox;": "\u2a8a", + "gnE;": "\u2269", + "gne;": "\u2a88", + "gneq;": "\u2a88", + "gneqq;": "\u2269", + "gnsim;": "\u22e7", + "Gopf;": "\U0001d53e", + "gopf;": "\U0001d558", + "grave;": "`", + "GreaterEqual;": "\u2265", + "GreaterEqualLess;": "\u22db", + "GreaterFullEqual;": "\u2267", + "GreaterGreater;": "\u2aa2", + "GreaterLess;": "\u2277", + "GreaterSlantEqual;": "\u2a7e", + "GreaterTilde;": "\u2273", + "Gscr;": "\U0001d4a2", + "gscr;": "\u210a", + "gsim;": "\u2273", + "gsime;": "\u2a8e", + "gsiml;": "\u2a90", + "GT": ">", + "gt": ">", + "GT;": ">", + "Gt;": "\u226b", + "gt;": ">", + "gtcc;": "\u2aa7", + "gtcir;": "\u2a7a", + "gtdot;": "\u22d7", + "gtlPar;": "\u2995", + "gtquest;": "\u2a7c", + "gtrapprox;": "\u2a86", + "gtrarr;": "\u2978", + "gtrdot;": "\u22d7", + "gtreqless;": "\u22db", + "gtreqqless;": "\u2a8c", + "gtrless;": "\u2277", + "gtrsim;": "\u2273", + "gvertneqq;": "\u2269\ufe00", + "gvnE;": "\u2269\ufe00", + "Hacek;": "\u02c7", + "hairsp;": "\u200a", + "half;": "\u00bd", + "hamilt;": "\u210b", + "HARDcy;": "\u042a", + "hardcy;": "\u044a", + "hArr;": "\u21d4", + "harr;": "\u2194", + "harrcir;": "\u2948", + "harrw;": "\u21ad", + "Hat;": "^", + "hbar;": "\u210f", + "Hcirc;": "\u0124", + "hcirc;": "\u0125", + "hearts;": "\u2665", + "heartsuit;": "\u2665", + "hellip;": "\u2026", + "hercon;": "\u22b9", + "Hfr;": "\u210c", + "hfr;": "\U0001d525", + "HilbertSpace;": "\u210b", + "hksearow;": "\u2925", + "hkswarow;": "\u2926", + "hoarr;": "\u21ff", + "homtht;": "\u223b", + "hookleftarrow;": "\u21a9", + "hookrightarrow;": "\u21aa", + "Hopf;": "\u210d", + "hopf;": "\U0001d559", + "horbar;": "\u2015", + "HorizontalLine;": "\u2500", + "Hscr;": "\u210b", + "hscr;": "\U0001d4bd", + "hslash;": "\u210f", + "Hstrok;": "\u0126", + "hstrok;": "\u0127", + "HumpDownHump;": "\u224e", + "HumpEqual;": "\u224f", + "hybull;": "\u2043", + "hyphen;": "\u2010", + "Iacute": "\u00cd", + "iacute": "\u00ed", + "Iacute;": "\u00cd", + "iacute;": "\u00ed", + "ic;": "\u2063", + "Icirc": "\u00ce", + "icirc": "\u00ee", + "Icirc;": "\u00ce", + "icirc;": "\u00ee", + "Icy;": "\u0418", + "icy;": "\u0438", + "Idot;": "\u0130", + "IEcy;": "\u0415", + "iecy;": "\u0435", + "iexcl": "\u00a1", + "iexcl;": "\u00a1", + "iff;": "\u21d4", + "Ifr;": "\u2111", + "ifr;": "\U0001d526", + "Igrave": "\u00cc", + "igrave": "\u00ec", + "Igrave;": "\u00cc", + "igrave;": "\u00ec", + "ii;": "\u2148", + "iiiint;": "\u2a0c", + "iiint;": "\u222d", + "iinfin;": "\u29dc", + "iiota;": "\u2129", + "IJlig;": "\u0132", + "ijlig;": "\u0133", + "Im;": "\u2111", + "Imacr;": "\u012a", + "imacr;": "\u012b", + "image;": "\u2111", + "ImaginaryI;": "\u2148", + "imagline;": "\u2110", + "imagpart;": "\u2111", + "imath;": "\u0131", + "imof;": "\u22b7", + "imped;": "\u01b5", + "Implies;": "\u21d2", + "in;": "\u2208", + "incare;": "\u2105", + "infin;": "\u221e", + "infintie;": "\u29dd", + "inodot;": "\u0131", + "Int;": "\u222c", + "int;": "\u222b", + "intcal;": "\u22ba", + "integers;": "\u2124", + "Integral;": "\u222b", + "intercal;": "\u22ba", + "Intersection;": "\u22c2", + "intlarhk;": "\u2a17", + "intprod;": "\u2a3c", + "InvisibleComma;": "\u2063", + "InvisibleTimes;": "\u2062", + "IOcy;": "\u0401", + "iocy;": "\u0451", + "Iogon;": "\u012e", + "iogon;": "\u012f", + "Iopf;": "\U0001d540", + "iopf;": "\U0001d55a", + "Iota;": "\u0399", + "iota;": "\u03b9", + "iprod;": "\u2a3c", + "iquest": "\u00bf", + "iquest;": "\u00bf", + "Iscr;": "\u2110", + "iscr;": "\U0001d4be", + "isin;": "\u2208", + "isindot;": "\u22f5", + "isinE;": "\u22f9", + "isins;": "\u22f4", + "isinsv;": "\u22f3", + "isinv;": "\u2208", + "it;": "\u2062", + "Itilde;": "\u0128", + "itilde;": "\u0129", + "Iukcy;": "\u0406", + "iukcy;": "\u0456", + "Iuml": "\u00cf", + "iuml": "\u00ef", + "Iuml;": "\u00cf", + "iuml;": "\u00ef", + "Jcirc;": "\u0134", + "jcirc;": "\u0135", + "Jcy;": "\u0419", + "jcy;": "\u0439", + "Jfr;": "\U0001d50d", + "jfr;": "\U0001d527", + "jmath;": "\u0237", + "Jopf;": "\U0001d541", + "jopf;": "\U0001d55b", + "Jscr;": "\U0001d4a5", + "jscr;": "\U0001d4bf", + "Jsercy;": "\u0408", + "jsercy;": "\u0458", + "Jukcy;": "\u0404", + "jukcy;": "\u0454", + "Kappa;": "\u039a", + "kappa;": "\u03ba", + "kappav;": "\u03f0", + "Kcedil;": "\u0136", + "kcedil;": "\u0137", + "Kcy;": "\u041a", + "kcy;": "\u043a", + "Kfr;": "\U0001d50e", + "kfr;": "\U0001d528", + "kgreen;": "\u0138", + "KHcy;": "\u0425", + "khcy;": "\u0445", + "KJcy;": "\u040c", + "kjcy;": "\u045c", + "Kopf;": "\U0001d542", + "kopf;": "\U0001d55c", + "Kscr;": "\U0001d4a6", + "kscr;": "\U0001d4c0", + "lAarr;": "\u21da", + "Lacute;": "\u0139", + "lacute;": "\u013a", + "laemptyv;": "\u29b4", + "lagran;": "\u2112", + "Lambda;": "\u039b", + "lambda;": "\u03bb", + "Lang;": "\u27ea", + "lang;": "\u27e8", + "langd;": "\u2991", + "langle;": "\u27e8", + "lap;": "\u2a85", + "Laplacetrf;": "\u2112", + "laquo": "\u00ab", + "laquo;": "\u00ab", + "Larr;": "\u219e", + "lArr;": "\u21d0", + "larr;": "\u2190", + "larrb;": "\u21e4", + "larrbfs;": "\u291f", + "larrfs;": "\u291d", + "larrhk;": "\u21a9", + "larrlp;": "\u21ab", + "larrpl;": "\u2939", + "larrsim;": "\u2973", + "larrtl;": "\u21a2", + "lat;": "\u2aab", + "lAtail;": "\u291b", + "latail;": "\u2919", + "late;": "\u2aad", + "lates;": "\u2aad\ufe00", + "lBarr;": "\u290e", + "lbarr;": "\u290c", + "lbbrk;": "\u2772", + "lbrace;": "{", + "lbrack;": "[", + "lbrke;": "\u298b", + "lbrksld;": "\u298f", + "lbrkslu;": "\u298d", + "Lcaron;": "\u013d", + "lcaron;": "\u013e", + "Lcedil;": "\u013b", + "lcedil;": "\u013c", + "lceil;": "\u2308", + "lcub;": "{", + "Lcy;": "\u041b", + "lcy;": "\u043b", + "ldca;": "\u2936", + "ldquo;": "\u201c", + "ldquor;": "\u201e", + "ldrdhar;": "\u2967", + "ldrushar;": "\u294b", + "ldsh;": "\u21b2", + "lE;": "\u2266", + "le;": "\u2264", + "LeftAngleBracket;": "\u27e8", + "LeftArrow;": "\u2190", + "Leftarrow;": "\u21d0", + "leftarrow;": "\u2190", + "LeftArrowBar;": "\u21e4", + "LeftArrowRightArrow;": "\u21c6", + "leftarrowtail;": "\u21a2", + "LeftCeiling;": "\u2308", + "LeftDoubleBracket;": "\u27e6", + "LeftDownTeeVector;": "\u2961", + "LeftDownVector;": "\u21c3", + "LeftDownVectorBar;": "\u2959", + "LeftFloor;": "\u230a", + "leftharpoondown;": "\u21bd", + "leftharpoonup;": "\u21bc", + "leftleftarrows;": "\u21c7", + "LeftRightArrow;": "\u2194", + "Leftrightarrow;": "\u21d4", + "leftrightarrow;": "\u2194", + "leftrightarrows;": "\u21c6", + "leftrightharpoons;": "\u21cb", + "leftrightsquigarrow;": "\u21ad", + "LeftRightVector;": "\u294e", + "LeftTee;": "\u22a3", + "LeftTeeArrow;": "\u21a4", + "LeftTeeVector;": "\u295a", + "leftthreetimes;": "\u22cb", + "LeftTriangle;": "\u22b2", + "LeftTriangleBar;": "\u29cf", + "LeftTriangleEqual;": "\u22b4", + "LeftUpDownVector;": "\u2951", + "LeftUpTeeVector;": "\u2960", + "LeftUpVector;": "\u21bf", + "LeftUpVectorBar;": "\u2958", + "LeftVector;": "\u21bc", + "LeftVectorBar;": "\u2952", + "lEg;": "\u2a8b", + "leg;": "\u22da", + "leq;": "\u2264", + "leqq;": "\u2266", + "leqslant;": "\u2a7d", + "les;": "\u2a7d", + "lescc;": "\u2aa8", + "lesdot;": "\u2a7f", + "lesdoto;": "\u2a81", + "lesdotor;": "\u2a83", + "lesg;": "\u22da\ufe00", + "lesges;": "\u2a93", + "lessapprox;": "\u2a85", + "lessdot;": "\u22d6", + "lesseqgtr;": "\u22da", + "lesseqqgtr;": "\u2a8b", + "LessEqualGreater;": "\u22da", + "LessFullEqual;": "\u2266", + "LessGreater;": "\u2276", + "lessgtr;": "\u2276", + "LessLess;": "\u2aa1", + "lesssim;": "\u2272", + "LessSlantEqual;": "\u2a7d", + "LessTilde;": "\u2272", + "lfisht;": "\u297c", + "lfloor;": "\u230a", + "Lfr;": "\U0001d50f", + "lfr;": "\U0001d529", + "lg;": "\u2276", + "lgE;": "\u2a91", + "lHar;": "\u2962", + "lhard;": "\u21bd", + "lharu;": "\u21bc", + "lharul;": "\u296a", + "lhblk;": "\u2584", + "LJcy;": "\u0409", + "ljcy;": "\u0459", + "Ll;": "\u22d8", + "ll;": "\u226a", + "llarr;": "\u21c7", + "llcorner;": "\u231e", + "Lleftarrow;": "\u21da", + "llhard;": "\u296b", + "lltri;": "\u25fa", + "Lmidot;": "\u013f", + "lmidot;": "\u0140", + "lmoust;": "\u23b0", + "lmoustache;": "\u23b0", + "lnap;": "\u2a89", + "lnapprox;": "\u2a89", + "lnE;": "\u2268", + "lne;": "\u2a87", + "lneq;": "\u2a87", + "lneqq;": "\u2268", + "lnsim;": "\u22e6", + "loang;": "\u27ec", + "loarr;": "\u21fd", + "lobrk;": "\u27e6", + "LongLeftArrow;": "\u27f5", + "Longleftarrow;": "\u27f8", + "longleftarrow;": "\u27f5", + "LongLeftRightArrow;": "\u27f7", + "Longleftrightarrow;": "\u27fa", + "longleftrightarrow;": "\u27f7", + "longmapsto;": "\u27fc", + "LongRightArrow;": "\u27f6", + "Longrightarrow;": "\u27f9", + "longrightarrow;": "\u27f6", + "looparrowleft;": "\u21ab", + "looparrowright;": "\u21ac", + "lopar;": "\u2985", + "Lopf;": "\U0001d543", + "lopf;": "\U0001d55d", + "loplus;": "\u2a2d", + "lotimes;": "\u2a34", + "lowast;": "\u2217", + "lowbar;": "_", + "LowerLeftArrow;": "\u2199", + "LowerRightArrow;": "\u2198", + "loz;": "\u25ca", + "lozenge;": "\u25ca", + "lozf;": "\u29eb", + "lpar;": "(", + "lparlt;": "\u2993", + "lrarr;": "\u21c6", + "lrcorner;": "\u231f", + "lrhar;": "\u21cb", + "lrhard;": "\u296d", + "lrm;": "\u200e", + "lrtri;": "\u22bf", + "lsaquo;": "\u2039", + "Lscr;": "\u2112", + "lscr;": "\U0001d4c1", + "Lsh;": "\u21b0", + "lsh;": "\u21b0", + "lsim;": "\u2272", + "lsime;": "\u2a8d", + "lsimg;": "\u2a8f", + "lsqb;": "[", + "lsquo;": "\u2018", + "lsquor;": "\u201a", + "Lstrok;": "\u0141", + "lstrok;": "\u0142", + "LT": "<", + "lt": "<", + "LT;": "<", + "Lt;": "\u226a", + "lt;": "<", + "ltcc;": "\u2aa6", + "ltcir;": "\u2a79", + "ltdot;": "\u22d6", + "lthree;": "\u22cb", + "ltimes;": "\u22c9", + "ltlarr;": "\u2976", + "ltquest;": "\u2a7b", + "ltri;": "\u25c3", + "ltrie;": "\u22b4", + "ltrif;": "\u25c2", + "ltrPar;": "\u2996", + "lurdshar;": "\u294a", + "luruhar;": "\u2966", + "lvertneqq;": "\u2268\ufe00", + "lvnE;": "\u2268\ufe00", + "macr": "\u00af", + "macr;": "\u00af", + "male;": "\u2642", + "malt;": "\u2720", + "maltese;": "\u2720", + "Map;": "\u2905", + "map;": "\u21a6", + "mapsto;": "\u21a6", + "mapstodown;": "\u21a7", + "mapstoleft;": "\u21a4", + "mapstoup;": "\u21a5", + "marker;": "\u25ae", + "mcomma;": "\u2a29", + "Mcy;": "\u041c", + "mcy;": "\u043c", + "mdash;": "\u2014", + "mDDot;": "\u223a", + "measuredangle;": "\u2221", + "MediumSpace;": "\u205f", + "Mellintrf;": "\u2133", + "Mfr;": "\U0001d510", + "mfr;": "\U0001d52a", + "mho;": "\u2127", + "micro": "\u00b5", + "micro;": "\u00b5", + "mid;": "\u2223", + "midast;": "*", + "midcir;": "\u2af0", + "middot": "\u00b7", + "middot;": "\u00b7", + "minus;": "\u2212", + "minusb;": "\u229f", + "minusd;": "\u2238", + "minusdu;": "\u2a2a", + "MinusPlus;": "\u2213", + "mlcp;": "\u2adb", + "mldr;": "\u2026", + "mnplus;": "\u2213", + "models;": "\u22a7", + "Mopf;": "\U0001d544", + "mopf;": "\U0001d55e", + "mp;": "\u2213", + "Mscr;": "\u2133", + "mscr;": "\U0001d4c2", + "mstpos;": "\u223e", + "Mu;": "\u039c", + "mu;": "\u03bc", + "multimap;": "\u22b8", + "mumap;": "\u22b8", + "nabla;": "\u2207", + "Nacute;": "\u0143", + "nacute;": "\u0144", + "nang;": "\u2220\u20d2", + "nap;": "\u2249", + "napE;": "\u2a70\u0338", + "napid;": "\u224b\u0338", + "napos;": "\u0149", + "napprox;": "\u2249", + "natur;": "\u266e", + "natural;": "\u266e", + "naturals;": "\u2115", + "nbsp": "\u00a0", + "nbsp;": "\u00a0", + "nbump;": "\u224e\u0338", + "nbumpe;": "\u224f\u0338", + "ncap;": "\u2a43", + "Ncaron;": "\u0147", + "ncaron;": "\u0148", + "Ncedil;": "\u0145", + "ncedil;": "\u0146", + "ncong;": "\u2247", + "ncongdot;": "\u2a6d\u0338", + "ncup;": "\u2a42", + "Ncy;": "\u041d", + "ncy;": "\u043d", + "ndash;": "\u2013", + "ne;": "\u2260", + "nearhk;": "\u2924", + "neArr;": "\u21d7", + "nearr;": "\u2197", + "nearrow;": "\u2197", + "nedot;": "\u2250\u0338", + "NegativeMediumSpace;": "\u200b", + "NegativeThickSpace;": "\u200b", + "NegativeThinSpace;": "\u200b", + "NegativeVeryThinSpace;": "\u200b", + "nequiv;": "\u2262", + "nesear;": "\u2928", + "nesim;": "\u2242\u0338", + "NestedGreaterGreater;": "\u226b", + "NestedLessLess;": "\u226a", + "NewLine;": "\u000a", + "nexist;": "\u2204", + "nexists;": "\u2204", + "Nfr;": "\U0001d511", + "nfr;": "\U0001d52b", + "ngE;": "\u2267\u0338", + "nge;": "\u2271", + "ngeq;": "\u2271", + "ngeqq;": "\u2267\u0338", + "ngeqslant;": "\u2a7e\u0338", + "nges;": "\u2a7e\u0338", + "nGg;": "\u22d9\u0338", + "ngsim;": "\u2275", + "nGt;": "\u226b\u20d2", + "ngt;": "\u226f", + "ngtr;": "\u226f", + "nGtv;": "\u226b\u0338", + "nhArr;": "\u21ce", + "nharr;": "\u21ae", + "nhpar;": "\u2af2", + "ni;": "\u220b", + "nis;": "\u22fc", + "nisd;": "\u22fa", + "niv;": "\u220b", + "NJcy;": "\u040a", + "njcy;": "\u045a", + "nlArr;": "\u21cd", + "nlarr;": "\u219a", + "nldr;": "\u2025", + "nlE;": "\u2266\u0338", + "nle;": "\u2270", + "nLeftarrow;": "\u21cd", + "nleftarrow;": "\u219a", + "nLeftrightarrow;": "\u21ce", + "nleftrightarrow;": "\u21ae", + "nleq;": "\u2270", + "nleqq;": "\u2266\u0338", + "nleqslant;": "\u2a7d\u0338", + "nles;": "\u2a7d\u0338", + "nless;": "\u226e", + "nLl;": "\u22d8\u0338", + "nlsim;": "\u2274", + "nLt;": "\u226a\u20d2", + "nlt;": "\u226e", + "nltri;": "\u22ea", + "nltrie;": "\u22ec", + "nLtv;": "\u226a\u0338", + "nmid;": "\u2224", + "NoBreak;": "\u2060", + "NonBreakingSpace;": "\u00a0", + "Nopf;": "\u2115", + "nopf;": "\U0001d55f", + "not": "\u00ac", + "Not;": "\u2aec", + "not;": "\u00ac", + "NotCongruent;": "\u2262", + "NotCupCap;": "\u226d", + "NotDoubleVerticalBar;": "\u2226", + "NotElement;": "\u2209", + "NotEqual;": "\u2260", + "NotEqualTilde;": "\u2242\u0338", + "NotExists;": "\u2204", + "NotGreater;": "\u226f", + "NotGreaterEqual;": "\u2271", + "NotGreaterFullEqual;": "\u2267\u0338", + "NotGreaterGreater;": "\u226b\u0338", + "NotGreaterLess;": "\u2279", + "NotGreaterSlantEqual;": "\u2a7e\u0338", + "NotGreaterTilde;": "\u2275", + "NotHumpDownHump;": "\u224e\u0338", + "NotHumpEqual;": "\u224f\u0338", + "notin;": "\u2209", + "notindot;": "\u22f5\u0338", + "notinE;": "\u22f9\u0338", + "notinva;": "\u2209", + "notinvb;": "\u22f7", + "notinvc;": "\u22f6", + "NotLeftTriangle;": "\u22ea", + "NotLeftTriangleBar;": "\u29cf\u0338", + "NotLeftTriangleEqual;": "\u22ec", + "NotLess;": "\u226e", + "NotLessEqual;": "\u2270", + "NotLessGreater;": "\u2278", + "NotLessLess;": "\u226a\u0338", + "NotLessSlantEqual;": "\u2a7d\u0338", + "NotLessTilde;": "\u2274", + "NotNestedGreaterGreater;": "\u2aa2\u0338", + "NotNestedLessLess;": "\u2aa1\u0338", + "notni;": "\u220c", + "notniva;": "\u220c", + "notnivb;": "\u22fe", + "notnivc;": "\u22fd", + "NotPrecedes;": "\u2280", + "NotPrecedesEqual;": "\u2aaf\u0338", + "NotPrecedesSlantEqual;": "\u22e0", + "NotReverseElement;": "\u220c", + "NotRightTriangle;": "\u22eb", + "NotRightTriangleBar;": "\u29d0\u0338", + "NotRightTriangleEqual;": "\u22ed", + "NotSquareSubset;": "\u228f\u0338", + "NotSquareSubsetEqual;": "\u22e2", + "NotSquareSuperset;": "\u2290\u0338", + "NotSquareSupersetEqual;": "\u22e3", + "NotSubset;": "\u2282\u20d2", + "NotSubsetEqual;": "\u2288", + "NotSucceeds;": "\u2281", + "NotSucceedsEqual;": "\u2ab0\u0338", + "NotSucceedsSlantEqual;": "\u22e1", + "NotSucceedsTilde;": "\u227f\u0338", + "NotSuperset;": "\u2283\u20d2", + "NotSupersetEqual;": "\u2289", + "NotTilde;": "\u2241", + "NotTildeEqual;": "\u2244", + "NotTildeFullEqual;": "\u2247", + "NotTildeTilde;": "\u2249", + "NotVerticalBar;": "\u2224", + "npar;": "\u2226", + "nparallel;": "\u2226", + "nparsl;": "\u2afd\u20e5", + "npart;": "\u2202\u0338", + "npolint;": "\u2a14", + "npr;": "\u2280", + "nprcue;": "\u22e0", + "npre;": "\u2aaf\u0338", + "nprec;": "\u2280", + "npreceq;": "\u2aaf\u0338", + "nrArr;": "\u21cf", + "nrarr;": "\u219b", + "nrarrc;": "\u2933\u0338", + "nrarrw;": "\u219d\u0338", + "nRightarrow;": "\u21cf", + "nrightarrow;": "\u219b", + "nrtri;": "\u22eb", + "nrtrie;": "\u22ed", + "nsc;": "\u2281", + "nsccue;": "\u22e1", + "nsce;": "\u2ab0\u0338", + "Nscr;": "\U0001d4a9", + "nscr;": "\U0001d4c3", + "nshortmid;": "\u2224", + "nshortparallel;": "\u2226", + "nsim;": "\u2241", + "nsime;": "\u2244", + "nsimeq;": "\u2244", + "nsmid;": "\u2224", + "nspar;": "\u2226", + "nsqsube;": "\u22e2", + "nsqsupe;": "\u22e3", + "nsub;": "\u2284", + "nsubE;": "\u2ac5\u0338", + "nsube;": "\u2288", + "nsubset;": "\u2282\u20d2", + "nsubseteq;": "\u2288", + "nsubseteqq;": "\u2ac5\u0338", + "nsucc;": "\u2281", + "nsucceq;": "\u2ab0\u0338", + "nsup;": "\u2285", + "nsupE;": "\u2ac6\u0338", + "nsupe;": "\u2289", + "nsupset;": "\u2283\u20d2", + "nsupseteq;": "\u2289", + "nsupseteqq;": "\u2ac6\u0338", + "ntgl;": "\u2279", + "Ntilde": "\u00d1", + "ntilde": "\u00f1", + "Ntilde;": "\u00d1", + "ntilde;": "\u00f1", + "ntlg;": "\u2278", + "ntriangleleft;": "\u22ea", + "ntrianglelefteq;": "\u22ec", + "ntriangleright;": "\u22eb", + "ntrianglerighteq;": "\u22ed", + "Nu;": "\u039d", + "nu;": "\u03bd", + "num;": "#", + "numero;": "\u2116", + "numsp;": "\u2007", + "nvap;": "\u224d\u20d2", + "nVDash;": "\u22af", + "nVdash;": "\u22ae", + "nvDash;": "\u22ad", + "nvdash;": "\u22ac", + "nvge;": "\u2265\u20d2", + "nvgt;": ">\u20d2", + "nvHarr;": "\u2904", + "nvinfin;": "\u29de", + "nvlArr;": "\u2902", + "nvle;": "\u2264\u20d2", + "nvlt;": "<\u20d2", + "nvltrie;": "\u22b4\u20d2", + "nvrArr;": "\u2903", + "nvrtrie;": "\u22b5\u20d2", + "nvsim;": "\u223c\u20d2", + "nwarhk;": "\u2923", + "nwArr;": "\u21d6", + "nwarr;": "\u2196", + "nwarrow;": "\u2196", + "nwnear;": "\u2927", + "Oacute": "\u00d3", + "oacute": "\u00f3", + "Oacute;": "\u00d3", + "oacute;": "\u00f3", + "oast;": "\u229b", + "ocir;": "\u229a", + "Ocirc": "\u00d4", + "ocirc": "\u00f4", + "Ocirc;": "\u00d4", + "ocirc;": "\u00f4", + "Ocy;": "\u041e", + "ocy;": "\u043e", + "odash;": "\u229d", + "Odblac;": "\u0150", + "odblac;": "\u0151", + "odiv;": "\u2a38", + "odot;": "\u2299", + "odsold;": "\u29bc", + "OElig;": "\u0152", + "oelig;": "\u0153", + "ofcir;": "\u29bf", + "Ofr;": "\U0001d512", + "ofr;": "\U0001d52c", + "ogon;": "\u02db", + "Ograve": "\u00d2", + "ograve": "\u00f2", + "Ograve;": "\u00d2", + "ograve;": "\u00f2", + "ogt;": "\u29c1", + "ohbar;": "\u29b5", + "ohm;": "\u03a9", + "oint;": "\u222e", + "olarr;": "\u21ba", + "olcir;": "\u29be", + "olcross;": "\u29bb", + "oline;": "\u203e", + "olt;": "\u29c0", + "Omacr;": "\u014c", + "omacr;": "\u014d", + "Omega;": "\u03a9", + "omega;": "\u03c9", + "Omicron;": "\u039f", + "omicron;": "\u03bf", + "omid;": "\u29b6", + "ominus;": "\u2296", + "Oopf;": "\U0001d546", + "oopf;": "\U0001d560", + "opar;": "\u29b7", + "OpenCurlyDoubleQuote;": "\u201c", + "OpenCurlyQuote;": "\u2018", + "operp;": "\u29b9", + "oplus;": "\u2295", + "Or;": "\u2a54", + "or;": "\u2228", + "orarr;": "\u21bb", + "ord;": "\u2a5d", + "order;": "\u2134", + "orderof;": "\u2134", + "ordf": "\u00aa", + "ordf;": "\u00aa", + "ordm": "\u00ba", + "ordm;": "\u00ba", + "origof;": "\u22b6", + "oror;": "\u2a56", + "orslope;": "\u2a57", + "orv;": "\u2a5b", + "oS;": "\u24c8", + "Oscr;": "\U0001d4aa", + "oscr;": "\u2134", + "Oslash": "\u00d8", + "oslash": "\u00f8", + "Oslash;": "\u00d8", + "oslash;": "\u00f8", + "osol;": "\u2298", + "Otilde": "\u00d5", + "otilde": "\u00f5", + "Otilde;": "\u00d5", + "otilde;": "\u00f5", + "Otimes;": "\u2a37", + "otimes;": "\u2297", + "otimesas;": "\u2a36", + "Ouml": "\u00d6", + "ouml": "\u00f6", + "Ouml;": "\u00d6", + "ouml;": "\u00f6", + "ovbar;": "\u233d", + "OverBar;": "\u203e", + "OverBrace;": "\u23de", + "OverBracket;": "\u23b4", + "OverParenthesis;": "\u23dc", + "par;": "\u2225", + "para": "\u00b6", + "para;": "\u00b6", + "parallel;": "\u2225", + "parsim;": "\u2af3", + "parsl;": "\u2afd", + "part;": "\u2202", + "PartialD;": "\u2202", + "Pcy;": "\u041f", + "pcy;": "\u043f", + "percnt;": "%", + "period;": ".", + "permil;": "\u2030", + "perp;": "\u22a5", + "pertenk;": "\u2031", + "Pfr;": "\U0001d513", + "pfr;": "\U0001d52d", + "Phi;": "\u03a6", + "phi;": "\u03c6", + "phiv;": "\u03d5", + "phmmat;": "\u2133", + "phone;": "\u260e", + "Pi;": "\u03a0", + "pi;": "\u03c0", + "pitchfork;": "\u22d4", + "piv;": "\u03d6", + "planck;": "\u210f", + "planckh;": "\u210e", + "plankv;": "\u210f", + "plus;": "+", + "plusacir;": "\u2a23", + "plusb;": "\u229e", + "pluscir;": "\u2a22", + "plusdo;": "\u2214", + "plusdu;": "\u2a25", + "pluse;": "\u2a72", + "PlusMinus;": "\u00b1", + "plusmn": "\u00b1", + "plusmn;": "\u00b1", + "plussim;": "\u2a26", + "plustwo;": "\u2a27", + "pm;": "\u00b1", + "Poincareplane;": "\u210c", + "pointint;": "\u2a15", + "Popf;": "\u2119", + "popf;": "\U0001d561", + "pound": "\u00a3", + "pound;": "\u00a3", + "Pr;": "\u2abb", + "pr;": "\u227a", + "prap;": "\u2ab7", + "prcue;": "\u227c", + "prE;": "\u2ab3", + "pre;": "\u2aaf", + "prec;": "\u227a", + "precapprox;": "\u2ab7", + "preccurlyeq;": "\u227c", + "Precedes;": "\u227a", + "PrecedesEqual;": "\u2aaf", + "PrecedesSlantEqual;": "\u227c", + "PrecedesTilde;": "\u227e", + "preceq;": "\u2aaf", + "precnapprox;": "\u2ab9", + "precneqq;": "\u2ab5", + "precnsim;": "\u22e8", + "precsim;": "\u227e", + "Prime;": "\u2033", + "prime;": "\u2032", + "primes;": "\u2119", + "prnap;": "\u2ab9", + "prnE;": "\u2ab5", + "prnsim;": "\u22e8", + "prod;": "\u220f", + "Product;": "\u220f", + "profalar;": "\u232e", + "profline;": "\u2312", + "profsurf;": "\u2313", + "prop;": "\u221d", + "Proportion;": "\u2237", + "Proportional;": "\u221d", + "propto;": "\u221d", + "prsim;": "\u227e", + "prurel;": "\u22b0", + "Pscr;": "\U0001d4ab", + "pscr;": "\U0001d4c5", + "Psi;": "\u03a8", + "psi;": "\u03c8", + "puncsp;": "\u2008", + "Qfr;": "\U0001d514", + "qfr;": "\U0001d52e", + "qint;": "\u2a0c", + "Qopf;": "\u211a", + "qopf;": "\U0001d562", + "qprime;": "\u2057", + "Qscr;": "\U0001d4ac", + "qscr;": "\U0001d4c6", + "quaternions;": "\u210d", + "quatint;": "\u2a16", + "quest;": "?", + "questeq;": "\u225f", + "QUOT": "\"", + "quot": "\"", + "QUOT;": "\"", + "quot;": "\"", + "rAarr;": "\u21db", + "race;": "\u223d\u0331", + "Racute;": "\u0154", + "racute;": "\u0155", + "radic;": "\u221a", + "raemptyv;": "\u29b3", + "Rang;": "\u27eb", + "rang;": "\u27e9", + "rangd;": "\u2992", + "range;": "\u29a5", + "rangle;": "\u27e9", + "raquo": "\u00bb", + "raquo;": "\u00bb", + "Rarr;": "\u21a0", + "rArr;": "\u21d2", + "rarr;": "\u2192", + "rarrap;": "\u2975", + "rarrb;": "\u21e5", + "rarrbfs;": "\u2920", + "rarrc;": "\u2933", + "rarrfs;": "\u291e", + "rarrhk;": "\u21aa", + "rarrlp;": "\u21ac", + "rarrpl;": "\u2945", + "rarrsim;": "\u2974", + "Rarrtl;": "\u2916", + "rarrtl;": "\u21a3", + "rarrw;": "\u219d", + "rAtail;": "\u291c", + "ratail;": "\u291a", + "ratio;": "\u2236", + "rationals;": "\u211a", + "RBarr;": "\u2910", + "rBarr;": "\u290f", + "rbarr;": "\u290d", + "rbbrk;": "\u2773", + "rbrace;": "}", + "rbrack;": "]", + "rbrke;": "\u298c", + "rbrksld;": "\u298e", + "rbrkslu;": "\u2990", + "Rcaron;": "\u0158", + "rcaron;": "\u0159", + "Rcedil;": "\u0156", + "rcedil;": "\u0157", + "rceil;": "\u2309", + "rcub;": "}", + "Rcy;": "\u0420", + "rcy;": "\u0440", + "rdca;": "\u2937", + "rdldhar;": "\u2969", + "rdquo;": "\u201d", + "rdquor;": "\u201d", + "rdsh;": "\u21b3", + "Re;": "\u211c", + "real;": "\u211c", + "realine;": "\u211b", + "realpart;": "\u211c", + "reals;": "\u211d", + "rect;": "\u25ad", + "REG": "\u00ae", + "reg": "\u00ae", + "REG;": "\u00ae", + "reg;": "\u00ae", + "ReverseElement;": "\u220b", + "ReverseEquilibrium;": "\u21cb", + "ReverseUpEquilibrium;": "\u296f", + "rfisht;": "\u297d", + "rfloor;": "\u230b", + "Rfr;": "\u211c", + "rfr;": "\U0001d52f", + "rHar;": "\u2964", + "rhard;": "\u21c1", + "rharu;": "\u21c0", + "rharul;": "\u296c", + "Rho;": "\u03a1", + "rho;": "\u03c1", + "rhov;": "\u03f1", + "RightAngleBracket;": "\u27e9", + "RightArrow;": "\u2192", + "Rightarrow;": "\u21d2", + "rightarrow;": "\u2192", + "RightArrowBar;": "\u21e5", + "RightArrowLeftArrow;": "\u21c4", + "rightarrowtail;": "\u21a3", + "RightCeiling;": "\u2309", + "RightDoubleBracket;": "\u27e7", + "RightDownTeeVector;": "\u295d", + "RightDownVector;": "\u21c2", + "RightDownVectorBar;": "\u2955", + "RightFloor;": "\u230b", + "rightharpoondown;": "\u21c1", + "rightharpoonup;": "\u21c0", + "rightleftarrows;": "\u21c4", + "rightleftharpoons;": "\u21cc", + "rightrightarrows;": "\u21c9", + "rightsquigarrow;": "\u219d", + "RightTee;": "\u22a2", + "RightTeeArrow;": "\u21a6", + "RightTeeVector;": "\u295b", + "rightthreetimes;": "\u22cc", + "RightTriangle;": "\u22b3", + "RightTriangleBar;": "\u29d0", + "RightTriangleEqual;": "\u22b5", + "RightUpDownVector;": "\u294f", + "RightUpTeeVector;": "\u295c", + "RightUpVector;": "\u21be", + "RightUpVectorBar;": "\u2954", + "RightVector;": "\u21c0", + "RightVectorBar;": "\u2953", + "ring;": "\u02da", + "risingdotseq;": "\u2253", + "rlarr;": "\u21c4", + "rlhar;": "\u21cc", + "rlm;": "\u200f", + "rmoust;": "\u23b1", + "rmoustache;": "\u23b1", + "rnmid;": "\u2aee", + "roang;": "\u27ed", + "roarr;": "\u21fe", + "robrk;": "\u27e7", + "ropar;": "\u2986", + "Ropf;": "\u211d", + "ropf;": "\U0001d563", + "roplus;": "\u2a2e", + "rotimes;": "\u2a35", + "RoundImplies;": "\u2970", + "rpar;": ")", + "rpargt;": "\u2994", + "rppolint;": "\u2a12", + "rrarr;": "\u21c9", + "Rrightarrow;": "\u21db", + "rsaquo;": "\u203a", + "Rscr;": "\u211b", + "rscr;": "\U0001d4c7", + "Rsh;": "\u21b1", + "rsh;": "\u21b1", + "rsqb;": "]", + "rsquo;": "\u2019", + "rsquor;": "\u2019", + "rthree;": "\u22cc", + "rtimes;": "\u22ca", + "rtri;": "\u25b9", + "rtrie;": "\u22b5", + "rtrif;": "\u25b8", + "rtriltri;": "\u29ce", + "RuleDelayed;": "\u29f4", + "ruluhar;": "\u2968", + "rx;": "\u211e", + "Sacute;": "\u015a", + "sacute;": "\u015b", + "sbquo;": "\u201a", + "Sc;": "\u2abc", + "sc;": "\u227b", + "scap;": "\u2ab8", + "Scaron;": "\u0160", + "scaron;": "\u0161", + "sccue;": "\u227d", + "scE;": "\u2ab4", + "sce;": "\u2ab0", + "Scedil;": "\u015e", + "scedil;": "\u015f", + "Scirc;": "\u015c", + "scirc;": "\u015d", + "scnap;": "\u2aba", + "scnE;": "\u2ab6", + "scnsim;": "\u22e9", + "scpolint;": "\u2a13", + "scsim;": "\u227f", + "Scy;": "\u0421", + "scy;": "\u0441", + "sdot;": "\u22c5", + "sdotb;": "\u22a1", + "sdote;": "\u2a66", + "searhk;": "\u2925", + "seArr;": "\u21d8", + "searr;": "\u2198", + "searrow;": "\u2198", + "sect": "\u00a7", + "sect;": "\u00a7", + "semi;": ";", + "seswar;": "\u2929", + "setminus;": "\u2216", + "setmn;": "\u2216", + "sext;": "\u2736", + "Sfr;": "\U0001d516", + "sfr;": "\U0001d530", + "sfrown;": "\u2322", + "sharp;": "\u266f", + "SHCHcy;": "\u0429", + "shchcy;": "\u0449", + "SHcy;": "\u0428", + "shcy;": "\u0448", + "ShortDownArrow;": "\u2193", + "ShortLeftArrow;": "\u2190", + "shortmid;": "\u2223", + "shortparallel;": "\u2225", + "ShortRightArrow;": "\u2192", + "ShortUpArrow;": "\u2191", + "shy": "\u00ad", + "shy;": "\u00ad", + "Sigma;": "\u03a3", + "sigma;": "\u03c3", + "sigmaf;": "\u03c2", + "sigmav;": "\u03c2", + "sim;": "\u223c", + "simdot;": "\u2a6a", + "sime;": "\u2243", + "simeq;": "\u2243", + "simg;": "\u2a9e", + "simgE;": "\u2aa0", + "siml;": "\u2a9d", + "simlE;": "\u2a9f", + "simne;": "\u2246", + "simplus;": "\u2a24", + "simrarr;": "\u2972", + "slarr;": "\u2190", + "SmallCircle;": "\u2218", + "smallsetminus;": "\u2216", + "smashp;": "\u2a33", + "smeparsl;": "\u29e4", + "smid;": "\u2223", + "smile;": "\u2323", + "smt;": "\u2aaa", + "smte;": "\u2aac", + "smtes;": "\u2aac\ufe00", + "SOFTcy;": "\u042c", + "softcy;": "\u044c", + "sol;": "/", + "solb;": "\u29c4", + "solbar;": "\u233f", + "Sopf;": "\U0001d54a", + "sopf;": "\U0001d564", + "spades;": "\u2660", + "spadesuit;": "\u2660", + "spar;": "\u2225", + "sqcap;": "\u2293", + "sqcaps;": "\u2293\ufe00", + "sqcup;": "\u2294", + "sqcups;": "\u2294\ufe00", + "Sqrt;": "\u221a", + "sqsub;": "\u228f", + "sqsube;": "\u2291", + "sqsubset;": "\u228f", + "sqsubseteq;": "\u2291", + "sqsup;": "\u2290", + "sqsupe;": "\u2292", + "sqsupset;": "\u2290", + "sqsupseteq;": "\u2292", + "squ;": "\u25a1", + "Square;": "\u25a1", + "square;": "\u25a1", + "SquareIntersection;": "\u2293", + "SquareSubset;": "\u228f", + "SquareSubsetEqual;": "\u2291", + "SquareSuperset;": "\u2290", + "SquareSupersetEqual;": "\u2292", + "SquareUnion;": "\u2294", + "squarf;": "\u25aa", + "squf;": "\u25aa", + "srarr;": "\u2192", + "Sscr;": "\U0001d4ae", + "sscr;": "\U0001d4c8", + "ssetmn;": "\u2216", + "ssmile;": "\u2323", + "sstarf;": "\u22c6", + "Star;": "\u22c6", + "star;": "\u2606", + "starf;": "\u2605", + "straightepsilon;": "\u03f5", + "straightphi;": "\u03d5", + "strns;": "\u00af", + "Sub;": "\u22d0", + "sub;": "\u2282", + "subdot;": "\u2abd", + "subE;": "\u2ac5", + "sube;": "\u2286", + "subedot;": "\u2ac3", + "submult;": "\u2ac1", + "subnE;": "\u2acb", + "subne;": "\u228a", + "subplus;": "\u2abf", + "subrarr;": "\u2979", + "Subset;": "\u22d0", + "subset;": "\u2282", + "subseteq;": "\u2286", + "subseteqq;": "\u2ac5", + "SubsetEqual;": "\u2286", + "subsetneq;": "\u228a", + "subsetneqq;": "\u2acb", + "subsim;": "\u2ac7", + "subsub;": "\u2ad5", + "subsup;": "\u2ad3", + "succ;": "\u227b", + "succapprox;": "\u2ab8", + "succcurlyeq;": "\u227d", + "Succeeds;": "\u227b", + "SucceedsEqual;": "\u2ab0", + "SucceedsSlantEqual;": "\u227d", + "SucceedsTilde;": "\u227f", + "succeq;": "\u2ab0", + "succnapprox;": "\u2aba", + "succneqq;": "\u2ab6", + "succnsim;": "\u22e9", + "succsim;": "\u227f", + "SuchThat;": "\u220b", + "Sum;": "\u2211", + "sum;": "\u2211", + "sung;": "\u266a", + "sup1": "\u00b9", + "sup1;": "\u00b9", + "sup2": "\u00b2", + "sup2;": "\u00b2", + "sup3": "\u00b3", + "sup3;": "\u00b3", + "Sup;": "\u22d1", + "sup;": "\u2283", + "supdot;": "\u2abe", + "supdsub;": "\u2ad8", + "supE;": "\u2ac6", + "supe;": "\u2287", + "supedot;": "\u2ac4", + "Superset;": "\u2283", + "SupersetEqual;": "\u2287", + "suphsol;": "\u27c9", + "suphsub;": "\u2ad7", + "suplarr;": "\u297b", + "supmult;": "\u2ac2", + "supnE;": "\u2acc", + "supne;": "\u228b", + "supplus;": "\u2ac0", + "Supset;": "\u22d1", + "supset;": "\u2283", + "supseteq;": "\u2287", + "supseteqq;": "\u2ac6", + "supsetneq;": "\u228b", + "supsetneqq;": "\u2acc", + "supsim;": "\u2ac8", + "supsub;": "\u2ad4", + "supsup;": "\u2ad6", + "swarhk;": "\u2926", + "swArr;": "\u21d9", + "swarr;": "\u2199", + "swarrow;": "\u2199", + "swnwar;": "\u292a", + "szlig": "\u00df", + "szlig;": "\u00df", + "Tab;": "\u0009", + "target;": "\u2316", + "Tau;": "\u03a4", + "tau;": "\u03c4", + "tbrk;": "\u23b4", + "Tcaron;": "\u0164", + "tcaron;": "\u0165", + "Tcedil;": "\u0162", + "tcedil;": "\u0163", + "Tcy;": "\u0422", + "tcy;": "\u0442", + "tdot;": "\u20db", + "telrec;": "\u2315", + "Tfr;": "\U0001d517", + "tfr;": "\U0001d531", + "there4;": "\u2234", + "Therefore;": "\u2234", + "therefore;": "\u2234", + "Theta;": "\u0398", + "theta;": "\u03b8", + "thetasym;": "\u03d1", + "thetav;": "\u03d1", + "thickapprox;": "\u2248", + "thicksim;": "\u223c", + "ThickSpace;": "\u205f\u200a", + "thinsp;": "\u2009", + "ThinSpace;": "\u2009", + "thkap;": "\u2248", + "thksim;": "\u223c", + "THORN": "\u00de", + "thorn": "\u00fe", + "THORN;": "\u00de", + "thorn;": "\u00fe", + "Tilde;": "\u223c", + "tilde;": "\u02dc", + "TildeEqual;": "\u2243", + "TildeFullEqual;": "\u2245", + "TildeTilde;": "\u2248", + "times": "\u00d7", + "times;": "\u00d7", + "timesb;": "\u22a0", + "timesbar;": "\u2a31", + "timesd;": "\u2a30", + "tint;": "\u222d", + "toea;": "\u2928", + "top;": "\u22a4", + "topbot;": "\u2336", + "topcir;": "\u2af1", + "Topf;": "\U0001d54b", + "topf;": "\U0001d565", + "topfork;": "\u2ada", + "tosa;": "\u2929", + "tprime;": "\u2034", + "TRADE;": "\u2122", + "trade;": "\u2122", + "triangle;": "\u25b5", + "triangledown;": "\u25bf", + "triangleleft;": "\u25c3", + "trianglelefteq;": "\u22b4", + "triangleq;": "\u225c", + "triangleright;": "\u25b9", + "trianglerighteq;": "\u22b5", + "tridot;": "\u25ec", + "trie;": "\u225c", + "triminus;": "\u2a3a", + "TripleDot;": "\u20db", + "triplus;": "\u2a39", + "trisb;": "\u29cd", + "tritime;": "\u2a3b", + "trpezium;": "\u23e2", + "Tscr;": "\U0001d4af", + "tscr;": "\U0001d4c9", + "TScy;": "\u0426", + "tscy;": "\u0446", + "TSHcy;": "\u040b", + "tshcy;": "\u045b", + "Tstrok;": "\u0166", + "tstrok;": "\u0167", + "twixt;": "\u226c", + "twoheadleftarrow;": "\u219e", + "twoheadrightarrow;": "\u21a0", + "Uacute": "\u00da", + "uacute": "\u00fa", + "Uacute;": "\u00da", + "uacute;": "\u00fa", + "Uarr;": "\u219f", + "uArr;": "\u21d1", + "uarr;": "\u2191", + "Uarrocir;": "\u2949", + "Ubrcy;": "\u040e", + "ubrcy;": "\u045e", + "Ubreve;": "\u016c", + "ubreve;": "\u016d", + "Ucirc": "\u00db", + "ucirc": "\u00fb", + "Ucirc;": "\u00db", + "ucirc;": "\u00fb", + "Ucy;": "\u0423", + "ucy;": "\u0443", + "udarr;": "\u21c5", + "Udblac;": "\u0170", + "udblac;": "\u0171", + "udhar;": "\u296e", + "ufisht;": "\u297e", + "Ufr;": "\U0001d518", + "ufr;": "\U0001d532", + "Ugrave": "\u00d9", + "ugrave": "\u00f9", + "Ugrave;": "\u00d9", + "ugrave;": "\u00f9", + "uHar;": "\u2963", + "uharl;": "\u21bf", + "uharr;": "\u21be", + "uhblk;": "\u2580", + "ulcorn;": "\u231c", + "ulcorner;": "\u231c", + "ulcrop;": "\u230f", + "ultri;": "\u25f8", + "Umacr;": "\u016a", + "umacr;": "\u016b", + "uml": "\u00a8", + "uml;": "\u00a8", + "UnderBar;": "_", + "UnderBrace;": "\u23df", + "UnderBracket;": "\u23b5", + "UnderParenthesis;": "\u23dd", + "Union;": "\u22c3", + "UnionPlus;": "\u228e", + "Uogon;": "\u0172", + "uogon;": "\u0173", + "Uopf;": "\U0001d54c", + "uopf;": "\U0001d566", + "UpArrow;": "\u2191", + "Uparrow;": "\u21d1", + "uparrow;": "\u2191", + "UpArrowBar;": "\u2912", + "UpArrowDownArrow;": "\u21c5", + "UpDownArrow;": "\u2195", + "Updownarrow;": "\u21d5", + "updownarrow;": "\u2195", + "UpEquilibrium;": "\u296e", + "upharpoonleft;": "\u21bf", + "upharpoonright;": "\u21be", + "uplus;": "\u228e", + "UpperLeftArrow;": "\u2196", + "UpperRightArrow;": "\u2197", + "Upsi;": "\u03d2", + "upsi;": "\u03c5", + "upsih;": "\u03d2", + "Upsilon;": "\u03a5", + "upsilon;": "\u03c5", + "UpTee;": "\u22a5", + "UpTeeArrow;": "\u21a5", + "upuparrows;": "\u21c8", + "urcorn;": "\u231d", + "urcorner;": "\u231d", + "urcrop;": "\u230e", + "Uring;": "\u016e", + "uring;": "\u016f", + "urtri;": "\u25f9", + "Uscr;": "\U0001d4b0", + "uscr;": "\U0001d4ca", + "utdot;": "\u22f0", + "Utilde;": "\u0168", + "utilde;": "\u0169", + "utri;": "\u25b5", + "utrif;": "\u25b4", + "uuarr;": "\u21c8", + "Uuml": "\u00dc", + "uuml": "\u00fc", + "Uuml;": "\u00dc", + "uuml;": "\u00fc", + "uwangle;": "\u29a7", + "vangrt;": "\u299c", + "varepsilon;": "\u03f5", + "varkappa;": "\u03f0", + "varnothing;": "\u2205", + "varphi;": "\u03d5", + "varpi;": "\u03d6", + "varpropto;": "\u221d", + "vArr;": "\u21d5", + "varr;": "\u2195", + "varrho;": "\u03f1", + "varsigma;": "\u03c2", + "varsubsetneq;": "\u228a\ufe00", + "varsubsetneqq;": "\u2acb\ufe00", + "varsupsetneq;": "\u228b\ufe00", + "varsupsetneqq;": "\u2acc\ufe00", + "vartheta;": "\u03d1", + "vartriangleleft;": "\u22b2", + "vartriangleright;": "\u22b3", + "Vbar;": "\u2aeb", + "vBar;": "\u2ae8", + "vBarv;": "\u2ae9", + "Vcy;": "\u0412", + "vcy;": "\u0432", + "VDash;": "\u22ab", + "Vdash;": "\u22a9", + "vDash;": "\u22a8", + "vdash;": "\u22a2", + "Vdashl;": "\u2ae6", + "Vee;": "\u22c1", + "vee;": "\u2228", + "veebar;": "\u22bb", + "veeeq;": "\u225a", + "vellip;": "\u22ee", + "Verbar;": "\u2016", + "verbar;": "|", + "Vert;": "\u2016", + "vert;": "|", + "VerticalBar;": "\u2223", + "VerticalLine;": "|", + "VerticalSeparator;": "\u2758", + "VerticalTilde;": "\u2240", + "VeryThinSpace;": "\u200a", + "Vfr;": "\U0001d519", + "vfr;": "\U0001d533", + "vltri;": "\u22b2", + "vnsub;": "\u2282\u20d2", + "vnsup;": "\u2283\u20d2", + "Vopf;": "\U0001d54d", + "vopf;": "\U0001d567", + "vprop;": "\u221d", + "vrtri;": "\u22b3", + "Vscr;": "\U0001d4b1", + "vscr;": "\U0001d4cb", + "vsubnE;": "\u2acb\ufe00", + "vsubne;": "\u228a\ufe00", + "vsupnE;": "\u2acc\ufe00", + "vsupne;": "\u228b\ufe00", + "Vvdash;": "\u22aa", + "vzigzag;": "\u299a", + "Wcirc;": "\u0174", + "wcirc;": "\u0175", + "wedbar;": "\u2a5f", + "Wedge;": "\u22c0", + "wedge;": "\u2227", + "wedgeq;": "\u2259", + "weierp;": "\u2118", + "Wfr;": "\U0001d51a", + "wfr;": "\U0001d534", + "Wopf;": "\U0001d54e", + "wopf;": "\U0001d568", + "wp;": "\u2118", + "wr;": "\u2240", + "wreath;": "\u2240", + "Wscr;": "\U0001d4b2", + "wscr;": "\U0001d4cc", + "xcap;": "\u22c2", + "xcirc;": "\u25ef", + "xcup;": "\u22c3", + "xdtri;": "\u25bd", + "Xfr;": "\U0001d51b", + "xfr;": "\U0001d535", + "xhArr;": "\u27fa", + "xharr;": "\u27f7", + "Xi;": "\u039e", + "xi;": "\u03be", + "xlArr;": "\u27f8", + "xlarr;": "\u27f5", + "xmap;": "\u27fc", + "xnis;": "\u22fb", + "xodot;": "\u2a00", + "Xopf;": "\U0001d54f", + "xopf;": "\U0001d569", + "xoplus;": "\u2a01", + "xotime;": "\u2a02", + "xrArr;": "\u27f9", + "xrarr;": "\u27f6", + "Xscr;": "\U0001d4b3", + "xscr;": "\U0001d4cd", + "xsqcup;": "\u2a06", + "xuplus;": "\u2a04", + "xutri;": "\u25b3", + "xvee;": "\u22c1", + "xwedge;": "\u22c0", + "Yacute": "\u00dd", + "yacute": "\u00fd", + "Yacute;": "\u00dd", + "yacute;": "\u00fd", + "YAcy;": "\u042f", + "yacy;": "\u044f", + "Ycirc;": "\u0176", + "ycirc;": "\u0177", + "Ycy;": "\u042b", + "ycy;": "\u044b", + "yen": "\u00a5", + "yen;": "\u00a5", + "Yfr;": "\U0001d51c", + "yfr;": "\U0001d536", + "YIcy;": "\u0407", + "yicy;": "\u0457", + "Yopf;": "\U0001d550", + "yopf;": "\U0001d56a", + "Yscr;": "\U0001d4b4", + "yscr;": "\U0001d4ce", + "YUcy;": "\u042e", + "yucy;": "\u044e", + "yuml": "\u00ff", + "Yuml;": "\u0178", + "yuml;": "\u00ff", + "Zacute;": "\u0179", + "zacute;": "\u017a", + "Zcaron;": "\u017d", + "zcaron;": "\u017e", + "Zcy;": "\u0417", + "zcy;": "\u0437", + "Zdot;": "\u017b", + "zdot;": "\u017c", + "zeetrf;": "\u2128", + "ZeroWidthSpace;": "\u200b", + "Zeta;": "\u0396", + "zeta;": "\u03b6", + "Zfr;": "\u2128", + "zfr;": "\U0001d537", + "ZHcy;": "\u0416", + "zhcy;": "\u0436", + "zigrarr;": "\u21dd", + "Zopf;": "\u2124", + "zopf;": "\U0001d56b", + "Zscr;": "\U0001d4b5", + "zscr;": "\U0001d4cf", + "zwj;": "\u200d", + "zwnj;": "\u200c", +} + +codepoint2name = {} +entitydefs = {} +for (name, codepoint) in name2codepoint.items(): + codepoint2name[codepoint] = name + entitydefs[name] = chr(codepoint) + +del name, codepoint diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/io.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/io.py new file mode 100644 index 00000000000..f30fd2379a2 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/io.py @@ -0,0 +1,730 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Streams: in-memory text and bytes, and files through open(). + +A file is read whole when it is opened and written whole when it is +flushed or closed: the device file system (`FileSystemStorage`) has no +random access to offer, and application files are small. A file opened +for appending writes only what was added. +""" + +import _cn1fs + +SEEK_SET = 0 +SEEK_CUR = 1 +SEEK_END = 2 +DEFAULT_BUFFER_SIZE = 8192 + +__all__ = ["open", "open_code", "IOBase", "RawIOBase", "BufferedIOBase", "TextIOBase", "StringIO", + "BytesIO", "TextIOWrapper", "BufferedReader", "BufferedWriter", "BufferedRandom", "FileIO", + "UnsupportedOperation", "SEEK_SET", "SEEK_CUR", "SEEK_END", "DEFAULT_BUFFER_SIZE", + "text_encoding"] + +_cwd = None + + +def _getcwd(): + global _cwd + if _cwd is None: + _cwd = _cn1fs.home() + return _cwd + + +def _setcwd(path): + global _cwd + _cwd = path + + +def _abspath(path): + """`path` made absolute against the working directory and normalised.""" + if not path.startswith("/"): + base = _getcwd() + path = base + "/" + path if base != "/" else "/" + path + parts = [] + for p in path.split("/"): + if p == "" or p == ".": + continue + if p == "..": + if parts: + parts.pop() + continue + parts.append(p) + return "/" + "/".join(parts) + + +def _fspath(path): + if isinstance(path, (str, bytes)): + return path.decode("utf-8") if isinstance(path, bytes) else path + fspath = getattr(type(path), "__fspath__", None) + if fspath is None: + raise TypeError("expected str, bytes or os.PathLike object, not " + type(path).__name__) + return _fspath(fspath(path)) + + +class UnsupportedOperation(OSError, ValueError): + pass + + +UnsupportedOperation.__module__ = "io" + + +class IOBase: + closed = False + + def close(self): + if not self.closed: + try: + self.flush() + finally: + self.closed = True + + def flush(self): + self._check() + + def readable(self): + return False + + def writable(self): + return False + + def seekable(self): + return False + + def isatty(self): + return False + + def fileno(self): + raise UnsupportedOperation("fileno") + + def __enter__(self): + self._check() + return self + + def __exit__(self, *exc): + self.close() + + def __iter__(self): + self._check() + return self + + def __next__(self): + line = self.readline() + if not line: + raise StopIteration + return line + + def readlines(self, hint=-1): + out = [] + total = 0 + for line in self: + out.append(line) + total += len(line) + if 0 < hint <= total: + break + return out + + def writelines(self, lines): + self._check() + for line in lines: + self.write(line) + + def _check(self): + if self.closed: + raise ValueError("I/O operation on closed file.") + + def __del__(self): + pass + + +class RawIOBase(IOBase): + pass + + +class BufferedIOBase(IOBase): + pass + + +class TextIOBase(IOBase): + encoding = None + errors = None + newlines = None + + +class _Buffer: + """A growable str or bytes with a cursor; appends are amortised.""" + + def __init__(self, initial, empty): + self._empty = empty + self._buf = initial + self._tail = [] + self._tail_len = 0 + self._pos = 0 + + def _flat(self): + if self._tail: + self._buf = self._buf + self._empty.join(self._tail) + self._tail = [] + self._tail_len = 0 + return self._buf + + def size(self): + return len(self._buf) + self._tail_len + + def value(self): + return self._flat() + + def write(self, s, pad): + pos = self._pos + n = len(s) + if pos == self.size(): + self._tail.append(s) + self._tail_len += n + else: + buf = self._flat() + if pos > len(buf): + buf = buf + pad(pos - len(buf)) + self._buf = buf[:pos] + s + buf[pos + n:] + self._pos = pos + n + return n + + def read(self, size): + buf = self._flat() + start = self._pos + end = len(buf) if size is None or size < 0 else min(len(buf), start + size) + self._pos = max(end, start) + return buf[start:end] + + def readline(self, size, nl): + buf = self._flat() + start = self._pos + i = buf.find(nl, start) + end = len(buf) if i < 0 else i + len(nl) + if size is not None and size >= 0: + end = min(end, start + size) + self._pos = max(end, start) + return buf[start:end] + + def seek(self, pos, whence): + if whence == 1: + pos += self._pos + elif whence == 2: + pos += self.size() + elif whence != 0: + raise ValueError("invalid whence (%r, should be 0, 1 or 2)" % whence) + if pos < 0: + raise ValueError("negative seek value %d" % pos) + self._pos = pos + return pos + + def truncate(self, size): + buf = self._flat() + if size is None: + size = self._pos + if size < 0: + raise ValueError("negative size value %d" % size) + self._buf = buf[:size] + return size + + +class StringIO(TextIOBase): + def __init__(self, initial_value="", newline="\n"): + if initial_value is not None and not isinstance(initial_value, str): + raise TypeError("initial_value must be str or None, not " + type(initial_value).__name__) + self._b = _Buffer(initial_value or "", "") + + def readable(self): + self._check() + return True + + def writable(self): + self._check() + return True + + def seekable(self): + self._check() + return True + + def getvalue(self): + self._check() + return self._b.value() + + def write(self, s): + self._check() + if not isinstance(s, str): + raise TypeError("string argument expected, got '" + type(s).__name__ + "'") + return self._b.write(s, lambda n: "\0" * n) + + def read(self, size=-1): + self._check() + return self._b.read(size) + + def readline(self, size=-1): + self._check() + return self._b.readline(size, "\n") + + def seek(self, pos, whence=0): + self._check() + return self._b.seek(pos, whence) + + def tell(self): + self._check() + return self._b._pos + + def truncate(self, size=None): + self._check() + return self._b.truncate(size) + + +class BytesIO(BufferedIOBase): + def __init__(self, initial_bytes=b""): + self._b = _Buffer(bytes(initial_bytes or b""), b"") + + def readable(self): + self._check() + return True + + def writable(self): + self._check() + return True + + def seekable(self): + self._check() + return True + + def getvalue(self): + self._check() + return self._b.value() + + def getbuffer(self): + self._check() + return memoryview(self._b.value()) + + def write(self, s): + self._check() + if isinstance(s, (bytearray, memoryview)): + s = bytes(s) + if not isinstance(s, bytes): + raise TypeError("a bytes-like object is required, not '" + type(s).__name__ + "'") + return self._b.write(s, bytes) + + def read(self, size=-1): + self._check() + return self._b.read(size) + + read1 = read + + def readline(self, size=-1): + self._check() + return self._b.readline(size, b"\n") + + def seek(self, pos, whence=0): + self._check() + return self._b.seek(pos, whence) + + def tell(self): + self._check() + return self._b._pos + + def truncate(self, size=None): + self._check() + return self._b.truncate(size) + + +class FileIO(RawIOBase): + """The bytes of one file, held in memory while it is open.""" + + def __init__(self, file, mode="r", closefd=True, opener=None): + name = _fspath(file) + path = _abspath(name) + self.name = file + self._path = path + kind = mode[0] + self._reading = kind == "r" or "+" in mode + self._writing = kind != "r" or "+" in mode + self._append = kind == "a" + self.mode = _raw_mode(mode) + exists = _cn1fs.exists(path) + if exists and _cn1fs.isdir(path): + raise IsADirectoryError(21, "Is a directory", name) + if kind == "x" and exists: + raise FileExistsError(17, "File exists", name) + if kind == "r" and not exists: + raise FileNotFoundError(2, "No such file or directory", name) + parent = path.rsplit("/", 1)[0] or "/" + if kind != "r" and not _cn1fs.isdir(parent): + raise FileNotFoundError(2, "No such file or directory", name) + initial = b"" + if kind == "r" or (kind == "a" and "+" in mode): + initial = _cn1fs.read(path) + if kind in "wx": + _cn1fs.write(path, b"", False) + self._b = _Buffer(initial, b"") + self._base = len(initial) if self._append else 0 + if self._append: + self._b._pos = self._b.size() + self._dirty = False + + def readable(self): + self._check() + return self._reading + + def writable(self): + self._check() + return self._writing + + def seekable(self): + self._check() + return True + + def read(self, size=-1): + self._check() + if not self._reading: + raise UnsupportedOperation("File not open for reading") + return self._b.read(size) + + def readall(self): + return self.read() + + def readline(self, size=-1): + self._check() + if not self._reading: + raise UnsupportedOperation("File not open for reading") + return self._b.readline(size, b"\n") + + def write(self, data): + self._check() + if not self._writing: + raise UnsupportedOperation("File not open for writing") + if isinstance(data, (bytearray, memoryview)): + data = bytes(data) + if not isinstance(data, bytes): + raise TypeError("a bytes-like object is required, not '" + type(data).__name__ + "'") + if self._append: + self._b._pos = self._b.size() + self._dirty = True + return self._b.write(data, bytes) + + def seek(self, pos, whence=0): + self._check() + return self._b.seek(pos, whence) + + def tell(self): + self._check() + return self._b._pos + + def truncate(self, size=None): + self._check() + if not self._writing: + raise UnsupportedOperation("File not open for writing") + self._dirty = True + return self._b.truncate(size) + + def flush(self): + self._check() + if self._dirty: + data = self._b.value() + if self._append: + _cn1fs.write(self._path, data[self._base:], True) + self._base = len(data) + else: + _cn1fs.write(self._path, data, False) + self._dirty = False + + def __repr__(self): + return "<_io.FileIO name=%r mode=%r closefd=True>" % (self.name, self.mode) + + +def _raw_mode(mode): + kind = mode[0] + if kind == "r": + return "rb+" if "+" in mode else "rb" + if kind == "a": + return "ab+" if "+" in mode else "ab" + if kind == "x": + return "xb+" if "+" in mode else "xb" + return "rb+" if "+" in mode else "wb" + + +class _BufferedFile(BufferedIOBase): + def __init__(self, raw): + self.raw = raw + + @property + def name(self): + return self.raw.name + + @property + def mode(self): + return self.raw.mode + + @property + def closed(self): + return self.raw.closed + + def _check(self): + self.raw._check() + + def close(self): + self.raw.close() + + def flush(self): + self.raw.flush() + + def readable(self): + return self.raw.readable() + + def writable(self): + return self.raw.writable() + + def seekable(self): + return self.raw.seekable() + + def read(self, size=-1): + return self.raw.read(size) + + read1 = read + + def peek(self, size=0): + pos = self.raw.tell() + data = self.raw.read(max(size, 1)) + self.raw.seek(pos) + return data + + def readline(self, size=-1): + return self.raw.readline(size) + + def write(self, data): + return self.raw.write(data) + + def seek(self, pos, whence=0): + return self.raw.seek(pos, whence) + + def tell(self): + return self.raw.tell() + + def truncate(self, size=None): + return self.raw.truncate(size) + + def detach(self): + raw = self.raw + self.raw = None + return raw + + def __repr__(self): + return "<_io.%s name=%r>" % (type(self).__name__, self.name) + + +class BufferedReader(_BufferedFile): + pass + + +class BufferedWriter(_BufferedFile): + pass + + +class BufferedRandom(_BufferedFile): + pass + + +for _c in (BufferedReader, BufferedWriter, BufferedRandom, FileIO): + _c.__module__ = "_io" + + +def text_encoding(encoding, stacklevel=2): + return "utf-8" if encoding is None else encoding + + +class TextIOWrapper(TextIOBase): + """Text over a binary stream: decoded whole on open, encoded on flush.""" + + def __init__(self, buffer, encoding=None, errors=None, newline=None, line_buffering=False, + write_through=False): + if newline not in (None, "", "\n", "\r", "\r\n"): + raise ValueError("illegal newline value: %r" % (newline,)) + self.buffer = buffer + self.encoding = "UTF-8" if encoding is None else encoding + self.errors = "strict" if errors is None else errors + self._newline = newline + self.line_buffering = line_buffering + text = "" + if buffer.readable(): + pos = buffer.tell() + buffer.seek(0) + raw = buffer.read() + buffer.seek(pos) + text = raw.decode(self.encoding, self.errors) + if newline is None: + text = text.replace("\r\n", "\n").replace("\r", "\n") + self._t = _Buffer(text, "") + if getattr(buffer, "raw", None) is not None and getattr(buffer.raw, "_append", False): + self._t._pos = self._t.size() + self._base = self._t.size() + else: + self._base = 0 + self._dirty = False + self._mode = None + + @property + def name(self): + return self.buffer.name + + @property + def mode(self): + if self._mode is not None: + return self._mode + return self.buffer.mode.replace("b", "") + + @property + def closed(self): + return self.buffer.closed + + def _check(self): + self.buffer._check() + + def readable(self): + return self.buffer.readable() + + def writable(self): + return self.buffer.writable() + + def seekable(self): + return self.buffer.seekable() + + def read(self, size=-1): + self._check() + if not self.buffer.readable(): + raise UnsupportedOperation("not readable") + return self._t.read(size) + + def readline(self, size=-1): + self._check() + if not self.buffer.readable(): + raise UnsupportedOperation("not readable") + return self._t.readline(size, "\n") + + def write(self, s): + self._check() + if not self.buffer.writable(): + raise UnsupportedOperation("not writable") + if not isinstance(s, str): + raise TypeError("write() argument must be str, not " + type(s).__name__) + if getattr(self.buffer.raw, "_append", False): + self._t._pos = self._t.size() + self._dirty = True + n = self._t.write(s, lambda k: "\0" * k) + if self.line_buffering and "\n" in s: + self.flush() + return n + + def seek(self, pos, whence=0): + self._check() + if whence == 1 and pos != 0: + raise UnsupportedOperation("can't do nonzero cur-relative seeks") + if whence == 2 and pos != 0: + raise UnsupportedOperation("can't do nonzero end-relative seeks") + return self._t.seek(pos, whence) + + def tell(self): + self._check() + return self._t._pos + + def truncate(self, size=None): + self._check() + self._dirty = True + return self._t.truncate(size) + + def flush(self): + self._check() + if self._dirty: + text = self._t.value() + nl = self._newline + if nl is not None and nl not in ("", "\n"): + text = text.replace("\n", nl) + raw = self.buffer.raw + if getattr(raw, "_append", False): + data = text[self._base:].encode(self.encoding, self.errors) + raw._b = _Buffer(data, b"") + raw._base = 0 + raw._dirty = True + raw.flush() + raw._b = _Buffer(b"", b"") + self._base = len(text) + else: + raw._b = _Buffer(text.encode(self.encoding, self.errors), b"") + raw._dirty = True + raw.flush() + self._dirty = False + + def close(self): + if not self.closed: + try: + self.flush() + finally: + self.buffer.close() + + def detach(self): + self.flush() + b = self.buffer + self.buffer = None + return b + + def reconfigure(self, *, encoding=None, errors=None, newline=None, line_buffering=None, + write_through=None): + if encoding is not None: + self.encoding = encoding + if errors is not None: + self.errors = errors + if line_buffering is not None: + self.line_buffering = line_buffering + + def __repr__(self): + return "<_io.TextIOWrapper name=%r mode=%r encoding=%r>" % (self.name, self.mode, self.encoding) + + +TextIOWrapper.__module__ = "_io" +StringIO.__module__ = "_io" +BytesIO.__module__ = "_io" + + +def open(file, mode="r", buffering=-1, encoding=None, errors=None, newline=None, closefd=True, + opener=None): + if not isinstance(mode, str): + raise TypeError("open() argument 'mode' must be str, not " + type(mode).__name__) + modes = set(mode) + if modes - set("axrwb+t") or len(mode) > len(modes): + raise ValueError("invalid mode: %r" % mode) + kinds = [c for c in mode if c in "rwax"] + if len(kinds) != 1: + raise ValueError("must have exactly one of create/read/write/append mode") + binary = "b" in modes + if binary and "t" in modes: + raise ValueError("can't have text and binary mode at once") + if binary and encoding is not None: + raise ValueError("binary mode doesn't take an encoding argument") + if binary and errors is not None: + raise ValueError("binary mode doesn't take an errors argument") + if binary and newline is not None: + raise ValueError("binary mode doesn't take a newline argument") + if isinstance(file, int): + raise OSError("file descriptors are not supported") + raw = FileIO(file, kinds[0] + ("+" if "+" in modes else "")) + if buffering == 0: + if not binary: + raise ValueError("can't have unbuffered text I/O") + return raw + if "+" in modes: + buf = BufferedRandom(raw) + elif kinds[0] == "r": + buf = BufferedReader(raw) + else: + buf = BufferedWriter(raw) + if binary: + return buf + w = TextIOWrapper(buf, encoding, errors, newline, line_buffering=buffering == 1) + w._mode = mode + return w + + +def open_code(path): + return open(path, "rb") diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/itertools.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/itertools.py new file mode 100644 index 00000000000..7670a389a9c --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/itertools.py @@ -0,0 +1,288 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Iterator building blocks.""" + + +def count(start=0, step=1): + n = start + while True: + yield n + n += step + + +def cycle(iterable): + saved = [] + for x in iterable: + yield x + saved.append(x) + while saved: + for x in saved: + yield x + + +def repeat(obj, times=None): + if times is None: + while True: + yield obj + else: + for _ in range(times): + yield obj + + +def accumulate(iterable, func=None, *, initial=None): + it = iter(iterable) + total = initial + if initial is None: + try: + total = next(it) + except StopIteration: + return + yield total + for x in it: + total = total + x if func is None else func(total, x) + yield total + + +def batched(iterable, n, *, strict=False): + if n < 1: + raise ValueError("n must be at least one") + it = iter(iterable) + while True: + batch = tuple(islice(it, n)) + if not batch: + return + if strict and len(batch) != n: + raise ValueError("batched(): incomplete batch") + yield batch + + +def chain(*iterables): + for it in iterables: + for x in it: + yield x + + +def _from_iterable(iterables): + for it in iterables: + for x in it: + yield x + + +chain.from_iterable = _from_iterable + + +def compress(data, selectors): + for d, s in zip(data, selectors): + if s: + yield d + + +def dropwhile(predicate, iterable): + it = iter(iterable) + for x in it: + if not predicate(x): + yield x + break + for x in it: + yield x + + +def filterfalse(predicate, iterable): + if predicate is None: + predicate = bool + for x in iterable: + if not predicate(x): + yield x + + +_NO_KEY = object() + + +class groupby: + def __init__(self, iterable, key=None): + self._key = key + self._it = iter(iterable) + self._have = False + self._cur_key = None + self._cur_val = None + self._tgt = _NO_KEY + + def __iter__(self): + return self + + def __next__(self): + while True: + if not self._have: + self._fetch() + if self._tgt is _NO_KEY or self._cur_key != self._tgt: + break + self._have = False + self._tgt = self._cur_key + return (self._cur_key, self._grouper(self._tgt)) + + def _fetch(self): + self._cur_val = next(self._it) + self._cur_key = self._cur_val if self._key is None else self._key(self._cur_val) + self._have = True + + def _grouper(self, tgt): + while True: + if not self._have: + try: + self._fetch() + except StopIteration: + return + if self._tgt is not tgt or self._cur_key != tgt: + return + self._have = False + yield self._cur_val + + +def islice(iterable, *args): + s = slice(*args) + start = 0 if s.start is None else s.start + stop = s.stop + step = 1 if s.step is None else s.step + if start < 0 or (stop is not None and stop < 0) or step <= 0: + raise ValueError("Indices for islice() must be None or an integer: 0 <= x <= sys.maxsize.") + it = iter(iterable) + i = 0 + nxt = start + while stop is None or i < stop: + try: + x = next(it) + except StopIteration: + return + if i == nxt: + yield x + nxt += step + i += 1 + + +def pairwise(iterable): + it = iter(iterable) + try: + a = next(it) + except StopIteration: + return + for b in it: + yield (a, b) + a = b + + +def starmap(function, iterable): + for args in iterable: + yield function(*args) + + +def takewhile(predicate, iterable): + for x in iterable: + if not predicate(x): + return + yield x + + +def tee(iterable, n=2): + it = iter(iterable) + buffers = [[] for _ in range(n)] + + def gen(mine): + while True: + if not mine: + try: + v = next(it) + except StopIteration: + return + for b in buffers: + b.append(v) + yield mine.pop(0) + return tuple(gen(b) for b in buffers) + + +def zip_longest(*iterables, fillvalue=None): + its = [iter(i) for i in iterables] + active = len(its) + if not active: + return + while True: + row = [] + for i in range(len(its)): + try: + row.append(next(its[i])) + except StopIteration: + active -= 1 + if not active: + return + its[i] = repeat(fillvalue) + row.append(fillvalue) + yield tuple(row) + + +def product(*iterables, repeat=1): + pools = [tuple(p) for p in iterables] * repeat + result = [[]] + for pool in pools: + result = [x + [y] for x in result for y in pool] + for prod in result: + yield tuple(prod) + + +def permutations(iterable, r=None): + pool = tuple(iterable) + n = len(pool) + r = n if r is None else r + if r > n: + return + indices = list(range(n)) + cycles = list(range(n, n - r, -1)) + yield tuple(pool[i] for i in indices[:r]) + while n: + for i in reversed(range(r)): + cycles[i] -= 1 + if cycles[i] == 0: + indices[i:] = indices[i + 1:] + indices[i:i + 1] + cycles[i] = n - i + else: + j = cycles[i] + indices[i], indices[-j] = indices[-j], indices[i] + yield tuple(pool[i] for i in indices[:r]) + break + else: + return + + +def combinations(iterable, r): + pool = tuple(iterable) + n = len(pool) + if r > n: + return + indices = list(range(r)) + yield tuple(pool[i] for i in indices) + while True: + for i in reversed(range(r)): + if indices[i] != i + n - r: + break + else: + return + indices[i] += 1 + for j in range(i + 1, r): + indices[j] = indices[j - 1] + 1 + yield tuple(pool[i] for i in indices) + + +def combinations_with_replacement(iterable, r): + pool = tuple(iterable) + n = len(pool) + if not n and r: + return + indices = [0] * r + yield tuple(pool[i] for i in indices) + while True: + for i in reversed(range(r)): + if indices[i] != n - 1: + break + else: + return + indices[i:] = [indices[i] + 1] * (r - i) + yield tuple(pool[i] for i in indices) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/json.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/json.py new file mode 100644 index 00000000000..03112722876 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/json.py @@ -0,0 +1,397 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""JSON encoding and decoding (RFC 8259), with the json module's interface.""" + +__all__ = ["dump", "dumps", "load", "loads", "JSONDecoder", "JSONDecodeError", "JSONEncoder"] + +_ESC = {'"': '\\"', "\\": "\\\\", "\n": "\\n", "\r": "\\r", "\t": "\\t", "\b": "\\b", "\f": "\\f"} +_HEX = "0123456789abcdef" + + +def _u4(n): + return "\\u" + _HEX[(n >> 12) & 15] + _HEX[(n >> 8) & 15] + _HEX[(n >> 4) & 15] + _HEX[n & 15] + + +def _quote(s, ensure_ascii): + out = ['"'] + for ch in s: + e = _ESC.get(ch) + if e is not None: + out.append(e) + continue + n = ord(ch) + if n < 0x20: + out.append(_u4(n)) + elif ensure_ascii and n > 0x7E: + if n > 0xFFFF: + n -= 0x10000 + out.append(_u4(0xD800 | (n >> 10))) + out.append(_u4(0xDC00 | (n & 0x3FF))) + else: + out.append(_u4(n)) + else: + out.append(ch) + out.append('"') + return "".join(out) + + +class JSONDecodeError(ValueError): + def __init__(self, msg, doc, pos): + lineno = doc.count("\n", 0, pos) + 1 + colno = pos - doc.rfind("\n", 0, pos) + ValueError.__init__(self, "%s: line %d column %d (char %d)" % (msg, lineno, colno, pos)) + self.msg = msg + self.doc = doc + self.pos = pos + self.lineno = lineno + self.colno = colno + + +class JSONEncoder: + item_separator = ", " + key_separator = ": " + + def __init__(self, *, skipkeys=False, ensure_ascii=True, check_circular=True, allow_nan=True, + sort_keys=False, indent=None, separators=None, default=None): + self.skipkeys = skipkeys + self.ensure_ascii = ensure_ascii + self.check_circular = check_circular + self.allow_nan = allow_nan + self.sort_keys = sort_keys + if indent is not None and not isinstance(indent, str): + indent = " " * indent + self.indent = indent + if separators is not None: + self.item_separator, self.key_separator = separators + elif indent is not None: + self.item_separator = "," + if default is not None: + self.default = default + + def default(self, o): + raise TypeError("Object of type %s is not JSON serializable" % type(o).__name__) + + def encode(self, o): + return "".join(self.iterencode(o)) + + def iterencode(self, o, _one_shot=False): + out = [] + self._enc(o, out, 0, {} if self.check_circular else None) + return out + + def _float(self, f): + if f != f: + text = "NaN" + elif f == float("inf"): + text = "Infinity" + elif f == -float("inf"): + text = "-Infinity" + else: + return float.__repr__(f) + if not self.allow_nan: + raise ValueError("Out of range float values are not JSON compliant: " + repr(f)) + return text + + def _key(self, k): + if isinstance(k, str): + return k + if isinstance(k, float): + return self._float(k) + if k is True: + return "true" + if k is False: + return "false" + if k is None: + return "null" + if isinstance(k, int): + return int.__repr__(k) + if self.skipkeys: + return None + raise TypeError("keys must be str, int, float, bool or None, not " + type(k).__name__) + + def _nl(self, out, level): + if self.indent is not None: + out.append("\n" + self.indent * level) + + def _enc(self, o, out, level, markers): + if isinstance(o, str): + out.append(_quote(o, self.ensure_ascii)) + elif o is None: + out.append("null") + elif o is True: + out.append("true") + elif o is False: + out.append("false") + elif isinstance(o, int): + out.append(int.__repr__(o)) + elif isinstance(o, float): + out.append(self._float(o)) + elif isinstance(o, (list, tuple)): + self._mark(o, markers) + if not o: + out.append("[]") + else: + out.append("[") + first = True + for v in o: + if not first: + out.append(self.item_separator) + first = False + self._nl(out, level + 1) + self._enc(v, out, level + 1, markers) + self._nl(out, level) + out.append("]") + self._unmark(o, markers) + elif isinstance(o, dict): + self._mark(o, markers) + if not o: + out.append("{}") + else: + out.append("{") + items = list(o.items()) + if self.sort_keys: + items.sort(key=lambda kv: kv[0]) + first = True + for k, v in items: + k = self._key(k) + if k is None: + continue + if not first: + out.append(self.item_separator) + first = False + self._nl(out, level + 1) + out.append(_quote(k, self.ensure_ascii)) + out.append(self.key_separator) + self._enc(v, out, level + 1, markers) + self._nl(out, level) + out.append("}") + self._unmark(o, markers) + else: + self._mark(o, markers) + self._enc(self.default(o), out, level, markers) + self._unmark(o, markers) + + def _mark(self, o, markers): + if markers is not None: + if id(o) in markers: + raise ValueError("Circular reference detected") + markers[id(o)] = o + + def _unmark(self, o, markers): + if markers is not None: + del markers[id(o)] + + +_WS = " \t\n\r" +_UNESC = {'"': '"', "\\": "\\", "/": "/", "b": "\b", "f": "\f", "n": "\n", "r": "\r", "t": "\t"} +_CONST = {"NaN": float("nan"), "Infinity": float("inf"), "-Infinity": -float("inf")} + + +class JSONDecoder: + def __init__(self, *, object_hook=None, parse_float=None, parse_int=None, parse_constant=None, + strict=True, object_pairs_hook=None): + self.object_hook = object_hook + self.parse_float = parse_float or float + self.parse_int = parse_int or int + self.parse_constant = parse_constant or _CONST.__getitem__ + self.strict = strict + self.object_pairs_hook = object_pairs_hook + + def decode(self, s): + obj, end = self.raw_decode(s, self._ws(s, 0)) + end = self._ws(s, end) + if end != len(s): + raise JSONDecodeError("Extra data", s, end) + return obj + + def raw_decode(self, s, idx=0): + return self._value(s, idx) + + def _ws(self, s, i): + n = len(s) + while i < n and s[i] in _WS: + i += 1 + return i + + def _value(self, s, i): + if i >= len(s): + raise JSONDecodeError("Expecting value", s, i) + c = s[i] + if c == '"': + return self._string(s, i + 1) + if c == "{": + return self._object(s, i + 1) + if c == "[": + return self._array(s, i + 1) + if c == "n" and s.startswith("null", i): + return None, i + 4 + if c == "t" and s.startswith("true", i): + return True, i + 4 + if c == "f" and s.startswith("false", i): + return False, i + 5 + if c == "N" and s.startswith("NaN", i): + return self.parse_constant("NaN"), i + 3 + if c == "I" and s.startswith("Infinity", i): + return self.parse_constant("Infinity"), i + 8 + if c == "-" and s.startswith("-Infinity", i): + return self.parse_constant("-Infinity"), i + 9 + return self._number(s, i) + + def _number(self, s, i): + n = len(s) + j = i + if j < n and s[j] == "-": + j += 1 + if j < n and s[j] == "0": + j += 1 + elif j < n and "1" <= s[j] <= "9": + while j < n and "0" <= s[j] <= "9": + j += 1 + else: + raise JSONDecodeError("Expecting value", s, i) + frac = False + if j + 1 < n and s[j] == "." and "0" <= s[j + 1] <= "9": + frac = True + j += 1 + while j < n and "0" <= s[j] <= "9": + j += 1 + if j < n and s[j] in "eE": + k = j + 1 + if k < n and s[k] in "+-": + k += 1 + if k < n and "0" <= s[k] <= "9": + while k < n and "0" <= s[k] <= "9": + k += 1 + frac = True + j = k + text = s[i:j] + return (self.parse_float(text) if frac else self.parse_int(text)), j + + def _string(self, s, i): + start = i - 1 + out = [] + n = len(s) + while True: + j = i + while j < n and s[j] != '"' and s[j] != "\\": + if self.strict and ord(s[j]) < 0x20: + raise JSONDecodeError("Invalid control character at", s, j) + j += 1 + out.append(s[i:j]) + if j >= n: + raise JSONDecodeError("Unterminated string starting at", s, start) + if s[j] == '"': + return "".join(out), j + 1 + j += 1 + if j >= n: + raise JSONDecodeError("Unterminated string starting at", s, start) + e = s[j] + if e == "u": + cp = self._hex4(s, j + 1) + j += 5 + if 0xD800 <= cp <= 0xDBFF and s.startswith("\\u", j): + lo = self._hex4(s, j + 2) + if 0xDC00 <= lo <= 0xDFFF: + cp = 0x10000 + (((cp - 0xD800) << 10) | (lo - 0xDC00)) + j += 6 + out.append(chr(cp)) + i = j + else: + r = _UNESC.get(e) + if r is None: + raise JSONDecodeError("Invalid \\escape", s, j - 1) + out.append(r) + i = j + 1 + + def _hex4(self, s, i): + h = s[i:i + 4] + if len(h) == 4: + v = 0 + for ch in h: + d = _HEX.find(ch.lower()) + if d < 0: + break + v = v * 16 + d + else: + return v + raise JSONDecodeError("Invalid \\uXXXX escape", s, i - 1) + + def _object(self, s, i): + pairs = [] + i = self._ws(s, i) + if i < len(s) and s[i] == "}": + i += 1 + else: + while True: + if i >= len(s) or s[i] != '"': + raise JSONDecodeError("Expecting property name enclosed in double quotes", s, i) + key, i = self._string(s, i + 1) + i = self._ws(s, i) + if i >= len(s) or s[i] != ":": + raise JSONDecodeError("Expecting ':' delimiter", s, i) + value, i = self._value(s, self._ws(s, i + 1)) + pairs.append((key, value)) + i = self._ws(s, i) + if i < len(s) and s[i] == ",": + comma = i + i = self._ws(s, i + 1) + if i < len(s) and s[i] == "}": + raise JSONDecodeError("Illegal trailing comma before end of object", s, comma) + continue + if i < len(s) and s[i] == "}": + i += 1 + break + raise JSONDecodeError("Expecting ',' delimiter", s, i) + if self.object_pairs_hook is not None: + return self.object_pairs_hook(pairs), i + d = dict(pairs) + if self.object_hook is not None: + d = self.object_hook(d) + return d, i + + def _array(self, s, i): + values = [] + i = self._ws(s, i) + if i < len(s) and s[i] == "]": + return values, i + 1 + while True: + v, i = self._value(s, i) + values.append(v) + i = self._ws(s, i) + if i < len(s) and s[i] == ",": + comma = i + i = self._ws(s, i + 1) + if i < len(s) and s[i] == "]": + raise JSONDecodeError("Illegal trailing comma before end of array", s, comma) + continue + if i < len(s) and s[i] == "]": + return values, i + 1 + raise JSONDecodeError("Expecting ',' delimiter", s, i) + + +def dumps(obj, *, skipkeys=False, ensure_ascii=True, check_circular=True, allow_nan=True, cls=None, + indent=None, separators=None, default=None, sort_keys=False, **kw): + if cls is None: + cls = JSONEncoder + return cls(skipkeys=skipkeys, ensure_ascii=ensure_ascii, check_circular=check_circular, + allow_nan=allow_nan, indent=indent, separators=separators, default=default, + sort_keys=sort_keys, **kw).encode(obj) + + +def dump(obj, fp, **kw): + fp.write(dumps(obj, **kw)) + + +def loads(s, *, cls=None, **kw): + if isinstance(s, bytes): + s = s.decode("utf-8") + if not isinstance(s, str): + raise TypeError("the JSON object must be str, bytes or bytearray, not " + type(s).__name__) + if cls is None: + cls = JSONDecoder + return cls(**kw).decode(s) + + +def load(fp, **kw): + return loads(fp.read(), **kw) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/logging/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/logging/__init__.py new file mode 100644 index 00000000000..8b93deca863 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/logging/__init__.py @@ -0,0 +1,1043 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Logging, with the API and output formats of CPython's logging package. + +Every application is single threaded on the EDT, so handlers take no +locks. A traceback is the exception's type and message only: there are +no frames to print. +""" + +import sys +import time +import os + +__all__ = ["BASIC_FORMAT", "CRITICAL", "DEBUG", "ERROR", "FATAL", "FileHandler", "Filter", "Formatter", "Handler", + "INFO", "LogRecord", "Logger", "LoggerAdapter", "NOTSET", "NullHandler", "StreamHandler", "WARN", + "WARNING", "addLevelName", "basicConfig", "critical", "debug", "disable", "error", "exception", "fatal", + "getLevelName", "getLogger", "getLoggerClass", "info", "log", "makeLogRecord", "setLoggerClass", + "shutdown", "warn", "warning", "getLevelNamesMapping", "getHandlerByName", "getHandlerNames", + "lastResort", "raiseExceptions", "captureWarnings", "PercentStyle", "StrFormatStyle", + "StringTemplateStyle", "BufferingFormatter", "root"] + +CRITICAL = 50 +FATAL = CRITICAL +ERROR = 40 +WARNING = 30 +WARN = WARNING +INFO = 20 +DEBUG = 10 +NOTSET = 0 + +raiseExceptions = True +logThreads = True +logMultiprocessing = True +logProcesses = True +logAsyncioTasks = True + +_startTime = time.time() + +_levelToName = {CRITICAL: "CRITICAL", ERROR: "ERROR", WARNING: "WARNING", INFO: "INFO", DEBUG: "DEBUG", + NOTSET: "NOTSET"} +_nameToLevel = {"CRITICAL": CRITICAL, "FATAL": FATAL, "ERROR": ERROR, "WARN": WARNING, "WARNING": WARNING, + "INFO": INFO, "DEBUG": DEBUG, "NOTSET": NOTSET} + + +def getLevelNamesMapping(): + return _nameToLevel.copy() + + +def getLevelName(level): + result = _levelToName.get(level) + if result is not None: + return result + result = _nameToLevel.get(level) + if result is not None: + return result + return "Level %s" % level + + +def addLevelName(level, levelName): + _levelToName[level] = levelName + _nameToLevel[levelName] = level + + +def _checkLevel(level): + if isinstance(level, int): + rv = level + elif str(level) == level: + if level not in _nameToLevel: + raise ValueError("Unknown level: %r" % level) + rv = _nameToLevel[level] + else: + raise TypeError("Level not an integer or a valid string: %r" % (level,)) + return rv + + +def _format_exception(ei): + etype, value = ei[0], ei[1] + lines = ["Traceback (most recent call last):"] + msg = str(value) + name = etype.__name__ + mod = getattr(etype, "__module__", "builtins") + if mod not in ("builtins", "__main__"): + name = mod + "." + name + lines.append(name + (": " + msg if msg else "")) + return "\n".join(lines) + + +class LogRecord: + def __init__(self, name, level, pathname, lineno, msg, args, exc_info, func=None, sinfo=None, **kwargs): + ct = time.time() + self.name = name + self.msg = msg + if (args and len(args) == 1 and isinstance(args[0], dict) and args[0]): + args = args[0] + self.args = args + self.levelname = getLevelName(level) + self.levelno = level + self.pathname = pathname + self.filename = os.path.basename(pathname) if pathname else pathname + self.module = os.path.splitext(self.filename)[0] if pathname else "Unknown module" + self.exc_info = exc_info + self.exc_text = None + self.stack_info = sinfo + self.lineno = lineno + self.funcName = func + self.created = ct + self.msecs = int((ct - int(ct)) * 1000) + 0.0 + self.relativeCreated = (ct - _startTime) * 1000 + self.thread = None + self.threadName = "MainThread" + self.processName = "MainProcess" + self.process = None + self.taskName = None + + def __repr__(self): + return '' % (self.name, self.levelno, self.pathname, self.lineno, self.msg) + + def getMessage(self): + msg = str(self.msg) + if self.args: + msg = msg % self.args + return msg + + +_logRecordFactory = LogRecord + + +def setLogRecordFactory(factory): + global _logRecordFactory + _logRecordFactory = factory + + +def getLogRecordFactory(): + return _logRecordFactory + + +def makeLogRecord(dict): + rv = _logRecordFactory(None, None, "", 0, "", (), None, None) + rv.__dict__.update(dict) + return rv + + +class _Mapping: + """Formats a record's fields by name, for the str.format and $ styles.""" + + def __init__(self, record): + self._record = record + + def __getitem__(self, key): + try: + return getattr(self._record, key) + except AttributeError: + raise KeyError(key) from None + + +class PercentStyle: + default_format = "%(message)s" + asctime_format = "%(asctime)s" + asctime_search = "%(asctime)" + + def __init__(self, fmt, *, defaults=None): + self._fmt = fmt or self.default_format + self._defaults = defaults + + def usesTime(self): + return self._fmt.find(self.asctime_search) >= 0 + + def validate(self): + import re + if not re.search(r"%\(\w+\)[#0+ -]*(\*|\d+)?(\.(\*|\d+))?[diouxefgcrsa%]", self._fmt, re.I): + raise ValueError("Invalid format '%s' for '%s' style" % (self._fmt, self.default_format[0])) + + def _values(self, record): + if self._defaults: + values = dict(self._defaults) + values.update(record.__dict__) + return values + return record.__dict__ + + def _format(self, record): + return self._fmt % self._values(record) + + def format(self, record): + try: + return self._format(record) + except KeyError as e: + raise ValueError("Formatting field not found in record: %s" % e) + + +class StrFormatStyle(PercentStyle): + default_format = "{message}" + asctime_format = "{asctime}" + asctime_search = "{asctime" + + def _format(self, record): + return self._fmt.format(**self._values(record)) + + def validate(self): + import string + fields = set() + for _, fieldname, spec, conversion in string.Formatter().parse(self._fmt): + if fieldname: + fields.add(fieldname) + if conversion and conversion not in "rsa": + raise ValueError("invalid conversion: %r" % conversion) + if not fields: + raise ValueError("invalid format: no fields") + + +class StringTemplateStyle(PercentStyle): + default_format = "${message}" + asctime_format = "${asctime}" + asctime_search = "${asctime}" + + def __init__(self, *args, **kwargs): + PercentStyle.__init__(self, *args, **kwargs) + from string import Template + self._tpl = Template(self._fmt) + + def usesTime(self): + fmt = self._fmt + return fmt.find("$asctime") >= 0 or fmt.find(self.asctime_search) >= 0 + + def validate(self): + if not self._tpl.is_valid(): + raise ValueError("invalid format: bare '$' not allowed") + if not self._tpl.get_identifiers(): + raise ValueError("invalid format: no fields") + + def _format(self, record): + return self._tpl.substitute(**self._values(record)) + + +BASIC_FORMAT = "%(levelname)s:%(name)s:%(message)s" + +_STYLES = {"%": (PercentStyle, BASIC_FORMAT), "{": (StrFormatStyle, "{levelname}:{name}:{message}"), + "$": (StringTemplateStyle, "${levelname}:${name}:${message}")} + + +class Formatter: + converter = time.localtime + default_time_format = "%Y-%m-%d %H:%M:%S" + default_msec_format = "%s,%03d" + + def __init__(self, fmt=None, datefmt=None, style="%", validate=True, *, defaults=None): + if style not in _STYLES: + raise ValueError("Style must be one of: %s" % ",".join(_STYLES.keys())) + self._style = _STYLES[style][0](fmt, defaults=defaults) + if validate: + self._style.validate() + self._fmt = self._style._fmt + self.datefmt = datefmt + + def formatTime(self, record, datefmt=None): + ct = self.converter(record.created) + if datefmt: + s = time.strftime(datefmt, ct) + else: + s = time.strftime(self.default_time_format, ct) + if self.default_msec_format: + s = self.default_msec_format % (s, record.msecs) + return s + + def formatException(self, ei): + return _format_exception(ei) + + def usesTime(self): + return self._style.usesTime() + + def formatMessage(self, record): + return self._style.format(record) + + def formatStack(self, stack_info): + return stack_info + + def format(self, record): + record.message = record.getMessage() + if self.usesTime(): + record.asctime = self.formatTime(record, self.datefmt) + s = self.formatMessage(record) + if record.exc_info: + if not record.exc_text: + record.exc_text = self.formatException(record.exc_info) + if record.exc_text: + if s[-1:] != "\n": + s = s + "\n" + s = s + record.exc_text + if record.stack_info: + if s[-1:] != "\n": + s = s + "\n" + s = s + self.formatStack(record.stack_info) + return s + + +_defaultFormatter = Formatter() + + +class BufferingFormatter: + def __init__(self, linefmt=None): + self.linefmt = linefmt if linefmt else _defaultFormatter + + def formatHeader(self, records): + return "" + + def formatFooter(self, records): + return "" + + def format(self, records): + rv = "" + if len(records) > 0: + rv = rv + self.formatHeader(records) + for record in records: + rv = rv + self.linefmt.format(record) + rv = rv + self.formatFooter(records) + return rv + + +class Filter: + def __init__(self, name=""): + self.name = name + self.nlen = len(name) + + def filter(self, record): + if self.nlen == 0: + return True + elif self.name == record.name: + return True + elif record.name.find(self.name, 0, self.nlen) != 0: + return False + return record.name[self.nlen] == "." + + +class Filterer: + def __init__(self): + self.filters = [] + + def addFilter(self, filter): + if filter not in self.filters: + self.filters.append(filter) + + def removeFilter(self, filter): + if filter in self.filters: + self.filters.remove(filter) + + def filter(self, record): + for f in self.filters: + if hasattr(f, "filter"): + result = f.filter(record) + else: + result = f(record) + if not result: + return False + if isinstance(result, LogRecord): + record = result + return record + + +_handlers = {} +_handlerList = [] + + +def getHandlerByName(name): + return _handlers.get(name) + + +def getHandlerNames(): + return frozenset(_handlers) + + +class Handler(Filterer): + def __init__(self, level=NOTSET): + Filterer.__init__(self) + self._name = None + self.level = _checkLevel(level) + self.formatter = None + self._closed = False + _handlerList.append(self) + + def get_name(self): + return self._name + + def set_name(self, name): + if self._name in _handlers: + del _handlers[self._name] + self._name = name + if name: + _handlers[name] = self + + name = property(get_name, set_name) + + def createLock(self): + self.lock = None + + def acquire(self): + pass + + def release(self): + pass + + def setLevel(self, level): + self.level = _checkLevel(level) + + def format(self, record): + fmt = self.formatter if self.formatter else _defaultFormatter + return fmt.format(record) + + def emit(self, record): + raise NotImplementedError("emit must be implemented by Handler subclasses") + + def handle(self, record): + rv = self.filter(record) + if isinstance(rv, LogRecord): + record = rv + if rv: + self.emit(record) + return rv + + def setFormatter(self, fmt): + self.formatter = fmt + + def flush(self): + pass + + def close(self): + self._closed = True + if self._name and self._name in _handlers: + del _handlers[self._name] + + def handleError(self, record): + if raiseExceptions and sys.stderr: + ei = sys.exc_info() + sys.stderr.write("--- Logging error ---\n") + sys.stderr.write(_format_exception(ei) + "\n") + sys.stderr.write("Message: %r\nArguments: %s\n" % (record.msg, record.args)) + + def __repr__(self): + level = getLevelName(self.level) + return "<%s (%s)>" % (self.__class__.__name__, level) + + +class StreamHandler(Handler): + terminator = "\n" + + def __init__(self, stream=None): + Handler.__init__(self) + if stream is None: + stream = sys.stderr + self.stream = stream + + def flush(self): + if self.stream and hasattr(self.stream, "flush"): + self.stream.flush() + + def emit(self, record): + try: + msg = self.format(record) + stream = self.stream + stream.write(msg + self.terminator) + self.flush() + except RecursionError: + raise + except Exception: + self.handleError(record) + + def setStream(self, stream): + if stream is self.stream: + result = None + else: + result = self.stream + self.flush() + self.stream = stream + return result + + def __repr__(self): + level = getLevelName(self.level) + name = getattr(self.stream, "name", "") + name = str(name) + if name: + name += " " + return "<%s %s(%s)>" % (self.__class__.__name__, name, level) + + +class FileHandler(StreamHandler): + def __init__(self, filename, mode="a", encoding=None, delay=False, errors=None): + filename = os.fspath(filename) + self.baseFilename = os.path.abspath(filename) + self.mode = mode + self.encoding = encoding + if "b" not in mode and encoding is None: + self.encoding = "utf-8" + self.errors = errors + self.delay = delay + if delay: + Handler.__init__(self) + self.stream = None + else: + StreamHandler.__init__(self, self._open()) + + def close(self): + if self.stream: + try: + self.flush() + finally: + stream = self.stream + self.stream = None + if hasattr(stream, "close"): + stream.close() + StreamHandler.close(self) + + def _open(self): + return open(self.baseFilename, self.mode, encoding=self.encoding, errors=self.errors) + + def emit(self, record): + if self.stream is None: + if self.mode != "w" or not self._closed: + self.stream = self._open() + if self.stream: + StreamHandler.emit(self, record) + + def __repr__(self): + level = getLevelName(self.level) + return "<%s %s (%s)>" % (self.__class__.__name__, self.baseFilename, level) + + +class _StderrHandler(StreamHandler): + def __init__(self, level=NOTSET): + Handler.__init__(self, level) + + @property + def stream(self): + return sys.stderr + + +_defaultLastResort = _StderrHandler(WARNING) +lastResort = _defaultLastResort + + +class NullHandler(Handler): + def handle(self, record): + pass + + def emit(self, record): + pass + + def createLock(self): + self.lock = None + + +class PlaceHolder: + def __init__(self, alogger): + self.loggerMap = {alogger: None} + + def append(self, alogger): + if alogger not in self.loggerMap: + self.loggerMap[alogger] = None + + +_loggerClass = None + + +def setLoggerClass(klass): + global _loggerClass + if klass != Logger: + if not issubclass(klass, Logger): + raise TypeError("logger not derived from logging.Logger: " + klass.__name__) + _loggerClass = klass + + +def getLoggerClass(): + return _loggerClass + + +class Manager: + def __init__(self, rootnode): + self.root = rootnode + self.disable = 0 + self.emittedNoHandlerWarning = False + self.loggerDict = {} + self.loggerClass = None + self.logRecordFactory = None + + @property + def disable(self): + return self._disable + + @disable.setter + def disable(self, value): + self._disable = _checkLevel(value) + + def getLogger(self, name): + rv = None + if not isinstance(name, str): + raise TypeError("A logger name must be a string") + if name in self.loggerDict: + rv = self.loggerDict[name] + if isinstance(rv, PlaceHolder): + ph = rv + rv = (self.loggerClass or _loggerClass)(name) + rv.manager = self + self.loggerDict[name] = rv + self._fixupChildren(ph, rv) + self._fixupParents(rv) + else: + rv = (self.loggerClass or _loggerClass)(name) + rv.manager = self + self.loggerDict[name] = rv + self._fixupParents(rv) + return rv + + def setLoggerClass(self, klass): + if klass != Logger: + if not issubclass(klass, Logger): + raise TypeError("logger not derived from logging.Logger: " + klass.__name__) + self.loggerClass = klass + + def setLogRecordFactory(self, factory): + self.logRecordFactory = factory + + def _fixupParents(self, alogger): + name = alogger.name + i = name.rfind(".") + rv = None + while (i > 0) and not rv: + substr = name[:i] + if substr not in self.loggerDict: + self.loggerDict[substr] = PlaceHolder(alogger) + else: + obj = self.loggerDict[substr] + if isinstance(obj, Logger): + rv = obj + else: + obj.append(alogger) + i = name.rfind(".", 0, i - 1) + if not rv: + rv = self.root + alogger.parent = rv + + def _fixupChildren(self, ph, alogger): + name = alogger.name + namelen = len(name) + for c in ph.loggerMap.keys(): + if c.parent.name[:namelen] != name: + alogger.parent = c.parent + c.parent = alogger + + def _clear_cache(self): + for logger in self.loggerDict.values(): + if isinstance(logger, Logger): + logger._cache.clear() + self.root._cache.clear() + + +class Logger(Filterer): + def __init__(self, name, level=NOTSET): + Filterer.__init__(self) + self.name = name + self.level = _checkLevel(level) + self.parent = None + self.propagate = True + self.handlers = [] + self.disabled = False + self._cache = {} + + def setLevel(self, level): + self.level = _checkLevel(level) + self.manager._clear_cache() + + def debug(self, msg, *args, **kwargs): + if self.isEnabledFor(DEBUG): + self._log(DEBUG, msg, args, **kwargs) + + def info(self, msg, *args, **kwargs): + if self.isEnabledFor(INFO): + self._log(INFO, msg, args, **kwargs) + + def warning(self, msg, *args, **kwargs): + if self.isEnabledFor(WARNING): + self._log(WARNING, msg, args, **kwargs) + + def warn(self, msg, *args, **kwargs): + import warnings + warnings.warn("The 'warn' method is deprecated, use 'warning' instead", DeprecationWarning, 2) + self.warning(msg, *args, **kwargs) + + def error(self, msg, *args, **kwargs): + if self.isEnabledFor(ERROR): + self._log(ERROR, msg, args, **kwargs) + + def exception(self, msg, *args, exc_info=True, **kwargs): + self.error(msg, *args, exc_info=exc_info, **kwargs) + + def critical(self, msg, *args, **kwargs): + if self.isEnabledFor(CRITICAL): + self._log(CRITICAL, msg, args, **kwargs) + + fatal = critical + + def log(self, level, msg, *args, **kwargs): + if not isinstance(level, int): + if raiseExceptions: + raise TypeError("level must be an integer") + else: + return + if self.isEnabledFor(level): + self._log(level, msg, args, **kwargs) + + def findCaller(self, stack_info=False, stacklevel=1): + return "(unknown file)", 0, "(unknown function)", None + + def makeRecord(self, name, level, fn, lno, msg, args, exc_info, func=None, extra=None, sinfo=None): + rv = _logRecordFactory(name, level, fn, lno, msg, args, exc_info, func, sinfo) + if extra is not None: + for key in extra: + if (key in ["message", "asctime"]) or (key in rv.__dict__): + raise KeyError("Attempt to overwrite %r in LogRecord" % key) + rv.__dict__[key] = extra[key] + return rv + + def _log(self, level, msg, args, exc_info=None, extra=None, stack_info=False, stacklevel=1): + fn, lno, func, sinfo = self.findCaller(stack_info, stacklevel) + if exc_info: + if isinstance(exc_info, BaseException): + exc_info = (type(exc_info), exc_info, exc_info.__traceback__) + elif not isinstance(exc_info, tuple): + exc_info = sys.exc_info() + record = self.makeRecord(self.name, level, fn, lno, msg, args, exc_info, func, extra, sinfo) + self.handle(record) + + def handle(self, record): + if self.disabled: + return + maybe_record = self.filter(record) + if not maybe_record: + return + if isinstance(maybe_record, LogRecord): + record = maybe_record + self.callHandlers(record) + + def addHandler(self, hdlr): + if not (hdlr in self.handlers): + self.handlers.append(hdlr) + + def removeHandler(self, hdlr): + if hdlr in self.handlers: + self.handlers.remove(hdlr) + + def hasHandlers(self): + c = self + rv = False + while c: + if c.handlers: + rv = True + break + if not c.propagate: + break + else: + c = c.parent + return rv + + def callHandlers(self, record): + c = self + found = 0 + while c: + for hdlr in c.handlers: + found = found + 1 + if record.levelno >= hdlr.level: + hdlr.handle(record) + if not c.propagate: + c = None + else: + c = c.parent + if found == 0: + if lastResort: + if record.levelno >= lastResort.level: + lastResort.handle(record) + elif raiseExceptions and not self.manager.emittedNoHandlerWarning: + sys.stderr.write("No handlers could be found for logger \"%s\"\n" % self.name) + self.manager.emittedNoHandlerWarning = True + + def getEffectiveLevel(self): + logger = self + while logger: + if logger.level: + return logger.level + logger = logger.parent + return NOTSET + + def isEnabledFor(self, level): + if self.disabled: + return False + try: + return self._cache[level] + except KeyError: + if self.manager.disable >= level: + is_enabled = self._cache[level] = False + else: + is_enabled = self._cache[level] = (level >= self.getEffectiveLevel()) + return is_enabled + + def getChild(self, suffix): + if self.root is not self: + suffix = ".".join((self.name, suffix)) + return self.manager.getLogger(suffix) + + def getChildren(self): + def _hierlevel(logger): + if logger is logger.manager.root: + return 0 + return 1 + logger.name.count(".") + + d = self.manager.loggerDict + return set(item for item in d.values() + if isinstance(item, Logger) and item.parent is self and _hierlevel(item) == 1 + _hierlevel(item.parent)) + + def __repr__(self): + level = getLevelName(self.getEffectiveLevel()) + return "<%s %s (%s)>" % (self.__class__.__name__, self.name, level) + + +class RootLogger(Logger): + def __init__(self, level): + Logger.__init__(self, "root", level) + + def __repr__(self): + return "" % getLevelName(self.getEffectiveLevel()) + + +_loggerClass = Logger + + +class LoggerAdapter: + def __init__(self, logger, extra=None, merge_extra=False): + self.logger = logger + self.extra = extra + self.merge_extra = merge_extra + + def process(self, msg, kwargs): + if self.merge_extra and "extra" in kwargs: + kwargs["extra"] = {**self.extra, **kwargs["extra"]} + else: + kwargs["extra"] = self.extra + return msg, kwargs + + def debug(self, msg, *args, **kwargs): + self.log(DEBUG, msg, *args, **kwargs) + + def info(self, msg, *args, **kwargs): + self.log(INFO, msg, *args, **kwargs) + + def warning(self, msg, *args, **kwargs): + self.log(WARNING, msg, *args, **kwargs) + + def error(self, msg, *args, **kwargs): + self.log(ERROR, msg, *args, **kwargs) + + def exception(self, msg, *args, exc_info=True, **kwargs): + self.log(ERROR, msg, *args, exc_info=exc_info, **kwargs) + + def critical(self, msg, *args, **kwargs): + self.log(CRITICAL, msg, *args, **kwargs) + + def log(self, level, msg, *args, **kwargs): + if self.isEnabledFor(level): + msg, kwargs = self.process(msg, kwargs) + self.logger.log(level, msg, *args, **kwargs) + + def isEnabledFor(self, level): + return self.logger.isEnabledFor(level) + + def setLevel(self, level): + self.logger.setLevel(level) + + def getEffectiveLevel(self): + return self.logger.getEffectiveLevel() + + def hasHandlers(self): + return self.logger.hasHandlers() + + @property + def manager(self): + return self.logger.manager + + @property + def name(self): + return self.logger.name + + def __repr__(self): + logger = self.logger + level = getLevelName(logger.getEffectiveLevel()) + return "<%s %s (%s)>" % (self.__class__.__name__, logger.name, level) + + +root = RootLogger(WARNING) +Logger.root = root +Logger.manager = Manager(Logger.root) + + +def basicConfig(**kwargs): + force = kwargs.pop("force", False) + encoding = kwargs.pop("encoding", None) + errors = kwargs.pop("errors", "backslashreplace") + if force: + for h in root.handlers[:]: + root.removeHandler(h) + h.close() + if len(root.handlers) == 0: + handlers = kwargs.pop("handlers", None) + if handlers is None: + if "stream" in kwargs and "filename" in kwargs: + raise ValueError("'stream' and 'filename' should not be specified together") + else: + if "stream" in kwargs or "filename" in kwargs: + raise ValueError("'stream' or 'filename' should not be specified together with 'handlers'") + if handlers is None: + filename = kwargs.pop("filename", None) + mode = kwargs.pop("filemode", "a") + if filename: + h = FileHandler(filename, mode, encoding=encoding, errors=errors) + else: + stream = kwargs.pop("stream", None) + h = StreamHandler(stream) + handlers = [h] + dfs = kwargs.pop("datefmt", None) + style = kwargs.pop("style", "%") + if style not in _STYLES: + raise ValueError("Style must be one of: %s" % ",".join(_STYLES.keys())) + fs = kwargs.pop("format", _STYLES[style][1]) + fmt = Formatter(fs, dfs, style) + for h in handlers: + if h.formatter is None: + h.setFormatter(fmt) + root.addHandler(h) + level = kwargs.pop("level", None) + if level is not None: + root.setLevel(level) + if kwargs: + keys = ", ".join(kwargs.keys()) + raise ValueError("Unrecognised argument(s): %s" % keys) + + +def getLogger(name=None): + if not name or isinstance(name, str) and name == root.name: + return root + return Logger.manager.getLogger(name) + + +def critical(msg, *args, **kwargs): + if len(root.handlers) == 0: + basicConfig() + root.critical(msg, *args, **kwargs) + + +fatal = critical + + +def error(msg, *args, **kwargs): + if len(root.handlers) == 0: + basicConfig() + root.error(msg, *args, **kwargs) + + +def exception(msg, *args, exc_info=True, **kwargs): + error(msg, *args, exc_info=exc_info, **kwargs) + + +def warning(msg, *args, **kwargs): + if len(root.handlers) == 0: + basicConfig() + root.warning(msg, *args, **kwargs) + + +def warn(msg, *args, **kwargs): + import warnings + warnings.warn("The 'warn' function is deprecated, use 'warning' instead", DeprecationWarning, 2) + warning(msg, *args, **kwargs) + + +def info(msg, *args, **kwargs): + if len(root.handlers) == 0: + basicConfig() + root.info(msg, *args, **kwargs) + + +def debug(msg, *args, **kwargs): + if len(root.handlers) == 0: + basicConfig() + root.debug(msg, *args, **kwargs) + + +def log(level, msg, *args, **kwargs): + if len(root.handlers) == 0: + basicConfig() + root.log(level, msg, *args, **kwargs) + + +def disable(level=CRITICAL): + root.manager.disable = level + root.manager._clear_cache() + + +def shutdown(handlerList=_handlerList): + for h in reversed(handlerList[:]): + try: + h.flush() + h.close() + except (OSError, ValueError): + pass + except Exception: + if raiseExceptions: + raise + + +_warnings_showwarning = None + + +def captureWarnings(capture): + global _warnings_showwarning + import warnings + if capture: + if _warnings_showwarning is None: + _warnings_showwarning = warnings.showwarning + + def _showwarning(message, category, filename, lineno, file=None, line=None): + if file is not None: + if _warnings_showwarning is not None: + _warnings_showwarning(message, category, filename, lineno, file, line) + else: + s = warnings.formatwarning(message, category, filename, lineno, line) + logger = getLogger("py.warnings") + if not logger.handlers: + logger.addHandler(NullHandler()) + logger.warning(str(s)) + + warnings.showwarning = _showwarning + else: + if _warnings_showwarning is not None: + warnings.showwarning = _warnings_showwarning + _warnings_showwarning = None + + +import atexit +atexit.register(shutdown) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/numbers.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/numbers.py new file mode 100644 index 00000000000..97a1564312d --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/numbers.py @@ -0,0 +1,257 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""The numeric tower: Number > Complex > Real > Rational > Integral.""" + +from abc import ABCMeta, abstractmethod + +__all__ = ["Number", "Complex", "Real", "Rational", "Integral"] + + +class Number(metaclass=ABCMeta): + __slots__ = () + __hash__ = None + + +class Complex(Number): + __slots__ = () + + @abstractmethod + def __complex__(self): + """Return a builtin complex instance.""" + + def __bool__(self): + return self != 0 + + @property + @abstractmethod + def real(self): + raise NotImplementedError + + @property + @abstractmethod + def imag(self): + raise NotImplementedError + + @abstractmethod + def __add__(self, other): + raise NotImplementedError + + @abstractmethod + def __radd__(self, other): + raise NotImplementedError + + @abstractmethod + def __neg__(self): + raise NotImplementedError + + @abstractmethod + def __pos__(self): + raise NotImplementedError + + def __sub__(self, other): + return self + -other + + def __rsub__(self, other): + return -self + other + + @abstractmethod + def __mul__(self, other): + raise NotImplementedError + + @abstractmethod + def __rmul__(self, other): + raise NotImplementedError + + @abstractmethod + def __truediv__(self, other): + raise NotImplementedError + + @abstractmethod + def __rtruediv__(self, other): + raise NotImplementedError + + @abstractmethod + def __pow__(self, exponent): + raise NotImplementedError + + @abstractmethod + def __rpow__(self, base): + raise NotImplementedError + + @abstractmethod + def __abs__(self): + raise NotImplementedError + + @abstractmethod + def conjugate(self): + raise NotImplementedError + + @abstractmethod + def __eq__(self, other): + raise NotImplementedError + + +Complex.register(complex) + + +class Real(Complex): + __slots__ = () + + @abstractmethod + def __float__(self): + raise NotImplementedError + + @abstractmethod + def __trunc__(self): + raise NotImplementedError + + @abstractmethod + def __floor__(self): + raise NotImplementedError + + @abstractmethod + def __ceil__(self): + raise NotImplementedError + + @abstractmethod + def __round__(self, ndigits=None): + raise NotImplementedError + + def __divmod__(self, other): + return (self // other, self % other) + + def __rdivmod__(self, other): + return (other // self, other % self) + + @abstractmethod + def __floordiv__(self, other): + raise NotImplementedError + + @abstractmethod + def __rfloordiv__(self, other): + raise NotImplementedError + + @abstractmethod + def __mod__(self, other): + raise NotImplementedError + + @abstractmethod + def __rmod__(self, other): + raise NotImplementedError + + @abstractmethod + def __lt__(self, other): + raise NotImplementedError + + @abstractmethod + def __le__(self, other): + raise NotImplementedError + + def __complex__(self): + return complex(float(self)) + + @property + def real(self): + return +self + + @property + def imag(self): + return 0 + + def conjugate(self): + return +self + + +Real.register(float) + + +class Rational(Real): + __slots__ = () + + @property + @abstractmethod + def numerator(self): + raise NotImplementedError + + @property + @abstractmethod + def denominator(self): + raise NotImplementedError + + def __float__(self): + return int(self.numerator) / int(self.denominator) + + +class Integral(Rational): + __slots__ = () + + @abstractmethod + def __int__(self): + raise NotImplementedError + + def __index__(self): + return int(self) + + @abstractmethod + def __pow__(self, exponent, modulus=None): + raise NotImplementedError + + @abstractmethod + def __lshift__(self, other): + raise NotImplementedError + + @abstractmethod + def __rlshift__(self, other): + raise NotImplementedError + + @abstractmethod + def __rshift__(self, other): + raise NotImplementedError + + @abstractmethod + def __rrshift__(self, other): + raise NotImplementedError + + @abstractmethod + def __and__(self, other): + raise NotImplementedError + + @abstractmethod + def __rand__(self, other): + raise NotImplementedError + + @abstractmethod + def __xor__(self, other): + raise NotImplementedError + + @abstractmethod + def __rxor__(self, other): + raise NotImplementedError + + @abstractmethod + def __or__(self, other): + raise NotImplementedError + + @abstractmethod + def __ror__(self, other): + raise NotImplementedError + + @abstractmethod + def __invert__(self): + raise NotImplementedError + + def __float__(self): + return float(int(self)) + + @property + def numerator(self): + return +self + + @property + def denominator(self): + return 1 + + +Integral.register(int) +Integral.register(bool) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/operator.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/operator.py new file mode 100644 index 00000000000..f9915947db3 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/operator.py @@ -0,0 +1,122 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Standard operators as functions.""" + + +def lt(a, b): return a < b +def le(a, b): return a <= b +def eq(a, b): return a == b +def ne(a, b): return a != b +def ge(a, b): return a >= b +def gt(a, b): return a > b +def not_(a): return not a +def truth(a): return bool(a) +def is_(a, b): return a is b +def is_not(a, b): return a is not b +def is_none(a): return a is None +def is_not_none(a): return a is not None +def abs(a): return a.__abs__() if hasattr(a, "__abs__") else (a if a >= 0 else -a) +def add(a, b): return a + b +def and_(a, b): return a & b +def floordiv(a, b): return a // b +def index(a): return a.__index__() +def inv(a): return ~a +invert = inv +def lshift(a, b): return a << b +def mod(a, b): return a % b +def mul(a, b): return a * b +def matmul(a, b): return a @ b +def neg(a): return -a +def or_(a, b): return a | b +def pos(a): return +a +def pow(a, b): return a ** b +def rshift(a, b): return a >> b +def sub(a, b): return a - b +def truediv(a, b): return a / b +def xor(a, b): return a ^ b +def concat(a, b): return a + b +def contains(a, b): return b in a +def countOf(a, b): return sum(1 for x in a if x is b or x == b) +def delitem(a, b): del a[b] +def getitem(a, b): return a[b] +def setitem(a, b, c): a[b] = c + + +def indexOf(a, b): + for i, x in enumerate(a): + if x is b or x == b: + return i + raise ValueError("sequence.index(x): x not in sequence") + + +def length_hint(obj, default=0): + try: + return len(obj) + except TypeError: + return default + + +def iadd(a, b): + a += b + return a + + +def isub(a, b): + a -= b + return a + + +def imul(a, b): + a *= b + return a + + +def iconcat(a, b): + a += b + return a + + +class attrgetter: + def __init__(self, attr, *attrs): + self._attrs = (attr,) + attrs + + def _one(self, obj, name): + for part in name.split("."): + obj = getattr(obj, part) + return obj + + def __call__(self, obj): + if len(self._attrs) == 1: + return self._one(obj, self._attrs[0]) + return tuple(self._one(obj, n) for n in self._attrs) + + def __repr__(self): + return "operator.attrgetter(" + ", ".join(repr(a) for a in self._attrs) + ")" + + +class itemgetter: + def __init__(self, item, *items): + self._items = (item,) + items + + def __call__(self, obj): + if len(self._items) == 1: + return obj[self._items[0]] + return tuple(obj[i] for i in self._items) + + def __repr__(self): + return "operator.itemgetter(" + ", ".join(repr(a) for a in self._items) + ")" + + +class methodcaller: + def __init__(self, name, *args, **kwargs): + self._name = name + self._args = args + self._kwargs = kwargs + + def __call__(self, obj): + return getattr(obj, self._name)(*self._args, **self._kwargs) + + +def call(obj, *args, **kwargs): + return obj(*args, **kwargs) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/os/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/os/__init__.py new file mode 100644 index 00000000000..a62e929f664 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/os/__init__.py @@ -0,0 +1,344 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Operating system services an application has: its files and environment. + +Files are the application's own (`FileSystemStorage`); the working +directory starts at the application's home. There are no processes, +users, file descriptors or links. +""" + +import _cn1fs +import io as _io +from os import path + +name = "posix" +sep = path.sep +altsep = path.altsep +extsep = path.extsep +pathsep = path.pathsep +curdir = path.curdir +pardir = path.pardir +defpath = path.defpath +devnull = path.devnull +linesep = "\n" +SEEK_SET = 0 +SEEK_CUR = 1 +SEEK_END = 2 +F_OK = 0 +R_OK = 4 +W_OK = 2 +X_OK = 1 + +environ = {} +error = OSError + + +def getenv(key, default=None): + return environ.get(key, default) + + +def putenv(key, value): + environ[key] = value + + +def unsetenv(key): + environ.pop(key, None) + + +class PathLike: + def __fspath__(self): + raise NotImplementedError + + @classmethod + def __subclasshook__(cls, C): + return hasattr(C, "__fspath__") or NotImplemented + + +def fspath(p): + if isinstance(p, (str, bytes)): + return p + f = getattr(type(p), "__fspath__", None) + if f is None: + raise TypeError("expected str, bytes or os.PathLike object, not " + type(p).__name__) + r = f(p) + if not isinstance(r, (str, bytes)): + raise TypeError("expected %s.__fspath__() to return str or bytes, not %s" + % (type(p).__name__, type(r).__name__)) + return r + + +def fsencode(filename): + filename = fspath(filename) + return filename.encode("utf-8") if isinstance(filename, str) else filename + + +def fsdecode(filename): + filename = fspath(filename) + return filename.decode("utf-8") if isinstance(filename, bytes) else filename + + +def getcwd(): + return _io._getcwd() + + +def getcwdb(): + return getcwd().encode("utf-8") + + +def _abs(p): + return _io._abspath(_io._fspath(p)) + + +def chdir(p): + a = _abs(p) + if not _cn1fs.exists(a): + raise FileNotFoundError(2, "No such file or directory", fspath(p)) + if not _cn1fs.isdir(a): + raise NotADirectoryError(20, "Not a directory", fspath(p)) + _io._setcwd(a) + + +def listdir(p="."): + a = _abs(p) + if not _cn1fs.exists(a): + raise FileNotFoundError(2, "No such file or directory", fspath(p)) + if not _cn1fs.isdir(a): + raise NotADirectoryError(20, "Not a directory", fspath(p)) + names = _cn1fs.listdir(a) + if isinstance(p, bytes): + return [n.encode("utf-8") for n in names] + return names + + +def mkdir(p, mode=0o777, *, dir_fd=None): + a = _abs(p) + if _cn1fs.exists(a): + raise FileExistsError(17, "File exists", fspath(p)) + parent = a.rsplit("/", 1)[0] or "/" + if not _cn1fs.isdir(parent): + raise FileNotFoundError(2, "No such file or directory", fspath(p)) + _cn1fs.mkdir(a) + + +def makedirs(name, mode=0o777, exist_ok=False): + a = _abs(name) + if _cn1fs.exists(a): + if exist_ok and _cn1fs.isdir(a): + return + raise FileExistsError(17, "File exists", fspath(name)) + head = a.rsplit("/", 1)[0] or "/" + if not _cn1fs.exists(head): + makedirs(head, mode, True) + _cn1fs.mkdir(a) + + +def remove(p, *, dir_fd=None): + a = _abs(p) + if not _cn1fs.exists(a): + raise FileNotFoundError(2, "No such file or directory", fspath(p)) + if _cn1fs.isdir(a): + raise IsADirectoryError(21, "Is a directory", fspath(p)) + _cn1fs.remove(a) + + +unlink = remove + + +def rmdir(p, *, dir_fd=None): + a = _abs(p) + if not _cn1fs.exists(a): + raise FileNotFoundError(2, "No such file or directory", fspath(p)) + if not _cn1fs.isdir(a): + raise NotADirectoryError(20, "Not a directory", fspath(p)) + if _cn1fs.listdir(a): + raise OSError(39, "Directory not empty", fspath(p)) + _cn1fs.remove(a) + + +def removedirs(name): + rmdir(name) + head, tail = path.split(name) + if not tail: + head, tail = path.split(head) + while head and tail: + try: + rmdir(head) + except OSError: + break + head, tail = path.split(head) + + +def rename(src, dst, *, src_dir_fd=None, dst_dir_fd=None): + a = _abs(src) + b = _abs(dst) + if not _cn1fs.exists(a): + raise FileNotFoundError(2, "No such file or directory", fspath(src), None, fspath(dst)) + if _cn1fs.isdir(b): + raise IsADirectoryError(21, "Is a directory", fspath(src), None, fspath(dst)) + _cn1fs.rename(a, b) + + +replace = rename + + +def renames(old, new): + head, tail = path.split(new) + if head and tail and not path.exists(head): + makedirs(head) + rename(old, new) + head, tail = path.split(old) + if head and tail: + try: + removedirs(head) + except OSError: + pass + + +class stat_result(tuple): + n_sequence_fields = 10 + + def __new__(cls, seq): + return tuple.__new__(cls, seq) + + st_mode = property(lambda self: self[0]) + st_ino = property(lambda self: self[1]) + st_dev = property(lambda self: self[2]) + st_nlink = property(lambda self: self[3]) + st_uid = property(lambda self: self[4]) + st_gid = property(lambda self: self[5]) + st_size = property(lambda self: self[6]) + st_atime = property(lambda self: self[10]) + st_mtime = property(lambda self: self[10]) + st_ctime = property(lambda self: self[10]) + + def __repr__(self): + return ("os.stat_result(st_mode=%d, st_ino=0, st_dev=0, st_nlink=1, st_uid=0, st_gid=0, " + "st_size=%d, st_atime=%d, st_mtime=%d, st_ctime=%d)" + % (self[0], self[6], self[7], self[8], self[9])) + + +def stat(p, *, dir_fd=None, follow_symlinks=True): + a = _abs(p) + if not _cn1fs.exists(a): + raise FileNotFoundError(2, "No such file or directory", fspath(p)) + d = _cn1fs.isdir(a) + t = _cn1fs.mtime(a) + size = 0 if d else _cn1fs.size(a) + it = int(t) + return stat_result((0o40755 if d else 0o100644, 0, 0, 1, 0, 0, size, it, it, it, t)) + + +lstat = stat + + +def access(p, mode, *, dir_fd=None, effective_ids=False, follow_symlinks=True): + return path.exists(p) + + +class DirEntry: + def __init__(self, parent, name): + self.name = name + self.path = path.join(parent, name) + + def is_dir(self, *, follow_symlinks=True): + return path.isdir(self.path) + + def is_file(self, *, follow_symlinks=True): + return path.isfile(self.path) + + def is_symlink(self): + return False + + def stat(self, *, follow_symlinks=True): + return stat(self.path) + + def inode(self): + return 0 + + def __fspath__(self): + return self.path + + def __repr__(self): + return "" % self.name + + +class _ScandirIterator: + def __init__(self, entries): + self._it = iter(entries) + + def __iter__(self): + return self + + def __next__(self): + return next(self._it) + + def __enter__(self): + return self + + def __exit__(self, *a): + self._it = iter(()) + + def close(self): + self._it = iter(()) + + +def scandir(p="."): + parent = _io._fspath(p) + return _ScandirIterator([DirEntry(parent, n) for n in listdir(parent)]) + + +def walk(top, topdown=True, onerror=None, followlinks=False): + top = fspath(top) + try: + names = listdir(top) + except OSError as err: + if onerror is not None: + onerror(err) + return + dirs = [] + nondirs = [] + for n in names: + if path.isdir(path.join(top, n)): + dirs.append(n) + else: + nondirs.append(n) + if topdown: + yield top, dirs, nondirs + for d in dirs: + yield from walk(path.join(top, d), topdown, onerror, followlinks) + else: + for d in dirs: + yield from walk(path.join(top, d), topdown, onerror, followlinks) + yield top, dirs, nondirs + + +def cpu_count(): + return 1 + + +def process_cpu_count(): + return 1 + + +def getpid(): + return 1 + + +def get_terminal_size(fd=1): + return terminal_size((80, 24)) + + +class terminal_size(tuple): + columns = property(lambda self: self[0]) + lines = property(lambda self: self[1]) + + +def urandom(n): + import random as _random + return _random.SystemRandom().randbytes(n) + + +def strerror(code): + return {2: "No such file or directory", 17: "File exists", 20: "Not a directory", + 21: "Is a directory", 39: "Directory not empty"}.get(code, "Unknown error %d" % code) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/os/path.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/os/path.py new file mode 100644 index 00000000000..094fcb613da --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/os/path.py @@ -0,0 +1,245 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""POSIX path names, over the application's file system.""" + +import _cn1fs +import io as _io + +sep = "/" +altsep = None +extsep = "." +pathsep = ":" +curdir = "." +pardir = ".." +defpath = "/bin:/usr/bin" +devnull = "/dev/null" +supports_unicode_filenames = False + +__all__ = ["join", "split", "splitext", "basename", "dirname", "exists", "lexists", "isfile", + "isdir", "islink", "ismount", "isabs", "abspath", "normpath", "realpath", "relpath", + "getsize", "getmtime", "getatime", "getctime", "expanduser", "expandvars", + "commonprefix", "commonpath", "splitdrive", "splitroot", "normcase", "samefile", + "sep", "altsep", "extsep", "pathsep", "curdir", "pardir", "defpath", "devnull"] + + +def _s(p): + return _io._fspath(p) + + +def _bytes_or(p, f): + if isinstance(p, bytes): + return f(p.decode("utf-8")).encode("utf-8") + return f(_s(p)) + + +def normcase(s): + return _s(s) + + +def isabs(s): + return _s(s).startswith("/") + + +def join(a, *p): + path = _s(a) + for b in p: + b = _s(b) + if b.startswith("/"): + path = b + elif not path or path.endswith("/"): + path += b + else: + path += "/" + b + return path + + +def split(p): + p = _s(p) + i = p.rfind("/") + 1 + head, tail = p[:i], p[i:] + if head and head != "/" * len(head): + head = head.rstrip("/") + return head, tail + + +def splitext(p): + p = _s(p) + sep_i = p.rfind("/") + dot = p.rfind(".") + if dot > sep_i: + f = sep_i + 1 + while f < dot: + if p[f] != ".": + return p[:dot], p[dot:] + f += 1 + return p, p[:0] + + +def splitdrive(p): + p = _s(p) + return p[:0], p + + +def splitroot(p): + p = _s(p) + if p[:1] != "/": + return "", "", p + if p[1:2] != "/" or p[2:3] == "/": + return "", "/", p[1:] + return "", p[:2], p[2:] + + +def basename(p): + p = _s(p) + return p[p.rfind("/") + 1:] + + +def dirname(p): + p = _s(p) + i = p.rfind("/") + 1 + head = p[:i] + if head and head != "/" * len(head): + head = head.rstrip("/") + return head + + +def normpath(path): + path = _s(path) + if path == "": + return "." + initial = 0 + if path.startswith("/"): + initial = 2 if path.startswith("//") and not path.startswith("///") else 1 + comps = path.split("/") + new = [] + for comp in comps: + if comp in ("", "."): + continue + if comp != ".." or (not initial and not new) or (new and new[-1] == ".."): + new.append(comp) + elif new: + new.pop() + path = "/" * initial + "/".join(new) + return path or "." + + +def abspath(path): + path = _s(path) + if not isabs(path): + path = join(_io._getcwd(), path) + return normpath(path) + + +realpath = abspath + + +def relpath(path, start=None): + path = _s(path) + if not path: + raise ValueError("no path specified") + start = curdir if start is None else _s(start) + start_list = [x for x in abspath(start).split("/") if x] + path_list = [x for x in abspath(path).split("/") if x] + i = len(commonprefix([start_list, path_list])) + rel = [pardir] * (len(start_list) - i) + path_list[i:] + if not rel: + return curdir + return join(*rel) + + +def commonprefix(m): + if not m: + return "" + s1 = min(m) + s2 = max(m) + for i, c in enumerate(s1): + if c != s2[i]: + return s1[:i] + return s1 + + +def commonpath(paths): + paths = [_s(p) for p in paths] + if not paths: + raise ValueError("commonpath() arg is an empty sequence") + abss = set(p.startswith("/") for p in paths) + if len(abss) != 1: + raise ValueError("Can't mix absolute and relative paths") + split_paths = [[c for c in p.split("/") if c and c != "."] for p in paths] + s1 = min(split_paths) + s2 = max(split_paths) + common = s1 + for i, c in enumerate(s1): + if c != s2[i]: + common = s1[:i] + break + prefix = "/" if paths[0].startswith("/") else "" + return prefix + "/".join(common) + + +def exists(path): + try: + return _cn1fs.exists(_io._abspath(_s(path))) + except (OSError, ValueError, TypeError): + return False + + +lexists = exists + + +def isdir(s): + try: + return _cn1fs.isdir(_io._abspath(_s(s))) + except (OSError, ValueError, TypeError): + return False + + +def isfile(path): + try: + p = _io._abspath(_s(path)) + return _cn1fs.exists(p) and not _cn1fs.isdir(p) + except (OSError, ValueError, TypeError): + return False + + +def islink(path): + return False + + +def ismount(path): + return abspath(path) == "/" + + +def _need(path): + p = _io._abspath(_s(path)) + if not _cn1fs.exists(p): + raise FileNotFoundError(2, "No such file or directory", path) + return p + + +def getsize(filename): + return _cn1fs.size(_need(filename)) + + +def getmtime(filename): + return _cn1fs.mtime(_need(filename)) + + +getatime = getmtime +getctime = getmtime + + +def samefile(f1, f2): + return _need(f1) == _need(f2) + + +def expanduser(path): + path = _s(path) + if path == "~" or path.startswith("~/"): + return _cn1fs.home() + path[1:] + return path + + +def expandvars(path): + return _s(path) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/pathlib.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/pathlib.py new file mode 100644 index 00000000000..64460b95314 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/pathlib.py @@ -0,0 +1,447 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Object-oriented paths over the application's file system (POSIX flavour).""" + +import os +import io as _io +from os import path as _p + +__all__ = ["PurePath", "PurePosixPath", "Path", "PosixPath", "UnsupportedOperation"] + +UnsupportedOperation = _io.UnsupportedOperation + + +class PurePath: + __slots__ = ("_raw", "_str", "_parts_cache") + + def __new__(cls, *args): + if cls is PurePath: + cls = PurePosixPath + return object.__new__(cls) + + def __init__(self, *args): + segs = [] + for a in args: + if isinstance(a, PurePath): + segs.append(str(a)) + else: + a = os.fspath(a) + if not isinstance(a, str): + raise TypeError("argument should be a str or an os.PathLike object where " + "__fspath__ returns a str, not %r" % type(a).__name__) + segs.append(a) + self._raw = _p.join(*segs) if segs else "" + self._str = None + self._parts_cache = None + + def with_segments(self, *args): + return type(self)(*args) + + def _split(self): + if self._parts_cache is None: + raw = self._raw + root = "" + if raw.startswith("/"): + root = "//" if raw.startswith("//") and not raw.startswith("///") else "/" + parts = [x for x in raw.split("/") if x and x != "."] + self._parts_cache = (root, parts) + return self._parts_cache + + def __str__(self): + if self._str is None: + root, parts = self._split() + self._str = (root + "/".join(parts)) or "." + return self._str + + def __fspath__(self): + return str(self) + + def as_posix(self): + return str(self) + + def __bytes__(self): + return os.fsencode(str(self)) + + def __repr__(self): + return "%s(%r)" % (type(self).__name__, self.as_posix()) + + def as_uri(self): + if not self.is_absolute(): + raise ValueError("relative path can't be expressed as a file URI") + return "file://" + str(self) + + def __hash__(self): + return hash(str(self)) + + def __eq__(self, other): + if not isinstance(other, PurePath): + return NotImplemented + return str(self) == str(other) + + def _key(self, other): + if not isinstance(other, PurePath): + raise TypeError + return self.parts, other.parts + + def __lt__(self, other): + if not isinstance(other, PurePath): + return NotImplemented + return self.parts < other.parts + + def __le__(self, other): + if not isinstance(other, PurePath): + return NotImplemented + return self.parts <= other.parts + + def __gt__(self, other): + if not isinstance(other, PurePath): + return NotImplemented + return self.parts > other.parts + + def __ge__(self, other): + if not isinstance(other, PurePath): + return NotImplemented + return self.parts >= other.parts + + def __truediv__(self, key): + try: + return self.with_segments(self, key) + except TypeError: + return NotImplemented + + def __rtruediv__(self, key): + try: + return self.with_segments(key, self) + except TypeError: + return NotImplemented + + def joinpath(self, *segments): + return self.with_segments(self, *segments) + + @property + def drive(self): + return "" + + @property + def root(self): + return self._split()[0] + + @property + def anchor(self): + return self.root + + @property + def parts(self): + root, parts = self._split() + return tuple([root] + parts) if root else tuple(parts) + + @property + def name(self): + parts = self._split()[1] + return parts[-1] if parts else "" + + @property + def suffix(self): + name = self.name + i = name.rfind(".") if name != ".." else -1 + if 0 < i: + return name[i:] + return "" + + @property + def suffixes(self): + name = self.name + name = name.lstrip(".") + return ["." + s for s in name.split(".")[1:]] + + @property + def stem(self): + name = self.name + i = name.rfind(".") if name != ".." else -1 + if 0 < i: + return name[:i] + return name + + @property + def parent(self): + root, parts = self._split() + if not parts: + return self + return self.with_segments(root + "/".join(parts[:-1]) or ".") + + @property + def parents(self): + out = [] + p = self + while True: + q = p.parent + if q == p: + break + out.append(q) + p = q + return tuple(out) + + def with_name(self, name): + if not self.name: + raise ValueError("%r has an empty name" % (self,)) + if not name or "/" in name or name == ".": + raise ValueError("Invalid name %r" % (name,)) + return self.parent / name if self.parent != self or self.root else self.with_segments(name) + + def with_stem(self, stem): + return self.with_name(stem + self.suffix) + + def with_suffix(self, suffix): + if suffix and (not suffix.startswith(".") or suffix == "." or "/" in suffix): + raise ValueError("Invalid suffix %r" % (suffix,)) + if not self.name: + raise ValueError("%r has an empty name" % (self,)) + return self.with_name(self.stem + suffix) + + def is_absolute(self): + return bool(self.root) + + def is_reserved(self): + return False + + def is_relative_to(self, other): + other = self.with_segments(other) + return other == self or other in self.parents + + def relative_to(self, other, *, walk_up=False): + other = self.with_segments(other) + if self.is_absolute() != other.is_absolute(): + raise ValueError("%r and %r have different anchors" % (str(self), str(other))) + mine = self.parts + theirs = other.parts + n = 0 + while n < len(mine) and n < len(theirs) and mine[n] == theirs[n]: + n += 1 + if n < len(theirs): + if not walk_up: + raise ValueError("%r is not in the subpath of %r" % (str(self), str(other))) + if ".." in theirs[n:]: + raise ValueError("'..' segment in %r cannot be walked" % (str(other),)) + rest = [".."] * (len(theirs) - n) + list(mine[n:]) + return self.with_segments(*rest) if rest else self.with_segments("") + + def match(self, pattern, *, case_sensitive=None): + pat = self.with_segments(pattern) + pparts = pat.parts + mparts = self.parts + if not pparts: + raise ValueError("empty pattern") + if pat.is_absolute(): + if len(pparts) != len(mparts): + return False + elif len(pparts) > len(mparts): + return False + for a, b in zip(reversed(mparts), reversed(pparts)): + if not _fnmatch(a, b): + return False + return True + + def full_match(self, pattern, *, case_sensitive=None): + pparts = self.with_segments(pattern).parts + return _match_parts(list(self.parts), list(pparts)) + + +def _fnmatch(name, pat): + import fnmatch + return fnmatch.fnmatchcase(name, pat) + + +def _match_parts(names, pats): + if not pats: + return not names + if pats[0] == "**": + for i in range(len(names) + 1): + if _match_parts(names[i:], pats[1:]): + return True + return False + if not names: + return False + return _fnmatch(names[0], pats[0]) and _match_parts(names[1:], pats[1:]) + + +class PurePosixPath(PurePath): + __slots__ = () + + +class Path(PurePath): + __slots__ = () + + def __new__(cls, *args, **kwargs): + if cls is Path: + cls = PosixPath + return object.__new__(cls) + + @classmethod + def cwd(cls): + return cls(os.getcwd()) + + @classmethod + def home(cls): + return cls(_p.expanduser("~")) + + def absolute(self): + if self.is_absolute(): + return self + return self.with_segments(os.getcwd(), self) + + def resolve(self, strict=False): + r = self.with_segments(_p.normpath(str(self.absolute()))) + if strict and not r.exists(): + raise FileNotFoundError(2, "No such file or directory", str(self)) + return r + + def expanduser(self): + return self.with_segments(_p.expanduser(str(self))) + + def stat(self, *, follow_symlinks=True): + return os.stat(self) + + lstat = stat + + def exists(self, *, follow_symlinks=True): + return _p.exists(self) + + def is_dir(self, *, follow_symlinks=True): + return _p.isdir(self) + + def is_file(self, *, follow_symlinks=True): + return _p.isfile(self) + + def is_symlink(self): + return False + + def is_mount(self): + return _p.ismount(self) + + def samefile(self, other): + return _p.samefile(self, other) + + def open(self, mode="r", buffering=-1, encoding=None, errors=None, newline=None): + return _io.open(self, mode, buffering, encoding, errors, newline) + + def read_text(self, encoding=None, errors=None, newline=None): + with self.open("r", encoding=encoding, errors=errors, newline=newline) as f: + return f.read() + + def read_bytes(self): + with self.open("rb") as f: + return f.read() + + def write_text(self, data, encoding=None, errors=None, newline=None): + if not isinstance(data, str): + raise TypeError("data must be str, not %s" % type(data).__name__) + with self.open("w", encoding=encoding, errors=errors, newline=newline) as f: + return f.write(data) + + def write_bytes(self, data): + data = bytes(memoryview(data)) if not isinstance(data, bytes) else data + with self.open("wb") as f: + return f.write(data) + + def iterdir(self): + for name in os.listdir(self): + yield self / name + + def glob(self, pattern, *, case_sensitive=None, recurse_symlinks=False): + pats = [x for x in pattern.split("/") if x and x != "."] + if not pats: + raise ValueError("Unacceptable pattern: %r" % (pattern,)) + yield from self._glob(pats) + + def _glob(self, pats): + head = pats[0] + rest = pats[1:] + if head == "**": + for d in self._walk_dirs(): + if rest: + yield from d._glob(rest) + else: + yield d + return + if head == "..": + p = self / ".." + if rest: + yield from p._glob(rest) + else: + yield p + return + if not self.is_dir(): + return + for name in sorted(os.listdir(self)): + if _fnmatch(name, head): + child = self / name + if rest: + if child.is_dir(): + yield from child._glob(rest) + else: + yield child + + def _walk_dirs(self): + yield self + if self.is_dir(): + for name in sorted(os.listdir(self)): + child = self / name + if child.is_dir(): + yield from child._walk_dirs() + + def rglob(self, pattern, *, case_sensitive=None, recurse_symlinks=False): + return self.glob("**/" + pattern) + + def walk(self, top_down=True, on_error=None, follow_symlinks=False): + for root, dirs, files in os.walk(self, top_down, on_error): + yield self.with_segments(root), dirs, files + + def touch(self, mode=0o666, exist_ok=True): + if self.exists(): + if not exist_ok: + raise FileExistsError(17, "File exists", str(self)) + return + with self.open("wb"): + pass + + def mkdir(self, mode=0o777, parents=False, exist_ok=False): + try: + if parents: + os.makedirs(self, mode, exist_ok) + else: + os.mkdir(self, mode) + except FileExistsError: + if not exist_ok or not self.is_dir(): + raise + + def unlink(self, missing_ok=False): + try: + os.unlink(self) + except FileNotFoundError: + if not missing_ok: + raise + + def rmdir(self): + os.rmdir(self) + + def rename(self, target): + os.rename(self, target) + return self.with_segments(target) + + def replace(self, target): + os.replace(self, target) + return self.with_segments(target) + + def owner(self): + raise UnsupportedOperation("owner() is not supported") + + def group(self): + raise UnsupportedOperation("group() is not supported") + + def chmod(self, mode, *, follow_symlinks=True): + pass + + +class PosixPath(Path, PurePosixPath): + __slots__ = () diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/pprint.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/pprint.py new file mode 100644 index 00000000000..9fbdcb8b631 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/pprint.py @@ -0,0 +1,552 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Pretty printing of data structures, laid out as CPython's pprint does.""" + +import collections as _collections +import dataclasses as _dataclasses +import re +import sys as _sys +import types as _types +from io import StringIO as _StringIO + +__all__ = ["pprint", "pformat", "isreadable", "isrecursive", "saferepr", "PrettyPrinter", "pp"] + + +def pprint(object, stream=None, indent=1, width=80, depth=None, *, compact=False, sort_dicts=True, + underscore_numbers=False): + printer = PrettyPrinter(stream=stream, indent=indent, width=width, depth=depth, compact=compact, + sort_dicts=sort_dicts, underscore_numbers=underscore_numbers) + printer.pprint(object) + + +def pformat(object, indent=1, width=80, depth=None, *, compact=False, sort_dicts=True, underscore_numbers=False): + return PrettyPrinter(indent=indent, width=width, depth=depth, compact=compact, sort_dicts=sort_dicts, + underscore_numbers=underscore_numbers).pformat(object) + + +def pp(object, *args, sort_dicts=False, **kwargs): + pprint(object, *args, sort_dicts=sort_dicts, **kwargs) + + +def saferepr(object): + return PrettyPrinter()._safe_repr(object, {}, None, 0)[0] + + +def isreadable(object): + return PrettyPrinter()._safe_repr(object, {}, None, 0)[1] + + +def isrecursive(object): + return PrettyPrinter()._safe_repr(object, {}, None, 0)[2] + + +class _safe_key: + __slots__ = ["obj"] + + def __init__(self, obj): + self.obj = obj + + def __lt__(self, other): + try: + return self.obj < other.obj + except TypeError: + return ((str(type(self.obj)), id(self.obj)) < (str(type(other.obj)), id(other.obj))) + + +def _safe_tuple(t): + return _safe_key(t[0]), _safe_key(t[1]) + + +class PrettyPrinter: + def __init__(self, indent=1, width=80, depth=None, stream=None, *, compact=False, sort_dicts=True, + underscore_numbers=False): + indent = int(indent) + width = int(width) + if indent < 0: + raise ValueError("indent must be >= 0") + if depth is not None and depth <= 0: + raise ValueError("depth must be > 0") + if not width: + raise ValueError("width must be != 0") + self._depth = depth + self._indent_per_level = indent + self._width = width + if stream is not None: + self._stream = stream + else: + self._stream = _sys.stdout + self._compact = bool(compact) + self._sort_dicts = sort_dicts + self._underscore_numbers = underscore_numbers + + def pprint(self, object): + if self._stream is not None: + self._format(object, self._stream, 0, 0, {}, 0) + self._stream.write("\n") + + def pformat(self, object): + sio = _StringIO() + self._format(object, sio, 0, 0, {}, 0) + return sio.getvalue() + + def isrecursive(self, object): + return self.format(object, {}, 0, 0)[2] + + def isreadable(self, object): + s, readable, recursive = self.format(object, {}, 0, 0) + return readable and not recursive + + def _dispatch_for(self, typ): + # The printer of the nearest builtin base, unless a class on the way + # writes its own __repr__ (CPython keys its table by __repr__). + for t in typ.__mro__: + p = self._dispatch.get(t) + if p is not None: + return p + if t is not object and "__repr__" in t.__dict__: + return None + return None + + def _format(self, object, stream, indent, allowance, context, level): + objid = id(object) + if objid in context: + stream.write(_recursion(object)) + self._recursive = True + self._readable = False + return + rep = self._repr(object, context, level) + max_width = self._width - indent - allowance + if len(rep) > max_width: + p = self._dispatch_for(type(object)) + if p is not None: + context[objid] = 1 + p(self, object, stream, indent, allowance, context, level + 1) + del context[objid] + return + elif (_dataclasses.is_dataclass(object) and not isinstance(object, type) + and getattr(type(object).__repr__, "_dataclass_repr", False)): + context[objid] = 1 + self._pprint_dataclass(object, stream, indent, allowance, context, level + 1) + del context[objid] + return + stream.write(rep) + + def _pprint_dataclass(self, object, stream, indent, allowance, context, level): + cls_name = object.__class__.__name__ + indent += len(cls_name) + 1 + items = [(f.name, getattr(object, f.name)) for f in _dataclasses.fields(object) if f.repr] + stream.write(cls_name + "(") + self._format_namespace_items(items, stream, indent, allowance, context, level) + stream.write(")") + + _dispatch = {} + + def _pprint_dict(self, object, stream, indent, allowance, context, level): + write = stream.write + write("{") + if self._indent_per_level > 1: + write((self._indent_per_level - 1) * " ") + length = len(object) + if length: + if self._sort_dicts: + items = sorted(object.items(), key=_safe_tuple) + else: + items = object.items() + self._format_dict_items(items, stream, indent, allowance + 1, context, level) + write("}") + + _dispatch[dict] = _pprint_dict + + def _pprint_ordered_dict(self, object, stream, indent, allowance, context, level): + if not len(object): + stream.write(repr(object)) + return + cls = object.__class__ + stream.write(cls.__name__ + "(") + self._format(list(object.items()), stream, indent + len(cls.__name__) + 1, allowance + 1, context, level) + stream.write(")") + + _dispatch[_collections.OrderedDict] = _pprint_ordered_dict + + def _pprint_list(self, object, stream, indent, allowance, context, level): + stream.write("[") + self._format_items(object, stream, indent, allowance + 1, context, level) + stream.write("]") + + _dispatch[list] = _pprint_list + + def _pprint_tuple(self, object, stream, indent, allowance, context, level): + stream.write("(") + endchar = ",)" if len(object) == 1 else ")" + self._format_items(object, stream, indent, allowance + len(endchar), context, level) + stream.write(endchar) + + _dispatch[tuple] = _pprint_tuple + + def _pprint_set(self, object, stream, indent, allowance, context, level): + if not len(object): + stream.write(repr(object)) + return + typ = object.__class__ + if typ is set: + stream.write("{") + endchar = "}" + else: + stream.write(typ.__name__ + "({") + endchar = "})" + indent += len(typ.__name__) + 1 + object = sorted(object, key=_safe_key) + self._format_items(object, stream, indent, allowance + len(endchar), context, level) + stream.write(endchar) + + _dispatch[set] = _pprint_set + _dispatch[frozenset] = _pprint_set + + def _pprint_str(self, object, stream, indent, allowance, context, level): + write = stream.write + if not len(object): + write(repr(object)) + return + chunks = [] + lines = object.splitlines(True) + if level == 1: + indent += 1 + allowance += 1 + max_width1 = max_width = self._width - indent + for i, line in enumerate(lines): + rep = repr(line) + if i == len(lines) - 1: + max_width1 -= allowance + if len(rep) <= max_width1: + chunks.append(rep) + else: + parts = re.findall(r"\S*\s*", line) + parts.pop() + max_width2 = max_width + current = "" + for j, part in enumerate(parts): + candidate = current + part + if j == len(parts) - 1 and i == len(lines) - 1: + max_width2 -= allowance + if len(repr(candidate)) > max_width2: + if current: + chunks.append(repr(current)) + current = part + else: + current = candidate + if current: + chunks.append(repr(current)) + if len(chunks) == 1: + write(rep) + return + if level == 1: + write("(") + for i, rep in enumerate(chunks): + if i > 0: + write("\n" + " " * indent) + write(rep) + if level == 1: + write(")") + + _dispatch[str] = _pprint_str + + def _pprint_bytes(self, object, stream, indent, allowance, context, level): + write = stream.write + if len(object) <= 4: + write(repr(object)) + return + parens = level == 1 + if parens: + indent += 1 + allowance += 1 + write("(") + delim = "" + for rep in _wrap_bytes_repr(object, self._width - indent, allowance): + write(delim) + write(rep) + if not delim: + delim = "\n" + " " * indent + if parens: + write(")") + + _dispatch[bytes] = _pprint_bytes + + def _pprint_bytearray(self, object, stream, indent, allowance, context, level): + write = stream.write + write("bytearray(") + self._pprint_bytes(bytes(object), stream, indent + 10, allowance + 1, context, level + 1) + write(")") + + _dispatch[bytearray] = _pprint_bytearray + + def _pprint_simplenamespace(self, object, stream, indent, allowance, context, level): + if type(object) is _types.SimpleNamespace: + cls_name = "namespace" + else: + cls_name = object.__class__.__name__ + indent += len(cls_name) + 1 + items = object.__dict__.items() + stream.write(cls_name + "(") + self._format_namespace_items(items, stream, indent, allowance, context, level) + stream.write(")") + + _dispatch[_types.SimpleNamespace] = _pprint_simplenamespace + + def _format_dict_items(self, items, stream, indent, allowance, context, level): + write = stream.write + indent += self._indent_per_level + delimnl = ",\n" + " " * indent + last_index = len(items) - 1 + for i, (key, ent) in enumerate(items): + last = i == last_index + rep = self._repr(key, context, level) + write(rep) + write(": ") + self._format(ent, stream, indent + len(rep) + 2, allowance if last else 1, context, level) + if not last: + write(delimnl) + + def _format_namespace_items(self, items, stream, indent, allowance, context, level): + write = stream.write + delimnl = ",\n" + " " * indent + last_index = len(items) - 1 + for i, (key, ent) in enumerate(items): + last = i == last_index + write(key) + write("=") + if id(ent) in context: + write("...") + else: + self._format(ent, stream, indent + len(key) + 1, allowance if last else 1, context, level) + if not last: + write(delimnl) + + def _format_items(self, items, stream, indent, allowance, context, level): + write = stream.write + indent += self._indent_per_level + if self._indent_per_level > 1: + write((self._indent_per_level - 1) * " ") + delimnl = ",\n" + " " * indent + delim = "" + width = max_width = self._width - indent + 1 + it = iter(items) + try: + next_ent = next(it) + except StopIteration: + return + last = False + while not last: + ent = next_ent + try: + next_ent = next(it) + except StopIteration: + last = True + max_width -= allowance + width -= allowance + if self._compact: + rep = self._repr(ent, context, level) + w = len(rep) + 2 + if width < w: + width = max_width + if delim: + delim = delimnl + if width >= w: + width -= w + write(delim) + delim = ", " + write(rep) + continue + write(delim) + delim = delimnl + self._format(ent, stream, indent, allowance if last else 1, context, level) + + def _repr(self, object, context, level): + repr, readable, recursive = self.format(object, context.copy(), self._depth, level) + if not readable: + self._readable = False + if recursive: + self._recursive = True + return repr + + def format(self, object, context, maxlevels, level): + return self._safe_repr(object, context, maxlevels, level) + + def _pprint_default_dict(self, object, stream, indent, allowance, context, level): + if not len(object): + stream.write(repr(object)) + return + rdf = self._repr(object.default_factory, context, level) + cls = object.__class__ + indent += len(cls.__name__) + 1 + stream.write("%s(%s,\n%s" % (cls.__name__, rdf, " " * indent)) + self._pprint_dict(object, stream, indent, allowance + 1, context, level) + stream.write(")") + + _dispatch[_collections.defaultdict] = _pprint_default_dict + + def _pprint_counter(self, object, stream, indent, allowance, context, level): + if not len(object): + stream.write(repr(object)) + return + cls = object.__class__ + stream.write(cls.__name__ + "({") + if self._indent_per_level > 1: + stream.write((self._indent_per_level - 1) * " ") + items = object.most_common() + self._format_dict_items(items, stream, indent + len(cls.__name__) + 1, allowance + 2, context, level) + stream.write("})") + + _dispatch[_collections.Counter] = _pprint_counter + + def _pprint_chain_map(self, object, stream, indent, allowance, context, level): + if not len(object.maps) or (len(object.maps) == 1 and not len(object.maps[0])): + stream.write(repr(object)) + return + cls = object.__class__ + stream.write(cls.__name__ + "(") + indent += len(cls.__name__) + 1 + for i, m in enumerate(object.maps): + if i == len(object.maps) - 1: + self._format(m, stream, indent, allowance + 1, context, level) + stream.write(")") + else: + self._format(m, stream, indent, 1, context, level) + stream.write(",\n" + " " * indent) + + _dispatch[_collections.ChainMap] = _pprint_chain_map + + def _pprint_deque(self, object, stream, indent, allowance, context, level): + if not len(object): + stream.write(repr(object)) + return + cls = object.__class__ + stream.write(cls.__name__ + "(") + indent += len(cls.__name__) + 1 + stream.write("[") + if object.maxlen is None: + self._format_items(object, stream, indent, allowance + 2, context, level) + stream.write("])") + else: + self._format_items(object, stream, indent, 2, context, level) + rml = self._repr(object.maxlen, context, level) + stream.write("],\n%smaxlen=%s)" % (" " * indent, rml)) + + _dispatch[_collections.deque] = _pprint_deque + + def _pprint_user_dict(self, object, stream, indent, allowance, context, level): + self._format(object.data, stream, indent, allowance, context, level - 1) + + _dispatch[_collections.UserDict] = _pprint_user_dict + + def _pprint_user_list(self, object, stream, indent, allowance, context, level): + self._format(object.data, stream, indent, allowance, context, level - 1) + + _dispatch[_collections.UserList] = _pprint_user_list + + def _pprint_user_string(self, object, stream, indent, allowance, context, level): + self._format(object.data, stream, indent, allowance, context, level - 1) + + _dispatch[_collections.UserString] = _pprint_user_string + + def _safe_repr(self, object, context, maxlevels, level): + typ = type(object) + if typ in _builtin_scalars: + return repr(object), True, False + if issubclass(typ, int) and _own_repr(typ, int): + if self._underscore_numbers: + return f"{object:_d}", True, False + return repr(object), True, False + if issubclass(typ, dict) and _own_repr(typ, dict): + if not object: + return "{}", True, False + objid = id(object) + if maxlevels and level >= maxlevels: + return "{...}", False, objid in context + if objid in context: + return _recursion(object), False, True + context[objid] = 1 + readable = True + recursive = False + components = [] + level += 1 + if self._sort_dicts: + items = sorted(object.items(), key=_safe_tuple) + else: + items = object.items() + for k, v in items: + krepr, kreadable, krecur = self.format(k, context, maxlevels, level) + vrepr, vreadable, vrecur = self.format(v, context, maxlevels, level) + components.append("%s: %s" % (krepr, vrepr)) + readable = readable and kreadable and vreadable + if krecur or vrecur: + recursive = True + del context[objid] + return "{%s}" % ", ".join(components), readable, recursive + if (issubclass(typ, list) and _own_repr(typ, list)) or (issubclass(typ, tuple) and _own_repr(typ, tuple)): + if issubclass(typ, list): + if not object: + return "[]", True, False + format = "[%s]" + elif len(object) == 1: + format = "(%s,)" + else: + if not object: + return "()", True, False + format = "(%s)" + objid = id(object) + if maxlevels and level >= maxlevels: + return format % "...", False, objid in context + if objid in context: + return _recursion(object), False, True + context[objid] = 1 + readable = True + recursive = False + components = [] + level += 1 + for o in object: + orepr, oreadable, orecur = self.format(o, context, maxlevels, level) + components.append(orepr) + if not oreadable: + readable = False + if orecur: + recursive = True + del context[objid] + return format % ", ".join(components), readable, recursive + rep = repr(object) + return rep, (rep and not rep.startswith("<")), False + + +def _own_repr(typ, base): + # True when typ prints as base does: nothing between them writes a __repr__. + for t in typ.__mro__: + if t is base: + return True + if "__repr__" in t.__dict__: + return False + return True + + +_builtin_scalars = frozenset({str, bytes, bytearray, float, complex, bool, type(None)}) + + +def _recursion(object): + return "" % (type(object).__name__, id(object)) + + +def _wrap_bytes_repr(object, width, allowance): + current = b"" + last = len(object) // 4 * 4 + for i in range(0, len(object), 4): + part = object[i:i + 4] + candidate = current + part + if i == last: + width -= allowance + if len(repr(candidate)) > width: + if current: + yield repr(current) + current = part + else: + current = candidate + if current: + yield repr(current) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/queue.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/queue.py new file mode 100644 index 00000000000..4e40f9951d5 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/queue.py @@ -0,0 +1,154 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Queues: FIFO, LIFO and priority. + +A translated program runs its Python code on one thread, so nothing can +fill an empty queue while get() waits on it. A get() on an empty queue or a +put() on a full one therefore raises Empty or Full at once, where CPython +would wait for another thread (forever, without one). +""" + +from collections import deque +import heapq + +__all__ = ["Empty", "Full", "ShutDown", "Queue", "PriorityQueue", "LifoQueue", "SimpleQueue"] + + +class Empty(Exception): + pass + + +class Full(Exception): + pass + + +class ShutDown(Exception): + pass + + +class Queue: + def __init__(self, maxsize=0): + self.maxsize = maxsize + self._init(maxsize) + self.unfinished_tasks = 0 + self.is_shutdown = False + + def task_done(self): + unfinished = self.unfinished_tasks - 1 + if unfinished < 0: + raise ValueError("task_done() called too many times") + self.unfinished_tasks = unfinished + + def join(self): + if self.unfinished_tasks: + raise RuntimeError("join() would wait forever: %d unfinished tasks" % self.unfinished_tasks) + + def qsize(self): + return self._qsize() + + def empty(self): + return not self._qsize() + + def full(self): + return 0 < self.maxsize <= self._qsize() + + def put(self, item, block=True, timeout=None): + if self.is_shutdown: + raise ShutDown + if block and timeout is not None and timeout < 0: + raise ValueError("'timeout' must be a non-negative number") + if 0 < self.maxsize <= self._qsize(): + raise Full + self._put(item) + self.unfinished_tasks += 1 + + def get(self, block=True, timeout=None): + if block and timeout is not None and timeout < 0: + raise ValueError("'timeout' must be a non-negative number") + if not self._qsize(): + if self.is_shutdown: + raise ShutDown + raise Empty + return self._get() + + def put_nowait(self, item): + return self.put(item, block=False) + + def get_nowait(self): + return self.get(block=False) + + def shutdown(self, immediate=False): + self.is_shutdown = True + if immediate: + while self._qsize(): + self._get() + if self.unfinished_tasks > 0: + self.unfinished_tasks -= 1 + + def _init(self, maxsize): + self.queue = deque() + + def _qsize(self): + return len(self.queue) + + def _put(self, item): + self.queue.append(item) + + def _get(self): + return self.queue.popleft() + + +class PriorityQueue(Queue): + def _init(self, maxsize): + self.queue = [] + + def _qsize(self): + return len(self.queue) + + def _put(self, item): + heapq.heappush(self.queue, item) + + def _get(self): + return heapq.heappop(self.queue) + + +class LifoQueue(Queue): + def _init(self, maxsize): + self.queue = [] + + def _qsize(self): + return len(self.queue) + + def _put(self, item): + self.queue.append(item) + + def _get(self): + return self.queue.pop() + + +class SimpleQueue: + def __init__(self): + self._queue = deque() + + def put(self, item, block=True, timeout=None): + self._queue.append(item) + + def get(self, block=True, timeout=None): + if timeout is not None and timeout < 0: + raise ValueError("'timeout' must be a non-negative number") + if not self._queue: + raise Empty + return self._queue.popleft() + + def put_nowait(self, item): + return self.put(item, block=False) + + def get_nowait(self): + return self.get(False) + + def empty(self): + return len(self._queue) == 0 + + def qsize(self): + return len(self._queue) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/random.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/random.py new file mode 100644 index 00000000000..aefbea6402e --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/random.py @@ -0,0 +1,486 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Pseudo-random numbers from a Mersenne Twister (MT19937). + +A seeded generator produces the same sequence CPython does: the seeding, +the 53-bit float construction and the rejection sampling of `_randbelow` +follow the published algorithms the reference implementation uses, so a +program that prints seeded draws prints the same values on every target. +""" + +from math import log as _log, exp as _exp, sqrt as _sqrt, cos as _cos, sin as _sin, pi as _pi +from math import floor as _floor, ceil as _ceil, isfinite as _isfinite, acos as _acos +from itertools import accumulate as _accumulate, repeat as _repeat +from bisect import bisect as _bisect + +__all__ = ["Random", "SystemRandom", "seed", "random", "uniform", "randint", "choice", "sample", + "randrange", "shuffle", "normalvariate", "lognormvariate", "expovariate", + "vonmisesvariate", "gammavariate", "triangular", "gauss", "betavariate", + "paretovariate", "weibullvariate", "getstate", "setstate", "getrandbits", "choices", + "randbytes", "binomialvariate"] + +NV_MAGICCONST = 4 * _exp(-0.5) / _sqrt(2.0) +TWOPI = 2.0 * _pi +LOG4 = _log(4.0) +SG_MAGICCONST = 1.0 + _log(4.5) +_N = 624 +_M = 397 +_MASK = 0xFFFFFFFF + + +def _index(v): + if isinstance(v, int): + return int(v) + if hasattr(type(v), "__index__"): + return v.__index__() + raise TypeError("'%s' object cannot be interpreted as an integer" % type(v).__name__) + + +class Random: + VERSION = 3 + + def __init__(self, x=None): + self._mt = [0] * _N + self._mti = _N + 1 + self.gauss_next = None + self.seed(x) + + def _init_genrand(self, s): + mt = self._mt + mt[0] = s & _MASK + for i in range(1, _N): + prev = mt[i - 1] + mt[i] = (1812433253 * (prev ^ (prev >> 30)) + i) & _MASK + self._mti = _N + + def _init_by_array(self, key): + self._init_genrand(19650218) + mt = self._mt + i = 1 + j = 0 + klen = len(key) + k = _N if _N > klen else klen + while k: + prev = mt[i - 1] + mt[i] = ((mt[i] ^ ((prev ^ (prev >> 30)) * 1664525)) + key[j] + j) & _MASK + i += 1 + j += 1 + if i >= _N: + mt[0] = mt[_N - 1] + i = 1 + if j >= klen: + j = 0 + k -= 1 + k = _N - 1 + while k: + prev = mt[i - 1] + mt[i] = ((mt[i] ^ ((prev ^ (prev >> 30)) * 1566083941)) - i) & _MASK + i += 1 + if i >= _N: + mt[0] = mt[_N - 1] + i = 1 + k -= 1 + mt[0] = 0x80000000 + + def _genrand(self): + mt = self._mt + if self._mti >= _N: + for kk in range(_N): + y = (mt[kk] & 0x80000000) | (mt[(kk + 1) % _N] & 0x7FFFFFFF) + v = mt[(kk + _M) % _N] ^ (y >> 1) + if y & 1: + v ^= 0x9908B0DF + mt[kk] = v + self._mti = 0 + y = mt[self._mti] + self._mti += 1 + y ^= y >> 11 + y ^= (y << 7) & 0x9D2C5680 + y ^= (y << 15) & 0xEFC60000 + y ^= y >> 18 + return y + + def seed(self, a=None, version=2): + if a is None: + import time + a = int(time.time() * 1000000) ^ id(self) + elif isinstance(a, (str, bytes)) and version == 2: + if isinstance(a, str): + a = a.encode("utf-8") + from hashlib import sha512 + a = int.from_bytes(a + sha512(a).digest(), "big") + elif isinstance(a, float): + a = hash(a) + elif not isinstance(a, int): + raise TypeError("The only supported seed types are:\nNone, int, float, str, bytes, " + "and bytearray.") + a = abs(int(a)) + key = [] + while a: + key.append(a & _MASK) + a >>= 32 + if not key: + key.append(0) + self._init_by_array(key) + self.gauss_next = None + + def getstate(self): + return (self.VERSION, tuple(self._mt) + (self._mti,), self.gauss_next) + + def setstate(self, state): + version, internal, self.gauss_next = state + if version != self.VERSION or len(internal) != _N + 1: + raise ValueError("state with version %s passed to Random.setstate() of version %s" + % (version, self.VERSION)) + self._mt = list(internal[:_N]) + self._mti = internal[_N] + + def random(self): + a = self._genrand() >> 5 + b = self._genrand() >> 6 + return (a * 67108864.0 + b) * (1.0 / 9007199254740992.0) + + def getrandbits(self, k): + if k < 0: + raise ValueError("number of bits must be non-negative") + if k == 0: + return 0 + if k <= 32: + return self._genrand() >> (32 - k) + result = 0 + shift = 0 + while k > 0: + r = self._genrand() + if k < 32: + r >>= 32 - k + result |= r << shift + shift += 32 + k -= 32 + return result + + def randbytes(self, n): + return self.getrandbits(n * 8).to_bytes(n, "little") + + def _randbelow(self, n): + k = n.bit_length() + r = self.getrandbits(k) + while r >= n: + r = self.getrandbits(k) + return r + + def randrange(self, start, stop=None, step=1): + istart = _index(start) + if stop is None: + if step != 1: + raise TypeError("Missing a non-None stop argument") + if istart > 0: + return self._randbelow(istart) + raise ValueError("empty range for randrange()") + istop = _index(stop) + width = istop - istart + istep = _index(step) + if istep == 1: + if width > 0: + return istart + self._randbelow(width) + raise ValueError("empty range in randrange(%s, %s)" % (start, stop)) + if istep > 0: + n = (width + istep - 1) // istep + elif istep < 0: + n = (width + istep + 1) // istep + else: + raise ValueError("zero step for randrange()") + if n <= 0: + raise ValueError("empty range in randrange(%s, %s, %s)" % (start, stop, step)) + return istart + istep * self._randbelow(n) + + def randint(self, a, b): + return self.randrange(a, b + 1) + + def choice(self, seq): + if not len(seq): + raise IndexError("Cannot choose from an empty sequence") + return seq[self._randbelow(len(seq))] + + def shuffle(self, x): + randbelow = self._randbelow + for i in reversed(range(1, len(x))): + j = randbelow(i + 1) + x[i], x[j] = x[j], x[i] + + def sample(self, population, k, *, counts=None): + if isinstance(population, (set, frozenset, dict)): + raise TypeError("Population must be a sequence. For dicts or sets, use sorted(d).") + n = len(population) + if counts is not None: + cum_counts = list(_accumulate(counts)) + if len(cum_counts) != n: + raise ValueError("The number of counts does not match the population") + total = cum_counts.pop() if cum_counts else 0 + if not isinstance(total, int): + raise TypeError("Counts must be integers") + if total < 0: + raise ValueError("Counts must be non-negative") + selections = self.sample(range(total), k=k) + return [population[_bisect(cum_counts, s)] for s in selections] + randbelow = self._randbelow + if not 0 <= k <= n: + raise ValueError("Sample larger than population or is negative") + result = [None] * k + setsize = 21 + if k > 5: + setsize += 4 ** _ceil(_log(k * 3, 4)) + if n <= setsize: + pool = list(population) + for i in range(k): + j = randbelow(n - i) + result[i] = pool[j] + pool[j] = pool[n - i - 1] + else: + selected = set() + for i in range(k): + j = randbelow(n) + while j in selected: + j = randbelow(n) + selected.add(j) + result[i] = population[j] + return result + + def choices(self, population, weights=None, *, cum_weights=None, k=1): + random = self.random + n = len(population) + if cum_weights is None: + if weights is None: + n += 0.0 + return [population[_floor(random() * n)] for i in _repeat(None, k)] + cum_weights = list(_accumulate(weights)) + elif weights is not None: + raise TypeError("Cannot specify both weights and cumulative weights") + if len(cum_weights) != n: + raise ValueError("The number of weights does not match the population") + total = cum_weights[-1] + 0.0 + if total <= 0.0: + raise ValueError("Total of weights must be greater than zero") + if not _isfinite(total): + raise ValueError("Total of weights must be finite") + hi = n - 1 + return [population[_bisect(cum_weights, random() * total, 0, hi)] for i in _repeat(None, k)] + + def uniform(self, a, b): + return a + (b - a) * self.random() + + def triangular(self, low=0.0, high=1.0, mode=None): + u = self.random() + try: + c = 0.5 if mode is None else (mode - low) / (high - low) + except ZeroDivisionError: + return low + if u > c: + u = 1.0 - u + c = 1.0 - c + low, high = high, low + return low + (high - low) * _sqrt(u * c) + + def normalvariate(self, mu=0.0, sigma=1.0): + random = self.random + while True: + u1 = random() + u2 = 1.0 - random() + z = NV_MAGICCONST * (u1 - 0.5) / u2 + zz = z * z / 4.0 + if zz <= -_log(u2): + break + return mu + z * sigma + + def gauss(self, mu=0.0, sigma=1.0): + random = self.random + z = self.gauss_next + self.gauss_next = None + if z is None: + x2pi = random() * TWOPI + g2rad = _sqrt(-2.0 * _log(1.0 - random())) + z = _cos(x2pi) * g2rad + self.gauss_next = _sin(x2pi) * g2rad + return mu + z * sigma + + def lognormvariate(self, mu, sigma): + return _exp(self.normalvariate(mu, sigma)) + + def expovariate(self, lambd=1.0): + return -_log(1.0 - self.random()) / lambd + + def vonmisesvariate(self, mu, kappa): + random = self.random + if kappa <= 1e-6: + return TWOPI * random() + s = 0.5 / kappa + r = s + _sqrt(1.0 + s * s) + while True: + u1 = random() + z = _cos(_pi * u1) + d = z / (r + z) + u2 = random() + if u2 < 1.0 - d * d or u2 <= (1.0 - d) * _exp(d): + break + q = 1.0 / r + f = (q + z) / (1.0 + q * z) + u3 = random() + if u3 > 0.5: + theta = (mu + _acos(f)) % TWOPI + else: + theta = (mu - _acos(f)) % TWOPI + return theta + + def gammavariate(self, alpha, beta): + if alpha <= 0.0 or beta <= 0.0: + raise ValueError("gammavariate: alpha and beta must be > 0.0") + random = self.random + if alpha > 1.0: + ainv = _sqrt(2.0 * alpha - 1.0) + bbb = alpha - LOG4 + ccc = alpha + ainv + while True: + u1 = random() + if not 1e-7 < u1 < 0.9999999: + continue + u2 = 1.0 - random() + v = _log(u1 / (1.0 - u1)) / ainv + x = alpha * _exp(v) + z = u1 * u1 * u2 + r = bbb + ccc * v - x + if r + SG_MAGICCONST - 4.5 * z >= 0.0 or r >= _log(z): + return x * beta + elif alpha == 1.0: + return -_log(1.0 - random()) * beta + else: + while True: + u = random() + b = (_exp(1.0) + alpha) / _exp(1.0) + p = b * u + if p <= 1.0: + x = p ** (1.0 / alpha) + else: + x = -_log((b - p) / alpha) + u1 = random() + if p > 1.0: + if u1 <= x ** (alpha - 1.0): + break + elif u1 <= _exp(-x): + break + return x * beta + + def betavariate(self, alpha, beta): + y = self.gammavariate(alpha, 1.0) + if y: + return y / (y + self.gammavariate(beta, 1.0)) + return 0.0 + + def paretovariate(self, alpha): + u = 1.0 - self.random() + return u ** (-1.0 / alpha) + + def weibullvariate(self, alpha, beta): + u = 1.0 - self.random() + return alpha * (-_log(u)) ** (1.0 / beta) + + def binomialvariate(self, n=1, p=0.5): + if n < 0: + raise ValueError("n must be non-negative") + if p <= 0.0 or p >= 1.0: + if p == 0.0: + return 0 + if p == 1.0: + return n + raise ValueError("p must be in the range 0.0 <= p <= 1.0") + random = self.random + if n == 1: + return int(random() < p) + if p > 0.5: + return n - self.binomialvariate(n, 1.0 - p) + if n * p < 10.0: + x = y = 0 + c = _log2(1.0 - p) + if not c: + return x + while True: + y += _floor(_log2(random()) / c) + 1 + if y > n: + return x + x += 1 + # Large n * p: the BTRS transformed rejection, as the reference uses. + spq = _sqrt(n * p * (1.0 - p)) + b = 1.15 + 2.53 * spq + a = -0.0873 + 0.0248 * b + 0.01 * p + c = n * p + 0.5 + vr = 0.92 - 4.2 / b + setup_complete = False + alpha = lpq = m = h = 0.0 + while True: + u = random() + u -= 0.5 + us = 0.5 - abs(u) + k = _floor((2.0 * a / us + b) * u + c) + if k < 0 or k > n: + continue + v = random() + if us >= 0.07 and v <= vr: + return k + if not setup_complete: + alpha = (2.83 + 5.1 / b) * spq + lpq = _log(p / (1.0 - p)) + m = _floor((n + 1) * p) + h = _lgamma(m + 1) + _lgamma(n - m + 1) + setup_complete = True + v *= alpha / (a / (us * us) + b) + if _log(v) <= h - _lgamma(k + 1) - _lgamma(n - k + 1) + (k - m) * lpq: + return k + + +def _log2(x): + from math import log2 + return log2(x) + + +def _lgamma(x): + from math import lgamma + return lgamma(x) + + +class SystemRandom(Random): + """Draws from the platform's entropy rather than a seeded sequence.""" + + def seed(self, *args, **kwds): + Random.seed(self, None) + + def getstate(self, *args, **kwds): + raise NotImplementedError("System entropy source does not have state.") + + def setstate(self, *args, **kwds): + raise NotImplementedError("System entropy source does not have state.") + + +_inst = Random() +seed = _inst.seed +random = _inst.random +uniform = _inst.uniform +triangular = _inst.triangular +randint = _inst.randint +choice = _inst.choice +randrange = _inst.randrange +sample = _inst.sample +shuffle = _inst.shuffle +choices = _inst.choices +normalvariate = _inst.normalvariate +lognormvariate = _inst.lognormvariate +expovariate = _inst.expovariate +vonmisesvariate = _inst.vonmisesvariate +gammavariate = _inst.gammavariate +gauss = _inst.gauss +betavariate = _inst.betavariate +binomialvariate = _inst.binomialvariate +paretovariate = _inst.paretovariate +weibullvariate = _inst.weibullvariate +getstate = _inst.getstate +setstate = _inst.setstate +getrandbits = _inst.getrandbits +randbytes = _inst.randbytes diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/re/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/re/__init__.py new file mode 100644 index 00000000000..8389e845337 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/re/__init__.py @@ -0,0 +1,455 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Regular expressions with the syntax and matching rules of Python's re. + +A pattern compiles to a small program for a backtracking machine +(`re._engine`). The machine keeps its choice points on an explicit stack +rather than the call stack, so a long subject cannot overflow a device's +thread stack, and undoes group captures through a trail when it backtracks. +""" + +from enum import IntFlag as _IntFlag +from re import _parser +from re import _engine + +__all__ = ["match", "fullmatch", "search", "sub", "subn", "split", "findall", "finditer", "compile", + "purge", "escape", "error", "Pattern", "Match", "A", "I", "L", "M", "S", "X", "U", + "ASCII", "IGNORECASE", "LOCALE", "MULTILINE", "DOTALL", "VERBOSE", "UNICODE", "NOFLAG", + "RegexFlag", "PatternError"] + + +class RegexFlag(_IntFlag): + NOFLAG = 0 + TEMPLATE = 1 + IGNORECASE = 2 + LOCALE = 4 + MULTILINE = 8 + DOTALL = 16 + UNICODE = 32 + VERBOSE = 64 + DEBUG = 128 + ASCII = 256 + + def __repr__(self): + v = int(self) + if v == 0: + return "re.NOFLAG" + names = [] + for name in ("TEMPLATE", "IGNORECASE", "LOCALE", "MULTILINE", "DOTALL", "UNICODE", + "VERBOSE", "DEBUG", "ASCII"): + bit = int(RegexFlag[name]) + if v & bit: + names.append("re." + name) + v &= ~bit + if v: + names.append(hex(v)) + return "|".join(names) + + __str__ = __repr__ + + +NOFLAG = RegexFlag.NOFLAG +A = ASCII = RegexFlag.ASCII +I = IGNORECASE = RegexFlag.IGNORECASE +L = LOCALE = RegexFlag.LOCALE +M = MULTILINE = RegexFlag.MULTILINE +S = DOTALL = RegexFlag.DOTALL +U = UNICODE = RegexFlag.UNICODE +X = VERBOSE = RegexFlag.VERBOSE +T = TEMPLATE = RegexFlag.TEMPLATE +DEBUG = RegexFlag.DEBUG + +error = _parser.error +PatternError = error + + +class Pattern: + def __init__(self, pattern, flags, code): + self.pattern = pattern + self._code = code + self.flags = flags + self.groups = code.groups + self.groupindex = dict(code.groupindex) + + def __repr__(self): + p = repr(self.pattern) + if len(p) > 200: + p = p[:200] + f = self.flags & ~(int(UNICODE) if isinstance(self.pattern, str) else 0) + if f: + return "re.compile(%s, %s)" % (p, repr(RegexFlag(f))) + return "re.compile(%s)" % p + + def __eq__(self, other): + if not isinstance(other, Pattern): + return NotImplemented + return self.pattern == other.pattern and self.flags == other.flags + + def __hash__(self): + return hash((self.pattern, self.flags)) + + def _check(self, string): + if isinstance(self.pattern, str): + if not isinstance(string, str): + raise TypeError("cannot use a string pattern on a bytes-like object") + elif isinstance(string, str): + raise TypeError("cannot use a bytes pattern on a string-like object") + + def _run(self, string, pos, endpos, mode): + self._check(string) + n = len(string) + if endpos is None or endpos > n: + endpos = n + if endpos < 0: + endpos = 0 + if pos < 0: + pos = 0 + if pos > endpos: + return None + text = string + if not isinstance(string, str): + text = string.decode("latin-1") + regs = _engine.run(self._code, text, pos, endpos, mode) + if regs is None: + return None + return Match(self, string, pos, endpos, regs) + + def match(self, string, pos=0, endpos=None): + return self._run(string, pos, endpos, 0) + + def fullmatch(self, string, pos=0, endpos=None): + return self._run(string, pos, endpos, 1) + + def search(self, string, pos=0, endpos=None): + return self._run(string, pos, endpos, 2) + + def finditer(self, string, pos=0, endpos=None): + self._check(string) + n = len(string) + if endpos is None or endpos > n: + endpos = n + text = string if isinstance(string, str) else string.decode("latin-1") + forbid = -1 + at = pos + while at <= endpos: + regs = _engine.run(self._code, text, at, endpos, 2, forbid) + if regs is None: + return + yield Match(self, string, pos, endpos, regs) + s, e = regs[0], regs[1] + # After an empty match the next one may not be empty at the + # same place; after a non-empty one it may. + forbid = e if s == e else -1 + at = e + + def findall(self, string, pos=0, endpos=None): + out = [] + for m in self.finditer(string, pos, endpos): + if self.groups == 0: + out.append(m.group()) + elif self.groups == 1: + out.append(m.group(1) if m.group(1) is not None else string[:0]) + else: + out.append(tuple(g if g is not None else string[:0] for g in m.groups())) + return out + + def split(self, string, maxsplit=0): + out = [] + last = 0 + n = 0 + for m in self.finditer(string): + if maxsplit and n >= maxsplit: + break + s, e = m.span() + out.append(string[last:s]) + out.extend(m.groups()) + last = e + n += 1 + out.append(string[last:]) + return out + + def subn(self, repl, string, count=0): + if callable(repl): + fn = repl + else: + template = _compile_template(self, repl) + if len(template) == 1 and isinstance(template[0], (str, bytes)): + literal = template[0] + fn = lambda m: literal + else: + fn = lambda m: _expand(template, m) + out = [] + last = 0 + n = 0 + for m in self.finditer(string): + if count and n >= count: + break + s, e = m.span() + out.append(string[last:s]) + out.append(fn(m)) + last = e + n += 1 + out.append(string[last:]) + return string[:0].join(out), n + + def sub(self, repl, string, count=0): + return self.subn(repl, string, count)[0] + + def scanner(self, string, pos=0, endpos=None): + raise NotImplementedError("Pattern.scanner is not supported") + + def __copy__(self): + return self + + def __deepcopy__(self, memo): + return self + + +class Match: + def __init__(self, pattern, string, pos, endpos, regs): + self.re = pattern + self.string = string + self.pos = pos + self.endpos = endpos + self._regs = regs + last = regs[-1] + self.lastindex = last if last > 0 else None + self.lastgroup = None + if self.lastindex is not None: + for name, idx in pattern.groupindex.items(): + if idx == self.lastindex: + self.lastgroup = name + + def _index(self, g): + if isinstance(g, int): + if 0 <= g <= self.re.groups: + return g + elif isinstance(g, str): + idx = self.re.groupindex.get(g) + if idx is not None: + return idx + raise IndexError("no such group") + + def _get(self, idx, default=None): + s = self._regs[2 * idx] + e = self._regs[2 * idx + 1] + if s < 0 or e < 0: + return default + return self.string[s:e] + + def group(self, *groups): + if not groups: + return self._get(0) + if len(groups) == 1: + return self._get(self._index(groups[0])) + return tuple(self._get(self._index(g)) for g in groups) + + def __getitem__(self, g): + return self._get(self._index(g)) + + def groups(self, default=None): + return tuple(self._get(i, default) for i in range(1, self.re.groups + 1)) + + def groupdict(self, default=None): + return {name: self._get(idx, default) for name, idx in self.re.groupindex.items()} + + def start(self, group=0): + return self._regs[2 * self._index(group)] + + def end(self, group=0): + return self._regs[2 * self._index(group) + 1] + + def span(self, group=0): + i = self._index(group) + return (self._regs[2 * i], self._regs[2 * i + 1]) + + @property + def regs(self): + return tuple((self._regs[2 * i], self._regs[2 * i + 1]) for i in range(self.re.groups + 1)) + + def expand(self, template): + return _expand(_compile_template(self.re, template), self) + + def __repr__(self): + return "" % (self.span(), self.group()) + + def __bool__(self): + return True + + def __copy__(self): + return self + + def __deepcopy__(self, memo): + return self + + +_cache = {} + + +def _compile(pattern, flags): + if isinstance(flags, RegexFlag): + flags = int(flags) + if isinstance(pattern, Pattern): + if flags: + raise ValueError("cannot process flags argument with a compiled pattern") + return pattern + if not isinstance(pattern, (str, bytes)): + raise TypeError("first argument must be string or compiled pattern") + key = (type(pattern), pattern, flags) + p = _cache.get(key) + if p is not None: + return p + text = pattern if isinstance(pattern, str) else pattern.decode("latin-1") + code = _parser.compile(text, flags, isinstance(pattern, str)) + p = Pattern(pattern, code.flags, code) + if len(_cache) >= 512: + _cache.clear() + _cache[key] = p + return p + + +def compile(pattern, flags=0): + return _compile(pattern, flags) + + +def purge(): + _cache.clear() + + +def match(pattern, string, flags=0): + return _compile(pattern, flags).match(string) + + +def fullmatch(pattern, string, flags=0): + return _compile(pattern, flags).fullmatch(string) + + +def search(pattern, string, flags=0): + return _compile(pattern, flags).search(string) + + +def sub(pattern, repl, string, count=0, flags=0): + return _compile(pattern, flags).sub(repl, string, count) + + +def subn(pattern, repl, string, count=0, flags=0): + return _compile(pattern, flags).subn(repl, string, count) + + +def split(pattern, string, maxsplit=0, flags=0): + return _compile(pattern, flags).split(string, maxsplit) + + +def findall(pattern, string, flags=0): + return _compile(pattern, flags).findall(string) + + +def finditer(pattern, string, flags=0): + return _compile(pattern, flags).finditer(string) + + +_SPECIAL = frozenset("()[]{}?*+-|^$\\.&~# \t\n\r\v\f") + + +def escape(pattern): + if isinstance(pattern, str): + return "".join("\\" + c if c in _SPECIAL else c for c in pattern) + text = pattern.decode("latin-1") + return "".join("\\" + c if c in _SPECIAL else c for c in text).encode("latin-1") + + +_TEMPLATE_ESCAPES = {"n": "\n", "t": "\t", "r": "\r", "f": "\f", "v": "\v", "a": "\a", "b": "\b", + "\\": "\\"} + + +def _compile_template(pattern, repl): + """Splits a replacement into literal text and group numbers.""" + is_bytes = isinstance(repl, bytes) + text = repl.decode("latin-1") if is_bytes else repl + parts = [] + lit = [] + i = 0 + n = len(text) + while i < n: + c = text[i] + if c != "\\": + lit.append(c) + i += 1 + continue + i += 1 + if i >= n: + raise error("bad escape (end of pattern)", text, i - 1) + c = text[i] + if c == "g": + if i + 1 >= n or text[i + 1] != "<": + raise error("missing <", text, i + 1) + close = text.find(">", i + 2) + if close < 0: + raise error("missing >, unterminated name", text, i + 2) + name = text[i + 2:close] + if not name: + raise error("missing group name", text, i + 2) + if name.isdigit(): + idx = int(name) + if idx > pattern.groups: + raise error("invalid group reference %d" % idx, text, i + 2) + else: + if not name.isidentifier(): + raise error("bad character in group name '%s'" % name, text, i + 2) + if name not in pattern.groupindex: + raise IndexError("unknown group name '%s'" % name) + idx = pattern.groupindex[name] + parts.append("".join(lit)) + lit = [] + parts.append(idx) + i = close + 1 + elif c == "0": + j = i + 1 + v = 0 + while j < n and j < i + 3 and text[j] in "01234567": + v = v * 8 + int(text[j]) + j += 1 + lit.append(chr(v)) + i = j + elif c.isdigit(): + j = i + while j < n and j < i + 2 and text[j].isdigit(): + j += 1 + if j < n and j == i + 2 and text[i] in "01234567" and text[i + 1] in "01234567" \ + and text[j] in "01234567": + lit.append(chr(int(text[i:j + 1], 8))) + i = j + 1 + continue + idx = int(text[i:j]) + if idx > pattern.groups: + raise error("invalid group reference %d" % idx, text, i) + parts.append("".join(lit)) + lit = [] + parts.append(idx) + i = j + elif c in _TEMPLATE_ESCAPES: + lit.append(_TEMPLATE_ESCAPES[c]) + i += 1 + elif c.isascii() and c.isalpha(): + raise error("bad escape \\" + c, text, i - 1) + else: + lit.append("\\" + c) + i += 1 + parts.append("".join(lit)) + if is_bytes: + return [p.encode("latin-1") if isinstance(p, str) else p for p in parts] + return parts + + +def _expand(template, m): + empty = m.string[:0] + out = [] + for p in template: + if isinstance(p, int): + g = m.group(p) + if g is not None: + out.append(g) + else: + out.append(p) + return empty.join(out) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/re/_engine.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/re/_engine.py new file mode 100644 index 00000000000..a2d920defad --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/re/_engine.py @@ -0,0 +1,314 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""The backtracking machine a compiled pattern runs on. + +Registers hold the group boundaries (2 per group, group 0 first), then +`lastindex`, then one slot per empty-loop guard. Every register write is +logged on a trail, and a choice point remembers the trail's length, so +backtracking restores exactly the captures that were live when the choice +was made. Choice points live on a list, never the call stack. +""" + +CHAR = 1 +CHARI = 2 +ANY = 3 +ANYALL = 4 +SET = 5 +AT = 6 +SPLIT = 7 +JMP = 8 +SAVE = 9 +BACKREF = 10 +LOOK = 11 +ATOMIC = 12 +MATCH = 13 +MARK = 14 +PROGRESS = 15 +GROUPREF_EXISTS = 16 +REPEAT_ONE = 17 +MIN_REPEAT_ONE = 18 + +_ASCII_WORD = frozenset("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_") +_ASCII_SPACE = frozenset(" \t\n\r\f\v") + + +def _cat(k, c, unicode): + lk = k.lower() + if lk == "d": + r = c.isdecimal() if unicode else "0" <= c <= "9" + elif lk == "w": + r = (c.isalnum() or c == "_") if unicode else c in _ASCII_WORD + else: + r = c.isspace() if unicode else c in _ASCII_SPACE + return r if k == lk else not r + + +def _one(op, c): + k = op[0] + if k == CHAR: + return c == op[1] + if k == CHARI: + return c == op[1] or c == op[2] or c.lower() == op[1] + if k == ANY: + return c != "\n" + if k == ANYALL: + return True + chars, ranges, cats, neg, rcase, unicode = op[1], op[2], op[3], op[4], op[5], op[6] + r = c in chars + if not r and ranges: + o = ord(c) + for lo, hi in ranges: + if lo <= o <= hi: + r = True + break + if not r and rcase: + for alt in (c.lower(), c.upper()): + if len(alt) == 1: + o = ord(alt) + for lo, hi in ranges: + if lo <= o <= hi: + r = True + break + if not r and cats: + for k2 in cats: + if _cat(k2, c, unicode): + r = True + break + return r != neg + + +def _word(text, i, endpos, unicode): + if i < 0 or i >= endpos: + return False + c = text[i] + return (c.isalnum() or c == "_") if unicode else c in _ASCII_WORD + + +def _at(kind, text, pos, endpos, unicode): + if kind == "begin": + return pos == 0 + if kind == "bol": + return pos == 0 or text[pos - 1] == "\n" + if kind == "end": + return pos == endpos + if kind == "eol": + return pos == endpos or (pos == endpos - 1 and text[pos] == "\n") + if kind == "eolm": + return pos == endpos or text[pos] == "\n" + b = _word(text, pos - 1, endpos, unicode) != _word(text, pos, endpos, unicode) + if kind == "boundary": + return b + return not b + + +def _exec(prog, text, start, endpos, regs, full, forbid, li): + """Runs `prog` from `start`; returns the end position, or -1.""" + stack = [] + trail = [] + pc = 0 + pos = start + while True: + op = prog[pc] + k = op[0] + ok = True + if k == CHAR: + if pos < endpos and text[pos] == op[1]: + pos += 1 + pc += 1 + continue + ok = False + elif k <= SET: + if pos < endpos and _one(op, text[pos]): + pos += 1 + pc += 1 + continue + ok = False + elif k == SPLIT: + stack.append((0, op[2], pos, len(trail))) + pc = op[1] + continue + elif k == JMP: + pc = op[1] + continue + elif k == SAVE: + n = op[1] + trail.append((n, regs[n])) + regs[n] = pos + if n & 1: + trail.append((li, regs[li])) + regs[li] = n >> 1 + pc += 1 + continue + elif k == REPEAT_ONE: + one = op[1] + lo = op[2] + hi = op[3] + limit = endpos if hi is None else min(endpos, pos + hi) + j = pos + while j < limit and _one(one, text[j]): + j += 1 + if j - pos < lo: + ok = False + else: + if j > pos + lo: + stack.append((1, pc + 1, pos + lo, j, len(trail))) + pos = j + pc += 1 + continue + elif k == MIN_REPEAT_ONE: + one = op[1] + lo = op[2] + j = pos + while j < pos + lo: + if j >= endpos or not _one(one, text[j]): + break + j += 1 + if j < pos + lo: + ok = False + else: + stack.append((2, pc + 1, j, lo, len(trail), one, op[3])) + pos = j + pc += 1 + continue + elif k == AT: + if _at(op[1], text, pos, endpos, op[2]): + pc += 1 + continue + ok = False + elif k == MATCH: + if (full and pos != endpos) or (forbid >= 0 and pos == start == forbid): + ok = False + else: + return pos + elif k == MARK: + n = li + 1 + op[1] + trail.append((n, regs[n])) + regs[n] = pos + pc += 1 + continue + elif k == PROGRESS: + if regs[li + 1 + op[1]] == pos: + pc = op[2] + else: + pc += 1 + continue + elif k == BACKREF: + s = regs[2 * op[1]] + e = regs[2 * op[1] + 1] + if s < 0 or e < 0: + ok = False + else: + seg = text[s:e] + n = e - s + got = text[pos:pos + n] if pos + n <= endpos else None + if got is not None and (got == seg or (op[2] and got.lower() == seg.lower())): + pos += n + pc += 1 + continue + ok = False + elif k == GROUPREF_EXISTS: + if regs[2 * op[1]] >= 0 and regs[2 * op[1] + 1] >= 0: + pc += 1 + else: + pc = op[2] + continue + elif k == LOOK: + sub = op[1] + inner = list(regs) + if op[2]: + s = pos - op[4] + end = -1 if s < 0 else _exec(sub, text, s, pos, inner, True, -1, li) + else: + end = _exec(sub, text, pos, endpos, inner, False, -1, li) + if op[3]: + if end < 0: + pc += 1 + continue + ok = False + elif end < 0: + ok = False + else: + _adopt(regs, inner, trail) + pc += 1 + continue + elif k == ATOMIC: + inner = list(regs) + end = _exec(op[1], text, pos, endpos, inner, False, -1, li) + if end < 0: + ok = False + else: + _adopt(regs, inner, trail) + pos = end + pc += 1 + continue + if ok: + raise RuntimeError("internal error: opcode %d" % k) + # Backtrack to the most recent choice point. + while True: + if not stack: + return -1 + e = stack.pop() + kind = e[0] + if kind == 0: + pc, pos, tl = e[1], e[2], e[3] + break + if kind == 1: + nxt, minpos, cur, tl = e[1], e[2], e[3], e[4] + cur -= 1 + # Only stop where the next literal could match. + nop = prog[nxt] + if nop[0] == CHAR: + c = nop[1] + while cur >= minpos and (cur >= endpos or text[cur] != c): + cur -= 1 + if cur < minpos: + continue + if cur > minpos: + stack.append((1, nxt, minpos, cur, tl)) + pc, pos = nxt, cur + break + nxt, cur, count, tl, one, hi = e[1], e[2], e[3], e[4], e[5], e[6] + if (hi is not None and count >= hi) or cur >= endpos or not _one(one, text[cur]): + continue + cur += 1 + stack.append((2, nxt, cur, count + 1, tl, one, hi)) + pc, pos = nxt, cur + break + while len(trail) > tl: + n, v = trail.pop() + regs[n] = v + + +def _adopt(regs, inner, trail): + for n in range(len(regs)): + if regs[n] != inner[n]: + trail.append((n, regs[n])) + regs[n] = inner[n] + + +def run(code, text, pos, endpos, mode, forbid=-1): + """mode 0 = match, 1 = fullmatch, 2 = search. Returns registers or None.""" + size = 2 * (code.groups + 1) + prog = code.prog + full = mode == 1 + start = pos + prefix = code.prefix + while start <= endpos: + if mode == 2 and prefix: + found = text.find(prefix, start, endpos) + if found < 0: + return None + start = found + regs = [-1] * (size + 1 + code.marks) + end = _exec(prog, text, start, endpos, regs, full, forbid, size) + if end >= 0: + regs[0] = start + regs[1] = end + out = regs[:size] + out.append(regs[size]) + return out + if mode != 2 or code.anchored: + return None + start += 1 + return None diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/re/_parser.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/re/_parser.py new file mode 100644 index 00000000000..412628b7327 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/re/_parser.py @@ -0,0 +1,724 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Parses a regular expression and compiles it to a backtracking program.""" + +from re import _engine as E + +F_IGNORECASE = 2 +F_LOCALE = 4 +F_MULTILINE = 8 +F_DOTALL = 16 +F_UNICODE = 32 +F_VERBOSE = 64 +F_ASCII = 256 + +_FLAG_LETTERS = {"a": F_ASCII, "i": F_IGNORECASE, "L": F_LOCALE, "m": F_MULTILINE, "s": F_DOTALL, + "u": F_UNICODE, "x": F_VERBOSE} +_DIGITS = "0123456789" +_OCT = "01234567" +_HEXD = "0123456789abcdefABCDEF" +_REPEAT = "*+?{" +_SIMPLE_ESC = {"a": "\a", "b": "\b", "f": "\f", "n": "\n", "r": "\r", "t": "\t", "v": "\v", + "\\": "\\"} +_CATS = "dDsSwW" +_MAXREPEAT = 4294967295 + + +class error(Exception): + def __init__(self, msg, pattern=None, pos=None): + self.msg = msg + self.pattern = pattern + self.pos = pos + if pattern is not None and pos is not None: + msg = "%s at position %d" % (msg, pos) + if isinstance(pattern, str): + newline = "\n" + else: + newline = b"\n" + self.lineno = pattern.count(newline, 0, pos) + 1 + self.colno = pos - pattern.rfind(newline, 0, pos) + if newline in pattern: + msg = "%s (line %d, column %d)" % (msg, self.lineno, self.colno) + else: + self.lineno = self.colno = None + Exception.__init__(self, msg) + + +error.__module__ = "re" +error.__qualname__ = "PatternError" +error.__name__ = "PatternError" + + +class Code: + def __init__(self, prog, groups, groupindex, flags, prefix, anchored): + self.prog = prog + self.groups = groups + self.groupindex = groupindex + self.flags = flags + self.prefix = prefix + self.anchored = anchored + + +class _State: + def __init__(self, text, flags, is_str): + self.text = text + self.i = 0 + self.flags = flags + self.is_str = is_str + self.groups = 0 + self.groupindex = {} + self.open_groups = set() + self.lookbehind_groups = None + + def err(self, msg, pos=None): + return error(msg, self.text, self.i if pos is None else pos) + + def peek(self): + return self.text[self.i] if self.i < len(self.text) else None + + def skip_ws(self, flags): + if not flags & F_VERBOSE: + return + t = self.text + while self.i < len(t): + c = t[self.i] + if c in " \t\n\r\v\f": + self.i += 1 + elif c == "#": + while self.i < len(t) and t[self.i] != "\n": + self.i += 1 + else: + break + + +def compile(text, flags, is_str): + if is_str: + if flags & F_LOCALE: + raise ValueError("cannot use LOCALE flag with a str pattern") + if flags & F_ASCII and flags & F_UNICODE: + raise ValueError("ASCII and UNICODE flags are incompatible") + elif flags & F_UNICODE: + raise ValueError("cannot use UNICODE flag with a bytes pattern") + st = _State(text, flags, is_str) + st.flags = _leading_flags(st, flags) + node = _parse_alt(st, st.flags, 0) + if st.i < len(text): + if text[st.i] == ")": + raise st.err("unbalanced parenthesis") + raise st.err("unexpected end of pattern") + final = st.flags + if is_str and not final & F_ASCII: + final |= F_UNICODE + unicode = is_str and not final & F_ASCII + prog = [] + c = _Compiler(prog, unicode) + c.emit(node) + prog.append((E.MATCH,)) + code = Code(prog, st.groups, st.groupindex, final, _prefix(node), _anchored(node)) + code.marks = c.marks + return code + + +def _leading_flags(st, flags): + # Global inline flags must open the pattern: "(?i)abc". + t = st.text + while t.startswith("(?", st.i): + j = st.i + 2 + add = 0 + while j < len(t) and t[j] in _FLAG_LETTERS: + add |= _FLAG_LETTERS[t[j]] + j += 1 + if j >= len(t) or t[j] != ")" or j == st.i + 2: + break + flags |= add + st.i = j + 1 + st.skip_ws(flags) + return flags + + +def _parse_alt(st, flags, depth): + branches = [_parse_seq(st, flags, depth)] + while st.peek() == "|": + st.i += 1 + branches.append(_parse_seq(st, flags, depth)) + if len(branches) == 1: + return branches[0] + return ("alt", branches) + + +def _parse_seq(st, flags, depth): + items = [] + t = st.text + while True: + st.skip_ws(flags) + if st.i >= len(t): + break + c = t[st.i] + if c == "|" or c == ")": + break + start = st.i + atom = _parse_atom(st, flags, depth) + if atom is None: + continue + st.skip_ws(flags) + atom = _parse_repeat(st, flags, atom, start) + items.append(atom) + if len(items) == 1: + return items[0] + return ("seq", items) + + +def _parse_repeat(st, flags, atom, start): + t = st.text + repeated = False + while st.i < len(t) and t[st.i] in _REPEAT: + c = t[st.i] + here = st.i + if c == "{": + r = _parse_braces(st) + if r is None: + break + lo, hi = r + else: + st.i += 1 + if c == "*": + lo, hi = 0, None + elif c == "+": + lo, hi = 1, None + else: + lo, hi = 0, 1 + if repeated: + raise st.err("multiple repeat", here) + if atom[0] in ("at", "empty") or atom[0] == "look": + raise st.err("nothing to repeat", here) + mode = "greedy" + if st.i < len(t) and t[st.i] == "?": + mode = "lazy" + st.i += 1 + elif st.i < len(t) and t[st.i] == "+": + mode = "possessive" + st.i += 1 + atom = ("rep", atom, lo, hi, mode) + repeated = True + st.skip_ws(flags) + return atom + + +def _parse_braces(st, probe=False): + t = st.text + j = st.i + 1 + if j < len(t) and t[j] == "}": + return None + a0 = j + while j < len(t) and t[j] in _DIGITS: + j += 1 + lo_s = t[a0:j] + hi_s = lo_s + if j < len(t) and t[j] == ",": + j += 1 + b0 = j + while j < len(t) and t[j] in _DIGITS: + j += 1 + hi_s = t[b0:j] + if j >= len(t) or t[j] != "}": + return None + if probe: + return True + lo = int(lo_s) if lo_s else 0 + hi = int(hi_s) if hi_s else None + if lo >= _MAXREPEAT or (hi is not None and hi >= _MAXREPEAT): + raise OverflowError("the repetition number is too large") + if hi is not None and hi < lo: + raise st.err("min repeat greater than max repeat", a0) + st.i = j + 1 + return lo, hi + + +def _parse_atom(st, flags, depth): + t = st.text + c = t[st.i] + if c == "(": + return _parse_group(st, flags, depth) + if c in "*+?": + raise st.err("nothing to repeat") + if c == "{" and _parse_braces(st, True): + raise st.err("nothing to repeat") + st.i += 1 + if c == ".": + return ("any", bool(flags & F_DOTALL)) + if c == "^": + return ("at", "bol" if flags & F_MULTILINE else "begin") + if c == "$": + return ("at", "eolm" if flags & F_MULTILINE else "eol") + if c == "[": + return _parse_class(st, flags) + if c == "\\": + return _parse_escape(st, flags, False) + return ("lit", c, bool(flags & F_IGNORECASE)) + + +def _parse_group(st, flags, depth): + t = st.text + start = st.i + st.i += 1 + name = None + capture = True + local_flags = flags + if t.startswith("?", st.i): + st.i += 1 + if st.i >= len(t): + raise st.err("unexpected end of pattern") + c = t[st.i] + if c == ":": + st.i += 1 + capture = False + elif c == "P": + st.i += 1 + if t.startswith("<", st.i): + st.i += 1 + end = t.find(">", st.i) + if end < 0: + raise st.err("missing >, unterminated name") + name = t[st.i:end] + if not name: + raise st.err("missing group name") + if not name.isidentifier(): + raise st.err("bad character in group name %r" % name) + if name in st.groupindex: + raise st.err("redefinition of group name %r as group %d; was group %d" + % (name, st.groups + 1, st.groupindex[name])) + st.i = end + 1 + elif t.startswith("=", st.i): + st.i += 1 + end = t.find(")", st.i) + if end < 0: + raise st.err("missing ), unterminated name") + ref = t[st.i:end] + if not ref: + raise st.err("missing group name") + if ref not in st.groupindex: + raise st.err("unknown group name %r" % ref) + idx = st.groupindex[ref] + if idx in st.open_groups: + raise st.err("cannot refer to an open group", start) + st.i = end + 1 + return ("backref", idx, bool(flags & F_IGNORECASE)) + else: + raise st.err("unknown extension ?P" + (t[st.i] if st.i < len(t) else ""), start + 1) + elif c == "#": + end = t.find(")", st.i) + if end < 0: + raise st.err("missing ), unterminated comment", start) + st.i = end + 1 + return None + elif c in "=!": + st.i += 1 + body = _parse_alt(st, flags, depth + 1) + _close(st, start) + return ("look", body, False, c == "!", 0) + elif c == "<": + st.i += 1 + if st.i < len(t) and t[st.i] in "=!": + neg = t[st.i] == "!" + st.i += 1 + body = _parse_alt(st, flags, depth + 1) + _close(st, start) + w = _width(body) + if w[0] != w[1]: + raise error("look-behind requires fixed-width pattern") + return ("look", body, True, neg, w[0]) + raise st.err("unknown extension ?<" + (t[st.i] if st.i < len(t) else ""), start + 1) + elif c == ">": + st.i += 1 + body = _parse_alt(st, flags, depth + 1) + _close(st, start) + return ("atomic", body) + elif c == "(": + st.i += 1 + end = t.find(")", st.i) + if end < 0: + raise st.err("missing ), unterminated name") + ref = t[st.i:end] + if ref.isdigit(): + idx = int(ref) + if idx == 0: + raise st.err("bad group number", st.i) + elif ref in st.groupindex: + idx = st.groupindex[ref] + elif ref.isidentifier(): + raise st.err("unknown group name %r" % ref, st.i) + else: + raise st.err("bad character in group name %r" % ref, st.i) + st.i = end + 1 + yes = _parse_seq(st, flags, depth + 1) + no = ("empty",) + if st.peek() == "|": + st.i += 1 + no = _parse_seq(st, flags, depth + 1) + if st.peek() == "|": + raise st.err("conditional backref with more than two branches") + _close(st, start) + if idx > st.groups: + raise st.err("invalid group reference %d" % idx, start + 3) + return ("cond", idx, yes, no) + elif c in _FLAG_LETTERS or c == "-": + on = 0 + off = 0 + while st.i < len(t) and t[st.i] in _FLAG_LETTERS: + on |= _FLAG_LETTERS[t[st.i]] + st.i += 1 + if st.i < len(t) and t[st.i] == "-": + st.i += 1 + while st.i < len(t) and t[st.i] in _FLAG_LETTERS: + off |= _FLAG_LETTERS[t[st.i]] + st.i += 1 + if st.i >= len(t): + raise st.err("missing -, : or )") + if t[st.i] == ")": + raise st.err("global flags not at the start of the expression", start) + if t[st.i] != ":": + raise st.err("unknown flag" if t[st.i].isalpha() else "missing -, : or )") + st.i += 1 + capture = False + local_flags = (flags | on) & ~off + else: + raise st.err("unknown extension ?" + c, start + 1) + idx = None + if capture: + st.groups += 1 + idx = st.groups + if name is not None: + st.groupindex[name] = idx + st.open_groups.add(idx) + body = _parse_alt(st, local_flags, depth + 1) + _close(st, start) + if idx is not None: + st.open_groups.discard(idx) + return ("group", idx, body) + return body if body is not None else ("empty",) + + +def _close(st, start): + if st.peek() != ")": + raise st.err("missing ), unterminated subpattern", start) + st.i += 1 + + +def _parse_escape(st, flags, in_class): + t = st.text + if st.i >= len(t): + raise st.err("bad escape (end of pattern)", st.i - 1) + start = st.i - 1 + c = t[st.i] + st.i += 1 + icase = bool(flags & F_IGNORECASE) + if c in _SIMPLE_ESC and not (c == "b" and not in_class): + return ("lit", _SIMPLE_ESC[c], icase) + if c in _CATS: + return ("cat", c) + if not in_class: + if c == "A": + return ("at", "begin") + if c == "Z" or c == "z": + return ("at", "end") + if c == "b": + return ("at", "boundary") + if c == "B": + return ("at", "nonboundary") + if c == "x": + h = t[st.i:st.i + 2] + if len(h) < 2 or h[0] not in _HEXD or h[1] not in _HEXD: + raise st.err("incomplete escape \\x" + h, start) + st.i += 2 + return ("lit", chr(int(h, 16)), icase) + if c in "uU" and st.is_str: + n = 4 if c == "u" else 8 + h = t[st.i:st.i + n] + if len(h) < n or any(ch not in _HEXD for ch in h): + raise st.err("incomplete escape \\" + c + h, start) + st.i += n + v = int(h, 16) + if v > 0x10FFFF: + raise st.err("bad escape \\" + c + h, start) + return ("lit", chr(v), icase) + if c == "0": + v = 0 + n = 0 + while n < 2 and st.i < len(t) and t[st.i] in _OCT: + v = v * 8 + int(t[st.i]) + st.i += 1 + n += 1 + return ("lit", chr(v), icase) + if c in _DIGITS: + j = st.i + if c in _OCT and j + 1 < len(t) and t[j] in _OCT and t[j + 1] in _OCT: + v = int(c + t[j:j + 2], 8) + if v > 0o377: + raise st.err("octal escape value \\%s outside of range 0-0o377" % (c + t[j:j + 2]), start) + st.i = j + 2 + return ("lit", chr(v), icase) + if in_class: + raise st.err("bad escape \\" + c, start) + digits = c + if j < len(t) and t[j] in _DIGITS: + digits += t[j] + st.i += 1 + idx = int(digits) + if idx > st.groups: + raise st.err("invalid group reference %d" % idx, start + 1) + if idx in st.open_groups: + raise st.err("cannot refer to an open group", start) + return ("backref", idx, icase) + if c == "N" and st.is_str: + raise st.err("bad escape \\N", start) + if c.isascii() and c.isalnum(): + raise st.err("bad escape \\" + c, start) + return ("lit", c, icase) + + +def _parse_class(st, flags): + t = st.text + start = st.i - 1 + neg = False + if st.i < len(t) and t[st.i] == "^": + neg = True + st.i += 1 + items = [] + first = True + while True: + if st.i >= len(t): + raise st.err("unterminated character set", start) + c = t[st.i] + if c == "]" and not first: + st.i += 1 + break + first = False + here = st.i + if c == "\\": + st.i += 1 + lo = _parse_escape(st, flags, True) + else: + st.i += 1 + lo = ("lit", c, False) + if lo[0] == "lit" and st.i + 1 < len(t) and t[st.i] == "-" and t[st.i + 1] != "]": + st.i += 1 + if t[st.i] == "\\": + st.i += 1 + hi = _parse_escape(st, flags, True) + else: + hi = ("lit", t[st.i], False) + st.i += 1 + if hi[0] != "lit": + raise st.err("bad character range %s" % t[here:st.i], here) + if ord(hi[1]) < ord(lo[1]): + raise st.err("bad character range %s" % t[here:st.i], here) + items.append(("range", lo[1], hi[1])) + elif lo[0] == "lit": + items.append(("ch", lo[1])) + else: + items.append(lo) + return ("set", items, neg, bool(flags & F_IGNORECASE)) + + +def _width(node): + """(min, max) characters a node can match; max is None when unbounded.""" + k = node[0] + if k in ("lit", "any", "set", "cat"): + return (1, 1) + if k in ("at", "look", "empty"): + return (0, 0) + if k == "group": + return _width(node[2]) + if k == "atomic": + return _width(node[1]) + if k == "seq": + lo = 0 + hi = 0 + for n in node[1]: + a, b = _width(n) + lo += a + hi = None if hi is None or b is None else hi + b + return (lo, hi) + if k == "alt": + ws = [_width(n) for n in node[1]] + lo = min(w[0] for w in ws) + hi = None if any(w[1] is None for w in ws) else max(w[1] for w in ws) + return (lo, hi) + if k == "rep": + a, b = _width(node[1]) + hi = None if node[3] is None or b is None else b * node[3] + return (a * node[2], hi) + if k == "cond": + a = _width(node[2]) + b = _width(node[3]) + return (min(a[0], b[0]), None if a[1] is None or b[1] is None else max(a[1], b[1])) + return (0, None) + + +def _prefix(node): + """The literal text every match starts with, for a quick scan.""" + out = [] + items = node[1] if node[0] == "seq" else [node] + for n in items: + if n[0] == "lit" and not n[2]: + out.append(n[1]) + elif n[0] == "group" and n[2][0] == "lit" and not n[2][2]: + out.append(n[2][1]) + break + else: + break + return "".join(out) + + +def _anchored(node): + first = node[1][0] if node[0] == "seq" and node[1] else node + return first[0] == "at" and first[1] == "begin" + + +class _Compiler: + def __init__(self, prog, unicode): + self.prog = prog + self.unicode = unicode + self.marks = 0 + + def emit(self, node): + p = self.prog + k = node[0] + if k == "lit": + if node[2]: + p.append((E.CHARI, node[1].lower(), node[1].upper())) + else: + p.append((E.CHAR, node[1])) + elif k == "any": + p.append((E.ANYALL,) if node[1] else (E.ANY,)) + elif k == "set": + p.append((E.SET,) + _compile_set(node[1], node[2], node[3], self.unicode)) + elif k == "cat": + p.append((E.SET,) + _compile_set([node], False, False, self.unicode)) + elif k == "at": + p.append((E.AT, node[1], self.unicode)) + elif k == "empty": + pass + elif k == "seq": + for n in node[1]: + self.emit(n) + elif k == "group": + p.append((E.SAVE, 2 * node[1])) + self.emit(node[2]) + p.append((E.SAVE, 2 * node[1] + 1)) + elif k == "alt": + jumps = [] + branches = node[1] + for i in range(len(branches)): + if i < len(branches) - 1: + split = len(p) + p.append(None) + self.emit(branches[i]) + jumps.append(len(p)) + p.append(None) + p[split] = (E.SPLIT, split + 1, len(p)) + else: + self.emit(branches[i]) + for j in jumps: + p[j] = (E.JMP, len(p)) + elif k == "backref": + p.append((E.BACKREF, node[1], node[2])) + elif k == "look": + sub = self._sub(node[1]) + p.append((E.LOOK, sub, node[2], node[3], node[4])) + elif k == "atomic": + p.append((E.ATOMIC, self._sub(node[1]))) + elif k == "cond": + at = len(p) + p.append(None) + self.emit(node[2]) + jmp = len(p) + p.append(None) + p[at] = (E.GROUPREF_EXISTS, node[1], len(p)) + self.emit(node[3]) + p[jmp] = (E.JMP, len(p)) + elif k == "rep": + self._rep(node) + else: + raise error("internal error: unknown node " + k) + + def _sub(self, node): + prog = [] + c = _Compiler(prog, self.unicode) + c.marks = self.marks + c.emit(node) + prog.append((E.MATCH,)) + self.marks = c.marks + return prog + + def _rep(self, node): + body, lo, hi, mode = node[1], node[2], node[3], node[4] + if mode == "possessive": + self.prog.append((E.ATOMIC, self._sub(("rep", body, lo, hi, "greedy")))) + return + p = self.prog + greedy = mode == "greedy" + single = body[0] in ("lit", "any", "set", "cat") + if single and greedy and (hi is None or hi - lo > 1): + # One character at a time: a counted loop with no per-step + # choice point, backing off one character on failure. + p.append((E.REPEAT_ONE, self._sub(body)[0], lo, hi)) + return + if single and not greedy and (hi is None or hi - lo > 1): + p.append((E.MIN_REPEAT_ONE, self._sub(body)[0], lo, hi)) + return + for _ in range(lo): + self.emit(body) + can_be_empty = _width(body)[0] == 0 + if hi is None: + loop = len(p) + p.append(None) + mark = None + if can_be_empty: + mark = self.marks + self.marks += 1 + p.append((E.MARK, mark)) + self.emit(body) + progress = None + if mark is not None: + progress = len(p) + p.append(None) + p.append((E.JMP, loop)) + out = len(p) + if progress is not None: + # An iteration that matched nothing ends the loop. + p[progress] = (E.PROGRESS, mark, out) + p[loop] = (E.SPLIT, loop + 1, out) if greedy else (E.SPLIT, out, loop + 1) + else: + holes = [] + for _ in range(hi - lo): + at = len(p) + p.append(None) + holes.append(at) + self.emit(body) + out = len(p) + for at in holes: + p[at] = (E.SPLIT, at + 1, out) if greedy else (E.SPLIT, out, at + 1) + + +def _compile_set(items, neg, icase, unicode): + chars = set() + ranges = [] + cats = [] + for it in items: + if it[0] == "ch": + chars.add(it[1]) + if icase: + chars.add(it[1].lower()) + chars.add(it[1].upper()) + elif it[0] == "range": + ranges.append((ord(it[1]), ord(it[2]))) + elif it[0] == "cat": + cats.append(it[1]) + elif it[0] == "lit": + chars.add(it[1]) + if icase: + chars.add(it[1].lower()) + chars.add(it[1].upper()) + return (frozenset(chars), tuple(ranges), "".join(cats), neg, icase and bool(ranges), unicode) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/secrets.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/secrets.py new file mode 100644 index 00000000000..0c61ee1c5b5 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/secrets.py @@ -0,0 +1,58 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Random numbers suitable for secrets: tokens, passwords, keys.""" + +import base64 +import random + +__all__ = ["choice", "randbelow", "randbits", "SystemRandom", "token_bytes", "token_hex", + "token_urlsafe", "compare_digest"] + +SystemRandom = random.SystemRandom +_sysrand = SystemRandom() +randbits = _sysrand.getrandbits +choice = _sysrand.choice + +DEFAULT_ENTROPY = 32 + + +def randbelow(exclusive_upper_bound): + if exclusive_upper_bound <= 0: + raise ValueError("Upper bound must be positive.") + return _sysrand._randbelow(exclusive_upper_bound) + + +def token_bytes(nbytes=None): + if nbytes is None: + nbytes = DEFAULT_ENTROPY + return _sysrand.randbytes(nbytes) + + +def token_hex(nbytes=None): + return token_bytes(nbytes).hex() + + +def token_urlsafe(nbytes=None): + tok = token_bytes(nbytes) + return base64.urlsafe_b64encode(tok).rstrip(b"=").decode("ascii") + + +def compare_digest(a, b): + if isinstance(a, str) and isinstance(b, str): + try: + a = a.encode("ascii") + b = b.encode("ascii") + except UnicodeEncodeError: + raise TypeError("comparing strings with non-ASCII characters is not supported") + elif not (isinstance(a, (bytes, bytearray, memoryview)) and isinstance(b, (bytes, bytearray, memoryview))): + raise TypeError("unsupported operand types(s) or combination of types: '%s' and '%s'" + % (type(a).__name__, type(b).__name__)) + a = bytes(a) + b = bytes(b) + result = len(a) ^ len(b) + if len(a) != len(b): + b = a + for x, y in zip(a, b): + result |= x ^ y + return result == 0 diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/shlex.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/shlex.py new file mode 100644 index 00000000000..5e165e1ad9b --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/shlex.py @@ -0,0 +1,229 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Lexical analysis of shell-style syntax.""" + +import re +from io import StringIO + +__all__ = ["shlex", "split", "quote", "join"] + + +class shlex: + def __init__(self, instream=None, infile=None, posix=False, punctuation_chars=False): + if isinstance(instream, str): + instream = StringIO(instream) + if instream is not None: + self.instream = instream + self.infile = infile + else: + import sys + self.instream = sys.stdin + self.infile = None + self.posix = posix + self.eof = None if posix else "" + self.commenters = "#" + self.wordchars = "abcdfeghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_" + if self.posix: + self.wordchars += ("ßàáâãäåæçèéêë" + "ìíîïðñòóôõöøù" + "úûüýþÿÀÁÂÃÄÅÆ" + "ÇÈÉÊËÌÍÎÏÐÑÒÓ" + "ÔÕÖØÙÚÛÜÝÞ") + self.whitespace = " \t\r\n" + self.whitespace_split = False + self.quotes = "'\"" + self.escape = "\\" + self.escapedquotes = '"' + self.state = " " + self.pushback = [] + self.lineno = 1 + self.debug = 0 + self.token = "" + self.filestack = [] + self.source = None + if not punctuation_chars: + punctuation_chars = "" + elif punctuation_chars is True: + punctuation_chars = "();<>|&" + self._punctuation_chars = punctuation_chars + if punctuation_chars: + self._pushback_chars = [] + self.wordchars += "~-./*?=" + t = self.wordchars.maketrans(dict.fromkeys(punctuation_chars)) + self.wordchars = self.wordchars.translate(t) + + @property + def punctuation_chars(self): + return self._punctuation_chars + + def push_token(self, tok): + self.pushback.insert(0, tok) + + def get_token(self): + if self.pushback: + return self.pushback.pop(0) + raw = self.read_token() + while raw == self.eof: + if not self.filestack: + return self.eof + self.instream, self.infile, self.lineno = self.filestack.pop(0) + raw = self.get_token() + return raw + + def read_token(self): + quoted = False + escapedstate = " " + while True: + if self.punctuation_chars and self._pushback_chars: + nextchar = self._pushback_chars.pop() + else: + nextchar = self.instream.read(1) + if nextchar == "\n": + self.lineno += 1 + if self.state is None: + self.token = "" + break + elif self.state == " ": + if not nextchar: + self.state = None + break + elif nextchar in self.whitespace: + if self.token or (self.posix and quoted): + break + else: + continue + elif nextchar in self.commenters: + self.instream.readline() + self.lineno += 1 + elif self.posix and nextchar in self.escape: + escapedstate = "a" + self.state = nextchar + elif nextchar in self.wordchars: + self.token = nextchar + self.state = "a" + elif nextchar in self.punctuation_chars: + self.token = nextchar + self.state = "c" + elif nextchar in self.quotes: + if not self.posix: + self.token = nextchar + self.state = nextchar + elif self.whitespace_split: + self.token = nextchar + self.state = "a" + else: + self.token = nextchar + if self.token or (self.posix and quoted): + break + else: + continue + elif self.state in self.quotes: + quoted = True + if not nextchar: + raise ValueError("No closing quotation") + if nextchar == self.state: + if not self.posix: + self.token += nextchar + self.state = " " + break + else: + self.state = "a" + elif (self.posix and nextchar in self.escape and self.state in self.escapedquotes): + escapedstate = self.state + self.state = nextchar + else: + self.token += nextchar + elif self.state in self.escape: + if not nextchar: + raise ValueError("No escaped character") + if (escapedstate in self.quotes and nextchar != self.state and nextchar != escapedstate): + self.token += self.state + self.token += nextchar + self.state = escapedstate + elif self.state in ("a", "c"): + if not nextchar: + self.state = None + break + elif nextchar in self.whitespace: + self.state = " " + if self.token or (self.posix and quoted): + break + else: + continue + elif nextchar in self.commenters: + self.instream.readline() + self.lineno += 1 + if self.posix: + self.state = " " + if self.token or (self.posix and quoted): + break + else: + continue + elif self.state == "c": + if nextchar in self.punctuation_chars: + self.token += nextchar + else: + if nextchar not in self.whitespace: + self._pushback_chars.append(nextchar) + self.state = " " + break + elif self.posix and nextchar in self.quotes: + self.state = nextchar + elif self.posix and nextchar in self.escape: + escapedstate = "a" + self.state = nextchar + elif (nextchar in self.wordchars or nextchar in self.quotes + or (self.whitespace_split and nextchar not in self.punctuation_chars)): + self.token += nextchar + else: + if self.punctuation_chars: + self._pushback_chars.append(nextchar) + else: + self.pushback.insert(0, nextchar) + self.state = " " + if self.token or (self.posix and quoted): + break + else: + continue + result = self.token + self.token = "" + if self.posix and not quoted and result == "": + result = None + return result + + def __iter__(self): + return self + + def __next__(self): + token = self.get_token() + if token == self.eof: + raise StopIteration + return token + + +def split(s, comments=False, posix=True): + if s is None: + raise ValueError("s argument must not be None") + lex = shlex(s, posix=posix) + lex.whitespace_split = True + if not comments: + lex.commenters = "" + return list(lex) + + +def join(split_command): + return " ".join(quote(arg) for arg in split_command) + + +_find_unsafe = re.compile(r"[^\w@%+=:,./-]", re.ASCII).search + + +def quote(s): + if not s: + return "''" + if not isinstance(s, str): + raise TypeError("expected string object, got %r" % type(s).__name__) + if _find_unsafe(s) is None: + return s + return "'" + s.replace("'", "'\"'\"'") + "'" diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/statistics.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/statistics.py new file mode 100644 index 00000000000..546f542ecd8 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/statistics.py @@ -0,0 +1,651 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Basic statistics: averages, spread and relations of numeric data. + +Sums are exact (over fractions) and converted once at the end, as +CPython's statistics module does, so results match it to the last bit. +""" + +import math +import numbers +from fractions import Fraction +from collections import Counter, namedtuple + +__all__ = ["NormalDist", "StatisticsError", "correlation", "covariance", "fmean", "geometric_mean", + "harmonic_mean", "linear_regression", "mean", "median", "median_grouped", "median_high", + "median_low", "mode", "multimode", "pstdev", "pvariance", "quantiles", "stdev", "variance"] + + +class StatisticsError(ValueError): + pass + + +def _exact_ratio(x): + try: + return x.as_integer_ratio() + except AttributeError: + pass + except (OverflowError, ValueError): + return (x, None) + try: + return (x.numerator, x.denominator) + except AttributeError: + raise TypeError("can't convert type '%s' to numerator/denominator" % type(x).__name__) + + +def _coerce(T, S): + if T is S: + return T + if S is int or S is bool: + return T + if T is int: + return S + if issubclass(S, T): + return S + if issubclass(T, S): + return T + if issubclass(T, int): + return S + if issubclass(S, int): + return T + if issubclass(T, Fraction) and issubclass(S, float): + return S + if issubclass(T, float) and issubclass(S, Fraction): + return T + raise TypeError("can't convert type '%s' to '%s'" % (T.__name__, S.__name__)) + + +def _sum(data): + count = 0 + types = set() + partials = {} + T = int + for typ, values in _groupby_type(data): + types.add(typ) + for n, d in map(_exact_ratio, values): + count += 1 + partials[d] = partials.get(d, 0) + n + if None in partials: + total = partials[None] + assert not _isfinite(total) + else: + total = sum(Fraction(n, d) for d, n in partials.items()) + for typ in types: + T = _coerce(T, typ) + return (T, total, count) + + +def _groupby_type(data): + out = [] + cur = None + group = [] + for x in data: + t = type(x) + if t is not cur: + if group: + out.append((cur, group)) + cur = t + group = [] + group.append(x) + if group: + out.append((cur, group)) + return out + + +def _isfinite(x): + try: + return x.is_finite() + except AttributeError: + return math.isfinite(x) + + +def _ss(data, c=None): + if c is not None: + T, ssd, count = _sum((d := x - c) * d for x in data) + return (T, ssd, c, count) + count = 0 + types = set() + sx_partials = {} + sxx_partials = {} + for x in data: + types.add(type(x)) + n, d = _exact_ratio(x) + count += 1 + sx_partials[d] = sx_partials.get(d, 0) + n + sxx_partials[d * d] = sxx_partials.get(d * d, 0) + n * n + if not count: + ssd = c = Fraction(0) + elif None in sx_partials: + ssd = c = sx_partials[None] + else: + sx = sum(Fraction(n, d) for d, n in sx_partials.items()) + sxx = sum(Fraction(n, d) for d, n in sxx_partials.items()) + ssd = (count * sxx - sx * sx) / count + c = sx / count + T = int + for typ in types: + T = _coerce(T, typ) + return (T, ssd, c, count) + + +def _convert(value, T): + if type(value) is T: + return value + if issubclass(T, int) and value.denominator != 1: + T = float + try: + return T(value) + except TypeError: + if issubclass(T, Decimal_type()): + return T(value.numerator) / T(value.denominator) + raise + + +def Decimal_type(): + try: + from decimal import Decimal + return Decimal + except ImportError: + return type(None) + + +def _fail_neg(values, errmsg="negative value"): + for x in values: + if x < 0: + raise StatisticsError(errmsg) + yield x + + +def _integer_sqrt_of_frac_rto(n, m): + a = math.isqrt(n // m) + return a | (a * a * m != n) + + +_sqrt_bit_width = 2 * 53 + 3 + + +def _float_sqrt_of_frac(n, m): + q = (n.bit_length() - m.bit_length() - _sqrt_bit_width) // 2 + if q >= 0: + numerator = _integer_sqrt_of_frac_rto(n, m << 2 * q) << q + denominator = 1 + else: + numerator = _integer_sqrt_of_frac_rto(n << -2 * q, m) + denominator = 1 << -q + return numerator / denominator + + +def mean(data): + T, total, n = _sum(data) + if n < 1: + raise StatisticsError("mean requires at least one data point") + return _convert(total / n, T) + + +def fmean(data, weights=None): + if weights is None: + n = 0 + total = 0.0 + vals = [] + for x in data: + n += 1 + vals.append(x) + if not n: + raise StatisticsError("fmean requires at least one data point") + return math.fsum(vals) / n + data = list(data) + weights = list(weights) + if len(weights) != len(data): + raise StatisticsError("data and weights must be the same length") + num = math.fsum(x * w for x, w in zip(data, weights)) + den = math.fsum(weights) + if not den: + raise StatisticsError("sum of weights must be non-zero") + return num / den + + +def geometric_mean(data): + n = 0 + found_zero = False + logs = [] + for x in data: + n += 1 + if x > 0.0: + logs.append(math.log(x)) + elif x == 0.0: + found_zero = True + else: + raise StatisticsError("No negative inputs allowed", x) + if not n: + raise StatisticsError("Must have a non-empty dataset") + if found_zero: + return 0.0 + return math.exp(math.fsum(logs) / n) + + +def harmonic_mean(data, weights=None): + data = list(data) + errmsg = "harmonic mean does not support negative values" + n = len(data) + if n < 1: + raise StatisticsError("harmonic_mean requires at least one data point") + elif n == 1 and weights is None: + x = data[0] + if isinstance(x, (numbers.Real, Decimal_type())): + if x < 0: + raise StatisticsError(errmsg) + return x + raise TypeError("unsupported type") + if weights is None: + weights = [1] * n + sum_weights = n + else: + weights = list(weights) + if len(weights) != n: + raise StatisticsError("Number of weights does not match data size") + _, sum_weights, _ = _sum(w for w in _fail_neg(weights, errmsg)) + try: + data = list(_fail_neg(data, errmsg)) + T, total, count = _sum(w / x if w else 0 for w, x in zip(weights, data)) + except ZeroDivisionError: + return 0 + if total <= 0: + raise StatisticsError("Weighted sum must be positive") + return _convert(sum_weights / total, T) + + +def median(data): + data = sorted(data) + n = len(data) + if n == 0: + raise StatisticsError("no median for empty data") + if n % 2 == 1: + return data[n // 2] + i = n // 2 + return (data[i - 1] + data[i]) / 2 + + +def median_low(data): + data = sorted(data) + n = len(data) + if n == 0: + raise StatisticsError("no median for empty data") + if n % 2 == 1: + return data[n // 2] + return data[n // 2 - 1] + + +def median_high(data): + data = sorted(data) + n = len(data) + if n == 0: + raise StatisticsError("no median for empty data") + return data[n // 2] + + +def median_grouped(data, interval=1.0): + data = sorted(data) + n = len(data) + if not n: + raise StatisticsError("no median for empty data") + x = data[n // 2] + i = j = n // 2 + while i > 0 and data[i - 1] == x: + i -= 1 + while j < n and data[j] == x: + j += 1 + try: + interval = float(interval) + x = float(x) + except ValueError: + raise TypeError("Value cannot be converted to a float") + L = x - interval / 2.0 + cf = i + f = j - i + return L + interval * (n / 2 - cf) / f + + +def mode(data): + pairs = Counter(iter(data)).most_common(1) + try: + return pairs[0][0] + except IndexError: + raise StatisticsError("no mode for empty data") + + +def multimode(data): + counts = Counter(iter(data)) + if not counts: + return [] + maxcount = max(counts.values()) + return [value for value, count in counts.items() if count == maxcount] + + +def quantiles(data, *, n=4, method="exclusive"): + if n < 1: + raise StatisticsError("n must be at least 1") + data = sorted(data) + ld = len(data) + if ld < 2: + if ld == 1: + return data * (n - 1) + raise StatisticsError("must have at least one data point") + if method == "inclusive": + m = ld - 1 + result = [] + for i in range(1, n): + j, delta = divmod(i * m, n) + interpolated = (data[j] * (n - delta) + data[j + 1] * delta) / n + result.append(interpolated) + return result + if method == "exclusive": + m = ld + 1 + result = [] + for i in range(1, n): + j = i * m // n + j = 1 if j < 1 else ld - 1 if j > ld - 1 else j + delta = i * m - j * n + interpolated = (data[j - 1] * (n - delta) + data[j] * delta) / n + result.append(interpolated) + return result + raise ValueError("Unknown method: %r" % (method,)) + + +def variance(data, xbar=None): + T, ss, c, n = _ss(data, xbar) + if n < 2: + raise StatisticsError("variance requires at least two data points") + return _convert(ss / (n - 1), T) + + +def pvariance(data, mu=None): + T, ss, c, n = _ss(data, mu) + if n < 1: + raise StatisticsError("pvariance requires at least one data point") + return _convert(ss / n, T) + + +def stdev(data, xbar=None): + T, ss, c, n = _ss(data, xbar) + if n < 2: + raise StatisticsError("stdev requires at least two data points") + mss = ss / (n - 1) + if issubclass(T, Decimal_type()): + return _convert(mss, T).sqrt() + return _float_sqrt_of_frac(mss.numerator, mss.denominator) + + +def pstdev(data, mu=None): + T, ss, c, n = _ss(data, mu) + if n < 1: + raise StatisticsError("pstdev requires at least one data point") + mss = ss / n + if issubclass(T, Decimal_type()): + return _convert(mss, T).sqrt() + return _float_sqrt_of_frac(mss.numerator, mss.denominator) + + +def _sqrtprod(x, y): + h = math.sqrt(x * y) + if not math.isfinite(h): + if math.isinf(h) and not math.isinf(x) and not math.isinf(y): + scale = 2.0 ** -512 + return _sqrtprod(scale * x, scale * y) / scale + return h + if not h: + if x and y: + scale = 2.0 ** 537 + return _sqrtprod(scale * x, scale * y) / scale + return h + d = Fraction(*h.as_integer_ratio()) ** 2 - Fraction(*x.as_integer_ratio()) * Fraction(*y.as_integer_ratio()) + return h - float(d) / (2.0 * h) + + +def covariance(x, y, /): + n = len(x) + if len(y) != n: + raise StatisticsError("covariance requires that both inputs have same number of data points") + if n < 2: + raise StatisticsError("covariance requires at least two data points") + xbar = fmean(x) + ybar = fmean(y) + sxy = math.fsum((xi - xbar) * (yi - ybar) for xi, yi in zip(x, y)) + return sxy / (n - 1) + + +def _ranks(data): + order = sorted(range(len(data)), key=lambda i: data[i]) + result = [0.0] * len(data) + i = 0 + while i < len(order): + j = i + while j + 1 < len(order) and data[order[j + 1]] == data[order[i]]: + j += 1 + rank = (i + j) / 2 + 1 + for k in range(i, j + 1): + result[order[k]] = rank + i = j + 1 + return result + + +def correlation(x, y, /, *, method="linear"): + n = len(x) + if len(y) != n: + raise StatisticsError("correlation requires that both inputs have same number of data points") + if n < 2: + raise StatisticsError("correlation requires at least two data points") + if method not in {"linear", "ranked"}: + raise ValueError("Unknown method: %r" % (method,)) + if method == "ranked": + start = (n - 1) / -2 + x = [r + start - 1 for r in _ranks(x)] + y = [r + start - 1 for r in _ranks(y)] + else: + xbar = fmean(x) + ybar = fmean(y) + x = [xi - xbar for xi in x] + y = [yi - ybar for yi in y] + sxy = math.sumprod(x, y) + sxx = math.sumprod(x, x) + syy = math.sumprod(y, y) + try: + return sxy / _sqrtprod(sxx, syy) + except ZeroDivisionError: + raise StatisticsError("at least one of the inputs is constant") + + +LinearRegression = namedtuple("LinearRegression", ("slope", "intercept")) + + +def linear_regression(x, y, /, *, proportional=False): + n = len(x) + if len(y) != n: + raise StatisticsError("linear regression requires that both inputs have same number of data points") + if n < 2: + raise StatisticsError("linear regression requires at least two data points") + if not proportional: + xbar = fmean(x) + ybar = fmean(y) + x = [xi - xbar for xi in x] + y = [yi - ybar for yi in y] + sxy = math.sumprod(x, y) + 0.0 + sxx = math.sumprod(x, x) + try: + slope = sxy / sxx + except ZeroDivisionError: + raise StatisticsError("x is constant") + intercept = 0.0 if proportional else ybar - slope * xbar + return LinearRegression(slope=slope, intercept=intercept) + + +def _normal_dist_inv_cdf(p, mu, sigma): + q = p - 0.5 + if math.fabs(q) <= 0.425: + r = 0.180625 - q * q + num = (((((((2.5090809287301226727e+3 * r + 3.3430575583588128105e+4) * r + 6.7265770927008700853e+4) * r + + 4.5921953931549871457e+4) * r + 1.3731693765509461125e+4) * r + 1.9715909503065514427e+3) * r + + 1.3314166789178437745e+2) * r + 3.3871328727963666080e+0) * q + den = (((((((5.2264952788528545610e+3 * r + 2.8729085735721942674e+4) * r + 3.9307895800092710610e+4) * r + + 2.1213794301586595867e+4) * r + 5.3941960214247511077e+3) * r + 6.8718700749205790830e+2) * r + + 4.2313330701600911252e+1) * r + 1.0) + x = num / den + return mu + (x * sigma) + r = p if q <= 0.0 else 1.0 - p + r = math.sqrt(-math.log(r)) + if r <= 5.0: + r = r - 1.6 + num = (((((((7.74545014278341407640e-4 * r + 2.27238449892691845833e-2) * r + 2.41780725177450611770e-1) * r + + 1.27045825245236838258e+0) * r + 3.64784832476320460504e+0) * r + 5.76949722146069140550e+0) * r + + 4.63033784615654529590e+0) * r + 1.42343711074968357734e+0) + den = (((((((1.05075007164441684324e-9 * r + 5.47593808499534494600e-4) * r + 1.51986665636164571966e-2) * r + + 1.48103976427480074590e-1) * r + 6.89767334985100004550e-1) * r + 1.67638483018380384940e+0) * r + + 2.05319162663775882187e+0) * r + 1.0) + else: + r = r - 5.0 + num = (((((((2.01033439929228813265e-7 * r + 2.71155556874348757815e-5) * r + 1.24266094738807843860e-3) * r + + 2.65321895265761230930e-2) * r + 2.96560571828504891230e-1) * r + 1.78482653991729133580e+0) * r + + 5.46378491116411436990e+0) * r + 6.65790464350110377720e+0) + den = (((((((2.04426310338993978564e-15 * r + 1.42151175831644588870e-7) * r + 1.84631831751005468180e-5) * r + + 7.86869131145613259100e-4) * r + 1.48753612908506148525e-2) * r + 1.36929880922735805310e-1) * r + + 5.99832206555887937690e-1) * r + 1.0) + x = num / den + if q < 0.0: + x = -x + return mu + (x * sigma) + + +class NormalDist: + __slots__ = {"_mu": "Arithmetic mean of a normal distribution", + "_sigma": "Standard deviation of a normal distribution"} + + def __init__(self, mu=0.0, sigma=1.0): + if sigma < 0.0: + raise StatisticsError("sigma must be non-negative") + self._mu = float(mu) + self._sigma = float(sigma) + + @classmethod + def from_samples(cls, data): + return cls(*_mean_stdev(data)) + + def samples(self, n, *, seed=None): + import random + rnd = random.random if seed is None else random.Random(seed).random + mu, sigma = self._mu, self._sigma + return [_normal_dist_inv_cdf(rnd(), mu, sigma) for i in range(n)] + + def pdf(self, x): + variance = self._sigma * self._sigma + if not variance: + raise StatisticsError("pdf() not defined when sigma is zero") + diff = x - self._mu + return math.exp(diff * diff / (-2.0 * variance)) / math.sqrt(2.0 * math.pi * variance) + + def cdf(self, x): + if not self._sigma: + raise StatisticsError("cdf() not defined when sigma is zero") + return 0.5 * math.erfc((self._mu - x) / (self._sigma * math.sqrt(2.0))) + + def inv_cdf(self, p): + if p <= 0.0 or p >= 1.0: + raise StatisticsError("p must be in the range 0.0 < p < 1.0") + return _normal_dist_inv_cdf(p, self._mu, self._sigma) + + def quantiles(self, n=4): + return [self.inv_cdf(i / n) for i in range(1, n)] + + def overlap(self, other): + if not isinstance(other, NormalDist): + raise TypeError("Expected another NormalDist instance") + X, Y = self, other + if (Y._sigma, Y._mu) < (X._sigma, X._mu): + X, Y = Y, X + X_var, Y_var = X.variance, Y.variance + if not X_var or not Y_var: + raise StatisticsError("overlap() not defined when sigma is zero") + dv = Y_var - X_var + dm = math.fabs(Y._mu - X._mu) + if not dv: + return 2.0 * NormalDist(dm, 2.0 * X._sigma).cdf(0) + a = X._mu * Y_var - Y._mu * X_var + b = X._sigma * Y._sigma * math.sqrt(dm * dm + dv * math.log(Y_var / X_var)) + x1 = (a + b) / dv + x2 = (a - b) / dv + return 1.0 - (math.fabs(Y.cdf(x1) - X.cdf(x1)) + math.fabs(Y.cdf(x2) - X.cdf(x2))) + + def zscore(self, x): + if not self._sigma: + raise StatisticsError("zscore() not defined when sigma is zero") + return (x - self._mu) / self._sigma + + @property + def mean(self): + return self._mu + + @property + def median(self): + return self._mu + + @property + def mode(self): + return self._mu + + @property + def stdev(self): + return self._sigma + + @property + def variance(self): + return self._sigma * self._sigma + + def __add__(x1, x2): + if isinstance(x2, NormalDist): + return NormalDist(x1._mu + x2._mu, math.hypot(x1._sigma, x2._sigma)) + return NormalDist(x1._mu + x2, x1._sigma) + + def __sub__(x1, x2): + if isinstance(x2, NormalDist): + return NormalDist(x1._mu - x2._mu, math.hypot(x1._sigma, x2._sigma)) + return NormalDist(x1._mu - x2, x1._sigma) + + def __mul__(x1, x2): + return NormalDist(x1._mu * x2, x1._sigma * math.fabs(x2)) + + def __truediv__(x1, x2): + return NormalDist(x1._mu / x2, x1._sigma / math.fabs(x2)) + + def __pos__(x1): + return NormalDist(x1._mu, x1._sigma) + + def __neg__(x1): + return NormalDist(-x1._mu, x1._sigma) + + __radd__ = __add__ + + def __rsub__(x1, x2): + return -(x1 - x2) + + __rmul__ = __mul__ + + def __eq__(x1, x2): + if not isinstance(x2, NormalDist): + return NotImplemented + return x1._mu == x2._mu and x1._sigma == x2._sigma + + def __hash__(self): + return hash((self._mu, self._sigma)) + + def __repr__(self): + return "%s(mu=%r, sigma=%r)" % (type(self).__name__, self._mu, self._sigma) + + +def _mean_stdev(data): + T, ss, xbar, n = _ss(data) + if n < 2: + raise StatisticsError("stdev requires at least two data points") + mss = ss / (n - 1) + try: + return float(xbar), _float_sqrt_of_frac(mss.numerator, mss.denominator) + except AttributeError: + return float(xbar), float(xbar) / float(ss) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/string.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/string.py new file mode 100644 index 00000000000..c36c9cb81d3 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/string.py @@ -0,0 +1,280 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Common string constants and the Formatter and Template classes.""" + +whitespace = " \t\n\r\x0b\x0c" +ascii_lowercase = "abcdefghijklmnopqrstuvwxyz" +ascii_uppercase = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" +ascii_letters = ascii_lowercase + ascii_uppercase +digits = "0123456789" +hexdigits = digits + "abcdef" + "ABCDEF" +octdigits = "01234567" +punctuation = "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~" +printable = digits + ascii_letters + punctuation + whitespace + + +def capwords(s, sep=None): + return (sep or " ").join(map(str.capitalize, s.split(sep))) + + +class Template: + delimiter = "$" + + def __init__(self, template): + self.template = template + + def _sub(self, mapping, safe): + t = self.template + d = self.delimiter + out = [] + i = 0 + n = len(t) + while i < n: + if not t.startswith(d, i): + out.append(t[i]) + i += 1 + continue + j = i + len(d) + if t.startswith(d, j): + out.append(d) + i = j + len(d) + continue + braced = j < n and t[j] == "{" + k = j + 1 if braced else j + e = k + while e < n and (t[e] == "_" or t[e].isalnum() and t[e].isascii()) and not (e == k and t[e].isdigit()): + e += 1 + name = t[k:e] + if not name or braced and (e >= n or t[e] != "}"): + if safe: + out.append(d) + i = j + continue + line = t.count("\n", 0, i) + 1 + col = i - (t.rfind("\n", 0, i) + 1) + 1 + raise ValueError("Invalid placeholder in string: line %d, col %d" % (line, col)) + end = e + 1 if braced else e + if name in mapping: + out.append(str(mapping[name])) + elif safe: + out.append(t[i:end]) + else: + raise KeyError(name) + i = end + return "".join(out) + + def substitute(self, mapping=None, /, **kws): + return self._sub(_merge(mapping, kws), False) + + def safe_substitute(self, mapping=None, /, **kws): + return self._sub(_merge(mapping, kws), True) + + def get_identifiers(self): + ids = [] + t = self.template + d = self.delimiter + i = t.find(d) + while i >= 0: + j = i + len(d) + if t.startswith(d, j): + i = t.find(d, j + len(d)) + continue + k = j + 1 if j < len(t) and t[j] == "{" else j + e = k + while e < len(t) and (t[e] == "_" or t[e].isalnum()): + e += 1 + name = t[k:e] + if name and name not in ids: + ids.append(name) + i = t.find(d, max(e, j)) + return ids + + def is_valid(self): + try: + self._sub({}, False) + except KeyError: + return True + except ValueError: + return False + return True + + +def _merge(mapping, kws): + if mapping is None: + return kws + if not kws: + return mapping + merged = dict(mapping) + merged.update(kws) + return merged + + +def _formatter_parser(s): + """Splits a format string as str.format reads it: (literal, field, spec, conversion) tuples.""" + i = 0 + n = len(s) + literal = [] + while i < n: + c = s[i] + if c == "{": + if i + 1 < n and s[i + 1] == "{": + literal.append("{") + i += 2 + # CPython ends a literal at each escaped brace. + yield ("".join(literal), None, None, None) + literal = [] + continue + depth = 1 + j = i + 1 + while j < n and depth: + if s[j] == "{": + depth += 1 + elif s[j] == "}": + depth -= 1 + j += 1 + if depth: + raise ValueError("expected '}' before end of string") + body = s[i + 1:j - 1] + k = 0 + bracket = False + while k < len(body): + ch = body[k] + if ch == "[": + bracket = True + elif ch == "]": + bracket = False + elif not bracket and (ch == "!" or ch == ":"): + break + k += 1 + name = body[:k] + conversion = None + spec = "" + if k < len(body) and body[k] == "!": + if k + 1 >= len(body): + raise ValueError("end of string while looking for conversion specifier") + conversion = body[k + 1] + k += 2 + if k < len(body) and body[k] != ":": + raise ValueError("expected ':' after conversion specifier") + if k < len(body) and body[k] == ":": + spec = body[k + 1:] + yield ("".join(literal), name, spec, conversion) + literal = [] + i = j + elif c == "}": + if i + 1 < n and s[i + 1] == "}": + literal.append("}") + i += 2 + # CPython ends a literal at each escaped brace. + yield ("".join(literal), None, None, None) + literal = [] + continue + raise ValueError("Single '}' encountered in format string") + else: + literal.append(c) + i += 1 + if literal: + yield ("".join(literal), None, None, None) + + +def _field_name_split(name): + i = 0 + while i < len(name) and name[i] not in ".[": + i += 1 + first = name[:i] + if first.isdigit(): + first = int(first) + rest = [] + while i < len(name): + if name[i] == ".": + j = i + 1 + while j < len(name) and name[j] not in ".[": + j += 1 + if j == i + 1: + raise ValueError("Empty attribute in format string") + rest.append((True, name[i + 1:j])) + i = j + elif name[i] == "[": + j = name.find("]", i) + if j < 0: + raise ValueError("Missing ']' in format string") + key = name[i + 1:j] + if key == "": + raise ValueError("Empty attribute in format string") + rest.append((False, int(key) if key.isdigit() else key)) + i = j + 1 + else: + raise ValueError("Only '.' or '[' may follow ']' in format field specifier") + return first, iter(rest) + + +class Formatter: + def format(self, format_string, /, *args, **kwargs): + return self.vformat(format_string, args, kwargs) + + def vformat(self, format_string, args, kwargs): + used_args = set() + result, _ = self._vformat(format_string, args, kwargs, used_args, 2) + self.check_unused_args(used_args, args, kwargs) + return result + + def _vformat(self, format_string, args, kwargs, used_args, recursion_depth, auto_arg_index=0): + if recursion_depth < 0: + raise ValueError("Max string recursion exceeded") + result = [] + for literal_text, field_name, format_spec, conversion in self.parse(format_string): + if literal_text: + result.append(literal_text) + if field_name is not None: + if field_name == "": + if auto_arg_index is False: + raise ValueError("cannot switch from manual field specification to automatic field numbering") + field_name = str(auto_arg_index) + auto_arg_index += 1 + elif field_name.isdigit(): + if auto_arg_index: + raise ValueError("cannot switch from automatic field numbering to manual field specification") + auto_arg_index = False + obj, arg_used = self.get_field(field_name, args, kwargs) + used_args.add(arg_used) + obj = self.convert_field(obj, conversion) + format_spec, auto_arg_index = self._vformat(format_spec, args, kwargs, used_args, + recursion_depth - 1, auto_arg_index=auto_arg_index) + result.append(self.format_field(obj, format_spec)) + return "".join(result), auto_arg_index + + def get_value(self, key, args, kwargs): + if isinstance(key, int): + return args[key] + return kwargs[key] + + def check_unused_args(self, used_args, args, kwargs): + pass + + def format_field(self, value, format_spec): + return format(value, format_spec) + + def convert_field(self, value, conversion): + if conversion is None: + return value + elif conversion == "s": + return str(value) + elif conversion == "r": + return repr(value) + elif conversion == "a": + return ascii(value) + raise ValueError("Unknown conversion specifier {0!s}".format(conversion)) + + def parse(self, format_string): + return _formatter_parser(format_string) + + def get_field(self, field_name, args, kwargs): + first, rest = _field_name_split(field_name) + obj = self.get_value(first, args, kwargs) + for is_attr, i in rest: + if is_attr: + obj = getattr(obj, i) + else: + obj = obj[i] + return obj, first diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/struct.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/struct.py new file mode 100644 index 00000000000..da2ddad8a23 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/struct.py @@ -0,0 +1,290 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Packing values into bytes as C structs lay them out. + +Native mode ('@', the default) follows a 64-bit little-endian device: +`long` and pointers are 8 bytes and fields are aligned to their size. +""" + +import math as _math + +__all__ = ["calcsize", "pack", "pack_into", "unpack", "unpack_from", "iter_unpack", "Struct", "error"] + + +class error(Exception): + pass + + +error.__module__ = "struct" + +_STD = {"x": 1, "c": 1, "b": 1, "B": 1, "?": 1, "h": 2, "H": 2, "i": 4, "I": 4, "l": 4, "L": 4, + "q": 8, "Q": 8, "e": 2, "f": 4, "d": 8, "s": 1, "p": 1} +_NATIVE = dict(_STD, l=8, L=8, n=8, N=8, P=8) +_SIGNED = "bhilqn" +_INTS = "bBhHiIlLqQnNP" + + +def _parse(fmt): + if isinstance(fmt, bytes): + fmt = fmt.decode("ascii") + if not isinstance(fmt, str): + raise TypeError("Struct() argument 1 must be a str or bytes object, not %s" % type(fmt).__name__) + order = "@" + i = 0 + if fmt[:1] in "@=<>!" and fmt: + order = fmt[0] + i = 1 + native = order == "@" + little = order in "<@=" + sizes = _NATIVE if native else _STD + items = [] + offset = 0 + n = len(fmt) + while i < n: + c = fmt[i] + if c.isspace(): + i += 1 + continue + count = None + while i < n and fmt[i].isdigit(): + count = (count or 0) * 10 + int(fmt[i]) + i += 1 + if i >= n: + raise error("repeat count given without format specifier") + c = fmt[i] + i += 1 + if c not in sizes: + raise error("bad char in struct format") + size = sizes[c] + if native and c not in "xcsp?bB": + offset = (offset + size - 1) // size * size + if count is None: + count = 1 + if c in "sp": + items.append((c, offset, count)) + offset += count + elif c == "x": + offset += count + else: + for _ in range(count): + items.append((c, offset, size)) + offset += size + return items, offset, little + + +def _pack_float(x, size, little): + ebits, mbits = {2: (5, 10), 4: (8, 23), 8: (11, 52)}[size] + bias = (1 << (ebits - 1)) - 1 + sign = 0 + if _math.copysign(1.0, x) < 0: + sign = 1 + x = -x + if _math.isnan(x): + e = (1 << ebits) - 1 + m = 1 << (mbits - 1) + elif _math.isinf(x): + e = (1 << ebits) - 1 + m = 0 + elif x == 0: + e = m = 0 + else: + f, ex = _math.frexp(x) + e = ex + bias - 1 + if e <= 0: + # Subnormal: the mantissa scaled to the fixed exponent. + scaled = _math.ldexp(f, ex + bias - 1 + mbits) + m = _round_even(scaled) + e = 0 + if m >> mbits: + e = 1 + m &= (1 << mbits) - 1 + else: + m = _round_even(_math.ldexp(f, mbits + 1)) - (1 << mbits) + if m >> mbits: + m = 0 + e += 1 + if e >= (1 << ebits) - 1: + raise OverflowError("float too large to pack with %s format" % {2: "e", 4: "f", 8: "d"}[size]) + bits = (sign << (size * 8 - 1)) | (e << mbits) | m + return bits.to_bytes(size, "little" if little else "big") + + +def _round_even(v): + fl = _math.floor(v) + d = v - fl + r = int(fl) + if d > 0.5 or (d == 0.5 and r % 2 == 1): + r += 1 + return r + + +def _unpack_float(b, little): + size = len(b) + ebits, mbits = {2: (5, 10), 4: (8, 23), 8: (11, 52)}[size] + bits = int.from_bytes(b, "little" if little else "big") + sign = bits >> (size * 8 - 1) + e = (bits >> mbits) & ((1 << ebits) - 1) + m = bits & ((1 << mbits) - 1) + bias = (1 << (ebits - 1)) - 1 + if e == (1 << ebits) - 1: + v = _math.nan if m else _math.inf + elif e == 0: + v = _math.ldexp(float(m), 1 - bias - mbits) + else: + v = _math.ldexp(float(m | (1 << mbits)), e - bias - mbits) + return -v if sign else v + + +def _int_arg(v, c): + if isinstance(v, bool): + return int(v) + if isinstance(v, int): + return v + idx = getattr(type(v), "__index__", None) + if idx is None: + raise error("required argument is not an integer") + return idx(v) + + +def _pack_items(items, total, little, args): + if len(args) != len(items): + raise error("pack expected %d items for packing (got %d)" % (len(items), len(args))) + out = bytearray(total) + for (c, off, size), v in zip(items, args): + if c in _INTS: + v = _int_arg(v, c) + signed = c in _SIGNED + bits = size * 8 + lo = -(1 << (bits - 1)) if signed else 0 + hi = (1 << (bits - 1)) - 1 if signed else (1 << bits) - 1 + if not lo <= v <= hi: + if signed: + raise error("'%s' format requires %d <= number <= %d" % (c, lo, hi)) + raise error("'%s' format requires 0 <= number <= %d" % (c, hi)) + out[off:off + size] = v.to_bytes(size, "little" if little else "big", signed=signed) + elif c == "?": + out[off] = 1 if v else 0 + elif c == "c": + if not isinstance(v, (bytes, bytearray)) or len(v) != 1: + raise error("char format requires a bytes object of length 1") + out[off] = v[0] + elif c in "efd": + if not isinstance(v, (int, float)): + try: + v = v.__float__() + except AttributeError: + raise error("required argument is not a float") + out[off:off + size] = _pack_float(float(v), size, little) + elif c == "s": + if not isinstance(v, (bytes, bytearray)): + raise error("argument for 's' must be a bytes object") + data = bytes(v[:size]) + out[off:off + len(data)] = data + elif c == "p": + if not isinstance(v, (bytes, bytearray)): + raise error("argument for 'p' must be a bytes object") + if size: + data = bytes(v[:size - 1]) + out[off] = min(len(data), 255) + out[off + 1:off + 1 + len(data)] = data + return bytes(out) + + +def _unpack_items(items, little, buf, offset): + out = [] + for c, off, size in items: + o = offset + off + if c in _INTS: + out.append(int.from_bytes(buf[o:o + size], "little" if little else "big", signed=c in _SIGNED)) + elif c == "?": + out.append(buf[o] != 0) + elif c == "c": + out.append(bytes(buf[o:o + 1])) + elif c in "efd": + out.append(_unpack_float(bytes(buf[o:o + size]), little)) + elif c == "s": + out.append(bytes(buf[o:o + size])) + elif c == "p": + n = min(buf[o], size - 1) if size else 0 + out.append(bytes(buf[o + 1:o + 1 + n])) + return tuple(out) + + +class Struct: + def __init__(self, format): + self._items, self.size, self._little = _parse(format) + self.format = format.decode("ascii") if isinstance(format, bytes) else format + + def pack(self, *args): + return _pack_items(self._items, self.size, self._little, args) + + def pack_into(self, buffer, offset, *args): + data = self.pack(*args) + if offset < 0: + offset += len(buffer) + if offset + len(data) > len(buffer): + raise error("pack_into requires a buffer of at least %d bytes for packing %d bytes at offset %d " + "(actual buffer size is %d)" % (offset + len(data), len(data), offset, len(buffer))) + buffer[offset:offset + len(data)] = data + + def unpack(self, buffer): + if len(buffer) != self.size: + raise error("unpack requires a buffer of %d bytes" % self.size) + return _unpack_items(self._items, self._little, buffer, 0) + + def unpack_from(self, buffer, offset=0): + if offset < 0: + offset += len(buffer) + if len(buffer) - offset < self.size: + raise error("unpack_from requires a buffer of at least %d bytes for unpacking %d bytes at offset %d " + "(actual buffer size is %d)" % (self.size + offset, self.size, offset, len(buffer))) + return _unpack_items(self._items, self._little, buffer, offset) + + def iter_unpack(self, buffer): + if self.size == 0: + raise error("cannot iteratively unpack with a struct of length 0") + if len(buffer) % self.size: + raise error("iterative unpacking requires a buffer of a multiple of %d bytes" % self.size) + for i in range(0, len(buffer), self.size): + yield _unpack_items(self._items, self._little, buffer, i) + + def __repr__(self): + return "Struct(%r)" % (self.format,) + + +_cache = {} + + +def _struct(fmt): + s = _cache.get(fmt) + if s is None: + s = Struct(fmt) + if len(_cache) > 100: + _cache.clear() + _cache[fmt] = s + return s + + +def calcsize(format): + return _struct(format).size + + +def pack(format, *args): + return _struct(format).pack(*args) + + +def pack_into(format, buffer, offset, *args): + _struct(format).pack_into(buffer, offset, *args) + + +def unpack(format, buffer): + return _struct(format).unpack(buffer) + + +def unpack_from(format, buffer, offset=0): + return _struct(format).unpack_from(buffer, offset) + + +def iter_unpack(format, buffer): + return _struct(format).iter_unpack(buffer) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/textwrap.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/textwrap.py new file mode 100644 index 00000000000..902001ae99e --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/textwrap.py @@ -0,0 +1,217 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Wrapping and filling text.""" + +import re + +__all__ = ["TextWrapper", "wrap", "fill", "dedent", "indent", "shorten"] + +_whitespace = "\t\n\x0b\x0c\r " + + +class TextWrapper: + unicode_whitespace_trans = dict.fromkeys(map(ord, _whitespace), ord(" ")) + + _word_punct = r"[\w!\"'&.,?]" + _letter = r"[^\d\W]" + _whitespace_re = r"[%s]" % re.escape(_whitespace) + _nowhitespace = "[^" + _whitespace_re[1:] + wordsep_re = re.compile(r""" + ( # any whitespace + %(ws)s+ + | # em-dash between words + (?<=%(wp)s) -{2,} (?=\w) + | # word, possibly hyphenated + %(nws)s+? (?: + # hyphenated word + -(?: (?<=%(lt)s{2}-) | (?<=%(lt)s-%(lt)s-)) + (?= %(lt)s -? %(lt)s) + | # end of word + (?=%(ws)s|\Z) + | # em-dash + (?<=%(wp)s) (?=-{2,}\w) + ) + )""" % {"wp": _word_punct, "lt": _letter, "ws": _whitespace_re, "nws": _nowhitespace}, + re.VERBOSE) + wordsep_simple_re = re.compile(r"(%s+)" % _whitespace_re) + sentence_end_re = re.compile(r"[a-z]" r"[\.\!\?]" r"[\"\']?" r"\Z") + + def __init__(self, width=70, initial_indent="", subsequent_indent="", expand_tabs=True, + replace_whitespace=True, fix_sentence_endings=False, break_long_words=True, + drop_whitespace=True, break_on_hyphens=True, tabsize=8, *, max_lines=None, + placeholder=" [...]"): + self.width = width + self.initial_indent = initial_indent + self.subsequent_indent = subsequent_indent + self.expand_tabs = expand_tabs + self.replace_whitespace = replace_whitespace + self.fix_sentence_endings = fix_sentence_endings + self.break_long_words = break_long_words + self.drop_whitespace = drop_whitespace + self.break_on_hyphens = break_on_hyphens + self.tabsize = tabsize + self.max_lines = max_lines + self.placeholder = placeholder + + def _munge_whitespace(self, text): + if self.expand_tabs: + text = text.expandtabs(self.tabsize) + if self.replace_whitespace: + text = text.translate(self.unicode_whitespace_trans) + return text + + def _split(self, text): + if self.break_on_hyphens is True: + chunks = self.wordsep_re.split(text) + else: + chunks = self.wordsep_simple_re.split(text) + return [c for c in chunks if c] + + def _fix_sentence_endings(self, chunks): + i = 0 + patsearch = self.sentence_end_re.search + while i < len(chunks) - 1: + if chunks[i + 1] == " " and patsearch(chunks[i]): + chunks[i + 1] = " " + i += 2 + else: + i += 1 + + def _handle_long_word(self, reversed_chunks, cur_line, cur_len, width): + if width < 1: + space_left = 1 + else: + space_left = width - cur_len + if self.break_long_words: + end = space_left + chunk = reversed_chunks[-1] + if self.break_on_hyphens and len(chunk) > space_left: + hyphen = chunk.rfind("-", 0, space_left) + if hyphen > 0 and any(c != "-" for c in chunk[:hyphen]): + end = hyphen + 1 + cur_line.append(chunk[:end]) + reversed_chunks[-1] = chunk[end:] + elif not cur_line: + cur_line.append(reversed_chunks.pop()) + + def _wrap_chunks(self, chunks): + lines = [] + if self.width <= 0: + raise ValueError("invalid width %r (must be > 0)" % self.width) + if self.max_lines is not None: + if self.max_lines > 1: + indent = self.subsequent_indent + else: + indent = self.initial_indent + if len(indent) + len(self.placeholder.lstrip()) > self.width: + raise ValueError("placeholder too large for max width") + chunks.reverse() + while chunks: + cur_line = [] + cur_len = 0 + indent = self.subsequent_indent if lines else self.initial_indent + width = self.width - len(indent) + if self.drop_whitespace and chunks[-1].strip() == "" and lines: + del chunks[-1] + while chunks: + n = len(chunks[-1]) + if cur_len + n <= width: + cur_line.append(chunks.pop()) + cur_len += n + else: + break + if chunks and len(chunks[-1]) > width: + self._handle_long_word(chunks, cur_line, cur_len, width) + cur_len = sum(map(len, cur_line)) + if self.drop_whitespace and cur_line and cur_line[-1].strip() == "": + cur_len -= len(cur_line[-1]) + del cur_line[-1] + if cur_line: + if (self.max_lines is None or len(lines) + 1 < self.max_lines + or (not chunks or self.drop_whitespace and len(chunks) == 1 and not chunks[0].strip()) + and cur_len <= width): + lines.append(indent + "".join(cur_line)) + else: + while cur_line: + if cur_line[-1].strip() and cur_len + len(self.placeholder) <= width: + cur_line.append(self.placeholder) + lines.append(indent + "".join(cur_line)) + break + cur_len -= len(cur_line[-1]) + del cur_line[-1] + else: + if lines: + prev_line = lines[-1].rstrip() + if len(prev_line) + len(self.placeholder) <= self.width: + lines[-1] = prev_line + self.placeholder + break + lines.append(indent + self.placeholder.lstrip()) + break + return lines + + def _split_chunks(self, text): + text = self._munge_whitespace(text) + return self._split(text) + + def wrap(self, text): + chunks = self._split_chunks(text) + if self.fix_sentence_endings: + self._fix_sentence_endings(chunks) + return self._wrap_chunks(chunks) + + def fill(self, text): + return "\n".join(self.wrap(text)) + + +def wrap(text, width=70, **kwargs): + return TextWrapper(width=width, **kwargs).wrap(text) + + +def fill(text, width=70, **kwargs): + return TextWrapper(width=width, **kwargs).fill(text) + + +def shorten(text, width, **kwargs): + w = TextWrapper(width=width, max_lines=1, **kwargs) + return w.fill(" ".join(text.strip().split())) + + +def dedent(text): + lines = text.split("\n") + margin = None + for line in lines: + stripped = line.lstrip(" \t") + if not stripped: + continue + ind = line[:len(line) - len(stripped)] + if margin is None: + margin = ind + elif ind.startswith(margin): + pass + elif margin.startswith(ind): + margin = ind + else: + k = 0 + while k < len(margin) and k < len(ind) and margin[k] == ind[k]: + k += 1 + margin = margin[:k] + out = [] + for line in lines: + if not line.strip(" \t"): + out.append(line.lstrip(" \t") if line.strip() == "" and line.strip(" \t") == "" else line) + elif margin: + out.append(line[len(margin):]) + else: + out.append(line) + return "\n".join(out) + + +def indent(text, prefix, predicate=None): + out = [] + for line in text.splitlines(True): + if (predicate(line) if predicate is not None else line.strip()): + out.append(prefix + line) + else: + out.append(line) + return "".join(out) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/time.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/time.py new file mode 100644 index 00000000000..0370006a4d5 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/time.py @@ -0,0 +1,327 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Time access and conversions. + +The clocks are the device's; the local zone is the device's default +zone, asked for its offset at each instant (daylight saving included). +""" + +import _time + +time = _time.time +time_ns = _time.time_ns +perf_counter = _time.perf_counter +perf_counter_ns = _time.perf_counter_ns +monotonic = _time.monotonic +sleep = _time.sleep + + +def monotonic_ns(): + return perf_counter_ns() + + +def process_time(): + return perf_counter() + + +def process_time_ns(): + return perf_counter_ns() + + +def thread_time(): + return perf_counter() + + +_DAYS = ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"] +_MONTHS = ["January", "February", "March", "April", "May", "June", "July", "August", "September", + "October", "November", "December"] + + +class struct_time(tuple): + n_sequence_fields = 9 + n_fields = 11 + n_unnamed_fields = 0 + + def __new__(cls, seq, zone=None, gmtoff=None): + seq = tuple(seq) + if len(seq) < 9: + raise TypeError("time.struct_time() takes an at least 9-sequence (%d-sequence given)" % len(seq)) + if len(seq) > 11: + raise TypeError("time.struct_time() takes an at most 11-sequence (%d-sequence given)" % len(seq)) + self = tuple.__new__(cls, seq[:9]) + self._zone = seq[9] if len(seq) > 9 else zone + self._gmtoff = seq[10] if len(seq) > 10 else gmtoff + return self + + tm_year = property(lambda self: self[0]) + tm_mon = property(lambda self: self[1]) + tm_mday = property(lambda self: self[2]) + tm_hour = property(lambda self: self[3]) + tm_min = property(lambda self: self[4]) + tm_sec = property(lambda self: self[5]) + tm_wday = property(lambda self: self[6]) + tm_yday = property(lambda self: self[7]) + tm_isdst = property(lambda self: self[8]) + tm_zone = property(lambda self: self._zone) + tm_gmtoff = property(lambda self: self._gmtoff) + + def __repr__(self): + return ("time.struct_time(tm_year=%d, tm_mon=%d, tm_mday=%d, tm_hour=%d, tm_min=%d, " + "tm_sec=%d, tm_wday=%d, tm_yday=%d, tm_isdst=%d)" % tuple(self)) + + def __reduce__(self): + return (struct_time, (tuple(self),)) + + +def _days_from_civil(y, m, d): + y -= m <= 2 + era = (y if y >= 0 else y - 399) // 400 + yoe = y - era * 400 + doy = (153 * (m + (-3 if m > 2 else 9)) + 2) // 5 + d - 1 + doe = yoe * 365 + yoe // 4 - yoe // 100 + doy + return era * 146097 + doe - 719468 + + +def _civil_from_days(z): + z += 719468 + era = (z if z >= 0 else z - 146096) // 146097 + doe = z - era * 146097 + yoe = (doe - doe // 1460 + doe // 36524 - doe // 146096) // 365 + y = yoe + era * 400 + doy = doe - (365 * yoe + yoe // 4 - yoe // 100) + mp = (5 * doy + 2) // 153 + d = doy - (153 * mp + 2) // 5 + 1 + m = mp + 3 if mp < 10 else mp - 9 + return y + (m <= 2), m, d + + +def _leap(y): + return y % 4 == 0 and (y % 100 != 0 or y % 400 == 0) + + +def _yday(y, m, d): + return _days_from_civil(y, m, d) - _days_from_civil(y, 1, 1) + 1 + + +def _tm(secs, off, zone, isdst): + secs = int(secs // 1) + off + days = secs // 86400 + rem = secs - days * 86400 + y, m, d = _civil_from_days(days) + return struct_time((y, m, d, rem // 3600, rem // 60 % 60, rem % 60, (days + 3) % 7, _yday(y, m, d), isdst), + zone, off) + + +def _secs(secs): + if secs is None: + return time() + if not isinstance(secs, (int, float)): + raise TypeError("'%s' object cannot be interpreted as an integer" % type(secs).__name__) + return secs + + +def gmtime(secs=None): + return _tm(_secs(secs), 0, "UTC", 0) + + +def localtime(secs=None): + secs = _secs(secs) + off = _time.utcoffset(secs) + return _tm(secs, off, _zone_name(off), 1 if off != timezone * -1 else 0) + + +def _zone_name(off): + return tzname[0] if off == -timezone else tzname[1] + + +def mktime(t): + if len(t) != 9 and not isinstance(t, struct_time): + raise TypeError("mktime(): illegal time tuple argument") + y, mo, d, h, mi, s = t[0], t[1], t[2], t[3], t[4], t[5] + local = _days_from_civil(y, 1, 1) + _yday_norm(y, mo, d) - 1 + secs = local * 86400 + h * 3600 + mi * 60 + s + off = _time.utcoffset(secs) + off = _time.utcoffset(secs - off) + return float(secs - off) + + +def _yday_norm(y, mo, d): + y += (mo - 1) // 12 + mo = (mo - 1) % 12 + 1 + return _days_from_civil(y, mo, 1) - _days_from_civil(y, 1, 1) + d + + +def _now_offset(): + return _time.utcoffset(time()) + + +def _jan_offset(): + y = gmtime()[0] + return _time.utcoffset(_days_from_civil(y, 1, 1) * 86400.0) + + +def _jul_offset(): + y = gmtime()[0] + return _time.utcoffset(_days_from_civil(y, 7, 1) * 86400.0) + + +_jan = _jan_offset() +_jul = _jul_offset() +timezone = -min(_jan, _jul) +altzone = -max(_jan, _jul) +daylight = 1 if _jan != _jul else 0 + + +def _abbrev(off): + if off == 0: + return "UTC" + sign = "+" if off >= 0 else "-" + off = abs(off) + h, m = off // 3600, off // 60 % 60 + return "%s%02d%s" % (sign, h, "%02d" % m if m else "") + + +tzname = (_abbrev(-timezone), _abbrev(-altzone)) + + +def asctime(t=None): + if t is None: + t = localtime() + return "%s %s %2d %02d:%02d:%02d %d" % (_DAYS[t[6]][:3], _MONTHS[t[1] - 1][:3], t[2], t[3], t[4], t[5], t[0]) + + +def ctime(secs=None): + return asctime(localtime(secs)) + + +def strftime(format, t=None): + if t is None: + t = localtime() + return _format(format, t, None, None, 0) + + +def _format(fmt, t, zone, offset, micro): + """`fmt` rendered for time tuple `t`; `offset` is seconds east of UTC + or None, and `zone` a name or None.""" + if zone is None and isinstance(t, struct_time): + zone = t.tm_zone + if offset is None: + offset = t.tm_gmtoff + out = [] + i = 0 + n = len(fmt) + while i < n: + c = fmt[i] + if c != "%" or i + 1 >= n: + out.append(c) + i += 1 + continue + k = fmt[i + 1] + i += 2 + y, mo, d, h, mi, s, wd, yd = t[0], t[1], t[2], t[3], t[4], t[5], t[6], t[7] + if k == "Y": + out.append(str(y) if y >= 1000 else "%04d" % y) + elif k == "y": + out.append("%02d" % (y % 100)) + elif k == "C": + out.append("%02d" % (y // 100)) + elif k == "m": + out.append("%02d" % mo) + elif k == "d": + out.append("%02d" % d) + elif k == "e": + out.append("%2d" % d) + elif k == "H": + out.append("%02d" % h) + elif k == "I": + out.append("%02d" % ((h + 11) % 12 + 1)) + elif k == "M": + out.append("%02d" % mi) + elif k == "S": + out.append("%02d" % s) + elif k == "f": + out.append("%06d" % micro) + elif k == "p": + out.append("AM" if h < 12 else "PM") + elif k == "a": + out.append(_DAYS[wd][:3]) + elif k == "A": + out.append(_DAYS[wd]) + elif k == "b" or k == "h": + out.append(_MONTHS[mo - 1][:3]) + elif k == "B": + out.append(_MONTHS[mo - 1]) + elif k == "w": + out.append(str((wd + 1) % 7)) + elif k == "u": + out.append(str(wd + 1)) + elif k == "j": + out.append("%03d" % yd) + elif k == "U": + out.append("%02d" % ((yd + 6 - (wd + 1) % 7) // 7)) + elif k == "W": + out.append("%02d" % ((yd + 6 - wd) // 7)) + elif k in "GVg": + iy, iw, _ = _iso(y, mo, d) + out.append(str(iy) if k == "G" else ("%02d" % iw if k == "V" else "%02d" % (iy % 100))) + elif k == "c": + out.append(_format("%a %b %e %H:%M:%S %Y", t, zone, offset, micro)) + elif k == "x": + out.append(_format("%m/%d/%y", t, zone, offset, micro)) + elif k == "X": + out.append(_format("%H:%M:%S", t, zone, offset, micro)) + elif k == "D": + out.append(_format("%m/%d/%y", t, zone, offset, micro)) + elif k == "F": + out.append(_format("%Y-%m-%d", t, zone, offset, micro)) + elif k == "T": + out.append(_format("%H:%M:%S", t, zone, offset, micro)) + elif k == "R": + out.append(_format("%H:%M", t, zone, offset, micro)) + elif k == "r": + out.append(_format("%I:%M:%S %p", t, zone, offset, micro)) + elif k == "n": + out.append("\n") + elif k == "t": + out.append("\t") + elif k == "z": + if offset is not None: + sign = "+" if offset >= 0 else "-" + o = abs(offset) + hh, rest = divmod(o, 3600) + mm, ss = divmod(rest, 60) + part = "%s%02d%02d" % (sign, hh, mm) + if ss: + part += "%02d" % int(ss) + us = int(round((ss - int(ss)) * 1000000)) if isinstance(ss, float) else 0 + if us: + part += ".%06d" % us + out.append(part) + elif k == "Z": + if zone is not None: + out.append(zone) + elif k == "%": + out.append("%") + else: + out.append("%" + k) + return "".join(out) + + +def _iso(y, m, d): + days = _days_from_civil(y, m, d) + wd = (days + 3) % 7 + thursday = days - wd + 3 + iy = _civil_from_days(thursday)[0] + week1 = _days_from_civil(iy, 1, 4) + week1 -= (week1 + 3) % 7 + return iy, (days - week1) // 7 + 1, wd + 1 + + +def strptime(string, format="%a %b %d %H:%M:%S %Y"): + import _strptime + return _strptime._strptime_time(string, format) + + +def get_clock_info(name): + raise ValueError("unknown clock") diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/types.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/types.py new file mode 100644 index 00000000000..80eb3e03880 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/types.py @@ -0,0 +1,140 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Names for built-in types.""" + + +def _f(): + pass + + +async def _c(): + pass + + +def _g(): + yield 1 + + +async def _ag(): + yield 1 + + +class _C: + def _m(self): + pass + + +FunctionType = type(_f) +LambdaType = type(lambda: None) +CodeType = None +MappingProxyType = dict +CellType = None +GeneratorType = type(_g()) +_co = _c() +CoroutineType = type(_co) +_co.close() +AsyncGeneratorType = type(_ag()) +MethodType = type(_C()._m) +BuiltinFunctionType = type(len) +BuiltinMethodType = type([].append) +NoneType = type(None) +NotImplementedType = type(NotImplemented) +EllipsisType = type(...) +GenericAlias = None +UnionType = None + + +class SimpleNamespace: + def __init__(self, mapping=None, /, **kwargs): + if mapping is not None: + kwargs = {**dict(mapping), **kwargs} + for k, v in kwargs.items(): + setattr(self, k, v) + + def __repr__(self): + items = ("%s=%r" % (k, v) for k, v in self.__dict__.items()) + return "namespace(" + ", ".join(items) + ")" + + def __eq__(self, other): + if isinstance(self, SimpleNamespace) and isinstance(other, SimpleNamespace): + return self.__dict__ == other.__dict__ + return NotImplemented + + __hash__ = None + + +def new_class(name, bases=(), kwds=None, exec_body=None): + ns = {} + if exec_body is not None: + exec_body(ns) + meta = (kwds or {}).get("metaclass", type) + return meta(name, tuple(bases), ns) + + +def resolve_bases(bases): + out = [] + for b in bases: + if isinstance(b, type): + out.append(b) + elif hasattr(b, "__mro_entries__"): + out.extend(b.__mro_entries__(tuple(bases))) + else: + out.append(b) + return tuple(out) + + +class DynamicClassAttribute: + """A data descriptor read through instances only; on the class it + defers to the metaclass's __getattr__.""" + + def __init__(self, fget=None, fset=None, fdel=None, doc=None): + self.fget = fget + self.fset = fset + self.fdel = fdel + self.__doc__ = doc or (fget.__doc__ if fget is not None else None) + self.overwrite_doc = doc is None + self.__isabstractmethod__ = bool(getattr(fget, "__isabstractmethod__", False)) + + def __get__(self, instance, ownerclass=None): + if instance is None: + if self.__isabstractmethod__: + return self + raise AttributeError() + elif self.fget is None: + raise AttributeError("unreadable attribute") + return self.fget(instance) + + def __set__(self, instance, value): + if self.fset is None: + raise AttributeError("can't set attribute") + self.fset(instance, value) + + def __delete__(self, instance): + if self.fdel is None: + raise AttributeError("can't delete attribute") + self.fdel(instance) + + def getter(self, fget): + fdoc = fget.__doc__ if self.overwrite_doc else None + result = type(self)(fget, self.fset, self.fdel, fdoc or self.__doc__) + result.overwrite_doc = self.overwrite_doc + return result + + def setter(self, fset): + result = type(self)(self.fget, fset, self.fdel, self.__doc__) + result.overwrite_doc = self.overwrite_doc + return result + + def deleter(self, fdel): + result = type(self)(self.fget, self.fset, fdel, self.__doc__) + result.overwrite_doc = self.overwrite_doc + return result + + +def coroutine(func): + return func + + +def get_original_bases(cls, /): + return cls.__dict__.get("__orig_bases__", cls.__bases__) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/typing.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/typing.py new file mode 100644 index 00000000000..6248ae013e1 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/typing.py @@ -0,0 +1,663 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Type hints: the names exist and subscript, nothing is checked at run time.""" + +import collections.abc as _abc +import functools as _functools + +TYPE_CHECKING = False + + +def _type_repr(obj): + if obj is None or obj is type(None): + return "None" + if isinstance(obj, type): + if obj.__module__ == "builtins": + return obj.__qualname__ + return obj.__module__ + "." + obj.__qualname__ + if obj is ...: + return "..." + if type(obj).__name__ == "function": + return obj.__name__ + if isinstance(obj, (list, tuple)): + return "[" + ", ".join(_type_repr(a) for a in obj) + "]" + return repr(obj) + + +class _Final: + __slots__ = () + + +class _SpecialForm(_Final): + def __init__(self, name, getitem=None): + self._name = name + self._getitem = getitem + + def __repr__(self): + return "typing." + self._name + + def __getitem__(self, params): + if self._getitem is None: + raise TypeError(self._name + " is not subscriptable") + return self._getitem(self, params) + + def __call__(self, *args, **kwds): + raise TypeError("Cannot instantiate " + repr(self)) + + def __or__(self, other): + return Union[self, other] + + def __ror__(self, other): + return Union[other, self] + + def __hash__(self): + return hash(self._name) + + def __eq__(self, other): + return isinstance(other, _SpecialForm) and other._name == self._name + + +class _GenericAlias(_Final): + def __init__(self, origin, args, name=None): + if not isinstance(args, tuple): + args = (args,) + self.__origin__ = origin + self.__args__ = args + self._name = name + self.__parameters__ = tuple(a for a in args if isinstance(a, TypeVar)) + + def __repr__(self): + if self._name: + name = "typing." + self._name + else: + name = _type_repr(self.__origin__) + if not self.__args__: + return name + "[()]" + return name + "[" + ", ".join(_type_repr(a) for a in self.__args__) + "]" + + def __eq__(self, other): + if not isinstance(other, _GenericAlias): + return NotImplemented + return self.__origin__ == other.__origin__ and self.__args__ == other.__args__ + + def __hash__(self): + return hash((self.__origin__, self.__args__)) + + def __call__(self, *args, **kwargs): + result = self.__origin__(*args, **kwargs) + try: + result.__orig_class__ = self + except Exception: + pass + return result + + def __getitem__(self, params): + if not isinstance(params, tuple): + params = (params,) + subst = dict(zip(self.__parameters__, params)) + return _GenericAlias(self.__origin__, tuple(subst.get(a, a) for a in self.__args__), self._name) + + def __or__(self, other): + return Union[self, other] + + def __ror__(self, other): + return Union[other, self] + + def __mro_entries__(self, bases): + return (self.__origin__,) + + def __instancecheck__(self, obj): + raise TypeError("Subscripted generics cannot be used with class and instance checks") + + +class _UnionGenericAlias(_GenericAlias): + def __repr__(self): + return " | ".join(_type_repr(a) for a in self.__args__) + + def __eq__(self, other): + if not isinstance(other, _UnionGenericAlias): + return NotImplemented + return set(self.__args__) == set(other.__args__) + + def __hash__(self): + return hash(frozenset(self.__args__)) + + +def _union(form, params): + if not isinstance(params, tuple): + params = (params,) + flat = [] + for p in params: + p = type(None) if p is None else p + if isinstance(p, _UnionGenericAlias): + items = p.__args__ + else: + items = (p,) + for i in items: + if i not in flat: + flat.append(i) + if len(flat) == 1: + return flat[0] + return _UnionGenericAlias(Union, tuple(flat), "Union") + + +def _optional(form, param): + return _union(form, (param, None)) + + +def _subscript(form, params): + return _GenericAlias(form, params, form._name) + + +def _literal(form, params): + if not isinstance(params, tuple): + params = (params,) + return _GenericAlias(form, params, "Literal") + + +Any = _SpecialForm("Any") +NoReturn = _SpecialForm("NoReturn") +Never = _SpecialForm("Never") +Self = _SpecialForm("Self") +LiteralString = _SpecialForm("LiteralString") +TypeAlias = _SpecialForm("TypeAlias") +Union = _SpecialForm("Union", _union) +Optional = _SpecialForm("Optional", _optional) +ClassVar = _SpecialForm("ClassVar", _subscript) +Final = _SpecialForm("Final", _subscript) +Literal = _SpecialForm("Literal", _literal) +TypeGuard = _SpecialForm("TypeGuard", _subscript) +TypeIs = _SpecialForm("TypeIs", _subscript) +Required = _SpecialForm("Required", _subscript) +NotRequired = _SpecialForm("NotRequired", _subscript) +ReadOnly = _SpecialForm("ReadOnly", _subscript) +Concatenate = _SpecialForm("Concatenate", _subscript) +Unpack = _SpecialForm("Unpack", _subscript) + + +class _AnnotatedAlias(_GenericAlias): + def __init__(self, origin, metadata): + super().__init__(origin, (origin,)) + self.__metadata__ = metadata + + def __repr__(self): + return "typing.Annotated[" + _type_repr(self.__origin__) + ", " + \ + ", ".join(repr(m) for m in self.__metadata__) + "]" + + +def _annotated(form, params): + if not isinstance(params, tuple) or len(params) < 2: + raise TypeError("Annotated[...] should be used with at least two arguments (a type and an annotation).") + return _AnnotatedAlias(params[0], tuple(params[1:])) + + +Annotated = _SpecialForm("Annotated", _annotated) + + +class _SpecialGenericAlias(_Final): + def __init__(self, origin, name, nparams=-1): + self.__origin__ = origin + self._name = name + self._nparams = nparams + + def __repr__(self): + return "typing." + self._name + + def __getitem__(self, params): + if self._name == "Callable" and isinstance(params, tuple) and len(params) == 2: + args, result = params + if args is ...: + return _CallableGenericAlias(self.__origin__, (..., result), self._name) + return _CallableGenericAlias(self.__origin__, tuple(args) + (result,), self._name) + return _GenericAlias(self.__origin__, params, self._name) + + def __call__(self, *args, **kwargs): + return self.__origin__(*args, **kwargs) + + def __mro_entries__(self, bases): + return (self.__origin__,) + + def __instancecheck__(self, obj): + return isinstance(obj, self.__origin__) + + def __subclasscheck__(self, cls): + return issubclass(cls, self.__origin__) + + def __or__(self, other): + return Union[self, other] + + def __ror__(self, other): + return Union[other, self] + + +class _CallableGenericAlias(_GenericAlias): + def __repr__(self): + args = self.__args__ + if len(args) == 2 and args[0] is ...: + return "typing.Callable[..., " + _type_repr(args[1]) + "]" + return "typing.Callable[[" + ", ".join(_type_repr(a) for a in args[:-1]) + "], " + \ + _type_repr(args[-1]) + "]" + + +List = _SpecialGenericAlias(list, "List", 1) +Dict = _SpecialGenericAlias(dict, "Dict", 2) +Set = _SpecialGenericAlias(set, "Set", 1) +FrozenSet = _SpecialGenericAlias(frozenset, "FrozenSet", 1) +Tuple = _SpecialGenericAlias(tuple, "Tuple") +Type = _SpecialGenericAlias(type, "Type", 1) +Callable = _SpecialGenericAlias(_abc.Callable, "Callable", 2) +Iterable = _SpecialGenericAlias(_abc.Iterable, "Iterable", 1) +Iterator = _SpecialGenericAlias(_abc.Iterator, "Iterator", 1) +Generator = _SpecialGenericAlias(_abc.Generator, "Generator", 3) +AsyncIterable = _SpecialGenericAlias(_abc.AsyncIterable, "AsyncIterable", 1) +AsyncIterator = _SpecialGenericAlias(_abc.AsyncIterator, "AsyncIterator", 1) +AsyncGenerator = _SpecialGenericAlias(_abc.AsyncGenerator, "AsyncGenerator", 2) +Awaitable = _SpecialGenericAlias(_abc.Awaitable, "Awaitable", 1) +Coroutine = _SpecialGenericAlias(_abc.Coroutine, "Coroutine", 3) +Sequence = _SpecialGenericAlias(_abc.Sequence, "Sequence", 1) +MutableSequence = _SpecialGenericAlias(_abc.MutableSequence, "MutableSequence", 1) +Mapping = _SpecialGenericAlias(_abc.Mapping, "Mapping", 2) +MutableMapping = _SpecialGenericAlias(_abc.MutableMapping, "MutableMapping", 2) +AbstractSet = _SpecialGenericAlias(_abc.Set, "AbstractSet", 1) +MutableSet = _SpecialGenericAlias(_abc.MutableSet, "MutableSet", 1) +Collection = _SpecialGenericAlias(_abc.Collection, "Collection", 1) +Container = _SpecialGenericAlias(_abc.Container, "Container", 1) +Sized = _SpecialGenericAlias(_abc.Sized, "Sized", 0) +Hashable = _SpecialGenericAlias(_abc.Hashable, "Hashable", 0) +Reversible = _SpecialGenericAlias(_abc.Reversible, "Reversible", 1) +KeysView = _SpecialGenericAlias(_abc.KeysView, "KeysView", 1) +ItemsView = _SpecialGenericAlias(_abc.ItemsView, "ItemsView", 2) +ValuesView = _SpecialGenericAlias(_abc.ValuesView, "ValuesView", 1) + + +def _lazy_collections(): + import collections + return collections + + +DefaultDict = _SpecialGenericAlias(_lazy_collections().defaultdict, "DefaultDict", 2) +OrderedDict = _SpecialGenericAlias(_lazy_collections().OrderedDict, "OrderedDict", 2) +Counter = _SpecialGenericAlias(_lazy_collections().Counter, "Counter", 1) +Deque = _SpecialGenericAlias(_lazy_collections().deque, "Deque", 1) +ChainMap = _SpecialGenericAlias(_lazy_collections().ChainMap, "ChainMap", 2) +Text = str +AnyStr = None + + +class TypeVar(_Final): + def __init__(self, name, *constraints, bound=None, covariant=False, contravariant=False, + infer_variance=False, default=None): + self.__name__ = name + self.__constraints__ = constraints + self.__bound__ = bound + self.__covariant__ = covariant + self.__contravariant__ = contravariant + self.__infer_variance__ = infer_variance + self.__default__ = default + + def __repr__(self): + if self.__infer_variance__: + prefix = "" + elif self.__covariant__: + prefix = "+" + elif self.__contravariant__: + prefix = "-" + else: + prefix = "~" + return prefix + self.__name__ + + def __or__(self, other): + return Union[self, other] + + def __ror__(self, other): + return Union[other, self] + + +AnyStr = TypeVar("AnyStr", bytes, str) + + +class ParamSpecArgs(_Final): + def __init__(self, origin): + self.__origin__ = origin + + def __repr__(self): + return self.__origin__.__name__ + ".args" + + +class ParamSpecKwargs(_Final): + def __init__(self, origin): + self.__origin__ = origin + + def __repr__(self): + return self.__origin__.__name__ + ".kwargs" + + +class ParamSpec(_Final): + def __init__(self, name, *, bound=None, covariant=False, contravariant=False, default=None): + self.__name__ = name + self.__bound__ = bound + self.args = ParamSpecArgs(self) + self.kwargs = ParamSpecKwargs(self) + + def __repr__(self): + return "~" + self.__name__ + + +class TypeVarTuple(_Final): + def __init__(self, name, *, default=None): + self.__name__ = name + + def __repr__(self): + return self.__name__ + + def __iter__(self): + yield Unpack[self] + + +class Generic: + __slots__ = () + + def __class_getitem__(cls, params): + if not isinstance(params, tuple): + params = (params,) + if cls is Generic or cls is Protocol: + return _GenericAlias(cls, params, cls.__name__) + return _GenericAlias(cls, params) + + def __init_subclass__(cls, *args, **kwargs): + super().__init_subclass__(*args, **kwargs) + + +class _ProtocolMeta(type(_abc.Sized)): + def __instancecheck__(cls, instance): + if not cls.__dict__.get("_is_runtime_protocol", False) and not cls.__dict__.get("_is_protocol", False): + return super().__instancecheck__(instance) + if super().__instancecheck__(instance): + return True + if cls.__dict__.get("_is_protocol", False) and not cls.__dict__.get("_is_runtime_protocol", False): + raise TypeError("Instance and class checks can only be used with @runtime_checkable protocols") + for attr in cls.__protocol_attrs__: + if not hasattr(instance, attr): + return False + return True + + +class Protocol(Generic, metaclass=_ProtocolMeta): + __slots__ = () + _is_protocol = True + + def __init_subclass__(cls, *args, **kwargs): + super().__init_subclass__(*args, **kwargs) + if not cls.__dict__.get("_is_protocol", False): + cls._is_protocol = any(b is Protocol for b in cls.__bases__) + attrs = set() + for base in cls.__mro__[:-1]: + if base.__name__ in ("Protocol", "Generic", "object"): + continue + for name in list(base.__dict__.keys()) + list(base.__dict__.get("__annotations__", {}).keys()): + if not name.startswith("_abc_") and name not in ("__module__", "__qualname__", "__doc__", + "__annotations__", "__dict__", "__weakref__", + "__init__", "__slots__", "_is_protocol", + "__abstractmethods__", "__parameters__", + "__orig_bases__", "__protocol_attrs__", + "_is_runtime_protocol", "__init_subclass__", + "__subclasshook__", "__match_args__", + "__firstlineno__", "__static_attributes__", + "_abc_registry"): + attrs.add(name) + cls.__protocol_attrs__ = attrs + + +def runtime_checkable(cls): + cls._is_runtime_protocol = True + return cls + + +def cast(typ, val): + return val + + +def assert_type(val, typ, /): + return val + + +def assert_never(arg, /): + raise AssertionError("Expected code to be unreachable, but got: " + repr(arg)[:100]) + + +def reveal_type(obj, /): + import sys + print("Runtime type is " + repr(type(obj).__name__), file=sys.stderr) + return obj + + +def overload(func): + def _overload_dummy(*args, **kwds): + raise NotImplementedError("You should not call an overloaded function. " + "A series of @typing.overload-decorated functions " + "outside a stub module should always be followed " + "by an implementation that is not @typing.overload-ed.") + return _overload_dummy + + +def get_overloads(func): + return [] + + +def clear_overloads(): + pass + + +def final(f): + try: + f.__final__ = True + except (AttributeError, TypeError): + pass + return f + + +def override(method, /): + try: + method.__override__ = True + except (AttributeError, TypeError): + pass + return method + + +def no_type_check(arg): + return arg + + +def no_type_check_decorator(decorator): + return decorator + + +def dataclass_transform(*, eq_default=True, order_default=False, kw_only_default=False, frozen_default=False, + field_specifiers=(), **kwargs): + def decorator(cls_or_fn): + cls_or_fn.__dataclass_transform__ = {"eq_default": eq_default, "order_default": order_default, + "kw_only_default": kw_only_default, "frozen_default": frozen_default, + "field_specifiers": field_specifiers, "kwargs": kwargs} + return cls_or_fn + return decorator + + +def get_origin(tp): + if isinstance(tp, _AnnotatedAlias): + return Annotated + if isinstance(tp, (_GenericAlias, _SpecialGenericAlias)): + return tp.__origin__ + if tp is Generic: + return Generic + return None + + +def get_args(tp): + if isinstance(tp, _AnnotatedAlias): + return (tp.__origin__,) + tp.__metadata__ + if isinstance(tp, _CallableGenericAlias): + args = tp.__args__ + if args[0] is ...: + return args + return (list(args[:-1]), args[-1]) + if isinstance(tp, _GenericAlias): + return tp.__args__ + return () + + +def get_type_hints(obj, globalns=None, localns=None, include_extras=False): + hints = {} + if isinstance(obj, type): + for base in reversed(obj.__mro__): + hints.update(base.__dict__.get("__annotations__", {})) + else: + hints.update(getattr(obj, "__annotations__", {})) + if not include_extras: + for k, v in list(hints.items()): + if isinstance(v, _AnnotatedAlias): + hints[k] = v.__origin__ + return hints + + +def is_typeddict(tp): + return getattr(tp, "__total__", None) is not None and isinstance(tp, type) and issubclass(tp, dict) + + +class NewType: + def __init__(self, name, tp): + self.__qualname__ = name + self.__name__ = name + self.__supertype__ = tp + + def __repr__(self): + return self.__name__ + + def __call__(self, x): + return x + + def __or__(self, other): + return Union[self, other] + + def __ror__(self, other): + return Union[other, self] + + +class TypeAliasType: + def __init__(self, name, value, *, type_params=()): + self.__name__ = name + self.__value__ = value + self.__type_params__ = type_params + + def __repr__(self): + return self.__name__ + + def __getitem__(self, params): + return _GenericAlias(self, params) + + +class _NamedTupleMeta(type): + def __new__(cls, typename, bases, ns): + import collections + types = ns.get("__annotations__", {}) + default_names = [n for n in types if n in ns] + nm_tpl = collections.namedtuple(typename, list(types.keys()), + defaults=[ns[n] for n in default_names]) + nm_tpl.__annotations__ = types + for key, val in ns.items(): + if key not in ("__annotations__", "__module__", "__qualname__") and key not in types and \ + key not in ("__slots__", "_fields", "__new__", "__init__"): + setattr(nm_tpl, key, val) + return nm_tpl + + +def NamedTuple(typename, fields=None, /, **kwargs): + import collections + if fields is None: + fields = kwargs.items() + fields = list(fields) + nt = collections.namedtuple(typename, [n for n, t in fields]) + nt.__annotations__ = dict(fields) + return nt + + +_NamedTuple = type.__new__(_NamedTupleMeta, "NamedTuple", (), {}) + + +def _namedtuple_mro_entries(bases): + return (_NamedTuple,) + + +NamedTuple.__mro_entries__ = _namedtuple_mro_entries + + +class _TypedDictMeta(type): + def __new__(cls, name, bases, ns, total=True): + annotations = {} + for base in bases: + annotations.update(base.__dict__.get("__annotations__", {})) + annotations.update(ns.get("__annotations__", {})) + tp_dict = type.__new__(_TypedDictMeta, name, (dict,), ns) + tp_dict.__annotations__ = annotations + tp_dict.__total__ = total + tp_dict.__required_keys__ = frozenset(annotations) if total else frozenset() + tp_dict.__optional_keys__ = frozenset() if total else frozenset(annotations) + return tp_dict + + def __call__(cls, *args, **kwargs): + # A TypedDict describes a plain dict; instances are exactly that. + return dict(*args, **kwargs) + + def __instancecheck__(cls, other): + raise TypeError("TypedDict does not support instance and class checks") + + __subclasscheck__ = __instancecheck__ + + +def TypedDict(typename, fields=None, /, *, total=True, **kwargs): + if fields is None: + fields = kwargs + ns = {"__annotations__": dict(fields)} + return _TypedDictMeta(typename, (), ns, total=total) + + +_TypedDict = type.__new__(_TypedDictMeta, "TypedDict", (), {}) + + +def _typeddict_mro_entries(bases): + return (_TypedDict,) + + +TypedDict.__mro_entries__ = _typeddict_mro_entries + + +class SupportsInt(Protocol): + def __int__(self): + pass + + +class SupportsFloat(Protocol): + def __float__(self): + pass + + +class SupportsIndex(Protocol): + def __index__(self): + pass + + +class SupportsAbs(Protocol): + def __abs__(self): + pass + + +class SupportsRound(Protocol): + def __round__(self, ndigits=0): + pass + + +runtime_checkable(SupportsInt) +runtime_checkable(SupportsFloat) +runtime_checkable(SupportsIndex) +runtime_checkable(SupportsAbs) +runtime_checkable(SupportsRound) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/urllib/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/urllib/__init__.py new file mode 100644 index 00000000000..a4e554a6034 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/urllib/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""URL handling: only urllib.parse is provided; network access goes through Codename One.""" diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/urllib/parse.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/urllib/parse.py new file mode 100644 index 00000000000..06bb94e6263 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/urllib/parse.py @@ -0,0 +1,564 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Parse URLs into components, and quote and unquote their parts.""" + +from collections import namedtuple + +__all__ = ["urlparse", "urlunparse", "urljoin", "urldefrag", "urlsplit", "urlunsplit", "urlencode", + "parse_qs", "parse_qsl", "quote", "quote_plus", "quote_from_bytes", "unquote", "unquote_plus", + "unquote_to_bytes", "DefragResult", "ParseResult", "SplitResult", "DefragResultBytes", + "ParseResultBytes", "SplitResultBytes"] + +uses_relative = ["", "ftp", "http", "gopher", "nntp", "imap", "wais", "file", "https", "shttp", "mms", + "prospero", "rtsp", "rtsps", "rtspu", "sftp", "svn", "svn+ssh", "ws", "wss"] +uses_netloc = ["", "ftp", "http", "gopher", "nntp", "telnet", "imap", "wais", "file", "mms", "https", "shttp", + "snews", "prospero", "rtsp", "rtsps", "rtspu", "rsync", "svn", "svn+ssh", "sftp", "nfs", "git", + "git+ssh", "ws", "wss", "itms-services"] +uses_params = ["", "ftp", "hdl", "prospero", "http", "imap", "https", "shttp", "rtsp", "rtsps", "rtspu", "sip", + "sips", "mms", "sftp", "tel"] + +scheme_chars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789+-." + +_UNSAFE_URL_BYTES_TO_REMOVE = ["\t", "\r", "\n"] +_WHATWG_C0_CONTROL_OR_SPACE = "".join(chr(c) for c in range(33)) + + +def _coerce_args(*args): + str_input = isinstance(args[0], str) + for arg in args[1:]: + if arg and isinstance(arg, str) != str_input: + raise TypeError("Cannot mix str and non-str arguments") + if str_input: + return args + (_noop,) + return tuple(_decode(x) for x in args) + (_encode_result,) + + +def _noop(obj): + return obj + + +def _decode(x): + if x is None or x == b"": + return "" + return bytes(x).decode("ascii", "strict") + + +def _encode_result(obj, encoding="ascii", errors="strict"): + if isinstance(obj, tuple): + enc = type(obj)._encoded + return enc(*(x.encode(encoding, errors) if isinstance(x, str) else x for x in obj)) + return obj.encode(encoding, errors) + + +class _NetlocResultMixinBase: + __slots__ = () + + @property + def username(self): + return self._userinfo[0] + + @property + def password(self): + return self._userinfo[1] + + @property + def hostname(self): + hostname = self._hostinfo[0] + if not hostname: + return None + separator = "%" if isinstance(hostname, str) else b"%" + hostname, percent, zone = hostname.partition(separator) + return hostname.lower() + percent + zone + + @property + def port(self): + port = self._hostinfo[1] + if port is not None: + if port.isdigit() and port.isascii(): + port = int(port) + else: + raise ValueError("Port could not be cast to integer value as %r" % (port,)) + if not (0 <= port <= 65535): + raise ValueError("Port out of range 0-65535") + return port + + @property + def _userinfo(self): + netloc = self.netloc + userinfo, have_info, hostinfo = netloc.rpartition("@" if isinstance(netloc, str) else b"@") + if have_info: + username, have_password, password = userinfo.partition(":" if isinstance(netloc, str) else b":") + if not have_password: + password = None + else: + username = password = None + return username, password + + @property + def _hostinfo(self): + netloc = self.netloc + s = isinstance(netloc, str) + _, _, hostinfo = netloc.rpartition("@" if s else b"@") + _, have_open_br, bracketed = hostinfo.partition("[" if s else b"[") + if have_open_br: + hostname, _, port = bracketed.partition("]" if s else b"]") + _, _, port = port.partition(":" if s else b":") + else: + hostname, _, port = hostinfo.partition(":" if s else b":") + if not port: + port = None + return hostname, port + + +_DefragResultBase = namedtuple("DefragResult", "url fragment") +_SplitResultBase = namedtuple("SplitResult", "scheme netloc path query fragment") +_ParseResultBase = namedtuple("ParseResult", "scheme netloc path params query fragment") + + +class DefragResult(_DefragResultBase): + __slots__ = () + + def geturl(self): + if self.fragment: + return self.url + "#" + self.fragment + return self.url + + def encode(self, encoding="ascii", errors="strict"): + return DefragResultBytes(*(x.encode(encoding, errors) for x in self)) + + +class SplitResult(_SplitResultBase, _NetlocResultMixinBase): + __slots__ = () + + def geturl(self): + return urlunsplit(self) + + def encode(self, encoding="ascii", errors="strict"): + return SplitResultBytes(*(x.encode(encoding, errors) for x in self)) + + +class ParseResult(_ParseResultBase, _NetlocResultMixinBase): + __slots__ = () + + def geturl(self): + return urlunparse(self) + + def encode(self, encoding="ascii", errors="strict"): + return ParseResultBytes(*(x.encode(encoding, errors) for x in self)) + + +class DefragResultBytes(_DefragResultBase): + __slots__ = () + + def geturl(self): + if self.fragment: + return self.url + b"#" + self.fragment + return self.url + + def decode(self, encoding="ascii", errors="strict"): + return DefragResult(*(x.decode(encoding, errors) for x in self)) + + +class SplitResultBytes(_SplitResultBase, _NetlocResultMixinBase): + __slots__ = () + + def geturl(self): + return urlunsplit(self) + + def decode(self, encoding="ascii", errors="strict"): + return SplitResult(*(x.decode(encoding, errors) for x in self)) + + +class ParseResultBytes(_ParseResultBase, _NetlocResultMixinBase): + __slots__ = () + + def geturl(self): + return urlunparse(self) + + def decode(self, encoding="ascii", errors="strict"): + return ParseResult(*(x.decode(encoding, errors) for x in self)) + + +DefragResult._encoded = DefragResultBytes +SplitResult._encoded = SplitResultBytes +ParseResult._encoded = ParseResultBytes + + +def urlparse(url, scheme="", allow_fragments=True): + url, scheme, _coerce_result = _coerce_args(url, scheme) + scheme, netloc, url, query, fragment = _urlsplit(url, scheme, allow_fragments) + if scheme in uses_params and ";" in url: + url, params = _splitparams(url) + else: + params = "" + return _coerce_result(ParseResult(scheme, netloc, url, params, query, fragment)) + + +def _splitparams(url): + if "/" in url: + i = url.find(";", url.rfind("/")) + if i < 0: + return url, "" + else: + i = url.find(";") + return url[:i], url[i + 1:] + + +def _splitnetloc(url, start=0): + delim = len(url) + for c in "/?#": + wdelim = url.find(c, start) + if wdelim >= 0: + delim = min(delim, wdelim) + return url[start:delim], url[delim:] + + +def _check_bracketed_netloc(netloc): + hostname_and_port = netloc.rpartition("@")[2] + before_bracket, have_open_br, bracketed = hostname_and_port.partition("[") + if have_open_br: + if before_bracket: + raise ValueError("Invalid IPv6 URL") + hostname, _, port = bracketed.partition("]") + if port and not port.startswith(":"): + raise ValueError("Invalid IPv6 URL") + + +def _urlsplit(url, scheme, allow_fragments): + url = url.lstrip(_WHATWG_C0_CONTROL_OR_SPACE) + scheme = scheme.strip(_WHATWG_C0_CONTROL_OR_SPACE) + for b in _UNSAFE_URL_BYTES_TO_REMOVE: + url = url.replace(b, "") + scheme = scheme.replace(b, "") + allow_fragments = bool(allow_fragments) + netloc = query = fragment = "" + i = url.find(":") + if i > 0 and url[0].isascii() and url[0].isalpha(): + for c in url[:i]: + if c not in scheme_chars: + break + else: + scheme, url = url[:i].lower(), url[i + 1:] + if url[:2] == "//": + netloc, url = _splitnetloc(url, 2) + if (("[" in netloc and "]" not in netloc) or ("]" in netloc and "[" not in netloc)): + raise ValueError("Invalid IPv6 URL") + if "[" in netloc and "]" in netloc: + _check_bracketed_netloc(netloc) + if allow_fragments and "#" in url: + url, fragment = url.split("#", 1) + if "?" in url: + url, query = url.split("?", 1) + return scheme, netloc, url, query, fragment + + +def urlsplit(url, scheme="", allow_fragments=True): + url, scheme, _coerce_result = _coerce_args(url, scheme) + return _coerce_result(SplitResult(*_urlsplit(url, scheme, allow_fragments))) + + +def urlunparse(components): + scheme, netloc, url, params, query, fragment, _coerce_result = _coerce_args(*components) + if params: + url = "%s;%s" % (url, params) + return _coerce_result(_urlunsplit(scheme, netloc, url, query, fragment)) + + +def urlunsplit(components): + scheme, netloc, url, query, fragment, _coerce_result = _coerce_args(*components) + return _coerce_result(_urlunsplit(scheme, netloc, url, query, fragment)) + + +def _urlunsplit(scheme, netloc, url, query, fragment): + if netloc: + if url and url[:1] != "/": + url = "/" + url + url = "//" + netloc + url + elif url[:2] == "//": + url = "//" + url + elif scheme and scheme in uses_netloc and (not url or url[:1] == "/"): + url = "//" + url + if scheme: + url = scheme + ":" + url + if query: + url = url + "?" + query + if fragment: + url = url + "#" + fragment + return url + + +def urljoin(base, url, allow_fragments=True): + if not base: + return url + if not url: + return base + base, url, _coerce_result = _coerce_args(base, url) + bscheme, bnetloc, bpath, bquery, bfragment = _urlsplit(base, "", allow_fragments) + scheme, netloc, path, query, fragment = _urlsplit(url, bscheme, allow_fragments) + if scheme != bscheme or scheme not in uses_relative: + return _coerce_result(url) + if scheme in uses_netloc: + if netloc: + return _coerce_result(_urlunsplit(scheme, netloc, path, query, fragment)) + netloc = bnetloc + if not path: + path = bpath + if not query: + query = bquery + if not fragment: + fragment = bfragment + return _coerce_result(_urlunsplit(scheme, netloc, path, query, fragment)) + base_parts = bpath.split("/") + if base_parts[-1] != "": + del base_parts[-1] + if path[:1] == "/": + segments = path.split("/") + else: + segments = base_parts + path.split("/") + segments[1:-1] = filter(None, segments[1:-1]) + resolved_path = [] + for seg in segments: + if seg == "..": + try: + resolved_path.pop() + except IndexError: + pass + elif seg == ".": + continue + else: + resolved_path.append(seg) + if segments[-1] in (".", ".."): + resolved_path.append("") + return _coerce_result(_urlunsplit(scheme, netloc, "/".join(resolved_path) or "/", query, fragment)) + + +def urldefrag(url): + url, _coerce_result = _coerce_args(url) + if "#" in url: + s, n, p, a, q, frag = urlparse(url) + defrag = urlunparse((s, n, p, a, q, "")) + else: + frag = "" + defrag = url + return _coerce_result(DefragResult(defrag, frag)) + + +_hexdig = "0123456789ABCDEFabcdef" +_hextobyte = None + + +def unquote_to_bytes(string): + if not string: + string.split + return b"" + if isinstance(string, str): + string = string.encode("utf-8") + bits = string.split(b"%") + if len(bits) == 1: + return bytes(string) + res = bytearray(bits[0]) + for item in bits[1:]: + h = item[:2] + if len(h) == 2 and all(c in b"0123456789ABCDEFabcdef" for c in h): + res.append(int(h, 16)) + res += item[2:] + else: + res += b"%" + res += item + return bytes(res) + + +def unquote(string, encoding="utf-8", errors="replace"): + if isinstance(string, bytes): + return unquote_to_bytes(string).decode(encoding, errors) + if "%" not in string: + string.split + return string + if encoding is None: + encoding = "utf-8" + if errors is None: + errors = "replace" + out = [] + i = 0 + n = len(string) + while i < n: + j = string.find("%", i) + if j < 0: + out.append(string[i:]) + break + out.append(string[i:j]) + k = j + raw = bytearray() + while k + 2 < n + 0 and string[k] == "%" and string[k + 1] in _hexdig and string[k + 2] in _hexdig: + raw.append(int(string[k + 1:k + 3], 16)) + k += 3 + if k >= n: + break + if raw: + out.append(bytes(raw).decode(encoding, errors)) + i = k + else: + out.append("%") + i = j + 1 + return "".join(out) + + +def parse_qs(qs, keep_blank_values=False, strict_parsing=False, encoding="utf-8", errors="replace", + max_num_fields=None, separator="&"): + parsed_result = {} + pairs = parse_qsl(qs, keep_blank_values, strict_parsing, encoding=encoding, errors=errors, + max_num_fields=max_num_fields, separator=separator) + for name, value in pairs: + if name in parsed_result: + parsed_result[name].append(value) + else: + parsed_result[name] = [value] + return parsed_result + + +def parse_qsl(qs, keep_blank_values=False, strict_parsing=False, encoding="utf-8", errors="replace", + max_num_fields=None, separator="&"): + if not qs: + return [] + if isinstance(qs, (bytes, bytearray, memoryview)): + qs = bytes(qs).decode("latin-1") + is_bytes = True + else: + is_bytes = False + if not separator or not isinstance(separator, (str, bytes)): + raise ValueError("Separator must be of type string or bytes.") + if isinstance(separator, bytes): + separator = separator.decode("ascii") + if max_num_fields is not None: + num_fields = 1 + qs.count(separator) + if max_num_fields < num_fields: + raise ValueError("Max number of fields exceeded") + r = [] + for name_value in qs.split(separator): + if not name_value and not strict_parsing: + continue + nv = name_value.split("=", 1) + if len(nv) != 2: + if strict_parsing: + raise ValueError("bad query field: %r" % (name_value,)) + if keep_blank_values: + nv.append("") + else: + continue + if len(nv[1]) or keep_blank_values: + name = nv[0].replace("+", " ") + value = nv[1].replace("+", " ") + if is_bytes: + name = unquote_to_bytes(name.encode("latin-1")) + value = unquote_to_bytes(value.encode("latin-1")) + else: + name = unquote(name, encoding=encoding, errors=errors) + value = unquote(value, encoding=encoding, errors=errors) + r.append((name, value)) + return r + + +def unquote_plus(string, encoding="utf-8", errors="replace"): + string = string.replace("+", " ") + return unquote(string, encoding, errors) + + +_ALWAYS_SAFE = frozenset(b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_.-~") +_safe_cache = {} + + +def quote(string, safe="/", encoding=None, errors=None): + if isinstance(string, str): + if not string: + return string + if encoding is None: + encoding = "utf-8" + if errors is None: + errors = "strict" + string = string.encode(encoding, errors) + else: + if encoding is not None: + raise TypeError("quote() doesn't support 'encoding' for bytes") + if errors is not None: + raise TypeError("quote() doesn't support 'errors' for bytes") + return quote_from_bytes(string, safe) + + +def quote_plus(string, safe="", encoding=None, errors=None): + if ((isinstance(string, str) and " " not in string) or (isinstance(string, bytes) and b" " not in string)): + return quote(string, safe, encoding, errors) + if isinstance(safe, str): + space = " " + else: + space = b" " + string = quote(string, safe + space, encoding, errors) + return string.replace(" ", "+") + + +def quote_from_bytes(bs, safe="/"): + if not isinstance(bs, (bytes, bytearray)): + raise TypeError("quote_from_bytes() expected bytes") + if not bs: + return "" + if isinstance(safe, str): + safe = safe.encode("ascii", "ignore") + else: + safe = bytes([c for c in safe if c < 128]) + allowed = _ALWAYS_SAFE.union(safe) + out = [] + for b in bs: + if b in allowed: + out.append(chr(b)) + else: + out.append("%{:02X}".format(b)) + return "".join(out) + + +def urlencode(query, doseq=False, safe="", encoding=None, errors=None, quote_via=quote_plus): + if hasattr(query, "items"): + query = query.items() + else: + try: + if len(query) and not isinstance(query[0], tuple): + raise TypeError + except TypeError: + raise TypeError("not a valid non-string sequence or mapping object") + l = [] + if not doseq: + for k, v in query: + if isinstance(k, bytes): + k = quote_via(k, safe) + else: + k = quote_via(str(k), safe, encoding, errors) + if isinstance(v, bytes): + v = quote_via(v, safe) + else: + v = quote_via(str(v), safe, encoding, errors) + l.append(k + "=" + v) + else: + for k, v in query: + if isinstance(k, bytes): + k = quote_via(k, safe) + else: + k = quote_via(str(k), safe, encoding, errors) + if isinstance(v, bytes): + v = quote_via(v, safe) + l.append(k + "=" + v) + elif isinstance(v, str): + v = quote_via(v, safe, encoding, errors) + l.append(k + "=" + v) + else: + try: + len(v) + except TypeError: + v = quote_via(str(v), safe, encoding, errors) + l.append(k + "=" + v) + else: + for elt in v: + if isinstance(elt, bytes): + elt = quote_via(elt, safe) + else: + elt = quote_via(str(elt), safe, encoding, errors) + l.append(k + "=" + elt) + return "&".join(l) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/uuid.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/uuid.py new file mode 100644 index 00000000000..782ae2de24a --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/uuid.py @@ -0,0 +1,375 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""UUID objects (RFC 9562) and the functions that make each version. + +There is no hardware address to read, so getnode() answers a random +48-bit number with the multicast bit set, as the RFC asks of a host +without one. +""" + +import os +import time + +__all__ = ["UUID", "SafeUUID", "getnode", "uuid1", "uuid3", "uuid4", "uuid5", "uuid6", "uuid7", "uuid8", + "NAMESPACE_DNS", "NAMESPACE_URL", "NAMESPACE_OID", "NAMESPACE_X500", "NIL", "MAX", + "RESERVED_NCS", "RFC_4122", "RESERVED_MICROSOFT", "RESERVED_FUTURE"] + +RESERVED_NCS, RFC_4122, RESERVED_MICROSOFT, RESERVED_FUTURE = [ + "reserved for NCS compatibility", "specified in RFC 4122", + "reserved for Microsoft compatibility", "reserved for future definition"] + +_UINT_128_MAX = (1 << 128) - 1 +_RFC_4122_CLEARFLAGS_MASK = ~((0xf000 << 64) | (0xc000 << 48)) +_RFC_4122_VERSION_1_FLAGS = ((1 << 76) | (0x8000 << 48)) +_RFC_4122_VERSION_3_FLAGS = ((3 << 76) | (0x8000 << 48)) +_RFC_4122_VERSION_4_FLAGS = ((4 << 76) | (0x8000 << 48)) +_RFC_4122_VERSION_5_FLAGS = ((5 << 76) | (0x8000 << 48)) +_RFC_4122_VERSION_6_FLAGS = ((6 << 76) | (0x8000 << 48)) +_RFC_4122_VERSION_7_FLAGS = ((7 << 76) | (0x8000 << 48)) +_RFC_4122_VERSION_8_FLAGS = ((8 << 76) | (0x8000 << 48)) + + +class SafeUUID: + safe = 0 + unsafe = -1 + unknown = None + + +class UUID: + __slots__ = ("int", "is_safe", "__weakref__") + + def __init__(self, hex=None, bytes=None, bytes_le=None, fields=None, int=None, version=None, *, + is_safe=SafeUUID.unknown): + if [hex, bytes, bytes_le, fields, int].count(None) != 4: + raise TypeError("one of the hex, bytes, bytes_le, fields, or int arguments must be given") + if hex is not None: + hex = hex.replace("urn:", "").replace("uuid:", "") + hex = hex.strip("{}").replace("-", "") + if len(hex) != 32: + raise ValueError("badly formed hexadecimal UUID string") + int = _int(hex, 16) + elif bytes_le is not None: + if len(bytes_le) != 16: + raise ValueError("bytes_le is not a 16-char string") + bytes = (bytes_le[4 - 1::-1] + bytes_le[6 - 1:4 - 1:-1] + bytes_le[8 - 1:6 - 1:-1] + bytes_le[8:]) + if bytes is not None: + if len(bytes) != 16: + raise ValueError("bytes is not a 16-char string") + int = _int.from_bytes(bytes, "big") + elif fields is not None: + if len(fields) != 6: + raise ValueError("fields is not a 6-tuple") + (time_low, time_mid, time_hi_version, clock_seq_hi_variant, clock_seq_low, node) = fields + if not 0 <= time_low < (1 << 32): + raise ValueError("field 1 out of range (need a 32-bit value)") + if not 0 <= time_mid < (1 << 16): + raise ValueError("field 2 out of range (need a 16-bit value)") + if not 0 <= time_hi_version < (1 << 16): + raise ValueError("field 3 out of range (need a 16-bit value)") + if not 0 <= clock_seq_hi_variant < (1 << 8): + raise ValueError("field 4 out of range (need an 8-bit value)") + if not 0 <= clock_seq_low < (1 << 8): + raise ValueError("field 5 out of range (need an 8-bit value)") + if not 0 <= node < (1 << 48): + raise ValueError("field 6 out of range (need a 48-bit value)") + clock_seq = (clock_seq_hi_variant << 8) | clock_seq_low + int = ((time_low << 96) | (time_mid << 80) | (time_hi_version << 64) | (clock_seq << 48) | node) + if not 0 <= int <= _UINT_128_MAX: + raise ValueError("int is out of range (need a 128-bit value)") + if version is not None: + if not 1 <= version <= 8: + raise ValueError("illegal version number") + int &= _RFC_4122_CLEARFLAGS_MASK + int |= 0x8000 << 48 + int |= version << 76 + object.__setattr__(self, "int", int) + object.__setattr__(self, "is_safe", is_safe) + + @classmethod + def _from_int(cls, value): + self = object.__new__(cls) + object.__setattr__(self, "int", value) + object.__setattr__(self, "is_safe", SafeUUID.unknown) + return self + + def __getstate__(self): + d = {"int": self.int} + if self.is_safe != SafeUUID.unknown: + d["is_safe"] = self.is_safe + return d + + def __eq__(self, other): + if isinstance(other, UUID): + return self.int == other.int + return NotImplemented + + def __lt__(self, other): + if isinstance(other, UUID): + return self.int < other.int + return NotImplemented + + def __gt__(self, other): + if isinstance(other, UUID): + return self.int > other.int + return NotImplemented + + def __le__(self, other): + if isinstance(other, UUID): + return self.int <= other.int + return NotImplemented + + def __ge__(self, other): + if isinstance(other, UUID): + return self.int >= other.int + return NotImplemented + + def __hash__(self): + return hash(self.int) + + def __int__(self): + return self.int + + def __repr__(self): + return "%s(%r)" % (self.__class__.__name__, str(self)) + + def __setattr__(self, name, value): + raise TypeError("UUID objects are immutable") + + def __str__(self): + x = self.hex + return "%s-%s-%s-%s-%s" % (x[:8], x[8:12], x[12:16], x[16:20], x[20:]) + + @property + def bytes(self): + return self.int.to_bytes(16, "big") + + @property + def bytes_le(self): + bytes = self.bytes + return (bytes[4 - 1::-1] + bytes[6 - 1:4 - 1:-1] + bytes[8 - 1:6 - 1:-1] + bytes[8:]) + + @property + def fields(self): + return (self.time_low, self.time_mid, self.time_hi_version, self.clock_seq_hi_variant, self.clock_seq_low, + self.node) + + @property + def time_low(self): + return self.int >> 96 + + @property + def time_mid(self): + return (self.int >> 80) & 0xffff + + @property + def time_hi_version(self): + return (self.int >> 64) & 0xffff + + @property + def clock_seq_hi_variant(self): + return (self.int >> 56) & 0xff + + @property + def clock_seq_low(self): + return (self.int >> 48) & 0xff + + @property + def time(self): + if self.version == 6: + time_hi = self.int >> 96 + time_lo = (self.int >> 64) & 0x0fff + return time_hi << 28 | (self.time_mid << 12) | time_lo + elif self.version == 7: + return self.int >> 80 + else: + time_hi = self.time_hi_version & 0x0fff + time_lo = self.int >> 96 + return time_hi << 48 | (self.time_mid << 32) | time_lo + + @property + def clock_seq(self): + return (((self.clock_seq_hi_variant & 0x3f) << 8) | self.clock_seq_low) + + @property + def node(self): + return self.int & 0xffffffffffff + + @property + def hex(self): + return "%032x" % self.int + + @property + def urn(self): + return "urn:uuid:" + str(self) + + @property + def variant(self): + if not self.int & (0x8000 << 48): + return RESERVED_NCS + elif not self.int & (0x4000 << 48): + return RFC_4122 + elif not self.int & (0x2000 << 48): + return RESERVED_MICROSOFT + else: + return RESERVED_FUTURE + + @property + def version(self): + if self.variant == RFC_4122: + return int((self.int >> 76) & 0xf) + + +_int = int + +_node = None + + +def getnode(): + global _node + if _node is None: + _node = _int.from_bytes(os.urandom(6), "big") | (1 << 40) + return _node + + +_last_timestamp = None + + +def uuid1(node=None, clock_seq=None): + global _last_timestamp + nanoseconds = time.time_ns() + timestamp = nanoseconds // 100 + 0x01b21dd213814000 + if _last_timestamp is not None and timestamp <= _last_timestamp: + timestamp = _last_timestamp + 1 + _last_timestamp = timestamp + if clock_seq is None: + clock_seq = _int.from_bytes(os.urandom(2), "big") & 0x3fff + time_low = timestamp & 0xffffffff + time_mid = (timestamp >> 32) & 0xffff + time_hi_version = (timestamp >> 48) & 0x0fff + clock_seq_low = clock_seq & 0xff + clock_seq_hi_variant = (clock_seq >> 8) & 0x3f + if node is None: + node = getnode() + return UUID(fields=(time_low, time_mid, time_hi_version, clock_seq_hi_variant, clock_seq_low, node), + version=1) + + +def uuid3(namespace, name): + if isinstance(name, str): + name = bytes(name, "utf-8") + import hashlib + h = hashlib.md5(namespace.bytes + name, usedforsecurity=False) + int_uuid_3 = _int.from_bytes(h.digest()) + int_uuid_3 &= _RFC_4122_CLEARFLAGS_MASK + int_uuid_3 |= _RFC_4122_VERSION_3_FLAGS + return UUID._from_int(int_uuid_3) + + +def uuid4(): + int_uuid_4 = _int.from_bytes(os.urandom(16)) + int_uuid_4 &= _RFC_4122_CLEARFLAGS_MASK + int_uuid_4 |= _RFC_4122_VERSION_4_FLAGS + return UUID._from_int(int_uuid_4) + + +def uuid5(namespace, name): + if isinstance(name, str): + name = bytes(name, "utf-8") + import hashlib + h = hashlib.sha1(namespace.bytes + name, usedforsecurity=False) + int_uuid_5 = _int.from_bytes(h.digest()[:16]) + int_uuid_5 &= _RFC_4122_CLEARFLAGS_MASK + int_uuid_5 |= _RFC_4122_VERSION_5_FLAGS + return UUID._from_int(int_uuid_5) + + +_last_timestamp_v6 = None + + +def uuid6(node=None, clock_seq=None): + global _last_timestamp_v6 + nanoseconds = time.time_ns() + timestamp = nanoseconds // 100 + 0x01b21dd213814000 + if _last_timestamp_v6 is not None and timestamp <= _last_timestamp_v6: + timestamp = _last_timestamp_v6 + 1 + _last_timestamp_v6 = timestamp + if clock_seq is None: + clock_seq = _int.from_bytes(os.urandom(2), "big") & 0x3fff + time_hi_and_mid = (timestamp >> 12) & 0xffff_ffff_ffff + time_lo = timestamp & 0x0fff + clock_s = clock_seq & 0x3fff + if node is None: + node = _int.from_bytes(os.urandom(6), "big") + int_uuid_6 = time_hi_and_mid << 80 + int_uuid_6 |= time_lo << 64 + int_uuid_6 |= clock_s << 48 + int_uuid_6 |= node & 0xffff_ffff_ffff + int_uuid_6 |= _RFC_4122_VERSION_6_FLAGS + return UUID._from_int(int_uuid_6) + + +_last_timestamp_v7 = None +_last_counter_v7 = 0 + + +def _uuid7_get_counter_and_tail(): + rand = _int.from_bytes(os.urandom(10)) + counter = (rand >> 32) & 0x1ff_ffff_ffff + tail = rand & 0xffff_ffff + return counter, tail + + +def uuid7(): + global _last_timestamp_v7, _last_counter_v7 + nanoseconds = time.time_ns() + timestamp_ms = nanoseconds // 1_000_000 + if _last_timestamp_v7 is None or timestamp_ms > _last_timestamp_v7: + counter, tail = _uuid7_get_counter_and_tail() + else: + if timestamp_ms < _last_timestamp_v7: + timestamp_ms = _last_timestamp_v7 + 1 + counter = _last_counter_v7 + 1 + if counter > 0x3ff_ffff_ffff: + timestamp_ms += 1 + counter, tail = _uuid7_get_counter_and_tail() + else: + tail = _int.from_bytes(os.urandom(4)) + unix_ts_ms = timestamp_ms & 0xffff_ffff_ffff + counter_msbs = counter >> 30 + counter_hi = counter_msbs & 0x0fff + counter_lo = counter & 0x3fff_ffff + tail &= 0xffff_ffff + int_uuid_7 = unix_ts_ms << 80 + int_uuid_7 |= counter_hi << 64 + int_uuid_7 |= counter_lo << 32 + int_uuid_7 |= tail + int_uuid_7 |= _RFC_4122_VERSION_7_FLAGS + _last_timestamp_v7 = timestamp_ms + _last_counter_v7 = counter + return UUID._from_int(int_uuid_7) + + +def uuid8(a=None, b=None, c=None): + if a is None: + import random + a = random.getrandbits(48) + if b is None: + import random + b = random.getrandbits(12) + if c is None: + import random + c = random.getrandbits(62) + int_uuid_8 = (a & 0xffff_ffff_ffff) << 80 + int_uuid_8 |= (b & 0xfff) << 64 + int_uuid_8 |= c & 0x3fff_ffff_ffff_ffff + int_uuid_8 |= _RFC_4122_VERSION_8_FLAGS + return UUID._from_int(int_uuid_8) + + +NAMESPACE_DNS = UUID("6ba7b810-9dad-11d1-80b4-00c04fd430c8") +NAMESPACE_URL = UUID("6ba7b811-9dad-11d1-80b4-00c04fd430c8") +NAMESPACE_OID = UUID("6ba7b812-9dad-11d1-80b4-00c04fd430c8") +NAMESPACE_X500 = UUID("6ba7b814-9dad-11d1-80b4-00c04fd430c8") +NIL = UUID("00000000-0000-0000-0000-000000000000") +MAX = UUID("ffffffff-ffff-ffff-ffff-ffffffffffff") diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/warnings.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/warnings.py new file mode 100644 index 00000000000..a7b7d9cb01a --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/warnings.py @@ -0,0 +1,284 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Warning control. + +A translated program has no frames to inspect, so a warning is reported +against the file "sys" at line 1, as CPython reports a warning whose +stacklevel runs past the top of the stack; the module registry used by +the "default" and "module" actions is therefore one per program. +""" + +import sys +import re + +__all__ = ["warn", "warn_explicit", "showwarning", "formatwarning", "filterwarnings", "simplefilter", + "resetwarnings", "catch_warnings", "deprecated"] + +filters = [] +defaultaction = "default" +onceregistry = {} +_registry = {} +_filters_version = 1 + +_ACTIONS = ("error", "ignore", "always", "all", "default", "module", "once") + + +def _filters_mutated(): + global _filters_version + _filters_version += 1 + _registry.clear() + + +def _check_action(action): + if action not in _ACTIONS: + raise ValueError("invalid action: %r" % (action,)) + + +def filterwarnings(action, message="", category=Warning, module="", lineno=0, append=False): + _check_action(action) + if not isinstance(message, str): + raise TypeError("message must be a string") + if not (isinstance(category, type) and issubclass(category, Warning)): + raise TypeError("category must be a Warning subclass") + if not isinstance(module, str): + raise TypeError("module must be a string") + if not isinstance(lineno, int) or lineno < 0: + raise ValueError("lineno must be an int >= 0") + msg = re.compile(message, re.I) if message else None + mod = re.compile(module) if module else None + _add_filter(action, msg, category, mod, lineno, append=append) + + +def simplefilter(action, category=Warning, lineno=0, append=False): + _check_action(action) + if not isinstance(lineno, int) or lineno < 0: + raise ValueError("lineno must be an int >= 0") + _add_filter(action, None, category, None, lineno, append=append) + + +def _add_filter(*item, append): + if not append: + try: + filters.remove(item) + except ValueError: + pass + filters.insert(0, item) + elif item not in filters: + filters.append(item) + _filters_mutated() + + +def resetwarnings(): + filters[:] = [] + _filters_mutated() + + +class WarningMessage: + _WARNING_DETAILS = ("message", "category", "filename", "lineno", "file", "line", "source") + + def __init__(self, message, category, filename, lineno, file=None, line=None, source=None): + self.message = message + self.category = category + self.filename = filename + self.lineno = lineno + self.file = file + self.line = line + self.source = source + self._category_name = category.__name__ if category else None + + def __str__(self): + return ("{message : %r, category : %r, filename : %r, lineno : %s, line : %r}" + % (self.message, self._category_name, self.filename, self.lineno, self.line)) + + +def formatwarning(message, category, filename, lineno, line=None): + s = "%s:%s: %s: %s\n" % (filename, lineno, category.__name__, message) + if line: + s += " %s\n" % line.strip() + return s + + +def _showwarnmsg_impl(msg): + f = msg.file + if f is None: + f = sys.stderr + if f is None: + return + text = formatwarning(msg.message, msg.category, msg.filename, msg.lineno, msg.line) + try: + f.write(text) + except OSError: + pass + + +def showwarning(message, category, filename, lineno, file=None, line=None): + _showwarnmsg_impl(WarningMessage(message, category, filename, lineno, file, line)) + + +_showwarning_orig = showwarning + + +def _showwarnmsg(msg): + sw = globals().get("showwarning") + if sw is not _showwarning_orig: + if not callable(sw): + raise TypeError("warnings.showwarning() must be set to a function or method") + sw(msg.message, msg.category, msg.filename, msg.lineno, msg.file, msg.line) + return + _showwarnmsg_impl(msg) + + +def warn(message, category=None, stacklevel=1, source=None, *, skip_file_prefixes=()): + if isinstance(message, Warning): + category = message.__class__ + if category is None: + category = UserWarning + if not (isinstance(category, type) and issubclass(category, Warning)): + raise TypeError("category must be a Warning subclass, not '%s'" % type(category).__name__) + warn_explicit(message, category, "sys", 1, "sys", _registry, source=source) + + +def warn_explicit(message, category, filename, lineno, module=None, registry=None, module_globals=None, + source=None): + lineno = int(lineno) + if module is None: + module = filename or "" + if module[-3:].lower() == ".py": + module = module[:-3] + if registry is None: + registry = {} + if registry.get("version", 0) != _filters_version: + registry.clear() + registry["version"] = _filters_version + if isinstance(message, Warning): + text = str(message) + category = message.__class__ + else: + text = message + message = category(message) + key = (text, category, lineno) + if registry.get(key): + return + for item in filters: + action, msg, cat, mod, ln = item + if ((msg is None or msg.match(text)) and issubclass(category, cat) + and (mod is None or mod.match(module)) and (ln == 0 or lineno == ln)): + break + else: + action = defaultaction + if action == "ignore": + return + if action == "error": + raise message + if action == "once": + registry[key] = 1 + oncekey = (text, category) + if onceregistry.get(oncekey): + return + onceregistry[oncekey] = 1 + elif action in ("always", "all"): + pass + elif action == "module": + registry[key] = 1 + altkey = (text, category, 0) + if registry.get(altkey): + return + registry[altkey] = 1 + elif action == "default": + registry[key] = 1 + else: + raise RuntimeError("Unrecognized action (%r) in warnings.filters:\n %s" % (action, item)) + _showwarnmsg(WarningMessage(message, category, filename, lineno, source=source)) + + +class catch_warnings: + def __init__(self, *, record=False, module=None, action=None, category=Warning, lineno=0, append=False): + self._record = record + self._entered = False + self._module = sys.modules["warnings"] if module is None else module + if action is None: + self._filter = None + else: + self._filter = (action, category, lineno, append) + + def __repr__(self): + args = [] + if self._record: + args.append("record=True") + return "%s(%s)" % (type(self).__name__, ", ".join(args)) + + def __enter__(self): + if self._entered: + raise RuntimeError("Cannot enter %r twice" % self) + self._entered = True + global showwarning + self._filters = filters[:] + self._showwarning = showwarning + self._onceregistry = dict(onceregistry) + _filters_mutated() + if self._filter is not None: + simplefilter(*self._filter) + if self._record: + log = [] + + def _record(message, category, filename, lineno, file=None, line=None): + log.append(WarningMessage(message, category, filename, lineno, file, line)) + + showwarning = _record + return log + return None + + def __exit__(self, *exc_info): + if not self._entered: + raise RuntimeError("Cannot exit %r without entering first" % self) + global showwarning + filters[:] = self._filters + showwarning = self._showwarning + onceregistry.clear() + onceregistry.update(self._onceregistry) + _filters_mutated() + + +class deprecated: + def __init__(self, message, /, *, category=DeprecationWarning, stacklevel=1): + if not isinstance(message, str): + raise TypeError("Expected an object of type str for 'message', not %r" % (type(message).__name__,)) + self.message = message + self.category = category + self.stacklevel = stacklevel + + def __call__(self, arg, /): + msg = self.message + category = self.category + if isinstance(arg, type): + original_init = arg.__init__ + + def __init__(self, *args, **kwargs): + warn(msg, category=category) + original_init(self, *args, **kwargs) + + arg.__init__ = __init__ + arg.__deprecated__ = msg + return arg + if callable(arg): + def wrapper(*args, **kwargs): + warn(msg, category=category) + return arg(*args, **kwargs) + + try: + wrapper.__name__ = arg.__name__ + wrapper.__doc__ = arg.__doc__ + except AttributeError: + pass + wrapper.__deprecated__ = msg + return wrapper + raise TypeError("@deprecated decorator with non-None category must be applied to a class or " + "callable, not %r" % (arg,)) + + +filters[:] = [("default", None, DeprecationWarning, re.compile("__main__"), 0), + ("ignore", None, DeprecationWarning, None, 0), + ("ignore", None, PendingDeprecationWarning, None, 0), + ("ignore", None, ImportWarning, None, 0), + ("ignore", None, ResourceWarning, None, 0)] diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/weakref.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/weakref.py new file mode 100644 index 00000000000..61486897575 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/weakref.py @@ -0,0 +1,495 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Weak references. + +A referent goes when the garbage collector runs, not when its last strong +reference is dropped as under CPython, so code must not expect a ref to +read None right after a del. Callbacks are not called (no port reports a +referent's death); the weak containers drop dead entries whenever they are +read instead, and finalize() runs its function only when called or at exit. +""" + +import atexit +from _weakref import ref, ReferenceType, referenceable as _referenceable + +__all__ = ["ref", "proxy", "getweakrefcount", "getweakrefs", "WeakKeyDictionary", "ReferenceType", + "ProxyType", "CallableProxyType", "ProxyTypes", "WeakValueDictionary", "WeakSet", "WeakMethod", + "finalize"] + + +def getweakrefcount(obj): + raise NotImplementedError("weakref.getweakrefcount is not available: references are not tracked per object") + + +def getweakrefs(obj): + raise NotImplementedError("weakref.getweakrefs is not available: references are not tracked per object") + + +class ProxyType: + __slots__ = ("_ref",) + + def __init__(self, obj, callback=None): + object.__setattr__(self, "_ref", ref(obj, callback)) + + def _target(self): + o = object.__getattribute__(self, "_ref")() + if o is None: + raise ReferenceError("weakly-referenced object no longer exists") + return o + + def __getattr__(self, name): + return getattr(ProxyType._target(self), name) + + def __setattr__(self, name, value): + setattr(ProxyType._target(self), name, value) + + def __delattr__(self, name): + delattr(ProxyType._target(self), name) + + def __repr__(self): + o = ProxyType._target(self) + return "" % (id(self), type(o).__name__, id(o)) + + def __str__(self): + return str(ProxyType._target(self)) + + def __bool__(self): + return bool(ProxyType._target(self)) + + def __len__(self): + return len(ProxyType._target(self)) + + def __iter__(self): + return iter(ProxyType._target(self)) + + def __getitem__(self, k): + return ProxyType._target(self)[k] + + def __setitem__(self, k, v): + ProxyType._target(self)[k] = v + + def __contains__(self, k): + return k in ProxyType._target(self) + + def __eq__(self, other): + return ProxyType._target(self) == other + + def __lt__(self, other): + return ProxyType._target(self) < other + + def __add__(self, other): + return ProxyType._target(self) + other + + def __hash__(self): + raise TypeError("unhashable type: 'weakref.ProxyType'") + + +class CallableProxyType(ProxyType): + __slots__ = () + + def __call__(self, *args, **kwargs): + return ProxyType._target(self)(*args, **kwargs) + + +ProxyTypes = (ProxyType, CallableProxyType) + + +def proxy(obj, callback=None): + if callable(obj): + return CallableProxyType(obj, callback) + return ProxyType(obj, callback) + + +class WeakMethod: + __slots__ = ("_self_ref", "_func", "_meth_type") + + def __init__(self, meth, callback=None): + try: + obj = meth.__self__ + func = meth.__func__ + except AttributeError: + raise TypeError("argument should be a bound method, not %s" % type(meth)) from None + self._self_ref = ref(obj) + self._func = ref(func) + self._meth_type = type(meth) + + def __call__(self): + obj = self._self_ref() + func = self._func() + if obj is None or func is None: + return None + return self._meth_type(func, obj) + + def __eq__(self, other): + if isinstance(other, WeakMethod): + a, b = self(), other() + if a is None or b is None: + return self is other + return a == b + return NotImplemented + + def __ne__(self, other): + r = self.__eq__(other) + return r if r is NotImplemented else not r + + def __hash__(self): + return hash((self._self_ref, self._func)) + + +class WeakValueDictionary: + def __init__(self, other=(), /, **kw): + self.data = {} + self.update(other, **kw) + + def _live(self): + dead = [k for k, r in self.data.items() if r() is None] + for k in dead: + del self.data[k] + return self.data + + def __getitem__(self, key): + o = self.data[key]() + if o is None: + del self.data[key] + raise KeyError(key) + return o + + def __delitem__(self, key): + del self.data[key] + + def __len__(self): + return len(self._live()) + + def __contains__(self, key): + try: + o = self.data[key]() + except KeyError: + return False + return o is not None + + def __repr__(self): + return "<%s at %#x>" % (self.__class__.__name__, id(self)) + + def __setitem__(self, key, value): + self.data[key] = ref(value) + + def copy(self): + new = WeakValueDictionary() + for key, r in self._live().items(): + o = r() + if o is not None: + new[key] = o + return new + + __copy__ = copy + + def get(self, key, default=None): + try: + r = self.data[key] + except KeyError: + return default + o = r() + return default if o is None else o + + def items(self): + for k, r in list(self._live().items()): + v = r() + if v is not None: + yield k, v + + def keys(self): + for k, v in self.items(): + yield k + + __iter__ = keys + + def itervaluerefs(self): + yield from list(self._live().values()) + + def valuerefs(self): + return list(self._live().values()) + + def values(self): + for k, v in self.items(): + yield v + + def popitem(self): + while True: + key, r = self.data.popitem() + o = r() + if o is not None: + return key, o + + def pop(self, key, *args): + try: + o = self.data.pop(key)() + except KeyError: + o = None + if o is None: + if args: + return args[0] + raise KeyError(key) + return o + + def setdefault(self, key, default=None): + try: + o = self.data[key]() + except KeyError: + o = None + if o is None: + self.data[key] = ref(default) + return default + return o + + def update(self, other=None, /, **kwargs): + d = self.data + if other is not None: + if not hasattr(other, "items"): + other = dict(other) + for key, o in other.items(): + d[key] = ref(o) + for key, o in kwargs.items(): + d[key] = ref(o) + + def __or__(self, other): + c = self.copy() + c.update(other) + return c + + def __ior__(self, other): + self.update(other) + return self + + +class WeakKeyDictionary: + def __init__(self, dict=None): + self.data = {} + if dict is not None: + self.update(dict) + + def _live(self): + dead = [k for k in self.data if k() is None] + for k in dead: + del self.data[k] + return self.data + + def __delitem__(self, key): + del self.data[ref(key)] + + def __getitem__(self, key): + return self.data[ref(key)] + + def __len__(self): + return len(self._live()) + + def __repr__(self): + return "<%s at %#x>" % (self.__class__.__name__, id(self)) + + def __setitem__(self, key, value): + self.data[ref(key)] = value + + def copy(self): + new = WeakKeyDictionary() + for key, value in self.items(): + new[key] = value + return new + + __copy__ = copy + + def get(self, key, default=None): + return self.data.get(ref(key), default) + + def __contains__(self, key): + try: + wr = ref(key) + except TypeError: + return False + return wr in self.data + + def items(self): + for wr, value in list(self._live().items()): + key = wr() + if key is not None: + yield key, value + + def keys(self): + for wr in list(self._live()): + obj = wr() + if obj is not None: + yield obj + + __iter__ = keys + + def values(self): + for wr, value in list(self._live().items()): + if wr() is not None: + yield value + + def keyrefs(self): + return list(self._live()) + + def popitem(self): + while True: + key, value = self.data.popitem() + o = key() + if o is not None: + return o, value + + def pop(self, key, *args): + return self.data.pop(ref(key), *args) + + def setdefault(self, key, default=None): + return self.data.setdefault(ref(key), default) + + def update(self, dict=None, /, **kwargs): + d = self.data + if dict is not None: + if not hasattr(dict, "items"): + dict = type({})(dict) + for key, value in dict.items(): + d[ref(key)] = value + if len(kwargs): + self.update(kwargs) + + +class WeakSet: + def __init__(self, data=None): + self.data = set() + if data is not None: + self.update(data) + + def _live(self): + dead = [r for r in self.data if r() is None] + for r in dead: + self.data.discard(r) + return self.data + + def __iter__(self): + for itemref in list(self._live()): + item = itemref() + if item is not None: + yield item + + def __len__(self): + return len(self._live()) + + def __contains__(self, item): + try: + wr = ref(item) + except TypeError: + return False + return wr in self.data + + def __repr__(self): + return "{%s(%r)}" % (type(self).__name__, list(self)) + + def add(self, item): + self.data.add(ref(item)) + + def clear(self): + self.data.clear() + + def copy(self): + return self.__class__(self) + + def pop(self): + while True: + try: + itemref = self.data.pop() + except KeyError: + raise KeyError("pop from empty WeakSet") from None + item = itemref() + if item is not None: + return item + + def remove(self, item): + self.data.remove(ref(item)) + + def discard(self, item): + self.data.discard(ref(item)) + + def update(self, other): + for element in other: + self.add(element) + + def __ior__(self, other): + self.update(other) + return self + + def difference(self, other): + return self.__class__(e for e in self if e not in other) + + __sub__ = difference + + def intersection(self, other): + return self.__class__(item for item in other if item in self) + + __and__ = intersection + + def union(self, other): + return self.__class__(list(self) + list(other)) + + __or__ = union + + def issubset(self, other): + return all(e in other for e in self) + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return NotImplemented + return set(self._live()) == set(other._live()) + + +class finalize: + __slots__ = ("_ref", "_func", "_args", "_kwargs", "_alive", "atexit") + _registry = [] + + def __init__(self, obj, func, /, *args, **kwargs): + self._ref = ref(obj) + self._func = func + self._args = args + self._kwargs = kwargs or {} + self._alive = True + self.atexit = True + finalize._registry.append(self) + + def __call__(self, _=None): + if self._alive: + self._alive = False + return self._func(*self._args, **self._kwargs) + return None + + def detach(self): + if self._alive: + obj = self._ref() + if obj is not None: + self._alive = False + return (obj, self._func, self._args, self._kwargs) + return None + + def peek(self): + if self._alive: + obj = self._ref() + if obj is not None: + return (obj, self._func, self._args, self._kwargs) + return None + + @property + def alive(self): + return self._alive + + def __repr__(self): + obj = self._ref() + if obj is None: + return "<%s object at %#x; dead>" % (type(self).__name__, id(self)) + return "<%s object at %#x; for %r at %#x>" % (type(self).__name__, id(self), type(obj).__name__, id(obj)) + + @staticmethod + def _exitfunc(): + for f in list(finalize._registry): + if f._alive and f.atexit: + f() + + +atexit.register(finalize._exitfunc) diff --git a/maven/python-transpiler/src/test/java/com/codename1/python/transpiler/ConformanceTest.java b/maven/python-transpiler/src/test/java/com/codename1/python/transpiler/ConformanceTest.java new file mode 100644 index 00000000000..4cc89b77428 --- /dev/null +++ b/maven/python-transpiler/src/test/java/com/codename1/python/transpiler/ConformanceTest.java @@ -0,0 +1,172 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.transpiler; + +import com.codename1.python.transpiler.api.PythonTranspiler; +import org.junit.jupiter.api.DynamicTest; +import org.junit.jupiter.api.TestFactory; + +import javax.tools.JavaCompiler; +import javax.tools.ToolProvider; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.PrintStream; +import java.lang.reflect.InvocationTargetException; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/// Transpiles each program under `conformance/`, compiles the Java against +/// the runtime, runs it and compares its output with what CPython printed +/// for the same program (the `.out` file beside it). A directory is a +/// multi-module program whose entry module is `main`. +/// +/// Every program runs in a class loader of its own that also holds the +/// runtime, so the module registry starts empty each time. +class ConformanceTest { + private static final Path ROOT = Paths.get("src/test/resources/conformance"); + + @TestFactory + List programs() throws IOException { + List tests = new ArrayList(); + try (Stream s = Files.list(ROOT)) { + for (Path p : s.sorted().collect(Collectors.toList())) { + String n = p.getFileName().toString(); + // -Dconformance.only=name runs that one program. + String only = System.getProperty("conformance.only"); + if (only != null && !only.isEmpty() && !n.equals(only) && !n.equals(only + ".py")) { + continue; + } + if (Files.isDirectory(p) || n.endsWith(".py")) { + final Path program = p; + tests.add(DynamicTest.dynamicTest(n, () -> run(program))); + } + } + } + return tests; + } + + private static void run(Path program) throws Exception { + String name = program.getFileName().toString(); + String stem = name.endsWith(".py") ? name.substring(0, name.length() - 3) : name; + Map sources = new TreeMap(); + if (Files.isDirectory(program)) { + try (Stream s = Files.walk(program)) { + for (Path p : s.filter(x -> x.toString().endsWith(".py")).collect(Collectors.toList())) { + sources.put(program.relativize(p).toString().replace(File.separatorChar, '/'), read(p)); + } + } + } else { + sources.put("main.py", read(program)); + } + Path work = Files.createTempDirectory("py-conf-" + stem); + Path gen = work.resolve("gen"); + Path cls = work.resolve("classes"); + Files.createDirectories(cls); + List args = new ArrayList(); + for (Map.Entry e : PythonTranspiler.transpile(sources, "main").entrySet()) { + Path f = gen.resolve(e.getKey()); + Files.createDirectories(f.getParent()); + Files.write(f, e.getValue().getBytes(StandardCharsets.US_ASCII)); + args.add(f.toString()); + } + String runtime = location(com.codename1.python.runtime.Py.class); + String core = location(com.codename1.io.Util.class); + args.add(0, "-nowarn"); + args.add(1, "--release"); + args.add(2, "17"); + args.add(3, "-cp"); + args.add(4, runtime + File.pathSeparator + core); + args.add(5, "-d"); + args.add(6, cls.toString()); + JavaCompiler javac = ToolProvider.getSystemJavaCompiler(); + ByteArrayOutputStream diagnostics = new ByteArrayOutputStream(); + int rc = javac.run(null, diagnostics, diagnostics, args.toArray(new String[0])); + assertEquals(0, rc, "generated Java did not compile:\n" + diagnostics.toString("UTF-8")); + + URL[] urls = {cls.toUri().toURL(), new File(runtime).toURI().toURL(), new File(core).toURI().toURL(), + new File(location(JvmFileSystem.class)).toURI().toURL()}; + Path fsRoot = work.resolve("fs"); + Files.createDirectories(fsRoot.resolve("home")); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + PrintStream saved = System.out; + String failure = null; + try (URLClassLoader loader = new URLClassLoader(urls, null)) { + System.setOut(new PrintStream(out, true, "UTF-8")); + try { + Class fsType = loader.loadClass("com.codename1.python.runtime.PyFileSystem"); + Object fs = loader.loadClass(JvmFileSystem.class.getName()).getConstructor(String.class) + .newInstance(fsRoot.toString()); + fsType.getMethod("install", fsType).invoke(null, fs); + loader.loadClass(PythonTranspiler.PACKAGE + ".PythonModules").getMethod("runMain").invoke(null); + } catch (InvocationTargetException e) { + failure = String.valueOf(e.getCause()); + if (!failure.startsWith("com.codename1.python.runtime.PyException")) { + // A Java failure: name the frames that recur, which is + // where a runaway recursion is. + StackTraceElement[] st = e.getCause().getStackTrace(); + Map seen = new TreeMap(); + for (int i = 0; i < Math.min(st.length, 400); i++) { + seen.merge(st[i].toString(), 1, Integer::sum); + } + StringBuilder b = new StringBuilder(failure); + for (Map.Entry en : seen.entrySet()) { + if (en.getValue() > 3 || st.length < 60) { + b.append("\n ").append(en.getValue()).append(" x ").append(en.getKey()); + } + } + failure = b.toString(); + } + } + } finally { + System.out.flush(); + System.setOut(saved); + } + String expected = read(ROOT.resolve(stem + ".out")); + String actual = out.toString("UTF-8"); + assertEquals(expected, actual, failure == null ? "output differs from CPython" : "raised " + failure); + assertTrue(failure == null, "raised " + failure); + } + + private static String location(Class c) throws Exception { + return new File(c.getProtectionDomain().getCodeSource().getLocation().toURI()).getPath(); + } + + private static String read(Path p) throws IOException { + return new String(Files.readAllBytes(p), StandardCharsets.UTF_8); + } +} diff --git a/maven/python-transpiler/src/test/java/com/codename1/python/transpiler/JvmFileSystem.java b/maven/python-transpiler/src/test/java/com/codename1/python/transpiler/JvmFileSystem.java new file mode 100644 index 00000000000..8d259535fbb --- /dev/null +++ b/maven/python-transpiler/src/test/java/com/codename1/python/transpiler/JvmFileSystem.java @@ -0,0 +1,118 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.transpiler; + +import com.codename1.python.runtime.PyFileSystem; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +/// The host file system under one root directory, for programs that +/// `open()` files in the conformance run. Loaded in each program's own +/// class loader, so it is public with a String constructor. +public final class JvmFileSystem extends PyFileSystem { + private final File root; + + public JvmFileSystem(String root) { + this.root = new File(root); + } + + private File f(String path) { + return new File(root, path); + } + + @Override + public String home() { + return "/home"; + } + + @Override + public InputStream openInput(String path) throws IOException { + return new FileInputStream(f(path)); + } + + @Override + public OutputStream openOutput(String path, boolean append) throws IOException { + return new FileOutputStream(f(path), append); + } + + @Override + public boolean exists(String path) { + return f(path).exists(); + } + + @Override + public boolean isDirectory(String path) { + return f(path).isDirectory(); + } + + @Override + public String[] list(String path) throws IOException { + String[] r = f(path).list(); + if (r == null) { + throw new IOException("not a directory"); + } + java.util.Arrays.sort(r); + return r; + } + + @Override + public void mkdir(String path) throws IOException { + if (!f(path).mkdir()) { + throw new IOException("mkdir failed"); + } + } + + @Override + public void delete(String path) throws IOException { + if (!f(path).delete()) { + throw new IOException("delete failed"); + } + } + + @Override + public void rename(String from, String to) throws IOException { + File t = f(to); + if (t.exists() && !t.isDirectory() && !t.delete()) { + throw new IOException("rename failed"); + } + if (!f(from).renameTo(t)) { + throw new IOException("rename failed"); + } + } + + @Override + public long size(String path) { + return f(path).length(); + } + + @Override + public long lastModified(String path) { + return f(path).lastModified(); + } +} diff --git a/maven/python-transpiler/src/test/java/com/codename1/python/transpiler/UnsupportedTest.java b/maven/python-transpiler/src/test/java/com/codename1/python/transpiler/UnsupportedTest.java new file mode 100644 index 00000000000..afeb499f3d7 --- /dev/null +++ b/maven/python-transpiler/src/test/java/com/codename1/python/transpiler/UnsupportedTest.java @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.transpiler; + +import com.codename1.python.transpiler.api.PythonSyntaxError; +import com.codename1.python.transpiler.api.PythonTranspiler; +import org.junit.jupiter.api.DynamicTest; +import org.junit.jupiter.api.TestFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/// What a translated program cannot do fails the build at the line that +/// does it, never silently or at run time on a device. +class UnsupportedTest { + /// Each case: the program, the line the error must name, and a fragment + /// of the reason. + private static final String[][] CASES = { + {"x = 1\nprint(eval('x + 1'))\n", "2", "eval()"}, + {"exec('print(1)')\n", "1", "exec()"}, + {"def f():\n return compile('1', 'f', 'eval')\n", "2", "compile()"}, + {"m = __import__('os')\n", "1", "__import__()"}, + {"run = eval\n", "1", "eval()"}, + {"import sys\n\ndef t(*a):\n pass\n\nsys.settrace(t)\n", "6", "sys.settrace"}, + {"import sys\nprint(sys._getframe().f_lineno)\n", "2", "sys._getframe"}, + {"def f():\n x = 1\n return locals()\n", "3", "locals()"}, + }; + + @TestFactory + List rejected() { + List tests = new ArrayList(); + for (final String[] c : CASES) { + tests.add(DynamicTest.dynamicTest(c[2] + " at line " + c[1], () -> { + Map sources = new TreeMap(); + sources.put("main.py", c[0]); + PythonSyntaxError e = assertThrows(PythonSyntaxError.class, + () -> PythonTranspiler.transpile(sources, "main")); + assertEquals("main.py", e.file); + assertEquals(Integer.parseInt(c[1]), e.line); + assertTrue(e.reason.contains(c[2]), e.getMessage()); + })); + } + return tests; + } + + /// A name the program binds itself is the program's own, not the builtin. + @TestFactory + List shadowed() { + List tests = new ArrayList(); + String[] programs = { + "def eval(s):\n return s\nprint(eval('x'))\n", + "import re\np = re.compile('a+')\n", + }; + for (final String p : programs) { + tests.add(DynamicTest.dynamicTest(p.substring(0, p.indexOf('\n')), () -> { + Map sources = new TreeMap(); + sources.put("main.py", p); + assertTrue(!PythonTranspiler.transpile(sources, "main").isEmpty()); + })); + } + return tests; + } +} diff --git a/maven/python-transpiler/src/test/resources/conformance/async_.out b/maven/python-transpiler/src/test/resources/conformance/async_.out new file mode 100644 index 00000000000..4dfab77a50f --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/async_.out @@ -0,0 +1,12 @@ +[1, 2, 3, 4, 5] +[1, 2, 3, 4, 5, 6, 7, 8] +[1, 3] [9, 7] +['aaa', 'bb'] ['a0', 'b0', 'a1', 'b1', 'a2'] +caught boom +[ValueError('boom'), 'c'] +sum 6 +timeout +event True +[1, 1, 2, 2] +cancelled True +42 diff --git a/maven/python-transpiler/src/test/resources/conformance/async_.py b/maven/python-transpiler/src/test/resources/conformance/async_.py new file mode 100644 index 00000000000..7b43f47622c --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/async_.py @@ -0,0 +1,76 @@ +import asyncio +import heapq + +h = [] +for x in [5, 1, 4, 2, 3]: + heapq.heappush(h, x) +print([heapq.heappop(h) for _ in range(5)]) +print(list(heapq.merge([1, 4, 7], [2, 5, 8], [3, 6]))) +print(heapq.nsmallest(2, [9, 3, 7, 1]), heapq.nlargest(2, [9, 3, 7, 1])) + +log = [] + +async def worker(name, n): + for i in range(n): + log.append(name + str(i)) + await asyncio.sleep(0) + return name * n + +async def fails(): + await asyncio.sleep(0) + raise ValueError("boom") + +async def producer(q): + for i in range(4): + await q.put(i) + await q.put(None) + +async def consumer(q): + total = 0 + while True: + v = await q.get() + if v is None: + return total + total += v + +async def main(): + r = await asyncio.gather(worker("a", 3), worker("b", 2)) + print(r, log) + try: + await asyncio.gather(fails()) + except ValueError as e: + print("caught", e) + print(await asyncio.gather(fails(), worker("c", 1), return_exceptions=True)) + q = asyncio.Queue(maxsize=2) + t = asyncio.create_task(producer(q)) + print("sum", await consumer(q)) + await t + try: + await asyncio.wait_for(asyncio.sleep(10), 0.01) + except TimeoutError: + print("timeout") + ev = asyncio.Event() + async def setter(): + await asyncio.sleep(0.001) + ev.set() + asyncio.create_task(setter()) + print("event", await ev.wait()) + lock = asyncio.Lock() + order = [] + async def crit(k): + async with lock: + order.append(k) + await asyncio.sleep(0) + order.append(k) + await asyncio.gather(crit(1), crit(2)) + print(order) + t2 = asyncio.create_task(asyncio.sleep(5)) + await asyncio.sleep(0) + t2.cancel() + try: + await t2 + except asyncio.CancelledError: + print("cancelled", t2.cancelled()) + return 42 + +print(asyncio.run(main())) diff --git a/maven/python-transpiler/src/test/resources/conformance/async_gen.out b/maven/python-transpiler/src/test/resources/conformance/async_gen.out new file mode 100644 index 00000000000..ab15d1779a7 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/async_gen.out @@ -0,0 +1,17 @@ +ticker closed 3 +[0, 1, 2] +ticker closed 4 +[0, 2, 4, 6] +ticker closed 3 +{0: 0, 1: 1, 2: 4} +ticker closed 5 +{0, 1} +[0, 1, 4, 9] +ticker closed 2 +async_generator [100, 101] +0 1 +ticker closed 10 +exhausted +None hi 5 +athrow x +function async_generator diff --git a/maven/python-transpiler/src/test/resources/conformance/async_gen.py b/maven/python-transpiler/src/test/resources/conformance/async_gen.py new file mode 100644 index 00000000000..541ebe185ff --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/async_gen.py @@ -0,0 +1,55 @@ +import asyncio + + +async def ticker(n): + try: + for i in range(n): + await asyncio.sleep(0) + yield i + finally: + print("ticker closed", n) + + +async def doubled(src): + async for v in src: + yield v * 2 + + +async def slow_sq(x): + await asyncio.sleep(0) + return x * x + + +async def main(): + out = [] + async for v in ticker(3): + out.append(v) + print(out) + print([v async for v in doubled(ticker(4))]) + print({v: await slow_sq(v) async for v in ticker(3)}) + print({v % 2 async for v in ticker(5) if v > 0}) + print([await slow_sq(x) for x in range(4)]) + g = (v + 100 async for v in ticker(2)) + print(type(g).__name__, [x async for x in g]) + agen = ticker(10) + print(await agen.__anext__(), await agen.__anext__()) + await agen.aclose() + try: + await agen.__anext__() + except StopAsyncIteration: + print("exhausted") + + async def echo(): + received = None + while True: + received = yield received + e = echo() + print(await e.asend(None), await e.asend("hi"), await e.asend(5)) + try: + await e.athrow(ValueError("x")) + except ValueError as ex: + print("athrow", ex) + print(type(ticker).__name__, type(ticker(1)).__name__) + + +asyncio.run(main()) diff --git a/maven/python-transpiler/src/test/resources/conformance/attrs_.out b/maven/python-transpiler/src/test/resources/conformance/attrs_.out new file mode 100644 index 00000000000..ea444e9c369 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/attrs_.out @@ -0,0 +1,15 @@ +[1, 2] ['m1', 'm2'] +3 m3 9 +shadowed m1 +m3 +replaced1 replaced3 +AttributeError 'P' object has no attribute 'x' +5 {'y': 9, 'x': 5} +q +prop prop +1 +base +base2 +kid base2 +15 +1 ['a', 'b'] 1 diff --git a/maven/python-transpiler/src/test/resources/conformance/attrs_.py b/maven/python-transpiler/src/test/resources/conformance/attrs_.py new file mode 100644 index 00000000000..9bfb6117097 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/attrs_.py @@ -0,0 +1,80 @@ +class P: + def __init__(self, x): + self.x = x + + def m(self): + return "m" + str(self.x) + + +def read(o): + return o.x + + +def call(o): + return o.m() + + +ps = [P(1), P(2)] +print([read(p) for p in ps], [call(p) for p in ps]) +q = P(3) +q.y = 9 +print(read(q), call(q), q.y) +q.m = lambda: "shadowed" +print(call(q), call(ps[0])) +del q.m +print(call(q)) +P.m = lambda self: "replaced" + str(self.x) +print(call(ps[0]), call(q)) +del q.x +try: + read(q) +except AttributeError as e: + print("AttributeError", e) +q.x = 5 +print(read(q), sorted(vars(q).items()) if False else q.__dict__) + + +class Q: + pass + + +r = Q() +r.x = "q" +print(read(r)) +P.x = property(lambda self: "prop") +print(read(ps[0]), read(ps[1])) +del P.x +print(read(ps[0])) + + +class Base: + def who(self): + return "base" + + +class Kid(Base): + pass + + +k = Kid() +print(call_who := k.who()) +Base.who = lambda self: "base2" +print(k.who()) +Kid.who = lambda self: "kid" +print(k.who(), Base().who()) + + +class Counter: + def __init__(self): + self.n = 0 + + def bump(self, by): + self.n += by + return self + + +c = Counter() +for i in range(5): + c.bump(i).bump(1) +print(c.n) +print([1, 2].index(2), "a,b".split(","), {"k": 1}.get("k")) diff --git a/maven/python-transpiler/src/test/resources/conformance/base64_.out b/maven/python-transpiler/src/test/resources/conformance/base64_.out new file mode 100644 index 00000000000..562903de796 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/base64_.out @@ -0,0 +1,23 @@ +b'' True b'' b'' b'' b'' b'' b'' +b'Zg==' True b'Zg==' b'MY======' b'CO======' b'66' b'W&' b'Ac' +b'Zm8=' True b'Zm8=' b'MZXQ====' b'CPNG====' b'666F' b'W^V' b'Ao@' +b'Zm9v' True b'Zm9v' b'MZXW6===' b'CPNMU===' b'666F6F' b'W^Zo' b'AoDS' +b'Zm9vYg==' True b'Zm9vYg==' b'MZXW6YQ=' b'CPNMUOG=' b'666F6F62' b'W^Zp|' b'AoDTs' +b'Zm9vYmE=' True b'Zm9vYmE=' b'MZXW6YTB' b'CPNMUOJ1' b'666F6F6261' b'W^Zp|VE' b'AoDTs@/' +b'Zm9vYmFy' True b'Zm9vYmFy' b'MZXW6YTBOI======' b'CPNMUOJ1E8======' b'666F6F626172' b'W^Zp|VR8' b'AoDTs@<)' +b'AAcOFRwjKjE4P0ZNVFtiaXB3foWMk5qhqK+2vcTL0tng5+71/A==' True b'AAcOFRwjKjE4P0ZNVFtiaXB3foWMk5qhqK-2vcTL0tng5-71_A==' b'AADQ4FI4EMVDCOB7IZGVIW3CNFYHO7UFRSJZVINIV63L3RGL2LM6BZ7O6X6A====' b'003GS58S4CL32E1V8P6L8MR2D5O7EVK5HI9PL8D8LURBRH6BQBCU1PVEUNU0====' b'00070E151C232A31383F464D545B626970777E858C939AA1A8AFB6BDC4CBD2D9E0E7EEF5FC' b'00#~g93v_*I6p>BR9j+caCd%%jFXz7sIRuY#LLpz;OFl3`~' b"!!`uK*$Zpf3'Tl,<*NgGE-HbbN0B^(W3' b'Hello World!' +b'HelloWorld' b'\x86O\xd2o\xb5Y\xf7[' +b'eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4\neHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eA==\n' b'xxx' b'foo' +Error: Invalid base64-encoded string: number of data characters (5) cannot be 1 more than a multiple of 4 +Error: Incorrect padding +Error: Invalid base64-encoded string: number of data characters (1) cannot be 1 more than a multiple of 4 +Only base64 data is allowed +a bytes-like object is required, not 'str' +Non-base16 digit found +b'01ab' b'01-ab-cd' b'\x01\xab' b'aGk=\n' +222957957 222957957 50018 +Odd-length string +True diff --git a/maven/python-transpiler/src/test/resources/conformance/base64_.py b/maven/python-transpiler/src/test/resources/conformance/base64_.py new file mode 100644 index 00000000000..5f4c034a1fd --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/base64_.py @@ -0,0 +1,44 @@ +# base64 and binascii. +import base64 +import binascii + +data = [b"", b"f", b"fo", b"foo", b"foob", b"fooba", b"foobar", bytes(range(256))[::7], b"\xff\xfe\xfd"] +for d in data: + e = base64.b64encode(d) + print(e, base64.b64decode(e) == d, base64.urlsafe_b64encode(d), base64.b32encode(d), base64.b32hexencode(d), + base64.b16encode(d), base64.b85encode(d), base64.a85encode(d)) + assert base64.b32decode(base64.b32encode(d)) == d + assert base64.b32hexdecode(base64.b32hexencode(d)) == d + assert base64.b16decode(base64.b16encode(d)) == d + assert base64.b85decode(base64.b85encode(d)) == d + assert base64.a85decode(base64.a85encode(d)) == d + assert base64.urlsafe_b64decode(base64.urlsafe_b64encode(d)) == d +print(base64.b64encode(b"\xfb\xff", altchars=b"-_"), base64.b64decode("+/8=", altchars="-_"), base64.b64decode("Zm9v\nYmFy")) +print(base64.b64decode(b"Zm9vYg"[:4]), base64.b85encode(b"abc", pad=True), base64.a85encode(b" ", foldspaces=True), + base64.a85encode(b"\0\0\0\0hello", adobe=True), base64.a85decode(b"<~87cURD]i,\"Ebo80~>", adobe=True)) +print(base64.z85encode(b"\x86\x4f\xd2\x6f\xb5\x59\xf7\x5b"), base64.z85decode(b"HelloWorld")) +print(base64.encodebytes(b"x" * 100), base64.decodebytes(b"eHh4\n"), base64.b32decode("mzxw6===", casefold=True)) +for bad in ["Zm9vY", "Zm9", "a===="]: + try: + base64.b64decode(bad) + except binascii.Error as e: + print("Error:", e) +try: + base64.b64decode("Zm9v!", validate=True) +except binascii.Error as e: + print(e) +try: + base64.b64encode("text") +except TypeError as e: + print(e) +try: + base64.b16decode("zz") +except binascii.Error as e: + print(e) +print(binascii.hexlify(b"\x01\xab"), binascii.hexlify(b"\x01\xab\xcd", "-"), binascii.unhexlify("01ab"), binascii.b2a_base64(b"hi")) +print(binascii.crc32(b"hello world"), binascii.crc32(b"world", binascii.crc32(b"hello ")), binascii.crc_hqx(b"hello", 0)) +try: + binascii.unhexlify("abc") +except binascii.Error as e: + print(e) +print(issubclass(binascii.Error, ValueError)) diff --git a/maven/python-transpiler/src/test/resources/conformance/basics.out b/maven/python-transpiler/src/test/resources/conformance/basics.out new file mode 100644 index 00000000000..ab185629c98 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/basics.out @@ -0,0 +1,20 @@ +3 3 -4 -2 1267650600228229401496703205376 2.5 +0.30000000000000004 1e+301 -0.0 3.0 1.5e-07 +2 None Ellipsis True +abcabcabc True [1, 2, 3] (1, 1, 1) +True False True True +x 2 [] None +11 é d dlr😀w olléh HÉLLO W😀RLD 'héllo w😀rld' +[1, 3, 5, 8] [8, 5, 3, 1] [3, 5] [1, 5] 1 8 17 +{'b': 2, 'a': 1, 'c': 3} ['b', 'a', 'c'] 0 True 3 +{'a': 1, 'c': 3} {'a': 2, 'c': 6} +{2, 3} [1, 2, 3] True +0 [1, 2, 3, 4] 5 +3.14 42 'x' 0xff 1,000,000 +a-5-2.500 1 two +297 1.5 12 False 3 +(3, 2) 2.67 4 0xff 0b101 A 65 +[(1, 'a'), (2, 'b')] [(1, 'x'), (2, 'y')] ['1', '2'] +True True [1, 2] [3, 2, 1] +0.5 14.0 1180591620717411303424 -1 -6 5 7 2 +True True float True diff --git a/maven/python-transpiler/src/test/resources/conformance/basics.py b/maven/python-transpiler/src/test/resources/conformance/basics.py new file mode 100644 index 00000000000..acfd1e63878 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/basics.py @@ -0,0 +1,28 @@ +# Core values, operators and builtins. +print(1 + 2, 7 // 2, -7 // 2, 7 % -3, 2 ** 100, 10 / 4) +print(0.1 + 0.2, 1e300 * 10, -0.0, 3.0, 1.5e-7) +print(True + True, None, ..., not 0) +print("abc" * 3, "a" < "b", [1, 2] + [3], (1,) * 3) +print(1 < 2 < 3, 3 > 2 > 2, 1 == 1.0, 2 is 2) +print(0 or "x", 1 and 2, [] or [], None and 1) +s = "héllo w\U0001f600rld" +print(len(s), s[1], s[-1], s[::-1], s.upper(), repr(s)) +x = [5, 3, 8, 1] +x.sort() +print(x, sorted(x, reverse=True), x[1:3], x[::2], min(x), max(x), sum(x)) +d = {"b": 2, "a": 1} +d["c"] = 3 +print(d, list(d), d.get("z", 0), "a" in d, len(d)) +del d["b"] +print(d, {k: v * 2 for k, v in d.items()}) +print({1, 2, 3} & {2, 3, 4}, sorted({3, 1, 2}), (1, 2) < (1, 3)) +a, *b, c = range(6) +print(a, b, c) +print(f"{3.14159:.2f} {42:>6} {'x'!r} {255:#x} {1000000:,}") +print("%s-%d-%.3f" % ("a", 5, 2.5), "{} {}".format(1, "two")) +print(int("42") + int("ff", 16), float("1.5"), str(12), bool(""), abs(-3)) +print(divmod(17, 5), round(2.675, 2), round(3.5), hex(255), bin(5), chr(65), ord("A")) +print(list(zip([1, 2], "ab")), list(enumerate("xy", 1)), list(map(str, [1, 2]))) +print(any([0, 1]), all([]), list(filter(None, [0, 1, 2])), list(reversed([1, 2, 3]))) +print(2 ** -1, 7 // 0.5, 1 << 70, -1 >> 3, ~5, 6 ^ 3, 6 | 3, 6 & 3) +print(isinstance(3, int), isinstance(True, int), type(3.0).__name__, type("") is str) diff --git a/maven/python-transpiler/src/test/resources/conformance/builtin_subclass.out b/maven/python-transpiler/src/test/resources/conformance/builtin_subclass.out new file mode 100644 index 00000000000..1b651e479df --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/builtin_subclass.out @@ -0,0 +1,12 @@ +[1, 2, 3, 4] 4 4 True Stack +Stack [1, 2, 3, 4, 5] 15 [5, 4, 3, 2, 1] [1, 2, 3, 4, 5] [2, 3] +1 3 {'a': 1, 'xyz': 3} {'a': 1, 'xyz': 3} [('a', 1), ('xyz', 3)] True +set k v + V 1 +100.0 212.0 101.0 True True 100.0 +red RED! True reddish 3 [Color('red')] True 1 +6 True False 7 12 0x6 6 6 +(3, 4) 3 4 2 True True +3 4 +['a', 'b'] 2 True ['a', 'b', 'c'] +TypeError type 'bool' is not an acceptable base type diff --git a/maven/python-transpiler/src/test/resources/conformance/builtin_subclass.py b/maven/python-transpiler/src/test/resources/conformance/builtin_subclass.py new file mode 100644 index 00000000000..136d62524cb --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/builtin_subclass.py @@ -0,0 +1,103 @@ +class Stack(list): + def push(self, x): + self.append(x) + return self + + def peek(self): + return self[-1] + + +s = Stack([1, 2]) +s.push(3).push(4) +print(s, len(s), s.peek(), isinstance(s, list), type(s).__name__) +s += [5] +print(type(s).__name__, s, sum(s), sorted(s, reverse=True), list(s), s[1:3]) + + +class Default(dict): + def __missing__(self, key): + v = self[key] = len(key) + return v + + +d = Default(a=1) +print(d["a"], d["xyz"], d, dict(d), sorted(d.items()), "xyz" in d) + + +class Logged(dict): + def __setitem__(self, key, value): + print("set", key, value) + super().__setitem__(key, value.upper()) + + def __getitem__(self, key): + return "<" + super().__getitem__(key) + ">" + + +lg = Logged() +lg["k"] = "v" +print(lg["k"], lg.get("k"), len(lg)) + + +class Celsius(float): + def __new__(cls, v): + return super().__new__(cls, v) + + def to_f(self): + return self * 9 / 5 + 32 + + +c = Celsius(100) +print(c, c.to_f(), c + 1, c > 50, isinstance(c, float), repr(c)) + + +class Color(str): + def shout(self): + return self.upper() + "!" + + def __repr__(self): + return "Color(" + str.__repr__(self) + ")" + + +r = Color("red") +print(r, r.shout(), r == "red", r + "dish", len(r), [r], r.startswith("re"), {r: 1}["red"]) + + +class Flags(int): + def has(self, bit): + return bool(self & bit) + + +f = Flags(6) +print(f, f.has(2), f.has(1), f + 1, f * 2, hex(f), int(f), [0, 1, 2, 3, 4, 5, 6, 7][f]) + + +class Point(tuple): + def __new__(cls, x, y): + return super().__new__(cls, (x, y)) + + @property + def x(self): + return self[0] + + +p = Point(3, 4) +print(p, p.x, p[1], len(p), p == (3, 4), hash(p) == hash((3, 4))) +x, y = p +print(x, y) + + +class Tags(set): + def add_all(self, *xs): + for x in xs: + self.add(x) + + +t = Tags() +t.add_all("a", "b", "a") +print(sorted(t), len(t), "a" in t, sorted(t | {"c"})) + +try: + class Bad(bool): + pass +except TypeError as e: + print("TypeError", e) diff --git a/maven/python-transpiler/src/test/resources/conformance/bytearray_.out b/maven/python-transpiler/src/test/resources/conformance/bytearray_.out new file mode 100644 index 00000000000..ebb356fb8b1 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/bytearray_.out @@ -0,0 +1,19 @@ +bytearray(b'hello') 5 104 bytearray(b'el') 111 [104, 101] bytearray(b'') bytearray(b'\x00\x00\x00') bytearray(b'AB') bytearray(b'hi') +bytearray(b'hello!??\x01') 2 62 bytearray(b'hello!??\x01') +bytearray(b'HEYlo!??!') bytearray(b'HEYLO!??!') 3 1 bytearray(b'HEYLo!??!') [bytearray(b'HEY'), bytearray(b'o!??!')] HEYlo!??! +True True b'HEYlo!??!' b'xHEYlo!??!' bytearray(b'HEYlo!??!x') bytearray(b'HEYlo!??!HEYlo!??!') True True 4845596c6f213f3f21 +bytearray(b'!??!olYEH') False bytearray(b'\xde\xad\xbe\xef') bytearray(b'olYEH') True +bytearray(b'') False True b'xyz' b'a-b' +bytearray(b'XdY') True bytearray +3 88 b'dY' [88, 100, 89] False 3 B 1 False b'XdY' +bytearray(b'Zzz') True [90, 122] 5a7a7a +b'abc' True +ValueError operation forbidden on released memoryview object +TypeError unhashable type: 'bytearray' +TypeError string argument without an encoding +ValueError byte must be in range(0, 256) +IndexError pop from empty bytearray +ValueError value not found in bytearray +TypeError cannot modify read-only memory +TypeError memoryview: a bytes-like object is required, not 'str' +IndexError bytearray index out of range diff --git a/maven/python-transpiler/src/test/resources/conformance/bytearray_.py b/maven/python-transpiler/src/test/resources/conformance/bytearray_.py new file mode 100644 index 00000000000..9ee79e889b1 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/bytearray_.py @@ -0,0 +1,40 @@ +b = bytearray(b"hello") +print(b, len(b), b[0], b[1:3], b[-1], list(b)[:2], bytearray(), bytearray(3), bytearray([65, 66]), bytearray("hi", "utf-8")) +b.append(33) +b.extend(b"??") +b.extend([1, 2]) +b.insert(0, 62) +print(b, b.pop(), b.pop(0), b) +b[0] = 72 +b[1:3] = b"EY" +del b[-1] +b += b"!" +print(b, b.upper(), b.find(b"l"), b.count(b"l"), b.replace(b"l", b"L"), b.split(b"l"), b.decode("ascii", "replace")) +print(b == b"HEYlo!??!", b"HEYlo!??!" == b, bytes(b), b"x" + b, b + b"x", b * 2, b"l" in b, 108 in b, b.hex()) +b.reverse() +print(b, b.copy() is b, bytearray.fromhex("de ad be ef"), b.strip(b"!?"), b.startswith(b"!")) +b.remove(ord("Y")) +b.clear() +print(b, bool(b), bytearray(b"a") < bytearray(b"b"), bytes(bytearray(b"xyz")), b"-".join([bytearray(b"a"), b"b"])) +c = bytearray(b"abcdef") +del c[::2] +c[::2] = b"XY" +print(c, isinstance(c, bytearray), type(c).__name__) +m = memoryview(c) +print(len(m), m[0], m[1:].tobytes(), m.tolist(), m.readonly, m.nbytes, m.format, m.itemsize, m == b"XdYf", bytes(m)) +m[0] = 90 +m[1:3] = b"zz" +print(c, m.obj is c, list(m[::2]), m.hex()) +with memoryview(b"abc") as v: + print(v.tobytes(), v.readonly) +try: + v.tobytes() +except ValueError as e: + print("ValueError", e) +for bad in [lambda: hash(bytearray()), lambda: bytearray("x"), lambda: bytearray(b"a").append(256), + lambda: bytearray().pop(), lambda: bytearray(b"a").remove(9), lambda: memoryview(b"a").__setitem__(0, 1), + lambda: memoryview("s"), lambda: bytearray(b"a")[5]]: + try: + bad() + except (TypeError, ValueError, IndexError) as e: + print(type(e).__name__, e) diff --git a/maven/python-transpiler/src/test/resources/conformance/classes.out b/maven/python-transpiler/src/test/resources/conformance/classes.out new file mode 100644 index 00000000000..f22767714d7 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/classes.out @@ -0,0 +1,13 @@ +[Rect('rect', area=6.0), Square('square', area=16.0), Square('square', area=1.0), Square('square', area=4.0)] 4 A shape. +5 25 True True +Vec(4, 6) Vec(8, 12) Vec(12, 18) True 2 6 [4, 6] {Vec(4, 6)} [Vec(1, 5), Vec(2, 0)] +enter a +enter b +body a +exit b None +exit a None +enter c +exit c KeyError +after +MyError code 7 7 True +2 1 True diff --git a/maven/python-transpiler/src/test/resources/conformance/classes.py b/maven/python-transpiler/src/test/resources/conformance/classes.py new file mode 100644 index 00000000000..5bf4414a8a8 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/classes.py @@ -0,0 +1,110 @@ +# Classes, inheritance, properties, dunders and context managers. +class Shape: + """A shape.""" + count = 0 + + def __init__(self, name): + self.name = name + Shape.count += 1 + + def area(self): + raise NotImplementedError + + def __repr__(self): + return f"{type(self).__name__}({self.name!r}, area={self.area():.1f})" + +class Rect(Shape): + def __init__(self, w, h): + super().__init__("rect") + self.w, self.h = w, h + + def area(self): + return self.w * self.h + +class Square(Rect): + def __init__(self, s): + super().__init__(s, s) + self.name = "square" + + @property + def side(self): + return self.w + + @side.setter + def side(self, v): + self.w = self.h = v + + @staticmethod + def unit(): + return Square(1) + + @classmethod + def of(cls, s): + return cls(s) + +shapes = [Rect(2, 3), Square(4), Square.unit(), Square.of(2)] +print(shapes, Shape.count, Shape.__doc__) +sq = shapes[1] +sq.side = 5 +print(sq.side, sq.area(), isinstance(sq, Rect), issubclass(Square, Shape), Square.__mro__ if False else "") + +class Vec: + def __init__(self, x, y): + self.x, self.y = x, y + def __add__(self, o): + return Vec(self.x + o.x, self.y + o.y) + def __mul__(self, k): + return Vec(self.x * k, self.y * k) + __rmul__ = __mul__ + def __eq__(self, o): + return isinstance(o, Vec) and (self.x, self.y) == (o.x, o.y) + def __hash__(self): + return hash((self.x, self.y)) + def __len__(self): + return 2 + def __getitem__(self, i): + return (self.x, self.y)[i] + def __iter__(self): + return iter((self.x, self.y)) + def __lt__(self, o): + return (self.x, self.y) < (o.x, o.y) + def __repr__(self): + return "Vec(%r, %r)" % (self.x, self.y) + +v = Vec(1, 2) + Vec(3, 4) +print(v, v * 2, 3 * v, v == Vec(4, 6), len(v), v[1], list(v), {v, Vec(4, 6)}, sorted([Vec(2, 0), Vec(1, 5)])) + +class Ctx: + def __init__(self, tag): + self.tag = tag + def __enter__(self): + print("enter", self.tag) + return self + def __exit__(self, t, v, tb): + print("exit", self.tag, t.__name__ if t else None) + return t is KeyError + +with Ctx("a") as a, Ctx("b"): + print("body", a.tag) +with Ctx("c"): + raise KeyError("swallowed") +print("after") + +class MyError(Exception): + def __init__(self, code): + super().__init__(f"code {code}") + self.code = code + +try: + raise MyError(7) +except Exception as e: + print(type(e).__name__, e, e.code, isinstance(e, MyError)) + +class Node: + def __init__(self, val, nxt=None): + self.val, self.nxt = val, nxt + def __getattr__(self, name): + return f"" + +n = Node(1, Node(2)) +print(n.nxt.val, n.color, getattr(n, "val"), hasattr(n, "nxt")) diff --git a/maven/python-transpiler/src/test/resources/conformance/collections_.out b/maven/python-transpiler/src/test/resources/conformance/collections_.out new file mode 100644 index 00000000000..49d9d231135 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/collections_.out @@ -0,0 +1,29 @@ +[0, 3, 1, 4, 1, 5, 9, 2, 6, 5, 3, 5] +5 0 5 2 [3, 1, 4, 1, 5, 9, 2, 6, 5, 3] +[3, 5, 6, 2, 9, 5, 1, 4, 3] +[9, 6, 3, 3, 4, 1, 5, 5, 2] [6, 2, 9, 5, 1] [3, 5, 6] [] [3, 5] +[3, 10, 20, 30, 2, 9, 5, 1, 4, 3] +[10, 30, 9, 1, 3] +False 5 [] [] [[0, 0], [0, 0]] +['fig', 'pear', 'apple', 'banana'] ['pear', 'fig', 'banana', 'apple'] fig +{'a': 1, 'c': 3, 'd': 4} 2 None ['a', 'c', 'd', 'e'] [1, 3, 4, 5] ('e', 5) +{'a': 1, 'c': 3, 'd': 4, 'x': 0} {'a': 1, 'b': 2} {'x': 0, 'y': 0} [('d', 4), ('c', 3), ('a', 1)] +{'d': 4} {1: 'c'} +{2, 3, 4} {9, 2, 3, 4} {3, 4} {2, 4, 7} True True True True +2 2 (1, 2, 3, 2, 4) (2, 3, 2, 1) True True ('a', 'b') +[[0, 0, 0, 0], [0, 1, 2, 3], [0, 2, 4, 6]] [0, 1, 2] [(0, 0, 0), (0, 1, 2), (0, 2, 4), (0, 3, 6)] [0, 0, 0, 0, 0, 1, 2, 3, 0, 2, 4, 6] +[10, 7, 4, 1] 4 15 True range(2, 6, 2) +3 {'k': [1, {'z': (2, 3)}]} [None, True, 1.0, 's'] +True [1] True False True True +[4, 10, 18] [(0, 'a'), (1, 'b')] 1.0 +[30, 40, 50] +[40, 50] +[50] +[10, 30, 50] 3 +[9.0, 3.5] [1.5, 2.5, 3.5] +[[0, 20, 30, 40, 50], [1, 20, 30, 40, 50], [2, 20, 30, 40, 50], [3, 20, 30, 40, 50]] +[[1, 2, 3]] [-1, 2, 3] +['0', '0-1', '0-1-2', '0-1-2-3'] +[7, 8, 9] [0, 2, 3] [5] +[('a', 2), ('b', 1), ('c', 1), (1, 2)] [('a', 101), ('b', 101), ('c', 101), (1, 101)] [1, 1] [2, 4] {'k': 'xy'} +-1 diff --git a/maven/python-transpiler/src/test/resources/conformance/collections_.py b/maven/python-transpiler/src/test/resources/conformance/collections_.py new file mode 100644 index 00000000000..fa2742ebdbf --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/collections_.py @@ -0,0 +1,155 @@ +# list, dict, set, tuple behaviour. +a = [3, 1, 4, 1, 5, 9, 2, 6] +a.append(5); a.extend([3, 5]); a.insert(0, 0); print(a) +print(a.pop(), a.pop(0), a.index(9), a.count(5), a) +a.remove(1); a.reverse(); print(a) +print(sorted(a, key=lambda v: (v % 3, -v)), a[2:-2], a[::-3], a[100:], a[-100:2]) +a[1:3] = [10, 20, 30]; print(a) +del a[::2]; print(a) +b = a.copy(); b.clear(); print(a == b, len(a), b, a * 0, [[0] * 2] * 2) +words = ["pear", "apple", "fig", "banana"] +print(sorted(words, key=len), sorted(words, key=str.upper, reverse=True), min(words, key=len)) +d = dict(a=1, b=2) +d.update({"c": 3}, d=4); d.setdefault("e", 5); d.setdefault("a", 100) +print(d, d.pop("b"), d.pop("zz", None), list(d.keys()), list(d.values()), d.popitem()) +print({**d, "x": 0}, dict(zip("ab", [1, 2])), dict.fromkeys("xy", 0), sorted(d.items(), key=lambda kv: -kv[1])) +for k in list(d): + if d[k] % 2: + del d[k] +print(d, {1: "a", True: "b", 1.0: "c"}) +s = {1, 2, 3} +s.add(4); s.discard(10); s.remove(1) +print(s, s | {9}, s - {2}, s ^ {3, 7}, s <= {2, 3, 4, 5}, {2, 3} < s, s.isdisjoint({8}), frozenset([1, 2]) == {1, 2}) +t = (1, 2, 3, 2) +print(t.count(2), t.index(3), t + (4,), t[::-1], (1, [2]) == (1, [2]), hash((1, 2)) == hash((1, 2)), tuple("ab")) +m = [[i * j for j in range(4)] for i in range(3)] +print(m, [row[1] for row in m], list(zip(*m)), sum(m, [])) +print(list(range(10, 0, -3)), range(5)[-1], len(range(0, 100, 7)), 3 in range(5), range(0, 10, 2)[1:3]) +nested = {"k": [1, {"z": (2, 3)}]} +print(nested["k"][1]["z"][1], repr(nested), str([None, True, 1.0, "s"])) +x = y = [] +x.append(1) +print(x is y, y, [1, 2] == [1, 2], [1, 2] is [1, 2], [] == [], not []) +print(list(map(lambda p, q: p * q, [1, 2, 3], [4, 5, 6])), list(enumerate(["a", "b"])), sum([0.1] * 10)) + + +# A local re-sliced in a loop: its list is reused, which must stay invisible. +def recycle(src): + out = [] + perm = None + for k in range(4): + perm = src[:] + perm[0] = k + out.append(perm[:]) + shorter = src[1:] + for k in range(3): + shorter = shorter[1:] + print(shorter[:]) + big = src[:] + big = src[::2] + print(big[:], len(big[:])) + floats = [1.5, 2.5, 3.5] + f = floats[:] + f = floats[1:] + f[0] = 9.0 + print(f[:], floats) + return out + + +print(recycle([10, 20, 30, 40, 50])) + + +def leaks(src): + keep = [] + p = src[:] + keep.append(p) + p = src[:] + p[0] = -1 + print(keep, p[:]) + + +leaks([1, 2, 3]) + + +def rows(n): + parts = [] + for i in range(n): + row = [] + for j in range(i + 1): + row.append(str(j)) + parts.append("-".join(row)) + row[0] = "x" + return parts + + +print(rows(4)) + +SHARED = [7, 8, 9] + + +class Lender: + def __getitem__(self, k): + return SHARED + + +def borrowed(): + p = Lender()[:] + p = [1, 2, 3][:] + p[0] = 0 + q = Lender()[1:] + q = [] + q.append(5) + print(SHARED, p[:], q[:]) + + +borrowed() + + +def fused_counts(): + class Loud: + def __init__(self): + self.d = {} + + def get(self, k, d=None): + return 100 + + def __setitem__(self, k, v): + self.d[k] = v + + def items(self): + return self.d.items() + words = ["a", "b", "a", 1, 1.0, "c"] + plain = {} + loud = Loud() + for w in words: + plain[w] = plain.get(w, 0) + 1 + loud[w] = loud.get(w, 0) + 1 + t = [] + for w in "xy": + t = t + t.append(len(w)) + + class Bag: + def __init__(self): + self.items = [] + + def append(self, x): + self.items.append(x * 2) + b = Bag() + for w in [1, 2]: + b.append(w) + acc = {"k": "x"} + k = "k" + acc[k] = acc.get(k, "") + "y" + print(sorted(plain.items(), key=str), sorted(loud.items(), key=str), t, b.items, acc) + + +def shadowed_len(): + def len(x): + return -1 + w = "abc" + print(len(w)) + + +fused_counts() +shadowed_len() diff --git a/maven/python-transpiler/src/test/resources/conformance/complex_.out b/maven/python-transpiler/src/test/resources/conformance/complex_.out new file mode 100644 index 00000000000..f8eef07336e --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/complex_.out @@ -0,0 +1,15 @@ +(3+4j) (1-2j) 1j (-0-1j) 0j 2.5j 0j (2+0j) -0j (-0+0j) +(4+2j) (2+6j) (11-2j) (-1+2j) (-7+24j) (0.12-0.16j) (2+1j) +[(-7.461496615+2.885492726j), (-0.222517157-0.100709131j)] +5.0 3.0 4.0 (3-4j) (-3-4j) (3+4j) True True True True +(4+4j) (7.5+10j) (6+8j) (0.12-0.16j) (2+4j) (7-4j) False True complex True +(1+2j) (3-4j) 5j -1j (2.5+0j) (1000-0.01j) +True 1000003 1000004 1 +(inf+nanj) (1e+300+1e+300j) True +[0j, (1+1j), (2+2j)] (5.5+1j) (-3+5j) +ZeroDivisionError division by zero +TypeError '<' not supported between instances of 'complex' and 'complex' +ValueError complex() arg is a malformed string +TypeError complex() argument 'real' must be a real number, not str +ZeroDivisionError zero to a negative or complex power +TypeError complex() argument must be a string or a number, not list diff --git a/maven/python-transpiler/src/test/resources/conformance/complex_.py b/maven/python-transpiler/src/test/resources/conformance/complex_.py new file mode 100644 index 00000000000..63394bbb2ae --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/complex_.py @@ -0,0 +1,17 @@ +z = 3 + 4j +w = complex(1, -2) +print(z, w, 1j, -1j, 0j, 2.5j, complex(), complex(2), complex(0, -0.0), complex(-0.0, 0)) +print(z + w, z - w, z * w, z / w, z ** 2, z ** -1, z ** 0.5) +print([round(v.real, 9) + round(v.imag, 9) * 1j for v in (2 ** z, w ** w)]) +print(abs(z), z.real, z.imag, z.conjugate(), -z, +z, z == complex(3, 4), z != 3, complex(3) == 3, complex(3) == 3.0) +print(1 + z, 2.5 * z, z * 2, 1 / z, z - 1, 10 - z, bool(0j), bool(1j), type(z).__name__, isinstance(z, complex)) +print(complex("1+2j"), complex(" (3-4j) "), complex("5j"), complex("-j"), complex("2.5"), complex("1e3-1e-2j")) +print(hash(complex(3, 0)) == hash(3), hash(1j), hash(complex(1, 1)), {z: 1}[complex(3, 4)]) +print(repr(complex(float("inf"), float("nan"))), complex(1e300, 1e300) * 1, z == z.conjugate().conjugate()) +print([complex(i, i) for i in range(3)], sum([1j, 2, 3.5]), complex(complex(1, 2), complex(3, 4))) +for bad in [lambda: z / 0, lambda: z < w, lambda: complex("1+"), lambda: complex("1", 2), lambda: 0j ** -1, + lambda: complex([])]: + try: + print(bad()) + except (ZeroDivisionError, TypeError, ValueError) as e: + print(type(e).__name__, e) diff --git a/maven/python-transpiler/src/test/resources/conformance/control.out b/maven/python-transpiler/src/test/resources/conformance/control.out new file mode 100644 index 00000000000..e360e60e71c --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/control.out @@ -0,0 +1,20 @@ +[0, 1, 1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233, 377] 354224848179261915075 +7 +(1, 2, (), 3, 4, []) (1, 2, (3, 4), 5, 4, [('e', 6), ('f', 7)]) (1, 2, (), 9, 4, []) +30 12 7 +12 +while-else 3 +for-else on empty +finally 0 +finally 5 +value:zero ok:2 +index list index out of range +key KeyError('k') +re-raised ZeroDivisionError division by zero +name 'undefined_name' is not defined +3 +{0: 0, 1: 1, 3: 9, 4: 16} [1, 2, 3] True +285 (3, 'bbb') +walrus 6 +[0, 1, 2] 2 +assert boom diff --git a/maven/python-transpiler/src/test/resources/conformance/control.py b/maven/python-transpiler/src/test/resources/conformance/control.py new file mode 100644 index 00000000000..fad158856fe --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/control.py @@ -0,0 +1,104 @@ +# Control flow, functions, closures and exceptions. +def fib(n): + a, b = 0, 1 + for _ in range(n): + a, b = b, a + b + return a + +print([fib(i) for i in range(15)], fib(100)) + +def counter(): + count = 0 + def inc(by=1): + nonlocal count + count += by + return count + return inc + +c = counter() +c(); c(5) +print(c()) + +def kw(a, b=2, *args, c, d=4, **kw): + return (a, b, args, c, d, sorted(kw.items())) + +print(kw(1, c=3), kw(1, 2, 3, 4, c=5, e=6, f=7), kw(*[1, 2], **{"c": 9})) +f = lambda x, y=10: x * y +print(f(3), f(3, 4), (lambda: 7)()) + +total = 0 +for i in range(10): + if i % 2: + continue + if i > 6: + break + total += i +else: + total = -1 +print(total) + +n = 0 +while n < 3: + n += 1 +else: + print("while-else", n) + +for i in []: + pass +else: + print("for-else on empty") + +def risky(x): + try: + if x == 0: + raise ValueError("zero") + r = 10 // x + except ValueError as e: + return "value:" + str(e) + except ZeroDivisionError: + return "zdiv" + else: + return "ok:" + str(r) + finally: + print("finally", x) + +print(risky(0), risky(5)) +try: + [][1] +except IndexError as e: + print("index", e) +try: + {}["k"] +except KeyError as e: + print("key", repr(e)) +try: + try: + 1 / 0 + except ZeroDivisionError: + raise +except ArithmeticError as e: + print("re-raised", type(e).__name__, e) +try: + undefined_name +except NameError as e: + print(e) + +def g(): + x = 1 + def h(): + return x + y + y = 2 + return h() +print(g()) + +squares = {x: x * x for x in range(5) if x != 2} +print(squares, [y for x in [[1, 2], [3]] for y in x], {c for c in "hello"} == set("helo")) +print(sum(i * i for i in range(10)), max((len(w), w) for w in ["a", "bbb", "cc"])) +if (m := len("walrus")) > 3: + print("walrus", m) +print([last := v for v in range(3)], last) +assert 1 + 1 == 2, "math" +try: + assert False, "boom" +except AssertionError as e: + print("assert", e) diff --git a/maven/python-transpiler/src/test/resources/conformance/csv_.out b/maven/python-transpiler/src/test/resources/conformance/csv_.out new file mode 100644 index 00000000000..c711f9a6e75 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/csv_.out @@ -0,0 +1,26 @@ +[['name', 'age', 'note'], ['Ada', '36', 'likes "math", engines'], ['Alan', '41', 'multi\nline'], ['', '', '']] +[['a', 'b', 'c;d'], ['e', 'f', 'g']] + [[1.0, 2.5, 3.0]] +[['a,b', 'c']] [['a', 'b"c']] +[['x', 'y'], [], ['z']] [['unterminated']] +Error ',' expected after '"' +[['a\x00b']] +'plain,"with,comma","with ""quote""","multi\nline",3,2.5,,\r\n1,2\r\na b, lead\r\n""\r\n' +1 '"s","1","2.0","",""\n' +2 '"s",1,2.0,"",""\n' +4 '"s",1,2.0,,""\n' +5 '"s","1","2.0",,""\n' +'a\\,b,q\\",c\n' +Error need to escape, but no escapechar set +'id,name\n1,x\n2,\n,only\n' +dict contains fields not in fieldnames: 'bogus' +['id', 'name'] [{'id': '1', 'name': 'a'}, {'id': '2', 'name': 'b', 'rest': ['extra']}, {'id': '3', 'name': '?'}] 4 +['excel', 'excel-tab', 'pipes', 'unix'] | [['a', 'b']] +unknown dialect +TypeError "delimiter" must be a unicode character, not a string of length 2 +TypeError "quotechar" must be a unicode character or None, not int +TypeError bad "quoting" value +ValueError bad delimiter value +; " True True +1 , 131072 131072 10 +field larger than field limit (10) diff --git a/maven/python-transpiler/src/test/resources/conformance/csv_.py b/maven/python-transpiler/src/test/resources/conformance/csv_.py new file mode 100644 index 00000000000..078df880db9 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/csv_.py @@ -0,0 +1,67 @@ +# csv reading, writing, dialects and the sniffer. +import csv +from io import StringIO + +text = 'name,age,note\r\nAda,36,"likes ""math"", engines"\r\nAlan,41,"multi\nline"\r\n"",,\r\n' +print(list(csv.reader(StringIO(text, newline="")))) +print(list(csv.reader(['a;b;"c;d"', "e; f;g"], delimiter=";", skipinitialspace=True))) +print(list(csv.reader(['1,"2",x'], quoting=csv.QUOTE_NONNUMERIC)) if False else "", list(csv.reader(["1,2.5,3"], quoting=csv.QUOTE_NONNUMERIC))) +print(list(csv.reader(['a\\,b,c'], escapechar="\\")), list(csv.reader(['a,"b\\"c"'], escapechar="\\", doublequote=False))) +print(list(csv.reader(["x,y", "", "z"])), list(csv.reader(['"unterminated']))) +for bad in [['a,"b"c'], ["a\0b"]]: + try: + print(list(csv.reader(bad, strict=True))) + except csv.Error as e: + print("Error", e) +out = StringIO() +w = csv.writer(out) +w.writerow(["plain", "with,comma", 'with "quote"', "multi\nline", 3, 2.5, None, ""]) +w.writerows([[1, 2], ["a b", " lead"]]) +w.writerow([""]) +print(repr(out.getvalue())) +for q in [csv.QUOTE_ALL, csv.QUOTE_NONNUMERIC, csv.QUOTE_STRINGS, csv.QUOTE_NOTNULL]: + o = StringIO() + csv.writer(o, quoting=q, lineterminator="\n").writerow(["s", 1, 2.0, None, ""]) + print(q, repr(o.getvalue())) +o = StringIO() +csv.writer(o, quoting=csv.QUOTE_NONE, escapechar="\\", lineterminator="\n").writerow(["a,b", 'q"', "c"]) +print(repr(o.getvalue())) +try: + csv.writer(StringIO(), quoting=csv.QUOTE_NONE).writerow(["a,b"]) +except csv.Error as e: + print("Error", e) +o = StringIO() +dw = csv.DictWriter(o, fieldnames=["id", "name"], lineterminator="\n") +dw.writeheader() +dw.writerow({"id": 1, "name": "x"}) +dw.writerows([{"id": 2}, {"name": "only"}]) +print(repr(o.getvalue())) +try: + dw.writerow({"id": 1, "bogus": 2}) +except ValueError as e: + print(e) +dr = csv.DictReader(StringIO("id,name\n1,a\n2,b,extra\n3\n"), restkey="rest", restval="?") +print(dr.fieldnames, list(dr), dr.line_num) +csv.register_dialect("pipes", delimiter="|", quoting=csv.QUOTE_ALL) +print(sorted(csv.list_dialects()), csv.get_dialect("pipes").delimiter, list(csv.reader(['"a"|"b"'], "pipes"))) +csv.unregister_dialect("pipes") +try: + csv.get_dialect("pipes") +except csv.Error as e: + print(e) +for kw in [{"delimiter": "ab"}, {"quotechar": 5}, {"quoting": 9}, {"delimiter": "\n"}]: + try: + csv.reader([], **kw) + except (TypeError, ValueError) as e: + print(type(e).__name__, e) +s = csv.Sniffer() +sample = "a;b;c\n1;2;3\n4;5;6\n" +d = s.sniff(sample) +print(d.delimiter, d.quotechar, s.has_header(sample), s.has_header("x,y\nname,1\nother,2\n")) +r = csv.reader(StringIO("a,b\nc,d\n")) +next(r) +print(r.line_num, r.dialect.delimiter, csv.field_size_limit(), csv.field_size_limit(10), csv.field_size_limit()) +try: + list(csv.reader(["0123456789abc"])) +except csv.Error as e: + print(e) diff --git a/maven/python-transpiler/src/test/resources/conformance/dataclasses_.out b/maven/python-transpiler/src/test/resources/conformance/dataclasses_.out new file mode 100644 index 00000000000..ca57e0154f1 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/dataclasses_.out @@ -0,0 +1,17 @@ +Point(x=3, y=4) 25 Point(x=1, y=0) Point(x=1, y=2) True True +['x', 'y'] {'x': 3, 'y': 4} (3, 4) Point(x=3, y=9) True False +TypeError Point.__init__() missing 1 required positional argument: 'x' +TypeError Point.__init__() takes from 2 to 3 positional arguments but 4 were given +TypeError unhashable type: 'Point' +[Version(major=1, minor=5, label='x'), Version(major=2, minor=0, label=''), Version(major=2, minor=1, label='')] True True True +FrozenInstanceError cannot assign to field 'major' +Inventory(items=['apple', 'apple'], owner=None) Inventory(items=[], owner='bo') {'apple': 2} 2 ['items', 'counts', 'owner'] +Rect(w=2, h=3, area=6) 600 +Child(a=1, b='b', c=1.5, d=4) Child(a=1, b='x', c=2.5, d=7) ['a', 'b', 'c', 'd'] +Opts(verbose=False, level=3) Opts(verbose=True, level=1) +False True True +Node(val=1, children=[Node(val=2, children=[]), Node(val=3, children=[Node(val=4, children=[])])]) +{'val': 1, 'children': [{'val': 2, 'children': []}, {'val': 3, 'children': [{'val': 4, 'children': []}]}]} +on y axis at 5 +Pair(left=1, right=0) True +ValueError mutable default for field xs is not allowed: use default_factory diff --git a/maven/python-transpiler/src/test/resources/conformance/dataclasses_.py b/maven/python-transpiler/src/test/resources/conformance/dataclasses_.py new file mode 100644 index 00000000000..8a1a402887b --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/dataclasses_.py @@ -0,0 +1,138 @@ +from dataclasses import dataclass, field, fields, asdict, astuple, replace, is_dataclass, FrozenInstanceError, \ + InitVar, KW_ONLY, make_dataclass +from typing import ClassVar, Optional, List + + +@dataclass +class Point: + x: int + y: int = 0 + + def norm2(self): + return self.x * self.x + self.y * self.y + + +p = Point(3, 4) +print(p, p.norm2(), Point(1), Point(y=2, x=1), p == Point(3, 4), p != Point(0, 0)) +print([f.name for f in fields(p)], asdict(p), astuple(p), replace(p, y=9), is_dataclass(p), is_dataclass(int)) +try: + Point() +except TypeError as e: + print("TypeError", e) +try: + Point(1, 2, 3) +except TypeError as e: + print("TypeError", e) +try: + hash(p) +except TypeError as e: + print("TypeError", e) + + +@dataclass(order=True, frozen=True) +class Version: + major: int + minor: int = 0 + label: str = field(default="", compare=False) + + +vs = sorted([Version(2, 1), Version(1, 5, "x"), Version(2, 0)]) +print(vs, Version(1) < Version(1, 1), Version(3, 0, "a") == Version(3, 0, "b"), hash(Version(1)) == hash(Version(1))) +try: + vs[0].major = 9 +except FrozenInstanceError as e: + print("FrozenInstanceError", e) + + +@dataclass +class Inventory: + items: List[str] = field(default_factory=list) + counts: dict = field(default_factory=dict, repr=False) + owner: Optional[str] = None + total: ClassVar[int] = 0 + + def add(self, name): + self.items.append(name) + self.counts[name] = self.counts.get(name, 0) + 1 + Inventory.total += 1 + + +a = Inventory() +b = Inventory(owner="bo") +a.add("apple") +a.add("apple") +print(a, b, a.counts, Inventory.total, [f.name for f in fields(Inventory)]) + + +@dataclass +class Rect: + w: float + h: float + area: float = field(init=False) + scale: InitVar[int] = 1 + + def __post_init__(self, scale): + self.w *= scale + self.h *= scale + self.area = self.w * self.h + + +print(Rect(2, 3), Rect(2, 3, 10).area) + + +@dataclass +class Base: + a: int + b: str = "b" + + +@dataclass +class Child(Base): + c: float = 1.5 + _: KW_ONLY + d: int = 4 + + +print(Child(1), Child(1, "x", 2.5, d=7), [f.name for f in fields(Child)]) + + +@dataclass(kw_only=True) +class Opts: + verbose: bool = False + level: int + + +print(Opts(level=3), Opts(level=1, verbose=True)) + + +@dataclass(eq=False) +class Ident: + v: int + + +i1 = Ident(1) +print(i1 == Ident(1), i1 == i1, hash(i1) == hash(i1)) + + +@dataclass +class Node: + val: int + children: list = field(default_factory=list) + + +tree = Node(1, [Node(2), Node(3, [Node(4)])]) +print(tree) +print(asdict(tree)) + +match Point(0, 5): + case Point(0, y): + print("on y axis at", y) + +Pair = make_dataclass("Pair", [("left", int), ("right", int, field(default=0))]) +print(Pair(1), Pair(1, 2) == Pair(1, 2)) +try: + @dataclass + class Bad: + xs: list = [] +except ValueError as e: + print("ValueError", e) diff --git a/maven/python-transpiler/src/test/resources/conformance/datetime_.out b/maven/python-transpiler/src/test/resources/conformance/datetime_.out new file mode 100644 index 00000000000..b67ad08fb7a --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/datetime_.out @@ -0,0 +1,43 @@ +1 day, 2:03:04.506007 datetime.timedelta(days=1, seconds=7384, microseconds=506007) 1 7384 506007 93784.506007 +0:00:00 datetime.timedelta(0) -1 day, 23:59:59 datetime.timedelta(days=-1, seconds=86399) 14 days, 0:00:00 +-2 days, 23:00:00 1 day, 12:00:00 14.4 14:24:00 +0:00:25 0:03:00 0:00:01 (2, datetime.timedelta(seconds=1)) 1 day, 0:00:00 +0:00:00 0:00:00.000002 0:00:00.333333 +True True False 999999999 days, 23:59:59.999999 -999999999 days, 0:00:00 +days=1000000000; must have magnitude <= 999999999 +2024-02-29 datetime.date(2024, 2, 29) 2024 2 29 3 4 738945 datetime.IsoCalendarDate(year=2024, week=9, weekday=4) +9 (2024, 9, 4) 2024-03-01 2023-12-31 424 days, 0:00:00 +0001-01-01 2021-07-29 2021-12-31 2021-12-31 2021-01-04 2021-01-03 +2024-02-01 Thu Feb 29 00:00:00 2024 Thursday 29 February 2024, day 060, week 08/09, Thu Feb 24 02 2024 2024-02-29 +time.struct_time(tm_year=2024, tm_mon=2, tm_mday=29, tm_hour=0, tm_min=0, tm_sec=0, tm_wday=3, tm_yday=60, tm_isdst=-1) 0001-01-01 9999-12-31 True True +day 29 must be in range 1..28 for month 2 in year 2023 +year must be in 1..9999, not 0 +month must be in 1..12, not 13 +Invalid isoformat string: '2021-1-1' +13:05:09.001500 datetime.time(13, 5, 9, 1500) datetime.time(1, 2) datetime.time(1, 2, 3) 13:05:09.001 13:05 01:05:09 PM 001500 10:11:12.500000 10:11:12 +01:00:00+05:30 datetime.time(1, 0, tzinfo=datetime.timezone(datetime.timedelta(seconds=19800))) 5:30:00 UTC+05:30 +True True 00:00:00 23:59:59.999999 +can't compare offset-naive and offset-aware times +UTC+05:30 datetime.timezone(datetime.timedelta(seconds=19800)) datetime.timezone.utc UTC datetime.timezone(datetime.timedelta(days=-1, seconds=75600), 'BRT') True UTC-01:30 True +offset must be a timedelta strictly between -timedelta(hours=24) and timedelta(hours=24), not datetime.timedelta(days=1) +2024-03-10 14:30:15.250000 datetime.datetime(2024, 3, 10, 14, 30, 15, 250000) 2024-03-10 14:30:15.250000 2024-03-10T14:30:15.250000 2024-03-10 14:30:15 Sun Mar 10 14:30:15 2024 6 +datetime.datetime(2020, 1, 1, 0, 0) datetime.datetime(2020, 1, 1, 5, 0) datetime.datetime(2020, 1, 1, 0, 0, 7) +2024-01-01 12:00:00+00:00 datetime.datetime(2024, 1, 1, 12, 0, tzinfo=datetime.timezone.utc) True 0:00:00 2024-01-01 12:00:00+00:00 2024-01-01 17:30:00+05:30 1704110400.0 5:30:00 +1970-01-01 00:00:00+00:00 2023-11-15 03:43:20.500000+05:30 1971-01-01T06:00:00+00:00 +2024-02-01 00:00:00 6 days, 18:00:00 2023-12-31 23:59:59 +2024-05-06 07:08:09.123000+02:00 2024-05-06 07:08:00 2024-05-06 00:00:00 2024-05-06 07:08:09+00:00 2024-05-07 00:00:00 +2020-05-05 08:09:00+05:30 +0530 UTC+05:30 00 [][] +2023-07-04 09:15:30 2023-07-04 15:00:00 2023-07-04 09:15:30.500000+01:30 2023-12-01 +2024-02-29 00:00:00 2024-03-04 00:00:00 +unconverted data remains: x +time data 'July' does not match format '%Y' +can't compare offset-naive and offset-aware datetimes +False False 2000-03-10 14:30:15.250000+05:30 +time.struct_time(tm_year=2024, tm_mon=3, tm_mday=10, tm_hour=14, tm_min=30, tm_sec=15, tm_wday=6, tm_yday=70, tm_isdst=-1) time.struct_time(tm_year=2024, tm_mon=1, tm_mday=1, tm_hour=12, tm_min=0, tm_sec=0, tm_wday=0, tm_yday=1, tm_isdst=0) 14:30 2024 2000-01-01 00:00:00 +0001-01-01 00:00:00 9999-12-31 23:59:59.999999 True 1 +time.struct_time(tm_year=1970, tm_mon=1, tm_mday=1, tm_hour=0, tm_min=0, tm_sec=0, tm_wday=3, tm_yday=1, tm_isdst=0) time.struct_time(tm_year=2023, tm_mon=11, tm_mday=14, tm_hour=22, tm_min=13, tm_sec=20, tm_wday=1, tm_yday=318, tm_isdst=0) 2009-02-13 23:31:30 +Thu Jan 1 00:00:00 1970 2 3 +(2020, 2, 3, 0, 0, 0, 0, 34) +float True +True True +2001-09-09 01:46:40+00:00 2001-09-09 01:46:40+00:00 diff --git a/maven/python-transpiler/src/test/resources/conformance/datetime_.py b/maven/python-transpiler/src/test/resources/conformance/datetime_.py new file mode 100644 index 00000000000..173ba3bc964 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/datetime_.py @@ -0,0 +1,93 @@ +# datetime, time and _strptime, kept independent of the local zone. +import datetime as dt +from datetime import date, datetime, time, timedelta, timezone +import time as tm + +td = timedelta(days=1, hours=2, minutes=3, seconds=4.5, milliseconds=6, microseconds=7) +print(td, repr(td), td.days, td.seconds, td.microseconds, td.total_seconds()) +print(timedelta(), repr(timedelta()), -timedelta(seconds=1), repr(-timedelta(seconds=1)), timedelta(weeks=2)) +print(timedelta(hours=-25), timedelta(days=3) / 2, timedelta(days=3) / timedelta(hours=5), timedelta(days=3) // 5) +print(timedelta(seconds=10) * 2.5, 3 * timedelta(minutes=1), timedelta(seconds=7) % timedelta(seconds=3), + divmod(timedelta(seconds=7), timedelta(seconds=3)), abs(timedelta(-1))) +print(timedelta(microseconds=0.5), timedelta(microseconds=1.5), timedelta(seconds=1 / 3)) +print(timedelta(1) > timedelta(0), timedelta(hours=24) == timedelta(1), bool(timedelta(0)), timedelta.max, timedelta.min) +try: + timedelta(days=10**9) +except OverflowError as e: + print(e) + +d = date(2024, 2, 29) +print(d, repr(d), d.year, d.month, d.day, d.weekday(), d.isoweekday(), d.toordinal(), d.isocalendar()) +print(d.isocalendar().week, tuple(d.isocalendar()), d + timedelta(days=1), d - timedelta(days=60), d - date(2023, 1, 1)) +print(date.fromordinal(1), date.fromordinal(738000), date.fromisoformat("2021-12-31"), date.fromisoformat("20211231"), + date.fromisoformat("2021-W01-1"), date.fromisocalendar(2020, 53, 7)) +print(d.replace(day=1), d.ctime(), d.strftime("%A %d %B %Y, day %j, week %U/%W, %a %b %y %m"), format(d, "%Y"), f"{d}") +print(d.timetuple(), date.min, date.max, date(2020, 1, 1) < date(2020, 1, 2), date(2020, 1, 1) == date(2020, 1, 1)) +for args in [(2023, 2, 29), (0, 1, 1), (2020, 13, 1)]: + try: + date(*args) + except ValueError as e: + print(e) +try: + date.fromisoformat("2021-1-1") +except ValueError as e: + print(e) + +t = time(13, 5, 9, 1500) +print(t, repr(t), repr(time(1, 2)), repr(time(1, 2, 3)), t.isoformat("milliseconds"), t.isoformat("minutes"), + t.strftime("%I:%M:%S %p %f"), time.fromisoformat("10:11:12.5"), time.fromisoformat("T101112")) +tz = timezone(timedelta(hours=5, minutes=30)) +print(time(1, tzinfo=tz), repr(time(1, tzinfo=tz)), time(1, tzinfo=tz).utcoffset(), time(1, tzinfo=tz).tzname()) +print(time(1, tzinfo=timezone.utc) == time(6, 30, tzinfo=tz), time(0) < time(1), time.min, time.max) +try: + time(1) < time(1, tzinfo=tz) +except TypeError as e: + print(e) + +print(tz, repr(tz), repr(timezone.utc), timezone.utc, repr(timezone(timedelta(hours=-3), "BRT")), + timezone(timedelta(0)) is timezone.utc, timezone(timedelta(minutes=-90)), dt.UTC is timezone.utc) +try: + timezone(timedelta(hours=24)) +except ValueError as e: + print(e) + +x = datetime(2024, 3, 10, 14, 30, 15, 250000) +print(x, repr(x), x.date(), x.time(), x.isoformat(), x.isoformat(" ", "seconds"), x.ctime(), x.weekday()) +print(repr(datetime(2020, 1, 1)), repr(datetime(2020, 1, 1, 5)), repr(datetime(2020, 1, 1, 0, 0, 7))) +a = datetime(2024, 1, 1, 12, tzinfo=timezone.utc) +b = datetime(2024, 1, 1, 17, 30, tzinfo=tz) +print(a, repr(a), a == b, a - b, b.astimezone(timezone.utc), a.astimezone(tz), a.timestamp(), b.utcoffset()) +print(datetime.fromtimestamp(0, timezone.utc), datetime.fromtimestamp(1700000000.5, tz), + datetime.fromtimestamp(86400 * 365.25, timezone.utc).isoformat()) +print(datetime(2024, 1, 31) + timedelta(days=1), datetime(2024, 1, 1) - datetime(2023, 12, 25, 6), + datetime(2024, 1, 1) - timedelta(seconds=1)) +print(datetime.fromisoformat("2024-05-06T07:08:09.123+02:00"), datetime.fromisoformat("2024-05-06 07:08"), + datetime.fromisoformat("2024-05-06"), datetime.fromisoformat("20240506T070809Z"), + datetime.fromisoformat("2024-05-06T24:00")) +print(datetime.combine(date(2020, 5, 5), time(8, 9, tzinfo=tz)), datetime(2020, 5, 5, tzinfo=tz).strftime("%z %Z %H"), + datetime(2020, 5, 5).strftime("[%z][%Z]")) +print(datetime.strptime("2023-07-04 09:15:30", "%Y-%m-%d %H:%M:%S"), datetime.strptime("4 Jul 23 3PM", "%d %b %y %I%p"), + datetime.strptime("2023-07-04T09:15:30.5+0130", "%Y-%m-%dT%H:%M:%S.%f%z"), date.strptime("2023/12/01", "%Y/%m/%d")) +print(datetime.strptime("2024 060", "%Y %j"), datetime.strptime("2024 10 1", "%Y %W %w")) +try: + datetime.strptime("2023-07-04x", "%Y-%m-%d") +except ValueError as e: + print(e) +try: + datetime.strptime("July", "%Y") +except ValueError as e: + print(e) +try: + a < datetime(2024, 1, 1) +except TypeError as e: + print(e) +print(a == datetime(2024, 1, 1, 12), date(2024, 1, 1) == datetime(2024, 1, 1), x.replace(year=2000, tzinfo=tz)) +print(x.timetuple(), a.utctimetuple(), f"{x:%H:%M}", "{:%Y}".format(x), sorted([x, datetime(2000, 1, 1)])[0]) +print(datetime.min, datetime.max, isinstance(x, date), {a: 1}[b]) +print(tm.gmtime(0), tm.gmtime(1700000000), tm.strftime("%Y-%m-%d %H:%M:%S", tm.gmtime(1234567890))) +print(tm.asctime(tm.gmtime(0)), tm.struct_time((2020, 1, 2, 3, 4, 5, 3, 2, 0)).tm_mday, tm.gmtime(0).tm_wday) +print(tm.strptime("2020-02-03", "%Y-%m-%d")[:8]) +print(type(tm.time()).__name__, tm.monotonic() <= tm.monotonic()) +lt = datetime.now() +print(abs((datetime.fromtimestamp(lt.timestamp()) - lt).total_seconds()) < 1e-3, abs(tm.mktime(tm.localtime(1e9)) - 1e9) < 1) +print(datetime.fromtimestamp(1e9).astimezone().astimezone(timezone.utc), datetime.fromtimestamp(1e9, timezone.utc)) diff --git a/maven/python-transpiler/src/test/resources/conformance/decimal_.out b/maven/python-transpiler/src/test/resources/conformance/decimal_.out new file mode 100644 index 00000000000..ea8c0486ad3 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/decimal_.out @@ -0,0 +1,56 @@ +3.30 1.690 0.3333333333333333333333333333 0.5 2E+2 1.0E+2 +3 -3 1.5 -1.5 (Decimal('-3'), Decimal('-1')) +0.1000000000000000055511151231257827021181583404541015625 1.5 -0 1000000000000000000000000000000 -0.123400 1000.5 -3.14 +Decimal('-0') Infinity -Infinity NaN sNaN12 True 1E+2 0.000001 +1E-7 1.23E-8 1.23E+5 100 123E+3 0.0E-6 12.3E-9 +1.414213562373095048801688724 4 0.5 2.718281828459045235360287471 2.302585092994045684017991455 2 0.3010299956639811952137388947 -0.6931471805599453094172321215 +1024 3.375 0.25 2.000000000000000000000000000 1.414213562373095048801688724 -8 1.000000000000000000000000000E+30 5.153775207320113310364611298E+47 1 +3.14 2.68 -2 3 1.23 +ROUND_UP ['3', '-3', '2', '-2', '1', '6', '-1'] +ROUND_DOWN ['2', '-2', '1', '-1', '0', '5', '-0'] +ROUND_CEILING ['3', '-2', '2', '-1', '1', '6', '-0'] +ROUND_FLOOR ['2', '-3', '1', '-2', '0', '5', '-1'] +ROUND_HALF_UP ['3', '-3', '1', '-1', '1', '5', '-0'] +ROUND_HALF_DOWN ['2', '-2', '1', '-1', '0', '5', '-0'] +ROUND_HALF_EVEN ['2', '-2', '1', '-1', '0', '5', '-0'] +ROUND_05UP ['2', '-2', '1', '-1', '1', '6', '-1'] +2 4 1.23 -7 1.25 1 -1 DecimalTuple(sign=0, digits=(1, 2, 5, 0), exponent=-2) (3, 4) (-3, 1) +2.5 1.2E+3 0 -1E+2 0 7 1 1 -1 +True True True False True True False True [Decimal('-2'), Decimal('1.5'), Decimal('1.50'), Decimal('3')] +True True True 314159 a True +28 ROUND_HALF_EVEN Context(prec=28, rounding=ROUND_HALF_EVEN, Emin=-999999, Emax=999999, capitals=1, clamp=0, flags=[Inexact, FloatOperation, Rounded], traps=[InvalidOperation, DivisionByZero, Overflow]) +0.14286 1.2346E+8 1.2346 5 +0.666 9.99 +28 3.14159 0.3333 +True True False Context(prec=28, rounding=ROUND_HALF_EVEN, Emin=-999999, Emax=999999, capitals=1, clamp=0, flags=[Inexact, Rounded], traps=[InvalidOperation, DivisionByZero, Overflow]) +DivisionByZero [] +InvalidOperation [] +InvalidOperation [] +Overflow [] +InvalidOperation [] +InvalidOperation [] +InvalidOperation [] +InvalidOperation [] +ValueError cannot convert NaN to integer +OverflowError cannot convert Infinity to integer +Infinity NaN NaN True True +'1,234,567.89' '-0.00' '12.50' '10,000,000,000.00' '0.00' +'1.234567891e+6' '-1.23e-4' '1.25e+1' '1e+10' '0e+0' +'1.235e+6' '-1.230e-4' '1.250e+1' '1.000e+10' '0.000e+3' +'123456789.1%' '-0.0123%' '1250%' '1000000000000%' '0%' +'1234567.891' '-0.000123' '12.5' '1E+10' '0' +'01,234,567.9' '-0,000,000.0' '00,000,012.5' '10,000,000,000.0' '00,000,000.0' +'1234567.891' '-0.000123 ' ' 12.5 ' ' 1E+10 ' ' 0 ' +'+1234567.9' '-0.0' '+12.5' '+10000000000.0' '+0.0' +'1234567.891' '-0.000123' '12.5' '1e+10' '0' +'1.2e+6' '-0.00012' '12' '1e+10' '0' +'1_234_567.9' '-0.0' '12.5' '10_000_000_000.0' '0.0' +'1.234567891E+6' '-1.23E-4' '1.25E+1' '1E+10' '0E+0' +'1234567.89' '-0.00 ' '12.50 ' '10000000000.00' '0.00 ' +'1234567.891' '-000.000' '0012.500' '10000000000.000' '0000.000' +3.142 0.0 Infinity +NaN +False True True +Normal -Infinity True -3 3.25 -1 0.18 3 +3 3 2.5 3.0 0.125 3 1 -5 2.5 2.25 8 False True +TypeError +0.25 10000000000000000159028911097599180468360808563945281389781327557747838772170381060813469985856815104 True True True +True False 1 0 diff --git a/maven/python-transpiler/src/test/resources/conformance/decimal_.py b/maven/python-transpiler/src/test/resources/conformance/decimal_.py new file mode 100644 index 00000000000..c3d526e6947 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/decimal_.py @@ -0,0 +1,71 @@ +# decimal: construction, arithmetic, rounding, contexts, signals and formatting. +import decimal +from decimal import Decimal as D, getcontext, localcontext, Context +from fractions import Fraction + +print(D("1.10") + D("2.20"), D("1.3") * D("1.30"), D(1) / D(3), D(2) / D(4), D("1e3") / D(5), D(10) / D("0.1")) +print(D("7.5") // D(2), D("-7.5") // D(2), D("7.5") % D(2), D("-7.5") % D(2), divmod(D(-7), D(2))) +print(D(0.1), D(1.5), D(-0.0), D(10 ** 30), D(" -12.3400e-2 "), D("1_000.5"), D((1, (3, 1, 4), -2))) +print(repr(D("-0")), D("Infinity"), D("-inf"), D("nan"), D("sNaN12"), D("NaN").is_nan(), D("1E+2"), D("0.000001")) +print(D("0.0000001"), D("123E-10"), D("1.23E+5"), D("100").to_eng_string(), D("1.23E+5").to_eng_string(), + D("0E-7").to_eng_string(), D("123E-10").to_eng_string()) +print(D(2).sqrt(), D(16).sqrt(), D("0.25").sqrt(), D(1).exp(), D(10).ln(), D(100).log10(), D(2).log10(), D("0.5").ln()) +print(D(2) ** 10, D("1.5") ** 3, D(2) ** -2, D(4) ** D("0.5"), D(2) ** D("0.5"), D(-2) ** 3, D(10) ** 30, + D(3) ** 100, pow(D(3), 4, 5)) +print(D("3.14159").quantize(D("0.01")), D("2.675").quantize(D("0.01")), D("-2.5").quantize(D(1)), + D("2.5").quantize(D(1), rounding=decimal.ROUND_HALF_UP), D("1.2345").quantize(D("1e-2"), decimal.ROUND_DOWN)) +for r in [decimal.ROUND_UP, decimal.ROUND_DOWN, decimal.ROUND_CEILING, decimal.ROUND_FLOOR, decimal.ROUND_HALF_UP, + decimal.ROUND_HALF_DOWN, decimal.ROUND_HALF_EVEN, decimal.ROUND_05UP]: + print(r, [str(D(x).quantize(D(1), rounding=r)) for x in ["2.5", "-2.5", "1.1", "-1.1", "0.5", "5.01", "-0.4"]]) +print(round(D("2.5")), round(D("3.5")), round(D("1.2345"), 2), int(D("-7.9")), float(D("1.25")), D("1.5").__floor__(), + D("-1.5").__ceil__(), D("12.50").as_tuple(), D("0.75").as_integer_ratio(), D(-3).as_integer_ratio()) +print(D("2.50").normalize(), D("1200").normalize(), D("0.00").normalize(), D("-1.0E+2").normalize(), + D("1.23").adjusted(), D("123E5").adjusted(), D(5).compare(D(3)), D(1).max(D("1.0")), D(-1).min(D(2))) +print(D(1) == 1, D("1.0") == D(1), D("0.5") == 0.5, D("0.1") == 0.1, D("0.5") == Fraction(1, 2), D(2) < 3, + D("NaN") == D("NaN"), D(1) < D("Infinity"), sorted([D(3), D("1.5"), D(-2), D("1.50")])) +print(hash(D(1)) == hash(1), hash(D("0.5")) == hash(0.5), hash(D("1.10")) == hash(D("1.1")), hash(D("Inf")), + {D("1.0"): "a"}[D(1)], hash(D("-7.25")) == hash(-7.25)) +c = getcontext() +print(c.prec, c.rounding, c) +with localcontext() as ctx: + ctx.prec = 5 + print(D(1) / D(7), D("123456789") + 0, +D("1.23456789"), ctx.prec) +with localcontext(prec=3, rounding=decimal.ROUND_DOWN): + print(D(2) / D(3), D("9.999") * 1) +print(getcontext().prec, Context(prec=6).create_decimal("3.14159265"), Context(prec=4).divide(1, 3)) +c.clear_flags() +x = D(1) / D(3) +print(c.flags[decimal.Inexact], c.flags[decimal.Rounded], c.flags[decimal.DivisionByZero], c) +c.clear_flags() +for expr in [lambda: D(1) / 0, lambda: D(0) / 0, lambda: D("abc"), lambda: D("9e999999") * 10, + lambda: D(-1).sqrt(), lambda: D("NaN") < 1, lambda: D(1).quantize(D("1e-30")), lambda: D(1) % 0, + lambda: int(D("NaN")), lambda: int(D("Inf"))]: + try: + print(expr()) + except (decimal.DecimalException, ValueError, OverflowError) as e: + print(type(e).__name__, e) +c.clear_flags() +ext = decimal.ExtendedContext.copy() +print(ext.divide(1, 0), ext.divide(0, 0), ext.sqrt(-1), ext.flags[decimal.DivisionByZero], + ext.flags[decimal.InvalidOperation]) +for spec in [",.2f", "e", ".3e", "%", "", "012,.1f", "^10", "+.1f", "g", ".2g", "_.1f", "E", "<8.2f|", "08.3f"]: + for v in ["1234567.891", "-0.000123", "12.5", "1e10", "0"]: + try: + print(repr(format(D(v), spec.rstrip("|"))), end=" ") + except ValueError as e: + print("ValueError", end=" ") + print() +print(f"{D('3.14159'):.3f}", f"{D('-0.0'):z.1f}", f"{D('Infinity'):>10}", f"{D('NaN'):+}") +print(D("1.5").is_signed(), D(-0).is_zero(), D("1e-1000000").is_subnormal(), D(5).number_class(), + D("-Inf").number_class(), D("0.1").same_quantum(D("0.2")), D(3).copy_sign(D(-1)), D("1.5").fma(2, D("0.25")), + D(7).remainder_near(D(4)), D(18).scaleb(-2), D(1234).logb()) +print(D(2) + 1, 1 + D(2), 3 - D("0.5"), 2 * D("1.5"), 1 / D(8), 7 // D(2), 7 % D(3), -D(5), abs(D("-2.5")), + D("1.5") ** 2, 2 ** D(3), bool(D(0)), bool(D("0.1"))) +try: + D(1) + 1.5 +except TypeError as e: + print("TypeError") +print(D.from_float(0.25), D.from_float(1e100), isinstance(D(1), decimal.Decimal), + issubclass(decimal.DivisionByZero, ZeroDivisionError), issubclass(decimal.FloatOperation, TypeError)) +import numbers +print(isinstance(D(1), numbers.Number), isinstance(D(1), numbers.Real), D(1).real, D(2).imag) diff --git a/maven/python-transpiler/src/test/resources/conformance/enum_.out b/maven/python-transpiler/src/test/resources/conformance/enum_.out new file mode 100644 index 00000000000..030d6eba686 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/enum_.out @@ -0,0 +1,17 @@ +[, , ] C.GREEN C.BLUE True 3 True green ['RED', 'GREEN', 'BLUE', 'ALIAS'] + P.R|W [, ] True False +2 2 1 2 True +M.A M.A +x X + 3 3 +5.97e+24 (5.97e+24, 6370000.0) +[, , ] 2 +9 is not a valid C +duplicate values found in : b -> a +Red Color.GREEN r GREEN True +1 True True True +['GREEN', 'RED'] Color Color.GREEN +12 4 +AttributeError cannot reassign member 'RED' +True [, ] NOT_FOUND 200 +200 True 200 404 diff --git a/maven/python-transpiler/src/test/resources/conformance/enum_.py b/maven/python-transpiler/src/test/resources/conformance/enum_.py new file mode 100644 index 00000000000..1363418830c --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/enum_.py @@ -0,0 +1,79 @@ + +from enum import * +class C(Enum): + RED=1; GREEN=auto(); BLUE=auto(); ALIAS=1 + def describe(self): return self.name.lower() +print(list(C), C(2), C['BLUE'], repr(C.RED), C.ALIAS is C.RED, len(C), C.RED in C, C.GREEN.describe(), list(C.__members__)) +class P(Flag): + R=auto(); W=auto(); X=auto() +print(repr(P.R|P.W), P.R|P.W, list(P.R|P.X), P.W in (P.R|P.W), repr(~P.R), bool(P(0)), repr(P(0))) +class N(IntEnum): + A=1; B=2 +print(N.A+1, str(N.B), f'{N.A}', repr(N.B), N(2), N.A==1) +class M(int, Enum): + A=1 +print(str(M.A), repr(M.A), f'{M.A}') +class S(StrEnum): + X=auto(); Y='yy' +print(S.X, repr(S.Y), S.X.upper()) +class I(IntFlag): + A=1; B=2 +print(repr(I.A|I.B), I.A|I.B, repr(I(8)), I(3)) +class Planet(Enum): + EARTH=(5.97e24, 6.37e6) + def __init__(self, mass, radius): self.mass=mass; self.radius=radius +print(Planet.EARTH.mass, Planet.EARTH.value) +A=Enum('A','x y z'); print(list(A), A.y.value) +try: C(9) +except ValueError as e: print(e) +try: + @unique + class U(Enum): + a=1; b=1 +except ValueError as e: print(e) + + +class Color(Enum): + RED = "r" + GREEN = "g" + + @property + def pretty(self): + return self.name.title() + + @classmethod + def parse(cls, s): + return cls(s) + + +print(Color.RED.pretty, Color.parse("g"), Color.RED.value, Color.GREEN.name, isinstance(Color.RED, Color)) +print({Color.RED: 1}[Color.RED], Color.RED == Color.RED, Color.RED != Color.GREEN, Color.RED is Color("r")) +print([c.name for c in reversed(Color)], type(Color.RED).__name__, Color.__members__["GREEN"]) + + +class Shape(Enum): + CIRCLE = auto() + SQUARE = auto() + + def area(self, s): + match self: + case Shape.CIRCLE: + return 3 * s * s + case Shape.SQUARE: + return s * s + + +print(Shape.CIRCLE.area(2), Shape.SQUARE.area(2)) +try: + Color.RED = 5 +except AttributeError as e: + print("AttributeError", e) + + +class Status(IntEnum): + OK = 200 + NOT_FOUND = 404 + + +print(Status.OK < Status.NOT_FOUND, sorted([Status.NOT_FOUND, Status.OK]), Status(404).name, Status.OK + 0) +print(int(Status.OK), Status.OK == 200, "%d" % Status.OK, max(Status)) diff --git a/maven/python-transpiler/src/test/resources/conformance/errors_.out b/maven/python-transpiler/src/test/resources/conformance/errors_.out new file mode 100644 index 00000000000..02f75b0b508 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/errors_.out @@ -0,0 +1,72 @@ +RuntimeError | generator raised StopIteration | StopIteration StopIteration True +ZeroDivisionError | division by zero | None None False +ZeroDivisionError | division by zero | None None False +ZeroDivisionError | division by zero | None None False +ZeroDivisionError | division by zero | None None False +IndexError | list index out of range | None None False +IndexError | tuple index out of range | None None False +IndexError | string index out of range | None None False +KeyError | 'k' | None None False +KeyError | ('t', 1) | None None False +ValueError | invalid literal for int() with base 10: '12x' | None None False +ValueError | invalid literal for int() with base 8: '9' | None None False +ValueError | could not convert string to float: 'nan?' | None None False +TypeError | can only concatenate str (not "int") to str | None None False +TypeError | unsupported operand type(s) for +: 'int' and 'str' | None None False +TypeError | can't multiply sequence by non-int of type 'str' | None None False +TypeError | object of type 'int' has no len() | None None False +AttributeError | 'NoneType' object has no attribute 'foo' | None None False +AttributeError | 'object' object has no attribute 'bar' | None None False +NameError | name 'undefined_name' is not defined | None None False +IndexError | pop from empty list | None None False +KeyError | 'popitem(): dictionary is empty' | None None False +KeyError | 'pop from an empty set' | None None False +ValueError | list.index(x): x not in list | None None False +ValueError | substring not found | None None False +StopIteration | | None None False +TypeError | 'int' object is not iterable | None None False +TypeError | unhashable type: 'list' | None None False +TypeError | cannot use 'list' as a dict key (unhashable type: 'list') | None None False +TypeError | '<' not supported between instances of 'str' and 'int' | None None False +ValueError | max() iterable argument is empty | None None False +ValueError | chr() arg not in range(0x110000) | None None False +TypeError | ord() expected a character, but string of length 2 found | None None False +TypeError | can't concat str to bytes | None None False +UnicodeDecodeError | 'utf-8' codec can't decode byte 0xff in position 0: invalid start byte | None None False +UnicodeEncodeError | 'ascii' codec can't encode character '\xe9' in position 0: ordinal not in range(128) | None None False +ZeroDivisionError | division by zero | None None False +ZeroDivisionError | zero to a negative power | None None False +OverflowError | int too large to convert to float | None None False +OverflowError | cannot convert float infinity to integer | None None False +ValueError | cannot convert float NaN to integer | None None False +ValueError | range() arg 3 must not be zero | None None False +ValueError | slice step cannot be zero | None None False +TypeError | tuple.__new__(X): X is not a type object (int) | None None False +TypeError | ..() missing 1 required positional argument: 'x' | None None False +TypeError | ..() takes 1 positional argument but 2 were given | None None False +TypeError | ..() got an unexpected keyword argument 'y' | None None False +TypeError | ..() takes 0 positional arguments but 1 was given | None None False +TypeError | f() missing 1 required keyword-only argument: 'c' | None None False +TypeError | f() takes from 1 to 2 positional arguments but 3 were given | None None False +TypeError | f() got multiple values for argument 'a' | None None False +AttributeError | 'Thing' object has no attribute 'y' and no __dict__ for setting new attributes | None None False +UnboundLocalError | cannot access local variable 'v' where it is not associated with a value | None None False +ValueError | bad | KeyError KeyError True +TypeError | during | None ZeroDivisionError False +TypeError | quiet | None ZeroDivisionError True +FileNotFoundError | [Errno 2] No such file: 'f.txt' | None None False +FileNotFoundError | [Errno 2] No such file or directory: '/definitely/not/here.txt' | None None False +Boom | ('boom', 7) | None None False +('boom', 7) 7 Boom('boom', 7) +AssertionError | math | None None False +ValueError | byteorder must be either 'little' or 'big' | None None False +ok None +ValueError | dictionary update sequence element #0 has length 3; 2 is required | None None False +IndexError | Replacement index 1 out of range for positional args tuple | None None False +KeyError | 'x' | None None False +TypeError | %d format: a real number is required, not str | None None False +TypeError | not enough arguments for format string | None None False +ValueError | Unknown format code 'q' for object of type 'int' | None None False +ValueError | bytes must be in range(0, 256) | None None False +ValueError | value not found in bytearray | None None False +AttributeError | 'frozenset' object has no attribute 'add' | None None False diff --git a/maven/python-transpiler/src/test/resources/conformance/errors_.py b/maven/python-transpiler/src/test/resources/conformance/errors_.py new file mode 100644 index 00000000000..91e2cd36b40 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/errors_.py @@ -0,0 +1,177 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. + +# Negative behaviour: what a program does wrong must fail exactly as in CPython, +# with the same exception type, message, cause and context. + + +def show(fn, *a): + try: + r = fn(*a) + except BaseException as e: + cause = type(e.__cause__).__name__ if e.__cause__ is not None else None + ctx = type(e.__context__).__name__ if e.__context__ is not None else None + print(type(e).__name__, "|", e, "|", cause, ctx, e.__suppress_context__) + else: + print("ok", repr(r)) + + +def gen_stop(): + yield 1 + raise StopIteration("inner") + + +show(lambda: list(gen_stop())) +show(lambda: 1 / 0) +show(lambda: 1 // 0) +show(lambda: 1 % 0) +show(lambda: 1.0 / 0) +show(lambda: [][0]) +show(lambda: (1, 2)[5]) +show(lambda: "abc"[10]) +show(lambda: {}["k"]) +show(lambda: {"a": 1}[("t", 1)]) +show(lambda: int("12x")) +show(lambda: int("9", 8)) +show(lambda: float("nan?")) +show(lambda: "a" + 1) +show(lambda: 1 + "a") +show(lambda: [1] * "x") +show(lambda: len(5)) +show(lambda: None.foo) +show(lambda: object().bar) +show(lambda: undefined_name) +show(lambda: [].pop()) +show(lambda: {}.popitem()) +show(lambda: set().pop()) +show(lambda: [1, 2].index(3)) +show(lambda: "abc".index("z")) +show(lambda: next(iter([]))) +show(lambda: iter(5)) +show(lambda: hash([])) +show(lambda: {[]: 1}) +show(lambda: sorted([1, "a"])) +show(lambda: max([])) +show(lambda: chr(-1)) +show(lambda: ord("ab")) +show(lambda: "x".encode("ascii") + "y") +show(lambda: b"\xff".decode("utf-8")) +show(lambda: "é".encode("ascii")) +show(lambda: divmod(1, 0)) +show(lambda: pow(0, -1)) +show(lambda: 2 ** 10000 * 1.0) +show(lambda: round(float("inf"))) +show(lambda: int(float("nan"))) +show(lambda: range(0, 10, 0)) +show(lambda: [1, 2, 3][::0]) +show(lambda: tuple.__new__(5)) +show(lambda: (lambda x: x)()) +show(lambda: (lambda x: x)(1, 2)) +show(lambda: (lambda x: x)(y=1)) +show(lambda: (lambda *, k: k)(1)) + + +def f(a, b=2, *, c): + return a, b, c + + +show(f, 1) +show(lambda: f(1, 2, 3)) +show(lambda: f(1, a=1, c=3)) + + +class Thing: + __slots__ = ("x",) + + +def set_y(): + t = Thing() + t.y = 1 + + +show(set_y) + + +def unbound(): + print(v) + v = 1 + + +show(unbound) + + +def chained(): + try: + {}["missing"] + except KeyError as e: + raise ValueError("bad") from e + + +show(chained) + + +def implicit(): + try: + 1 / 0 + except ZeroDivisionError: + raise TypeError("during") + + +show(implicit) + + +def suppressed(): + try: + 1 / 0 + except ZeroDivisionError: + raise TypeError("quiet") from None + + +show(suppressed) + + +def reraise(): + try: + raise OSError(2, "No such file", "f.txt") + except OSError: + raise + + +show(reraise) +show(lambda: open("/definitely/not/here.txt")) + + +class Boom(Exception): + def __init__(self, code): + super().__init__("boom", code) + self.code = code + + +def custom(): + raise Boom(7) + + +show(custom) +try: + custom() +except Boom as e: + print(e.args, e.code, repr(e)) + + +def assert_fail(): + assert 1 == 2, "math" + + +show(assert_fail) +show(lambda: int.from_bytes(b"\x01", "middle")) +show(lambda: [].sort(key=1)) +show(lambda: dict([(1, 2, 3)])) +show(lambda: "{}{}".format(1)) +show(lambda: "{x}".format(y=1)) +show(lambda: "%d" % "s") +show(lambda: "%s %s" % (1,)) +show(lambda: f"{1:q}") +show(lambda: bytes([256])) +show(lambda: bytearray(b"ab").remove(99)) +show(lambda: frozenset([1]).add) diff --git a/maven/python-transpiler/src/test/resources/conformance/except_star.out b/maven/python-transpiler/src/test/resources/conformance/except_star.out new file mode 100644 index 00000000000..55fb66cf747 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/except_star.out @@ -0,0 +1,54 @@ +ExceptionGroup('many', [ValueError(1), TypeError(2), ExceptionGroup('inner', [ValueError(3), KeyError(4)])]) +ExceptionGroup: many (3 sub-exceptions) + ValueError(1) + TypeError(2) + ExceptionGroup: inner (2 sub-exceptions) + ValueError(3) + KeyError(4) +ExceptionGroup: many (2 sub-exceptions) + ValueError(1) + ExceptionGroup: inner (1 sub-exception) + ValueError(3) +ExceptionGroup: many (2 sub-exceptions) + TypeError(2) + ExceptionGroup: inner (1 sub-exception) + KeyError(4) +ExceptionGroup: many (1 sub-exception) + ExceptionGroup: inner (1 sub-exception) + KeyError(4) +None +ExceptionGroup BaseExceptionGroup +values: +ExceptionGroup: many (2 sub-exceptions) + ValueError(1) + ExceptionGroup: inner (1 sub-exception) + ValueError(3) +types: +ExceptionGroup: many (1 sub-exception) + TypeError(2) +keys +only values +left: +ExceptionGroup: many (2 sub-exceptions) + TypeError(2) + ExceptionGroup: inner (1 sub-exception) + KeyError(4) +naked caught: +ExceptionGroup: (1 sub-exception) + OSError('naked') +unchanged OSError('plain') +raised in handler: +RuntimeError('from handler') +raised with rest: +ExceptionGroup: (2 sub-exceptions) + RuntimeError('from handler 2') + ExceptionGroup: g (1 sub-exception) + TypeError(2) +bare raise: +ExceptionGroup: g (2 sub-exceptions) + ValueError(1) + TypeError(2) +TypeError catching ExceptionGroup with except* is not allowed. Use except instead. +in f +finally +ok diff --git a/maven/python-transpiler/src/test/resources/conformance/except_star.py b/maven/python-transpiler/src/test/resources/conformance/except_star.py new file mode 100644 index 00000000000..3e074910bcc --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/except_star.py @@ -0,0 +1,103 @@ +def show(e, depth=0): + pad = " " * depth + if isinstance(e, BaseExceptionGroup): + print(pad + type(e).__name__ + ": " + str(e)) + for x in e.exceptions: + show(x, depth + 1) + else: + print(pad + repr(e)) + + +eg = ExceptionGroup("many", [ValueError(1), TypeError(2), ExceptionGroup("inner", [ValueError(3), KeyError(4)])]) +print(repr(eg)) +show(eg) +m, r = eg.split(ValueError) +show(m) +show(r) +show(eg.subgroup(lambda e: isinstance(e, KeyError))) +print(eg.subgroup(OSError)) +print(type(BaseExceptionGroup("x", [ValueError()])).__name__, type(BaseExceptionGroup("x", [KeyboardInterrupt()])).__name__) + +try: + raise eg +except* ValueError as e: + print("values:") + show(e) +except* TypeError as e: + print("types:") + show(e) +except* KeyError: + print("keys") + +try: + try: + raise eg + except* ValueError: + print("only values") +except ExceptionGroup as e: + print("left:") + show(e) + +try: + raise OSError("naked") +except* OSError as e: + print("naked caught:") + show(e) + +try: + try: + raise OSError("plain") + except* ValueError: + print("no") +except OSError as e: + print("unchanged", repr(e)) + +try: + try: + raise ExceptionGroup("g", [ValueError(1)]) + except* ValueError: + raise RuntimeError("from handler") +except Exception as e: + print("raised in handler:") + show(e) + +try: + try: + raise ExceptionGroup("g", [ValueError(1), TypeError(2)]) + except* ValueError: + raise RuntimeError("from handler 2") +except Exception as e: + print("raised with rest:") + show(e) + +try: + try: + raise ExceptionGroup("g", [ValueError(1), TypeError(2)]) + except* ValueError: + raise +except ExceptionGroup as e: + print("bare raise:") + show(e) + +try: + try: + raise ValueError(5) + except* ExceptionGroup: + pass +except TypeError as e: + print("TypeError", e) + + +def f(): + try: + raise ExceptionGroup("g", [ValueError(1)]) + except* ValueError: + print("in f") + else: + print("no else") + finally: + print("finally") + return "ok" + + +print(f()) diff --git a/maven/python-transpiler/src/test/resources/conformance/fractions_.out b/maven/python-transpiler/src/test/resources/conformance/fractions_.out new file mode 100644 index 00000000000..08c622cf135 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/fractions_.out @@ -0,0 +1,18 @@ +3/4 Fraction(-3, 4) 5 3/7 -125 2001/2 3602879701896397/36028797018963968 3/2 1/3 2/3 +1/2 1/6 1/18 2 2 0 (5, Fraction(1, 6)) -1/3 1/3 2/5 +4/3 4/3 -2/3 5/3 1 9 0.8333333333333333 0.8333333333333333 0.3333333333333333j 1/9 9 -27/8 2.0 8 1.4142135623730951 +True False True True False True True True True True +-3 3 -4 -3 2 4 -2 123/10 1200 +0.3333333333333333 1 3 (5, 2) True False +355/113 22/7 1/3 1/4 7 +True True 1537228672809129301 -1537228672809129301 2097686951299046259 +0.33333 0.67| -12.50% 1,234,567.0 1/3 2.5 | +0.333 +[Fraction(1, 3), 0.4, Fraction(1, 2), 1] 1 2/3 +True True True False True True True False +ZeroDivisionError Fraction(1, 0) +ValueError Invalid literal for Fraction: 'abc' +ZeroDivisionError Fraction(1, 0) +TypeError both arguments should be Rational instances +OverflowError cannot convert Infinity to integer ratio +ValueError cannot convert NaN to integer ratio +Fraction(1, 0) diff --git a/maven/python-transpiler/src/test/resources/conformance/fractions_.py b/maven/python-transpiler/src/test/resources/conformance/fractions_.py new file mode 100644 index 00000000000..2730f1a7e29 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/fractions_.py @@ -0,0 +1,27 @@ +# fractions and numbers. +from fractions import Fraction as F +import numbers +import math + +print(F(3, 4), repr(F(6, -8)), F(5), F("3/7"), F(" -1.25e2 "), F("1_000.5"), F(0.1), F(1.5), F(F(1, 3)), F(F(1, 2), F(3, 4))) +a, b = F(1, 3), F(1, 6) +print(a + b, a - b, a * b, a / b, a // b, a % b, divmod(F(7, 2), F(2, 3)), -a, +a, abs(F(-2, 5))) +print(a + 1, 1 + a, a - 1, 2 - a, a * 3, 3 / a, a + 0.5, 0.5 + a, a * 1j, a ** 2, a ** -2, F(-2, 3) ** -3, F(4) ** F(1, 2), 2 ** F(3), 2 ** F(1, 2)) +print(a == F(2, 6), a == 1 / 3, F(1, 2) == 0.5, F(1, 2) < 0.6, a < b, a > b, F(1) == 1, F(3, 2) >= 1, F(1, 2) != F(1, 3), 1 < F(3, 2)) +print(int(F(-7, 2)), math.trunc(F(7, 2)), math.floor(F(-7, 2)), math.ceil(F(-7, 2)), round(F(5, 2)), round(F(7, 2)), round(F(-5, 2)), round(F(1234, 100), 1), round(F(1234), -2)) +print(float(F(1, 3)), F(1, 3).numerator, F(1, 3).denominator, F(10, 4).as_integer_ratio(), F(4, 2).is_integer(), bool(F(0))) +print(F(3.141592653589793).limit_denominator(1000), F(3.141592653589793).limit_denominator(10), F(1, 3).limit_denominator(5), F.from_float(0.25), F.from_number(7)) +print(hash(F(1, 2)) == hash(0.5), hash(F(3)) == hash(3), hash(F(1, 3)), hash(F(-1, 3)), hash(F(10**20, 7))) +print(f"{F(1, 3):.5f}", f"{F(2, 3):10.2f}|", f"{F(-1, 8):.2%}", f"{F(1234567, 1):,.1f}", f"{F(1, 3)}", f"{F(5, 2):<8.1f}|", f"{F(1, 3):+.3f}") +print(sorted([F(1, 2), F(1, 3), 0.4, 1]), sum([F(1, 2), F(1, 3), F(1, 6)]), max(F(1, 2), F(2, 3))) +print(isinstance(F(1, 2), numbers.Rational), isinstance(1, numbers.Integral), isinstance(1.5, numbers.Real), isinstance(1.5, numbers.Rational), + isinstance(1j, numbers.Complex), isinstance(True, numbers.Number), issubclass(int, numbers.Real), isinstance("x", numbers.Number)) +for args in [(1, 0), ("abc",), ("1/0",), (1.5, 2), (float("inf"),), (float("nan"),)]: + try: + F(*args) + except (ValueError, ZeroDivisionError, TypeError, OverflowError) as e: + print(type(e).__name__, e) +try: + F(1, 2) / 0 +except ZeroDivisionError as e: + print(e) diff --git a/maven/python-transpiler/src/test/resources/conformance/generators.out b/maven/python-transpiler/src/test/resources/conformance/generators.out new file mode 100644 index 00000000000..0e3b1484750 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/generators.out @@ -0,0 +1,48 @@ +[0, 1, 2, 3, 4] +285 +0 1 2 +[0, 1, 2, 3] +None +a +caught boom +stopped done +1 +cleanup ran +cleanup ran +[1, 2] +i1 +inner got hello +i2 +outer got inner result +[10, 20, 'end'] +[(0, 0), (0, 1), (2, 0), (2, 1), 'else'] +fin 0 +fin 1 +fin 2 +[0, 1, 'after'] +finally before return +returned 5 +enter a +enter b +exit b KeyError +exit a None +['ab', 'suppressed'] +first +11 +p +q +['P', 'Q'] +c +got 1 +error inside +pep479 generator raised StopIteration +[0, 1, 1, 2, 3, 5, 8, 13, 21, 34] +0 [1, 4, 9, 16] +True False +{'a': 'aa', 'b': 'bb'} +[0, 10, 20] +generator +0 2 4 +[] +[0, 1, 2] [0, 1, 2] +[(1, 'b'), (2, 'b')] diff --git a/maven/python-transpiler/src/test/resources/conformance/generators.py b/maven/python-transpiler/src/test/resources/conformance/generators.py new file mode 100644 index 00000000000..f362c27dc24 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/generators.py @@ -0,0 +1,222 @@ +# Generators, yield from, send/throw/close, generator expressions. + + +def count(n): + i = 0 + while i < n: + yield i + i += 1 + + +print(list(count(5))) +print(sum(x * x for x in range(10))) + + +def nat(): + n = 0 + while True: + yield n + n += 1 + + +g = nat() +print(next(g), next(g), next(g)) +print([v for _, v in zip(range(4), nat())]) + + +def echo(): + got = None + while True: + try: + got = yield got + except ValueError as e: + got = "caught " + str(e) + if got == "stop": + return "done" + + +e = echo() +print(e.send(None)) +print(e.send("a")) +print(e.throw(ValueError("boom"))) +try: + e.send("stop") +except StopIteration as s: + print("stopped", s.value) + + +def cleanup(): + try: + yield 1 + yield 2 + finally: + print("cleanup ran") + + +c = cleanup() +print(next(c)) +c.close() +print(list(cleanup())) + + +def inner(): + x = yield "i1" + print("inner got", x) + yield "i2" + return "inner result" + + +def outer(): + r = yield from inner() + print("outer got", r) + yield from [10, 20] + yield "end" + + +o = outer() +print(next(o)) +print(o.send("hello")) +print(list(o)) + + +def loops(): + for i in range(3): + for j in range(3): + if j == 2: + break + if i == 1: + continue + yield i, j + else: + yield "else" + + +print(list(loops())) + + +def fin_break(): + for i in range(5): + try: + if i == 2: + break + yield i + finally: + print("fin", i) + yield "after" + + +print(list(fin_break())) + + +def fin_return(): + try: + yield 1 + return 5 + finally: + print("finally before return") + + +try: + f = fin_return() + next(f) + next(f) +except StopIteration as s: + print("returned", s.value) + + +class Ctx: + def __init__(self, name): + self.name = name + + def __enter__(self): + print("enter", self.name) + return self.name + + def __exit__(self, t, v, tb): + print("exit", self.name, t.__name__ if t else None) + return t is KeyError + + +def with_gen(): + with Ctx("a") as a, Ctx("b") as b: + yield a + b + raise KeyError("k") + yield "suppressed" + + +print(list(with_gen())) + + +def nested_expr(): + x = 1 + (yield "first") * 2 + yield x + y = [(yield "p"), (yield "q")] + yield y + z = (yield "c") or "default" + yield z + + +n = nested_expr() +print(next(n)) +print(n.send(5)) +print(next(n)) +print(n.send("P")) +print(n.send("Q")) +print(n.send("")) + + +def raiser(): + yield 1 + raise RuntimeError("inside") + + +try: + for v in raiser(): + print("got", v) +except RuntimeError as err: + print("error", err) + + +def stop_iter(): + yield 1 + raise StopIteration + + +try: + list(stop_iter()) +except RuntimeError as err: + print("pep479", err) + + +def fib(): + a, b = 0, 1 + while True: + yield a + a, b = b, a + b + + +gen = fib() +print([next(gen) for _ in range(10)]) +squares = (i * i for i in range(5)) +print(next(squares), list(squares)) +print(any(x > 3 for x in [1, 2, 5]), all(x > 0 for x in [1, -1])) +print(dict((k, k * 2) for k in "ab")) + + +def closure_gen(k): + def scale(v): + return v * k + for i in range(3): + yield scale(i) + + +print(list(closure_gen(10))) +print(type(count(1)).__name__) +evens = (n for n in nat() if n % 2 == 0) +print(next(evens), next(evens), next(evens)) +side = [] +lazy = (side.append(i) or i for i in range(3)) +print(side) +print(list(lazy), side) +print(list(x + y for x in range(3) if x for y in "ab" if y != "a" for _ in [0]) if False else + list((x, y) for x in range(3) if x for y in "ab" if y != "a")) diff --git a/maven/python-transpiler/src/test/resources/conformance/imports_.out b/maven/python-transpiler/src/test/resources/conformance/imports_.out new file mode 100644 index 00000000000..0cd01488876 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/imports_.out @@ -0,0 +1,17 @@ +main __main__ +init pkg pkg +init a pkg.a +pkg 21 5 +init sub pkg.sub +pkg.sub.b 44 +True b in pkg.sub +42 pkg.a +46 +1 two +hidden not exported +yes +unlisted not exported +4 True +ModuleNotFoundError +ImportError from +['VALUE', 'a', 'allstar', 'helper', 'star', 'sub', 'twice'] diff --git a/maven/python-transpiler/src/test/resources/conformance/imports_/main.py b/maven/python-transpiler/src/test/resources/conformance/imports_/main.py new file mode 100644 index 00000000000..16de09753f8 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/imports_/main.py @@ -0,0 +1,34 @@ +print("main", __name__) +import pkg +print(pkg.__name__, pkg.VALUE, pkg.helper(2)) +import pkg.sub.b +print(pkg.sub.b.__name__, pkg.sub.b.combined()) +import pkg.sub.b as bb +print(bb is pkg.sub.b, bb.NAME) +from pkg import a, VALUE as V +print(a.twice(V), a.__name__) +from pkg.sub import b +print(b.combined()) +from pkg.star import * +print(public_one, public_two) +try: + print(_hidden) +except NameError as e: + print("hidden not exported") +from pkg.allstar import * +print(listed) +try: + print(unlisted) +except NameError: + print("unlisted not exported") +import pkg.a +print(pkg.a.counter, pkg.a is a) +try: + import pkg.missing +except ImportError as e: + print(type(e).__name__) +try: + from pkg import nothing_here +except ImportError as e: + print("ImportError from") +print(sorted(k for k in dir(pkg) if not k.startswith("__"))) diff --git a/maven/python-transpiler/src/test/resources/conformance/imports_/pkg/__init__.py b/maven/python-transpiler/src/test/resources/conformance/imports_/pkg/__init__.py new file mode 100644 index 00000000000..3f66a597563 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/imports_/pkg/__init__.py @@ -0,0 +1,5 @@ +print("init pkg", __name__) +VALUE = 21 +from .a import twice +def helper(x): + return twice(x) + 1 diff --git a/maven/python-transpiler/src/test/resources/conformance/imports_/pkg/a.py b/maven/python-transpiler/src/test/resources/conformance/imports_/pkg/a.py new file mode 100644 index 00000000000..031f3fcf401 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/imports_/pkg/a.py @@ -0,0 +1,6 @@ +print("init a", __name__) +counter = 0 +def twice(x): + global counter + counter += 1 + return x * 2 diff --git a/maven/python-transpiler/src/test/resources/conformance/imports_/pkg/allstar.py b/maven/python-transpiler/src/test/resources/conformance/imports_/pkg/allstar.py new file mode 100644 index 00000000000..dc9c1e1ccfc --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/imports_/pkg/allstar.py @@ -0,0 +1,3 @@ +__all__ = ["listed"] +listed = "yes" +unlisted = "no" diff --git a/maven/python-transpiler/src/test/resources/conformance/imports_/pkg/star.py b/maven/python-transpiler/src/test/resources/conformance/imports_/pkg/star.py new file mode 100644 index 00000000000..71f603c0e63 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/imports_/pkg/star.py @@ -0,0 +1,3 @@ +public_one = 1 +public_two = "two" +_hidden = 3 diff --git a/maven/python-transpiler/src/test/resources/conformance/imports_/pkg/sub/__init__.py b/maven/python-transpiler/src/test/resources/conformance/imports_/pkg/sub/__init__.py new file mode 100644 index 00000000000..1496734797a --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/imports_/pkg/sub/__init__.py @@ -0,0 +1 @@ +print("init sub", __name__) diff --git a/maven/python-transpiler/src/test/resources/conformance/imports_/pkg/sub/b.py b/maven/python-transpiler/src/test/resources/conformance/imports_/pkg/sub/b.py new file mode 100644 index 00000000000..37f7bc96f2a --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/imports_/pkg/sub/b.py @@ -0,0 +1,7 @@ +from .. import a +from ..a import twice +from . import __name__ as subname +from .. import VALUE +NAME = "b in " + subname +def combined(): + return twice(VALUE) + a.counter diff --git a/maven/python-transpiler/src/test/resources/conformance/itertools_.out b/maven/python-transpiler/src/test/resources/conformance/itertools_.out new file mode 100644 index 00000000000..9d53c4b53b9 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/itertools_.out @@ -0,0 +1,33 @@ +[5, 7, 9, 11] +['a', 'b', 'a', 'b', 'a'] +['x', 'x', 'x'] +[1, 3, 6, 10] [10, 10, 20, 60] +[(0, 1, 2), (3, 4, 5), (6,)] +['a', 'b', 1, 2] [1, 2, 3] +['a', 'c', 'e', 'f'] +[3, 1] [1, 2] +[0, ''] +[('a', ['a', 'a', 'a']), ('b', ['b']), ('c', ['c', 'c']), ('a', ['a', 'a'])] +[(1, 2), (0, 1), (1, 1), (0, 1), (1, 1), (0, 1), (1, 1)] +[2, 5, 8, 11, 14] [('a', 'b'), ('b', 'c'), ('c', 'd')] +[8, 9] +[1, 2, 3] [1, 2, 3] +[('a', 'x'), ('b', 'y'), ('-', 'z')] +[('a', 0), ('a', 1), ('b', 0), ('b', 1)] 27 +[('a', 'b'), ('a', 'c'), ('b', 'a'), ('b', 'c'), ('c', 'a'), ('c', 'b')] +[(0, 1), (0, 2), (0, 3), (1, 2), (1, 3), (2, 3)] [('a', 'a'), ('a', 'b'), ('b', 'b')] +10 7 +TypeError reduce() of empty iterable with no initial value +5 9 +1548008755920 CacheInfo(hits=58, misses=61, maxsize=None, currsize=61) +CacheInfo(hits=1, misses=4, maxsize=2, currsize=2) +42 answer Doc. 41 +True True True False +12 12 1 +int 1 list 2 obj int True +[3, 2, 1] +6 ('a', 'c') 2 +HI 5 -4 True +[(0, 'a'), (1, 'b')] +2 4 4 +[1, 2, 3, 4, 4, 8] 5 diff --git a/maven/python-transpiler/src/test/resources/conformance/itertools_.py b/maven/python-transpiler/src/test/resources/conformance/itertools_.py new file mode 100644 index 00000000000..1e3481cc904 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/itertools_.py @@ -0,0 +1,125 @@ +import itertools as it +import functools +import operator +import bisect +from functools import reduce, partial, lru_cache, wraps, total_ordering, cached_property, singledispatch, cmp_to_key + +print(list(it.islice(it.count(5, 2), 4))) +print(list(it.islice(it.cycle("ab"), 5))) +print(list(it.repeat("x", 3))) +print(list(it.accumulate([1, 2, 3, 4])), list(it.accumulate([1, 2, 3], operator.mul, initial=10))) +print(list(it.batched(range(7), 3))) +print(list(it.chain("ab", [1, 2])), list(it.chain.from_iterable([[1], [2, 3]]))) +print(list(it.compress("abcdef", [1, 0, 1, 0, 1, 1]))) +print(list(it.dropwhile(lambda x: x < 3, [1, 2, 3, 1])), list(it.takewhile(lambda x: x < 3, [1, 2, 3, 1]))) +print(list(it.filterfalse(None, [0, 1, "", "a"]))) +print([(k, list(g)) for k, g in it.groupby("aaabccaa")]) +print([(k, len(list(g))) for k, g in it.groupby(sorted([3, 1, 4, 1, 5, 9, 2, 6]), key=lambda v: v % 2)]) +print(list(it.islice(range(20), 2, 15, 3)), list(it.pairwise("abcd"))) +print(list(it.starmap(pow, [(2, 3), (3, 2)]))) +a, b = it.tee(iter([1, 2, 3])) +print(list(a), list(b)) +print(list(it.zip_longest("ab", "xyz", fillvalue="-"))) +print(list(it.product("ab", range(2))), len(list(it.product(range(3), repeat=3)))) +print(list(it.permutations("abc", 2))) +print(list(it.combinations(range(4), 2)), list(it.combinations_with_replacement("ab", 2))) + +print(reduce(operator.add, [1, 2, 3, 4]), reduce(lambda x, y: x * y, [], 7)) +try: + reduce(operator.add, []) +except TypeError as e: + print("TypeError", e) +p = partial(int, base=2) +print(p("101"), partial(partial(max, 3), 9)(5)) + + +@lru_cache(maxsize=None) +def fib(n): + return n if n < 2 else fib(n - 1) + fib(n - 2) + + +print(fib(60), fib.cache_info()) + + +@lru_cache(maxsize=2) +def sq(x): + return x * x + + +for v in (1, 2, 1, 3, 2): + sq(v) +print(sq.cache_info()) + + +def deco(f): + @wraps(f) + def inner(*a): + return f(*a) + 1 + return inner + + +@deco +def answer(): + """Doc.""" + return 41 + + +print(answer(), answer.__name__, answer.__doc__, answer.__wrapped__()) + + +@total_ordering +class V: + def __init__(self, v): + self.v = v + + def __eq__(self, o): + return self.v == o.v + + def __lt__(self, o): + return self.v < o.v + + +print(V(1) <= V(2), V(3) > V(2), V(2) >= V(2), V(1) > V(2)) + + +class Circle: + def __init__(self, r): + self.r = r + self.calls = 0 + + @cached_property + def area(self): + self.calls += 1 + return 3 * self.r * self.r + + +c = Circle(2) +print(c.area, c.area, c.calls) + + +@singledispatch +def show(x): + return "obj" + + +@show.register +def _(x: int): + return "int " + str(x) + + +@show.register(list) +def _(x): + return "list " + str(len(x)) + + +print(show(1), show([1, 2]), show("s"), show(True)) +print(sorted([3, 1, 2], key=cmp_to_key(lambda a, b: b - a))) + +print(operator.itemgetter(1)([5, 6]), operator.itemgetter(0, 2)("abc"), operator.attrgetter("r")(c)) +print(operator.methodcaller("upper")("hi"), operator.add(2, 3), operator.neg(4), operator.contains([1], 1)) +print(sorted([(1, "b"), (0, "a")], key=operator.itemgetter(1))) + +xs = [1, 2, 4, 4, 8] +print(bisect.bisect_left(xs, 4), bisect.bisect_right(xs, 4), bisect.bisect(xs, 5)) +bisect.insort(xs, 3) +print(xs, bisect.bisect_left(xs, 7, key=lambda v: v)) diff --git a/maven/python-transpiler/src/test/resources/conformance/json_.out b/maven/python-transpiler/src/test/resources/conformance/json_.out new file mode 100644 index 00000000000..986c7790f2b --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/json_.out @@ -0,0 +1,56 @@ +{"name": "caf\u00e9", "n": [1, 2.5, -3e-07, true, false, null], "nested": {"b": 1, "a": [{}]}, "big": 12345678901234567890, "esc": "tab\there \"q\" \\ \u0001 \ud83d\ude00"} +{"name": "café", "n": [1, 2.5, -3e-07, true, false, null], "nested": {"b": 1, "a": [{}]}, "big": 12345678901234567890, "esc": "tab\there \"q\" \\ \u0001 😀"} +{ + "big": 12345678901234567890, + "esc": "tab\there \"q\" \\ \u0001 \ud83d\ude00", + "n": [ + 1, + 2.5, + -3e-07, + true, + false, + null + ], + "name": "caf\u00e9", + "nested": { + "a": [ + {} + ], + "b": 1 + } +} +[1,[2,3]] +{"1": "d", "2.5": "b", "null": "c"} +NaN Infinity 1e+100 0.1 +[1, 2] [] {} "" +[ + 1, + {} +] +True True +{'a': [1, 2.0, 0, 1000.0, 'xé😀'], 'b': None} +[nan, inf, -inf] +{'a': 2} +[('x', 1)] +[('x', 1), ('y', 2)] +F1.5 14 +[1] +{"x": 1, "y": 2} +{"s": [1, 2, 3]} +JSONDecodeError True Expecting value: line 1 column 1 (char 0) 0 1 1 +JSONDecodeError True Illegal trailing comma before end of array: line 1 column 3 (char 2) 2 1 3 +JSONDecodeError True Expecting ':' delimiter: line 1 column 6 (char 5) 5 1 6 +JSONDecodeError True Illegal trailing comma before end of object: line 1 column 8 (char 7) 7 1 8 +JSONDecodeError True Expecting ',' delimiter: line 1 column 4 (char 3) 3 1 4 +JSONDecodeError True Unterminated string starting at: line 1 column 1 (char 0) 0 1 1 +JSONDecodeError True Expecting value: line 1 column 1 (char 0) 0 1 1 +JSONDecodeError True Expecting property name enclosed in double quotes: line 1 column 2 (char 1) 1 1 2 +JSONDecodeError True Extra data: line 1 column 5 (char 4) 4 1 5 +JSONDecodeError True Invalid \escape: line 1 column 2 (char 1) 1 1 2 +TypeError Object of type object is not JSON serializable +ValueError Out of range float values are not JSON compliant: nan +ValueError Circular reference detected +TypeError keys must be str, int, float, bool or None, not tuple +{"k": 4} +[1, 2] +Expecting value: line 2 column 3 (char 6) diff --git a/maven/python-transpiler/src/test/resources/conformance/json_.py b/maven/python-transpiler/src/test/resources/conformance/json_.py new file mode 100644 index 00000000000..b69490678a4 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/json_.py @@ -0,0 +1,71 @@ +import json + +data = {"name": "café", "n": [1, 2.5, -3e-7, True, False, None], "nested": {"b": 1, "a": [{}]}, + "big": 12345678901234567890, "esc": "tab\there \"q\" \\ \x01 \U0001F600"} +print(json.dumps(data)) +print(json.dumps(data, ensure_ascii=False)) +print(json.dumps(data, indent=2, sort_keys=True)) +print(json.dumps([1, [2, 3]], separators=(",", ":"))) +print(json.dumps({1: "a", 2.5: "b", None: "c", True: "d"})) +print(json.dumps(float("nan")), json.dumps(float("inf")), json.dumps(1e100), json.dumps(0.1)) +print(json.dumps((1, 2)), json.dumps([]), json.dumps({}), json.dumps("")) +print(json.dumps([1, {}], indent="\t")) + +back = json.loads(json.dumps(data)) +print(back == data, back["esc"] == data["esc"]) +print(json.loads(' {"a": [1, 2.0, -0, 1E3, "x\\u00e9\\ud83d\\ude00"], "b": null} ')) +print(json.loads("[NaN, Infinity, -Infinity]")) +print(json.loads('{"a": 1, "a": 2}')) +print(json.loads('{"x": 1}', object_hook=lambda d: sorted(d.items()))) +print(json.loads('{"x": 1, "y": 2}', object_pairs_hook=list)) +print(json.loads("1.5", parse_float=lambda s: "F" + s), json.loads("7", parse_int=lambda s: int(s) * 2)) +print(json.loads(b'[1]')) + + +class Point: + def __init__(self, x, y): + self.x, self.y = x, y + + +print(json.dumps(Point(1, 2), default=lambda o: o.__dict__)) + + +class Enc(json.JSONEncoder): + def default(self, o): + if isinstance(o, set): + return sorted(o) + return super().default(o) + + +print(json.dumps({"s": {3, 1, 2}}, cls=Enc)) + +for bad in ['', '[1,]', '{"a" 1}', '{"a": 1,}', '[1 2]', '"abc', 'nul', '{1: 2}', '[1] x', '"\\q"']: + try: + json.loads(bad) + except json.JSONDecodeError as e: + print(type(e).__name__, isinstance(e, ValueError), e, e.pos, e.lineno, e.colno) + +try: + json.dumps(object()) +except TypeError as e: + print("TypeError", e) +try: + json.dumps(float("nan"), allow_nan=False) +except ValueError as e: + print("ValueError", e) +a = [] +a.append(a) +try: + json.dumps(a) +except ValueError as e: + print("ValueError", e) +try: + json.dumps({(1, 2): 3}) +except TypeError as e: + print("TypeError", e) +print(json.dumps({(1, 2): 3, "k": 4}, skipkeys=True)) +print(json.loads("\n\n [1,\n x]".replace("x", "2"))) +try: + json.loads("[1,\n x]") +except ValueError as e: + print(e) diff --git a/maven/python-transpiler/src/test/resources/conformance/logging_.out b/maven/python-transpiler/src/test/resources/conformance/logging_.out new file mode 100644 index 00000000000..bd099bc9e72 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/logging_.out @@ -0,0 +1,23 @@ +True True True 10 +NOTICE 25 Level 7 40 +DEBUG|app|debug 1 +INFO|app.db|query select took 2.2ms +WARNING|app.db|dict args +Level 25|app|custom +NOTICE|app|named +WARNING app:both +CRITICALapp:templ +CRITICAL templ [ada] +ERROR app:n=1 +ERROR app:n=3 +a False; a.b True; a.b.c True; a.bc False; +failed ZeroDivisionError: division by zero +'req1 adapted\np [p] pref\n' + False True +False True +Unknown level: 'LOUD' +level must be an integer +hi there hi there +invalid format: no fields +'INFO:root info\nWARNING:propagated\n' 20 +[('CRITICAL', 50), ('DEBUG', 10), ('ERROR', 40)] app.db.x ['app.db'] diff --git a/maven/python-transpiler/src/test/resources/conformance/logging_.py b/maven/python-transpiler/src/test/resources/conformance/logging_.py new file mode 100644 index 00000000000..1727290f68a --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/logging_.py @@ -0,0 +1,104 @@ +# logging: levels, hierarchy, handlers, formatters, filters and adapters. +import logging +from io import StringIO + +buf = StringIO() +h = logging.StreamHandler(buf) +h.setFormatter(logging.Formatter("%(levelname)s|%(name)s|%(message)s")) +app = logging.getLogger("app") +app.addHandler(h) +app.setLevel(logging.DEBUG) +db = logging.getLogger("app.db") +print(db.parent is app, app.parent is logging.root, logging.getLogger("app") is app, db.getEffectiveLevel()) +app.debug("debug %d", 1) +db.info("query %s took %.1fms", "select", 2.25) +db.warning("dict %(a)s", {"a": "args"}) +app.log(25, "custom") +logging.addLevelName(25, "NOTICE") +app.log(25, "named") +print(logging.getLevelName(25), logging.getLevelName("NOTICE"), logging.getLevelName(7), logging.getLevelName("ERROR")) +print(buf.getvalue(), end="") + +buf2 = StringIO() +h2 = logging.StreamHandler(buf2) +h2.setFormatter(logging.Formatter("{levelname:<8}{name}:{message}", style="{")) +h2.setLevel(logging.WARNING) +app.addHandler(h2) +app.info("only first") +app.warning("both") +h3 = logging.StreamHandler(buf2) +h3.setFormatter(logging.Formatter("$levelname $message [${user}]", style="$")) +app.addHandler(h3) +app.critical("templ", extra={"user": "ada"}) +app.removeHandler(h3) + + +class OnlyOdd(logging.Filter): + def filter(self, record): + return record.args[0] % 2 == 1 + + +h2.addFilter(OnlyOdd()) +for i in range(4): + app.error("n=%d", i) +app.addFilter(lambda r: "secret" not in r.getMessage()) +app.error("secret %d", 1) +print(buf2.getvalue(), end="") + +f = logging.Filter("a.b") +for name in ["a", "a.b", "a.b.c", "a.bc"]: + print(name, bool(f.filter(logging.makeLogRecord({"name": name}))), end="; ") +print() + +buf3 = StringIO() +h4 = logging.StreamHandler(buf3) +h4.setFormatter(logging.Formatter("%(message)s")) +lg = logging.getLogger("exc") +lg.addHandler(h4) +lg.propagate = False +try: + 1 / 0 +except ZeroDivisionError: + lg.exception("failed") +print(buf3.getvalue().splitlines()[0], buf3.getvalue().splitlines()[-1]) + +ad = logging.LoggerAdapter(lg, {"ctx": "req1"}) +h4.setFormatter(logging.Formatter("%(ctx)s %(message)s")) + + +class Prefix(logging.LoggerAdapter): + def process(self, msg, kwargs): + msg, kwargs = super().process(msg, kwargs) + return "[%s] %s" % (self.extra["ctx"], msg), kwargs + + +buf3.seek(0) +buf3.truncate() +ad.warning("adapted") +Prefix(lg, {"ctx": "p"}).warning("pref") +print(repr(buf3.getvalue())) +print(repr(app), repr(logging.root), repr(ad), app.isEnabledFor(5), app.hasHandlers()) +logging.disable(logging.ERROR) +print(app.isEnabledFor(logging.ERROR), app.isEnabledFor(logging.CRITICAL)) +logging.disable(logging.NOTSET) +try: + app.setLevel("LOUD") +except ValueError as e: + print(e) +try: + app.log("x", "y") +except TypeError as e: + print(e) +rec = logging.makeLogRecord({"msg": "hi %s", "args": ("there",), "levelno": 20}) +print(rec.getMessage(), logging.Formatter().format(rec)) +try: + logging.Formatter("no fields", style="{") +except ValueError as e: + print(e) +out = StringIO() +logging.basicConfig(stream=out, level=logging.INFO, format="%(levelname)s:%(message)s") +logging.info("root info") +logging.debug("root debug") +logging.getLogger("other").warning("propagated") +print(repr(out.getvalue()), logging.root.level) +print(sorted(logging.getLevelNamesMapping().items())[:3], db.getChild("x").name, sorted(c.name for c in app.getChildren())) diff --git a/maven/python-transpiler/src/test/resources/conformance/match_.out b/maven/python-transpiler/src/test/resources/conformance/match_.out new file mode 100644 index 00000000000..b1a585f6831 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/match_.out @@ -0,0 +1,25 @@ +none +true +small 0 +small 1 +negative -5 +number 7 +number 2.5 +the blue +verb +str hi +empty +one 9 +long 1 [2, 3, 4] +3+ (1, 2, 3) +pair (1, 2) +circle 2 +shape sq {'w': 1, 'h': 2} +origin +on y 5 +diag 3 +point +other set +[1, -1, 2] +loading file.txt +done 5 diff --git a/maven/python-transpiler/src/test/resources/conformance/match_.py b/maven/python-transpiler/src/test/resources/conformance/match_.py new file mode 100644 index 00000000000..ee8c02440ba --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/match_.py @@ -0,0 +1,91 @@ + +class Point: + __match_args__ = ("x", "y") + + def __init__(self, x, y): + self.x = x + self.y = y + + +RED = "red" + + +class Colors: + BLUE = "blue" + + +def describe(v): + match v: + case None: + return "none" + case True: + return "true" + case 0 | 1 as small: + return "small " + str(small) + case int(n) if n < 0: + return "negative " + str(n) + case int() | float(): + return "number " + str(v) + case Colors.BLUE: + return "the blue" + case "go" | "run": + return "verb" + case str(s): + return "str " + s + case []: + return "empty" + case [x]: + return "one " + repr(x) + case [first, *rest] if len(rest) > 2: + return "long " + repr(first) + " " + repr(rest) + case [a, b, *_, c]: + return "3+ " + repr((a, b, c)) + case (a, b): + return "pair " + repr((a, b)) + case {"kind": "circle", "r": r}: + return "circle " + str(r) + case {"kind": kind, **others}: + return "shape " + kind + " " + repr(others) + case Point(0, 0): + return "origin" + case Point(x=0, y=yy): + return "on y " + str(yy) + case Point(xx, yy) if xx == yy: + return "diag " + str(xx) + case Point(): + return "point" + case _: + return "other " + type(v).__name__ + + +for v in [None, True, 0, 1, -5, 7, 2.5, "blue", "go", "hi", [], [9], [1, 2, 3, 4], [1, 2, 3], (1, 2), {"kind": "circle", "r": 2}, + {"kind": "sq", "w": 1, "h": 2}, Point(0, 0), Point(0, 5), Point(3, 3), Point(1, 2), {1, 2}]: + print(describe(v)) + + +def gen(items): + for it in items: + match it: + case ("add", n): + yield n + case ("skip",): + continue + case ("stop",): + return + case _: + yield -1 + + +print(list(gen([("add", 1), ("skip",), ("bad",), ("add", 2), ("stop",), ("add", 3)]))) + +cmd = "load file.txt" +match cmd.split(): + case ["load", fn]: + print("loading", fn) + case _: + print("?") +x = 5 +match x: + case 1: + print("one") +print("done", x) diff --git a/maven/python-transpiler/src/test/resources/conformance/math2_.out b/maven/python-transpiler/src/test/resources/conformance/math2_.out new file mode 100644 index 00000000000..2ee47666a28 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/math2_.out @@ -0,0 +1,35 @@ +sinh [-0.822316731936, -1e-09, 0.0, 1e-09, 0.304520293447, 1.02651672571] +cosh [1.29468328468, 1.0, 1.0, 1.0, 1.04533851413, 1.43308638545] +tanh [-0.635148952387, -1e-09, 0.0, 1e-09, 0.291312612452, 0.716297870199] +asinh [-0.69314718056, -1e-09, 0.0, 1e-09, 0.295673047563, 0.808866935653] +atanh [-0.972955074528, -1e-09, 0.0, 1e-09, 0.309519604203, 1.47221948958] +expm1 [-0.527633447259, -9.999999995e-10, 0.0, 1.0000000005e-09, 0.349858807576, 1.45960311116] +log1p [-1.38629436112, -1.0000000005e-09, 0.0, 9.999999995e-10, 0.262364264467, 0.641853886172] +erf [-0.711155633654, -1.1283791671e-09, 0.0, 1.1283791671e-09, 0.328626759459, 0.796908212423] +erfc [1.71115563365, 1.00000000113, 1.0, 0.999999998872, 0.671373240541, 0.203091787577] +cbrt [-0.908560296416, -0.001, 0.0, 0.001, 0.669432950082, 0.965489384606] +exp2 [0.594603557501, 0.999999999307, 1.0, 1.00000000069, 1.23114441334, 1.86606598307] +[0.0, 0.962423650119, 2.99322284613, 23.7189981105] +[0.976348383345, 0.999977909503, 1.0] [0.0236516166554, 2.20904969986e-05, 2.15197367125e-17, 1.99532226502] +[1.77245385091, 1.0, 24.0, 1871.2543058, -3.54490770181, -0.945308720483, 1.74100944459e+31] 6.0 +[0.572364942925, 0.0, 0.0, 0.69314718056, 13.9406252194, 359.13420537, 1.26551212348, -0.0562437164977] +(0.5, 4) (-0.625, -2) (0.0, 0) (0.5, -1073) 8.0 5e-324 6.0 +(0.75, 3.0) (-0.75, -3.0) (0.0, 2.0) (0.0, inf) +1.0 -1.0 1.0 1.0 -1.0 0.5 +1.0000000000000002 0.9999999999999999 5e-324 1.0000000000000007 2.220446049250313e-16 1.487016908477783e+284 5e-324 +5.0 5.0 12 1 0 12 +120 20 0 32 7.0 7.0 +1.0 -0.0 -0.0 -3.0 2.0 +acosh expected argument value not less than 1, got 0.5 +atanh expected a number between -1 and 1, got 1.0 +gamma expected a noninteger or positive integer, got 0.0 +gamma expected a noninteger or positive integer, got -3.0 +lgamma expected a noninteger or positive integer, got -2.0 +log1p expected argument value > -1, got -2.0 +sinh math range error +exp2 math range error +gamma math range error +Expected an int as second argument to ldexp. +factorial() not defined for negative values +[1e-323, 5e-324, 5e-324, 0.0, 1e-323, 1.0, 1.5657565312570098e-294, -3e-323, 4.4501477170144023e-308, 8.98846567431158e+307, 0.0] +OverflowError math range error diff --git a/maven/python-transpiler/src/test/resources/conformance/math2_.py b/maven/python-transpiler/src/test/resources/conformance/math2_.py new file mode 100644 index 00000000000..1949fe992d1 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/math2_.py @@ -0,0 +1,44 @@ +# The math functions beyond the elementary ones. +import math + +def r(x): + return float("%.12g" % x) + +for f in ["sinh", "cosh", "tanh", "asinh", "atanh", "expm1", "log1p", "erf", "erfc", "cbrt", "exp2"]: + print(f, [r(getattr(math, f)(x)) for x in [-0.75, -1e-9, 0.0, 1e-9, 0.3, 0.9]]) +print([r(math.acosh(x)) for x in [1, 1.5, 10, 1e10]]) +print([r(math.erf(x)) for x in [1.6, 3, 6]], [r(math.erfc(x)) for x in [1.6, 3, 6, -2]]) +print([r(math.gamma(x)) for x in [0.5, 1, 5, 7.5, -0.5, -2.5, 30.2]], math.gamma(4)) +print([r(math.lgamma(x)) for x in [0.5, 1, 2, 3, 10.5, 100, -0.5, -2.5]]) +print(math.frexp(8.0), math.frexp(-0.15625), math.frexp(0.0), math.frexp(5e-324), math.ldexp(0.5, 4), math.ldexp(1, -1074), math.ldexp(3, 1)) +print(math.modf(3.75), math.modf(-3.75), math.modf(2.0), math.modf(math.inf)) +print(math.remainder(10, 3), math.remainder(11, 3), math.remainder(-7, 2), math.remainder(5, 2), math.remainder(7, 2), math.remainder(4.5, 1)) +print(math.nextafter(1.0, 2.0), math.nextafter(1.0, 0.0), math.nextafter(0.0, 1.0), math.nextafter(1.0, 2.0, steps=3), math.ulp(1.0), math.ulp(1e300), math.ulp(0.0)) +print(math.dist((0, 0), (3, 4)), math.dist([1, 2, 3], [4, 6, 3]), math.lcm(4, 6), math.lcm(), math.lcm(3, 0), math.lcm(2, 3, 4)) +print(math.perm(5), math.perm(5, 2), math.perm(3, 5), math.sumprod([1, 2, 3], [4, 5, 6]), math.sumprod([1.5, 2], [2, 2]), math.fma(2.0, 3.0, 1.0)) +print(math.tanh(100), math.tanh(-0.0), math.sinh(-0.0), math.cbrt(-27), math.cbrt(8)) +for f, x in [(math.acosh, 0.5), (math.atanh, 1), (math.gamma, 0), (math.gamma, -3), (math.lgamma, -2), (math.log1p, -2)]: + try: + f(x) + except ValueError as e: + print(f.__name__, e) +for f, x in [(math.sinh, 1000), (math.exp2, 2000), (math.gamma, 200)]: + try: + f(x) + except OverflowError as e: + print(f.__name__, e) +try: + math.ldexp(1.0, 1.5) +except TypeError as e: + print(e) +try: + math.perm(-1) +except ValueError as e: + print(e) +print([math.ldexp(*a) for a in [(1.5, -1074), (0.75, -1074), (1.0000000000000002, -1075), (1.0, -1075), + (5e-324, 1), (5e-324, 1074), (1.7976931348623157e308, -2000), (-3.0, -1073), + (0.9999999999999999, -1021), (1.0, 1023), (2.5, -10**6)]]) +try: + math.ldexp(1.0, 1024) +except OverflowError as e: + print("OverflowError", e) diff --git a/maven/python-transpiler/src/test/resources/conformance/metaclasses.out b/maven/python-transpiler/src/test/resources/conformance/metaclasses.out new file mode 100644 index 00000000000..9b8db8a33fd --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/metaclasses.out @@ -0,0 +1,12 @@ +['Base', 'Child'] base child {'flavor': 'mint'} {} +Registry True True True +hello from Child! class Child CHILD +1 2 2 [0, 1] P3 True False +['B', 'alpha'] +_title Rec unset +Dune Dune ok +42 42 1 Lazy True +Box[int] True +5 dyn dyn Registry +True Plugin +TypeError Q.__init_subclass__() takes no keyword arguments diff --git a/maven/python-transpiler/src/test/resources/conformance/metaclasses.py b/maven/python-transpiler/src/test/resources/conformance/metaclasses.py new file mode 100644 index 00000000000..3c3f24bf6e3 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/metaclasses.py @@ -0,0 +1,157 @@ +class Registry(type): + classes = [] + + def __new__(mcs, name, bases, ns, **kw): + ns["tag"] = name.lower() + cls = super().__new__(mcs, name, bases, ns) + mcs.classes.append(name) + return cls + + def __init__(cls, name, bases, ns, **kw): + super().__init__(name, bases, ns) + cls.extra = kw + + def describe(cls): + return "class " + cls.__name__ + + @property + def upper(cls): + return cls.__name__.upper() + + +class Base(metaclass=Registry): + def hello(self): + return "hello from " + type(self).__name__ + + +class Child(Base, flavor="mint"): + def hello(self): + return super().hello() + "!" + + +print(Registry.classes, Base.tag, Child.tag, Child.extra, Base.extra) +print(type(Base).__name__, type(Child) is Registry, isinstance(Child, Registry), isinstance(Child, type)) +print(Child().hello(), Child.describe(), Child.upper) + + +class Counted(type): + def __call__(cls, *args, **kw): + cls.made = getattr(cls, "made", 0) + 1 + return super().__call__(*args, **kw) + + def __len__(cls): + return cls.made + + def __iter__(cls): + return iter(range(cls.made)) + + def __getitem__(cls, i): + return cls.__name__ + str(i) + + def __repr__(cls): + return "" + + def __contains__(cls, x): + return x == "yes" + + +class P(metaclass=Counted): + def __init__(self, x): + self.x = x + + +a = P(1) +b = P(x=2) +print(a.x, b.x, len(P), list(P), P[3], repr(P), "yes" in P, "no" in P) + + +class Plugin: + registry = {} + + def __init_subclass__(cls, name=None, **kw): + super().__init_subclass__(**kw) + Plugin.registry[name or cls.__name__] = cls + + +class A(Plugin, name="alpha"): + pass + + +class B(Plugin): + pass + + +print(sorted(Plugin.registry)) + + +class Field: + def __set_name__(self, owner, name): + self.name = "_" + name + self.owner = owner.__name__ + + def __get__(self, obj, objtype=None): + return getattr(obj, self.name, "unset") + + def __set__(self, obj, value): + setattr(obj, self.name, value) + + +class Rec: + title = Field() + + +r = Rec() +print(Rec.__dict__["title"].name, Rec.__dict__["title"].owner, r.title) +r.title = "Dune" +print(r.title, r._title, type(Rec.title).__name__ if False else "ok") + + +class Lazy: + def __init__(self, fn): + self.fn = fn + + def __get__(self, obj, objtype=None): + if obj is None: + return self + v = self.fn(obj) + obj.__dict__[self.fn.__name__] = v + return v + + +class Calc: + calls = 0 + + @Lazy + def answer(self): + Calc.calls += 1 + return 42 + + +cc = Calc() +print(cc.answer, cc.answer, Calc.calls, type(Calc.answer).__name__, "answer" in vars(cc)) + + +class Box: + def __class_getitem__(cls, item): + return cls.__name__ + "[" + item.__name__ + "]" + + +print(Box[int], list[int] is not None) + +Dyn = type("Dyn", (Base,), {"x": 5, "hello": lambda self: "dyn"}) +print(Dyn.x, Dyn().hello(), Dyn.tag, type(Dyn).__name__) + +bases = (Plugin,) + + +class S(*bases, name="star"): + pass + + +print("star" in Plugin.registry, S.__mro__[1].__name__) + +try: + class Q(x=1): + pass +except TypeError as e: + print("TypeError", e) diff --git a/maven/python-transpiler/src/test/resources/conformance/numbers.out b/maven/python-transpiler/src/test/resources/conformance/numbers.out new file mode 100644 index 00000000000..db528038dac --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/numbers.out @@ -0,0 +1,87 @@ +fused float float 4.038873605350878 +fused mixed float 14.75 +fused ints int 50 +int leaf float const float 13.0 +zero div ZeroDivisionError division by zero +int zero div ZeroDivisionError division by zero +pow overflow OverflowError (34, 'Result too large') +neg float 1.125 +float overflow inf float inf +big int add int 9223372036854775808 +big int mul int 18446744073709551616 +big int sub int -9223372036854775817 +floordiv neg int -4 +mod neg int 2 +mod neg divisor int -2 +mixed cmp tuple (True, True, True, False) +shift tuple (4611686018427387904, 18446744073709551616, -4, 0) +bits tuple (2, 7, 5, -6) +true div ints tuple (3.5, -3.5, 3.843071682022823e+17) +hash eq tuple ('a', 'b', True) +bool arith tuple (2, 2.5, -1) +slice list [2, 4, 6] +slice neg list [8, 5] +slice copy list [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] +const index tuple (0, 9, 9) +const index err IndexError list index out of range +same len assign [2, 1, 0, 3, 4, 5] +grow assign [2, 10, 11, 12, 0, 3, 4, 5] +shrink assign [0, 3, 4, 5] +extended assign [0, 3, 0, 5] +extended err attempt to assign sequence of size 1 to extended slice of size 2 +self assign [1, 2, 3] +self grow [1, 1, 2, 3] +tuple const index tuple (4, 6) +str slice str 'ell' +2193406845013623645 9223372030926249001 9223372037000250000 +-9223372036854775807 9223372036854775808 9223372036854775808 +1109335178 +1 None 7 1 +None [3] +['x', 'y', 'z'] ['a', 'b', '', 'c'] ABC abc +p-q p-q r-s +3 2 0 +TypeError sequence item 1: expected str instance, int found +[-10.069251117961866, 2.889287802318147] +[-6.211102550927978, 2.8320502943378436] +[-9.752906583803284, 2.813733471206735] +skip +TypeError 'tuple' object does not support item assignment +(0, -2) (2, 1) (4, 4) (6, 7) (8, 10) (10, 13) (12, 16) (14, 19) (16, 22) (18, 25) (21.0, 29.5) (23.0, 32.5) +8.75 14.0 17.25 +3.0 +UnboundLocalError cannot access local variable 'q' where it is not associated with a value +[-1.0, 2.5, 3.5, 4.5] 4 4.5 [2.5, 3.5] True 2 9.5 +[0.0, 2.25, 7] [0.0, 2.25, 7, 0.0, 2.25, 7] [0.0, 2.25, 7, -1.0, 2.5, 3.5, 4.5] +[-1.0, 2.5, 3.5, 4.5, 's'] +[9.0, 1.5, 1.0, 0.5] 0.0 True [18.0, 3.0, 2.0, 1.0] +0.25 0.75 [1.0, 2.0, 3.0] [2.0, 1.0] +[5.0, 6.0, 3.0] 1 6.0 3.0 +([1.0, 3.0], [0.5, -0.5]) ([1.0, 3.0], [0.5, -0.5]) ([1, 4.0], [0, -2.0]) +[1, 2, 3, 11, 12] +[2, 4] +[1, 2] [9, 9, 9] +empty else +1109335178 +[55, 23, 2, 0, 2, 16, -23, 6, 0.009523809523809525, 16.0] [6, -19, -5, 0, -5, 30, 19, -6, 0.017857142857142856, -12.0] [10, -23, -5, 0, -3, -32, 23, -2, 0.016666666666666666, -16.0] [2417851639230357861040128, 1208925819614629174706179, 0, 0, 366503875924, 7696581394431, -1208925819614629174706179, 3, 2.7572687085092564e-25, 4398046511102.5] +[42535295865117307937533511947398414336, 21267647932558653966460912964485513221, 0, 0, 922337203685477576, 32281802128991715327, -21267647932558653966460912964485513221, 5, 1.5673258010963833e-38, 2.7670116110564327e+19] [42535295865117307919086767873688862721, -9223372036854775809, 9223372036854775809, 0, 9223372036854775808, 9223372036854775815, 9223372036854775809, -9223372036854775801, 7.836629005481916e-39, 9.223372036854776e+18] +9223372036854775809 5316911983139663491615228241121378303 4611686018427387904 4 0 -1 -9223372036854775809 +3002399751580331.0 2.0 10 -11 -3 +4 3-43-4 13 1.5 4 +ZeroDivisionError division by zero +ZeroDivisionError division by zero +ValueError negative shift count +7.0 32.0 6.0 +0.0 (0.6361904761904762, 5.0625) (0.6361904761904762, 5.0625) (0, 1.0) +0.05555555555555555 -0.3333333333333333 2.350988701644575e-38 5.5 5.5 2.0 +nan [1.0, 4.0] +TypeError can't multiply sequence by non-int of type 'float' +(6.0, 3.375) +2.5 +(10.569346804511278, 4) (0.0, 'before') (6.378721804511279, 3) +TypeError can't multiply sequence by non-int of type 'float' +(6.917529027641082e+18, 4611686018427387906) +(0.011427895039841603, 20) +0.7416666666666667 +10.0 (6.0, 1) +(104.66666666666667, 5, [1.0, 2.0, 3.0, 4.0]) (8.10008100042e+16, 3, [1.0, 2.0, 3.0, 4.0]) (2.1922524559963544e+48, 3, [1.0, 2.0, 3.0, 4.0]) (5.602277094844144e+28, 2, [1.0, 2.0, 3.0, 4.0]) diff --git a/maven/python-transpiler/src/test/resources/conformance/numbers.py b/maven/python-transpiler/src/test/resources/conformance/numbers.py new file mode 100644 index 00000000000..f9e38a0de28 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/numbers.py @@ -0,0 +1,354 @@ +# Int and float fast paths, fused float expressions, slices and constant indexes. + +def show(label, f): + try: + r = f() + print(label, type(r).__name__, repr(r)) + except Exception as e: + print(label, type(e).__name__, e) + + +a, b, c = 1.5, 2.25, -3.0 +i, j = 7, -2 +show("fused float", lambda: (a * a + b * b + c * c) ** 0.5) +show("fused mixed", lambda: a * i + b - j) +show("fused ints", lambda: i * i + j * j - 3) +show("int leaf float const", lambda: i * 2.0 + j * 0.5) +show("zero div", lambda: a * b / (c - c)) +show("int zero div", lambda: i // (j - j)) +show("pow overflow", lambda: (a * 1e200) ** 2.0) +show("neg", lambda: -a * b + -(c - a)) +show("float overflow inf", lambda: a * 1e308 * 10.0) +show("big int add", lambda: 9223372036854775807 + 1) +show("big int mul", lambda: 4611686018427387904 * 4) +show("big int sub", lambda: -9223372036854775807 - 10) +show("floordiv neg", lambda: -7 // 2) +show("mod neg", lambda: -7 % 3) +show("mod neg divisor", lambda: 7 % -3) +show("mixed cmp", lambda: (1 < 1.5, 2 == 2.0, 3 >= 3.0, 2 ** 53 + 1 == float(2 ** 53 + 1))) +show("shift", lambda: (1 << 62, 1 << 64, -16 >> 2, 5 >> 70)) +show("bits", lambda: (6 & 3, 6 | 3, 6 ^ 3, ~5)) +show("true div ints", lambda: (7 / 2, -7 / 2, 2 ** 60 / 3)) +show("hash eq", lambda: ({1: "a"}[1.0], {2.0: "b"}[2], hash(1) == hash(1.0))) +show("bool arith", lambda: (True + True, True * 2.5, -True)) + +xs = list(range(10)) +show("slice", lambda: xs[2:8:2]) +show("slice neg", lambda: xs[8:2:-3]) +show("slice copy", lambda: xs[:]) +show("const index", lambda: (xs[0], xs[-1], xs[9])) +show("const index err", lambda: xs[10]) +ys = list(range(6)) +ys[:3] = ys[2::-1] +print("same len assign", ys) +ys[1:2] = [10, 11, 12] +print("grow assign", ys) +ys[0:4] = [] +print("shrink assign", ys) +ys[::2] = [0, 0] +print("extended assign", ys) +try: + ys[::2] = [1] +except ValueError as e: + print("extended err", e) +zs = [1, 2, 3] +zs[:] = zs +print("self assign", zs) +zs[1:] = zs +print("self grow", zs) +t = (4, 5, 6) +show("tuple const index", lambda: (t[0], t[-1])) +show("str slice", lambda: "hello"[1:4]) + +# Wide int products: exact in 64 bits, and the ones that overflow into big ints. +print(1103515245 * 1987654321, 3037000499 * 3037000499, 3037000500 * 3037000500) +print(-9223372036854775807 * 1, (-9223372036854775807 - 1) * -1, 4611686018427387904 * 2) +s = 12345 +for _ in range(5): + s = (s * 1103515245 + 12345) & 0x7fffffff +print(s) + +# Builtin methods called directly. +d = {"a": 1} +print(d.get("a"), d.get("b"), d.get("b", 7), d.get("a", 7)) +lst = [] +print(lst.append(3), lst) +print("x y z".split(), "a,b,,c".split(","), "AbC".upper(), "AbC".lower()) +print("-".join(["p", "q"]), "-".join(("p", "q")), "-".join(c for c in "rs"), "".join([])) +print(len("abc"), len([1, 2]), len({})) +try: + "-".join(["p", 1]) +except TypeError as e: + print("TypeError", e) + +# Float regions: straight-line float statements, their bail-outs and locals. +def region(a, b, v): + dx = a - b + dy = a * b + mag = (dx * dx + dy * dy) ** 0.5 + v[0] -= dx * mag + v[1] += dy / mag + return v + + +print(region(3.5, 1.25, [1.0, 2.0])) +print(region(3, 1, [1, 2])) # ints all the way: the generic path +print(region(3.5, 1, [1.0, 2])) # mixed: bails part way +print(region(3.5, 1.25, (1.0, 2.0)) if False else "skip") +try: + region(3.5, 1.25, (1.0, 2.0)) +except TypeError as e: + print("TypeError", e) +try: + region(1.5, 1.5, [0.0, 0.0]) # zero magnitude: ZeroDivisionError in the region +except ZeroDivisionError as e: + print("ZeroDivisionError", e) + + +def region_live(a, b): + t = a * b + u = t + a + w = u - b + return t, w # t is read after the region + + +for k in range(12): # ints first, then floats: the fast path turns itself off + print(region_live(k, 2) if k < 10 else region_live(k + 0.5, 2.0), end=" ") +print() + + +def region_loop(xs): + total = 0.0 + for x in xs: + sq = x * x + total += sq + return total + + +print(region_loop([0.5, 1.5, 2.5]), region_loop([1, 2, 3]), region_loop([1.0, 2, 3.5])) + + +def region_unbound(f): + if f: + q = 1.0 + r = q * 2.0 + s = r + 1.0 + return s + + +print(region_unbound(True)) +try: + region_unbound(False) +except UnboundLocalError as e: + print("UnboundLocalError", e) + +# Lists of floats, kept unboxed until something else goes in. +fl = [1.5, 2.5, 3.5] +fl.append(4.5) +fl[0] = -1.0 +print(fl, len(fl), fl[-1], fl[1:3], 2.5 in fl, fl.index(3.5), sum(fl)) +z = [0.0] * 3 +z[1] += 2.25 +z[2] = 7 # an int: the list becomes boxed +print(z, z * 2, z + fl) +fl.append("s") +print(fl) +g = [] +for q in range(4): + g.append(q * 0.5) +g.insert(0, 9.0) +g.sort() +g.reverse() +print(g, g.pop(), g == [9.0, 1.5, 1.0, 0.5], [x * 2 for x in g]) +a1, a2 = [0.25, 0.75] +print(a1, a2, sorted([3.0, 1.0, 2.0]), list(reversed([1.0, 2.0]))) +h = [1.0, 2.0, 3.0] +h[1:2] = [5.0, 6.0] +del h[0] +print(h, h.count(6.0), max(h), min(h)) + + +def region_list(v, w): + d = v[0] - w[0] + e = d * 2.0 + v[1] += e + w[1] -= e + return v, w + + +print(region_list([1.0, 2.0], [0.5, 0.5]), region_list([1.0, 2], [0.5, 0.5]), region_list([1, 2], [0, 0])) + +# A list changed while a for loop walks it. +grow = [1, 2, 3] +for x in grow: + if x < 3: + grow.append(x + 10) +print(grow) +shrink = [1, 2, 3, 4, 5] +for x in shrink: + shrink.remove(x) +print(shrink) +seen = [] +rebound = [1, 2] +for x in rebound: + rebound = [9, 9, 9] + seen.append(x) +print(seen, rebound) +for x in []: + print("never") +else: + print("empty else") + +# Fused int trees: every edge where long arithmetic leaves Python's ints. +def lcg(seed): + return (seed * 1103515245 + 12345) & 0x7fffffff + + +def int_tree(a, b, c): + return [(a + b) * (a + b + 1) >> 1, a * b + c, (a - b) // c, (a - b) % c, a // c - b % c, + ~a + (b << 3), -a * b - c, (a ^ b) | c & 7, 1.0 / (((a + b) * (a + b + 1) >> 1) + a * a + 1), a * c + b - 0.5 * c] + + +s = 12345 +for _ in range(5): + s = lcg(s) +print(s) +print(int_tree(7, 3, 2), int_tree(-7, 3, 2), int_tree(7, -3, -2), int_tree(2 ** 40, 2 ** 40, 3)) +print(int_tree(2 ** 62, 2 ** 62, 5), int_tree(-2 ** 63, 1, -1)) +big = 2 ** 61 +print(big * 4 + 1, big * big - 1, (big << 3) >> 2, (1 << 70) >> 68, 5 >> 100, -5 >> 100, (-big - big) * 2 - 1) +print((2 ** 53 + 1) / 3 + 0, (2 ** 60) / (2 ** 59 + 1) + 0, 7 // 2 * 3 + 1, -7 // 2 * 3 + 1, 7 % -3 * 2 + 1) +print(True + True * 3, "%d-%d" % (3, 4) * 2, 3 * 2 ** 2 + 1, 2 ** -1 + 1, 10 // 3 + 10 % 3) +for bad in [(1, 0), (0, 0)]: + try: + print(bad[0] // bad[1] + 1) + except ZeroDivisionError as e: + print("ZeroDivisionError", e) +try: + print((1 << -1) + 1) +except ValueError as e: + print("ValueError", e) + +# Pure float functions, and loops that keep their float locals unboxed. +def tri(i, j): + return 1.0 / (((i + j) * (i + j + 1) >> 1) + i + 1) + + +def scale(x, k): + return x * k - 0.5 + + +def dot(u, v, n): + s = 0.0 + for i in range(n): + s += u[i] * v[i] + return s + + +def accumulate(n, items): + s = 0 + t = 1.0 + for j in range(n): + s += tri(j, 3) * items[j] + t *= 1.5 + return s, t + + +def inf_minus(a, b): + return a * 2.0 - b + + +print(dot([1.0, 2.0, 3.0], [0.5, 0.25, 2.0], 3), dot([1, 2, 3], [4, 5, 6], 3), dot([1.0, 2, 3.0], [1.0, 1.0, 1.0], 3)) +print(dot([], [], 0), accumulate(4, [1.0, 2.0, 3.0, 4.0]), accumulate(4, [1.0, 2, 3.0, 4]), accumulate(0, [])) +print(tri(2, 3), tri(-5, 3), tri(2 ** 62, 2 ** 62), scale(3, 2), scale(3.0, 2.0), scale(True, 2.5)) +print(inf_minus(float("inf"), float("inf")), [inf_minus(x, 1.0) for x in (1.0, 2.5)]) +try: + accumulate(3, [1.0, "x", 2.0]) +except TypeError as e: + print("TypeError", e) +saved = tri +tri = lambda i, j: 2 +print(accumulate(3, [1.0, 1.0, 1.0])) +tri = saved +total = 0.0 +for k in range(5): + total += scale(k, 0.5) +print(total) + + +# A counted loop whose index a float region reads unboxed. +def k2(i, j): + return 1.0 / (((i + j) * (i + j + 1) >> 1) + i + 1) + + +def sweep(v, start, stop): + s = 0.0 + j = "before" + for j in range(start, stop): + s += k2(3, j) * v[j] + j + return s, j + + +vals = [0.5, 1.5, 2.5, 3.5, 4.5] +print(sweep(vals, 0, 5), sweep(vals, 2, 2), sweep(vals, 1, 4)) +mixed = [0.5, 1.5, "x", 3.5] +try: + print(sweep(mixed, 0, 4)) +except TypeError as e: + print("TypeError", e) + + +def sweep_big(n): + s = 0.0 + for j in range(2 ** 62, 2 ** 62 + n): + s += j * 0.5 + return s, j + + +print(sweep_big(3)) + + +def shadowed(): + def range(a, b): + return [10, 20] + s = 0.0 + for j in range(0, 2): + s += k2(j, j) * 2.0 + return s, j + + +print(shadowed()) + + +def rebound(n): + global k2 + s = 0.0 + for j in range(n): + s += k2(1, j) + if_rebind = 0 + return s + + +print(rebound(4)) +old_k2 = k2 +k2 = lambda i, j: 2.5 +print(rebound(4), sweep(vals, 0, 2)) +k2 = old_k2 + + +def int_parts(n, base): + s = 0.5 + for j in range(base, base + n): + s += j * j * j + s -= j / 3 + s += (j - base) ** 2 * 0.25 + t = 0 + for j in range(n): + t = j + 1 + lst = [0.5] * 4 + for j in range(4): + lst[j] = lst[j] * 2.0 + j + return s, t, lst + + +print(int_parts(5, 0), int_parts(3, 300000), int_parts(3, 2 ** 53 - 1), int_parts(2, 3037000499)) diff --git a/maven/python-transpiler/src/test/resources/conformance/open_.out b/maven/python-transpiler/src/test/resources/conformance/open_.out new file mode 100644 index 00000000000..4cdc0ea23af --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/open_.out @@ -0,0 +1,32 @@ +12 +w a.txt False +True +'hello\nworld\nline three' +'hello\n' +'world\n' +'line three' +['hello\n', 'world\n', 'line three'] +'hello\n' 'wor' 9 +'hello\nworld\nline three\nappended' +b'\x00\x01' b'\xffab' +b'\x01' 2 +b'\x00\x01\xffabZ' +b'caf\xc3\xa9 \xe2\x82\xac\n' +café € + +'a\nb\nc\n' +'a\r\nb\rc\n' +xyz +Ayz +['1', '2'] +FileNotFoundError 2 No such file or directory missing.txt +[Errno 2] No such file or directory: 'missing.txt' +[Errno 17] File exists: 'w.txt' +must have exactly one of create/read/write/append mode +invalid mode: 'q' +can't have text and binary mode at once +binary mode doesn't take an encoding argument +I/O operation on closed file. +UnsupportedOperation +UnsupportedOperation +write() argument must be str, not int diff --git a/maven/python-transpiler/src/test/resources/conformance/open_.py b/maven/python-transpiler/src/test/resources/conformance/open_.py new file mode 100644 index 00000000000..7b85707dbfa --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/open_.py @@ -0,0 +1,86 @@ +# open(): text and binary, modes, newlines, errors. +with open("a.txt", "w") as f: + print(f.write("hello\nworld\n")) + f.write("line three") + print(f.mode, f.name, f.closed) +print(f.closed) +with open("a.txt") as f: + print(repr(f.read())) +with open("a.txt") as f: + for line in f: + print(repr(line)) +with open("a.txt") as f: + print(f.readlines()) +with open("a.txt") as f: + print(repr(f.readline()), repr(f.read(3)), f.tell()) +with open("a.txt", "a") as f: + f.write("\nappended") +print(repr(open("a.txt").read())) +with open("b.bin", "wb") as f: + f.write(b"\x00\x01\xff") + f.write(bytearray(b"ab")) +with open("b.bin", "rb") as f: + print(f.read(2), f.read()) + f.seek(1) + print(f.read(1), f.tell()) +with open("b.bin", "ab") as f: + f.write(b"Z") +print(open("b.bin", "rb").read()) +with open("u.txt", "w", encoding="utf-8") as f: + f.write("café €\n") +print(open("u.txt", "rb").read()) +print(open("u.txt", encoding="utf-8").read()) +with open("n.txt", "wb") as f: + f.write(b"a\r\nb\rc\n") +print(repr(open("n.txt").read())) +print(repr(open("n.txt", newline="").read())) +with open("r.txt", "w+") as f: + f.write("xyz") + f.seek(0) + print(f.read()) +with open("r.txt", "r+") as f: + f.write("A") +print(open("r.txt").read()) +f = open("w.txt", "w") +f.writelines(["1\n", "2\n"]) +f.close() +print(open("w.txt").read().split()) +try: + open("missing.txt") +except FileNotFoundError as e: + print(type(e).__name__, e.errno, e.strerror, e.filename) + print(e) +try: + open("w.txt", "x") +except FileExistsError as e: + print(e) +for bad in ["rw", "q", "rbt"]: + try: + open("w.txt", bad) + except ValueError as e: + print(e) +try: + open("w.txt", "rb", encoding="utf-8") +except ValueError as e: + print(e) +f = open("w.txt") +f.close() +try: + f.read() +except ValueError as e: + print(e) +with open("w.txt") as f: + try: + f.write("x") + except OSError as e: + print(type(e).__name__) +with open("w.txt", "w") as f: + try: + f.read() + except OSError as e: + print(type(e).__name__) +try: + with open("w.txt", "w") as f: + f.write(5) +except TypeError as e: + print(e) diff --git a/maven/python-transpiler/src/test/resources/conformance/os_.out b/maven/python-transpiler/src/test/resources/conformance/os_.out new file mode 100644 index 00000000000..b2161a1782c --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/os_.out @@ -0,0 +1,37 @@ +/ posix . .. True +a/b/c.txt a/b /abs/x x +('/a/b', 'c') ('', 'c') ('/', '') ('a', 'b') +('x/y.tar', '.gz') ('.bashrc', '') ('a.b/c', '') ('f', '.') + /a/b +a/c/d ../x /x . //a +True False +/usr/l /usr +../b/c . +('', '/', 'a/b') ('', 'x/y') +True +True True False +['f.txt', 'x'] +5 5 True +['f.txt', 'x'] [False, True] +d1 ['x'] ['f.txt'] +d1/x ['y'] [] +d1/x/y [] [] +True ['f.txt', 'x'] +12345 +['g.txt', 'x'] +['h.txt', 'y'] +True True +False +FileNotFoundError [Errno 2] No such file or directory: 'nope' +FileNotFoundError [Errno 2] No such file or directory: 'nope' +FileNotFoundError [Errno 2] No such file or directory: 'nope' +FileNotFoundError [Errno 2] No such file or directory: 'nope' +FileNotFoundError [Errno 2] No such file or directory: 'nope' +[Errno 17] File exists: 'z' +[Errno 2] No such file or directory: 'p/q' +True +z True s b'a' +expected str, bytes or os.PathLike object, not int +None dflt +False getsize +5 diff --git a/maven/python-transpiler/src/test/resources/conformance/os_.py b/maven/python-transpiler/src/test/resources/conformance/os_.py new file mode 100644 index 00000000000..2672cd83526 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/os_.py @@ -0,0 +1,74 @@ +# os and os.path over the application's files. +import os +import os.path +from os.path import join, splitext + +print(os.sep, os.name, os.curdir, os.pardir, os.linesep == "\n") +print(join("a", "b", "c.txt"), join("a/", "b"), join("a", "/abs", "x"), join("", "x")) +print(os.path.split("/a/b/c"), os.path.split("c"), os.path.split("/"), os.path.split("a//b")) +print(splitext("x/y.tar.gz"), splitext(".bashrc"), splitext("a.b/c"), splitext("f.")) +print(os.path.basename("/a/b/"), os.path.dirname("/a/b/c"), os.path.dirname("c")) +print(os.path.normpath("a/./b/../c//d/"), os.path.normpath("../x"), os.path.normpath("/../x"), + os.path.normpath(""), os.path.normpath("//a")) +print(os.path.isabs("/x"), os.path.isabs("x")) +print(os.path.commonprefix(["/usr/lib", "/usr/local"]), os.path.commonpath(["/usr/lib", "/usr/local"])) +print(os.path.relpath("a/b/c", "a/d"), os.path.relpath("a", "a")) +print(os.path.splitroot("/a/b"), os.path.splitdrive("x/y")) +start = os.getcwd() +print(os.path.abspath("q") == join(start, "q")) +os.mkdir("d1") +print(os.path.exists("d1"), os.path.isdir("d1"), os.path.isfile("d1")) +os.makedirs("d1/x/y") +os.makedirs("d1/x/y", exist_ok=True) +with open("d1/f.txt", "w") as f: + f.write("12345") +print(sorted(os.listdir("d1"))) +print(os.path.getsize("d1/f.txt"), os.stat("d1/f.txt").st_size, os.path.isfile("d1/f.txt")) +print(sorted(e.name for e in os.scandir("d1")), [e.is_dir() for e in sorted(os.scandir("d1"), key=lambda e: e.name)]) +for root, dirs, files in os.walk("d1"): + print(root, sorted(dirs), sorted(files)) +os.chdir("d1") +print(os.getcwd() == join(start, "d1"), sorted(os.listdir())) +print(open("f.txt").read()) +os.rename("f.txt", "g.txt") +print(sorted(os.listdir("."))) +os.replace("g.txt", "x/h.txt") +print(os.listdir("x")) +os.chdir("..") +os.remove("d1/x/h.txt") +os.rmdir("d1/x/y") +try: + os.rmdir("d1") +except OSError as e: + print(isinstance(e, OSError), e.errno in (39, 66)) +os.removedirs("d1/x") +print(os.path.exists("d1")) +for f, arg in [(os.remove, "nope"), (os.listdir, "nope"), (os.stat, "nope"), (os.rmdir, "nope"), (os.chdir, "nope")]: + try: + f(arg) + except OSError as e: + print(type(e).__name__, e) +os.mkdir("z") +try: + os.mkdir("z") +except FileExistsError as e: + print(e) +try: + os.mkdir("p/q") +except FileNotFoundError as e: + print(e) +try: + os.remove("z") +except OSError as e: + print(isinstance(e, OSError)) +class P: + def __fspath__(self): + return "z" +print(os.fspath(P()), os.path.isdir(P()), os.fspath("s"), os.fsencode("a")) +try: + os.fspath(3) +except TypeError as e: + print(e) +print(os.getenv("CN1_NOT_SET_XYZ"), os.getenv("CN1_NOT_SET_XYZ", "dflt")) +print(os.path.exists("missing"), os.path.getsize.__name__) +print(len(os.urandom(5))) diff --git a/maven/python-transpiler/src/test/resources/conformance/pathlib_.out b/maven/python-transpiler/src/test/resources/conformance/pathlib_.out new file mode 100644 index 00000000000..e65cc9d3d7b --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/pathlib_.out @@ -0,0 +1,33 @@ +/usr/lib/python3.tar.gz PurePosixPath('/usr/lib/python3.tar.gz') python3.tar.gz .gz ['.tar', '.gz'] python3.tar /usr/lib / / ('/', 'usr', 'lib', 'python3.tar.gz') +['/usr/lib', '/usr', '/'] +/usr/lib/x.py /usr/lib/python3.tar.bz2 /usr/lib/y.gz /usr/lib/python3.tar +a/b/c/d a/b/c x/y . a/b/c +b/c True False +../b +'/a/b' is not in the subpath of '/c' +True True False +True True +True True True +[PurePosixPath('a'), PurePosixPath('a/z'), PurePosixPath('b')] + . / +file:///a True False +Invalid suffix 'py' +PosixPath PurePosixPath True +alpha b'print(1)' 5 +['work/.hidden.py', 'work/a.txt', 'work/b.py', 'work/sub'] +['work/.hidden.py', 'work/b.py'] +['work/.hidden.py', 'work/b.py', 'work/sub/c.py', 'work/sub/deep/d.py'] +['work/sub/c.py'] +True True True False +True True +work/z.txt True False +[Errno 2] No such file or directory: 'work/z.txt' +[Errno 17] File exists: 'work' +hi +work ['sub'] ['.hidden.py', 'b.py', 'w.txt'] +work/sub ['deep'] ['c.py'] +work/sub/deep [] ['d.py'] +True True True False False +['a.py', 'c.py'] (?s:.*\.py)\z (?s:[^a-c].)\z +['work/b.py'] ['work/b.py', 'work/sub/c.py', 'work/sub/deep/d.py'] +['b.py'] ['work/sub/'] True a[*]b[?] diff --git a/maven/python-transpiler/src/test/resources/conformance/pathlib_.py b/maven/python-transpiler/src/test/resources/conformance/pathlib_.py new file mode 100644 index 00000000000..493fa0d3c04 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/pathlib_.py @@ -0,0 +1,67 @@ +# pathlib, fnmatch and glob. +from pathlib import Path, PurePath, PurePosixPath +import fnmatch +import glob + +p = PurePosixPath("/usr/lib/python3.tar.gz") +print(p, repr(p), p.name, p.suffix, p.suffixes, p.stem, p.parent, p.root, p.anchor, p.parts) +print([str(x) for x in p.parents]) +print(p.with_name("x.py"), p.with_suffix(".bz2"), p.with_stem("y"), p.with_suffix("")) +print(PurePath("a", "b/c", "d"), PurePath("a") / "b" / "c", "x" / PurePath("y"), PurePath(), PurePath("a//b/./c/")) +print(PurePath("/a/b/c").relative_to("/a"), PurePath("/a/b").is_relative_to("/a"), PurePath("/a/b").is_relative_to("/c")) +print(PurePath("a/b").relative_to("a/c", walk_up=True)) +try: + PurePath("/a/b").relative_to("/c") +except ValueError as e: + print(e) +print(PurePath("a/b.py").match("*.py"), PurePath("/a/b/c.py").match("b/*.py"), PurePath("a/b.py").match("/*.py")) +print(PurePath("a/b/c.py").full_match("a/**/*.py"), PurePath("a.py").full_match("**/*.py")) +print(PurePath("a") == PurePath("a/"), PurePath("a") < PurePath("b"), hash(PurePath("a")) == hash(PurePath("a"))) +print(sorted([PurePath("b"), PurePath("a/z"), PurePath("a")])) +print(PurePath(".bashrc").suffix, PurePath("x.").suffix, PurePath("/").name, PurePath("/").parent) +print(Path("/a").as_uri(), PurePath("/a").is_absolute(), PurePath("a").is_absolute()) +try: + PurePath("a").with_suffix("py") +except ValueError as e: + print(e) +print(type(Path("x")).__name__, type(PurePath("x")).__name__, isinstance(Path("x"), PurePath)) + +base = Path("work") +base.mkdir() +(base / "sub" / "deep").mkdir(parents=True) +(base / "a.txt").write_text("alpha") +(base / "b.py").write_bytes(b"print(1)") +(base / "sub" / "c.py").write_text("c") +(base / "sub" / "deep" / "d.py").write_text("d") +(base / ".hidden.py").touch() +print((base / "a.txt").read_text(), (base / "b.py").read_bytes(), (base / "a.txt").stat().st_size) +print(sorted(str(x) for x in base.iterdir())) +print(sorted(str(x) for x in base.glob("*.py"))) +print(sorted(str(x) for x in base.rglob("*.py"))) +print(sorted(str(x) for x in base.glob("*/*.py"))) +print(base.exists(), base.is_dir(), (base / "a.txt").is_file(), (base / "zz").exists()) +print(Path("work/sub/../a.txt").resolve() == Path.cwd() / "work" / "a.txt", Path("w").absolute() == Path.cwd() / "w") +t = (base / "a.txt").rename(base / "z.txt") +print(t, t.exists(), (base / "a.txt").exists()) +(base / "z.txt").unlink() +(base / "z.txt").unlink(missing_ok=True) +try: + (base / "z.txt").unlink() +except FileNotFoundError as e: + print(e) +base.mkdir(exist_ok=True) +try: + base.mkdir() +except FileExistsError as e: + print(e) +with (base / "w.txt").open("w") as f: + f.write("hi") +print((base / "w.txt").read_text()) +for root, dirs, files in sorted(base.walk()): + print(root, sorted(dirs), sorted(files)) + +print(fnmatch.fnmatch("foo.py", "*.py"), fnmatch.fnmatch("foo.py", "f?o.*"), fnmatch.fnmatch("foo", "[a-f]*"), + fnmatch.fnmatch("zoo", "[!z]*"), fnmatch.fnmatchcase("A.PY", "*.py")) +print(fnmatch.filter(["a.py", "b.txt", "c.py"], "*.py"), fnmatch.translate("*.py"), fnmatch.translate("[!a-c]?")) +print(sorted(glob.glob("work/*.py")), sorted(glob.glob("work/**/*.py", recursive=True))) +print(sorted(glob.glob("*.py", root_dir="work")), sorted(glob.glob("work/s*/")), glob.has_magic("a*"), glob.escape("a*b?")) diff --git a/maven/python-transpiler/src/test/resources/conformance/pprint_.out b/maven/python-transpiler/src/test/resources/conformance/pprint_.out new file mode 100644 index 00000000000..4f1881c8828 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/pprint_.out @@ -0,0 +1,354 @@ +{3: 'int key', + 'bytes': b'\x00\x01binary data that goes on and on and on and on and on and o' + b'n and on', + 'name': 'codename one', + 'nested': {'alpha': [0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29], + 'beta': {'x': 1.5, 'y': (1, 2, 3), 'z': None}, + 'gamma': 'a short string'}, + 'targets': ['ios', 'android', 'javascript', 'windows', 'linux', 'mac']} +{ 3: 'int key', + 'bytes': b'\x00\x01binary data that g' + b'oes on and on and on and' + b' on and on and on and on', + 'name': 'codename one', + 'nested': { 'alpha': [ 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29], + 'beta': { 'x': 1.5, + 'y': (1, 2, 3), + 'z': None}, + 'gamma': 'a short ' + 'string'}, + 'targets': [ 'ios', + 'android', + 'javascript', + 'windows', + 'linux', + 'mac']} +{3: 'int key', + 'bytes': b'\x00\x01binary data that goes on and on and on and on and on and o' + b'n and on', + 'name': 'codename one', + 'nested': {...}, + 'targets': [...]} +{3: 'int key', + 'bytes': b'\x00\x01binary data that goes on a' + b'nd on and on and on and on and on an' + b'd on', + 'name': 'codename one', + 'nested': {'alpha': [0, 1, 2, 3, 4, 5, 6, 7, 8, + 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, + 23, 24, 25, 26, 27, 28, + 29], + 'beta': {'x': 1.5, + 'y': (1, 2, 3), + 'z': None}, + 'gamma': 'a short string'}, + 'targets': ['ios', 'android', 'javascript', + 'windows', 'linux', 'mac']} +{'b': 1, + 'a': 2, + 'c': [3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3]} +[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 16, + 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, + 31, 32, 33, 34, 35, 36, 37, + 38, 39] +('this is a long string that will need ' + 'to be wrapped across several lines ' + 'of output this is a long string that ' + 'will need to be wrapped across ' + 'several lines of output this is a ' + 'long string that will need to be ' + 'wrapped across several lines of ' + 'output ') +('one',) () [] {} set() +{1, + 3, + 5, + 'a', + (1, 2)} +frozenset({0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24}) +OrderedDict([('0', 0), + ('1', 1000), + ('2', 2000), + ('3', 3000), + ('4', 4000), + ('5', 5000), + ('6', 6000), + ('7', 7000), + ('8', 8000), + ('9', 9000), + ('10', 10000), + ('11', 11000)]) +defaultdict(, + {'key0': [0], + 'key1': [11111], + 'key10': [111110], + 'key11': [122221], + 'key2': [22222], + 'key3': [33333], + 'key4': [44444], + 'key5': [55555], + 'key6': [66666], + 'key7': [77777], + 'key8': [88888], + 'key9': [99999]}) +Counter({' ': 11, + 'e': 4, + 'o': 4, + 'm': 4, + 't': 3, + 'n': 3, + 'a': 3, + 'y': 3, + 'h': 2, + 'u': 2, + 'i': 2, + 'r': 2, + 's': 2, + 'q': 1, + 'c': 1, + 'k': 1, + 'b': 1, + 'w': 1, + 'f': 1, + 'x': 1, + 'j': 1, + 'p': 1, + 'v': 1, + 'l': 1, + 'z': 1, + 'd': 1, + 'g': 1}) +deque([0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29], + maxlen=40) deque([0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29]) +ChainMap({'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa': 1}, + {'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbb': 2}, + {'cccccccccccccccccccccccccccccc': 3}) +Point(x=1.0, + y=2.0, + tags=['tag0', + 'tag1', + 'tag2', + 'tag3', + 'tag4', + 'tag5', + 'tag6', + 'tag7', + 'tag8', + 'tag9', + 'tag10', + 'tag11']) +namespace(alpha='aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', + beta=[0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19], + gamma={'k': 'v'}) +True False True False +[1, 2, \\w+)\\s(?P=word)' (0, 11) 'hello hello' ('hello',) 1 word + True False ['hello'] +'\\d{2,3}' (1, 4) '123' () None None + False False ['123', '45'] +'\\d{,2}x' (1, 4) '23x' () None None + False False ['23x'] +'x{2}' (0, 2) 'xx' () None None + True False ['xx'] +'a{,}' (0, 3) 'aaa' () None None + True True ['aaa', ''] +'[a-c]+' (2, 7) 'abcab' () None None + False False ['abcab'] +'[^a-c]+' (3, 6) 'xyz' () None None + False False ['xyz'] +'(?i)hello' (0, 5) 'HeLLo' () None None + True True ['HeLLo'] +'^abc$' (0, 3) 'abc' () None None + True False ['abc'] +'(?m)^b$' (2, 3) 'b' () None None + False False ['b'] +'a.c' None + False False [] +'(?s)a.c' (0, 3) 'a\nc' () None None + True True ['a\nc'] +'\\bfoo\\b' (2, 5) 'foo' () None None + False False ['foo'] +'\\Bfoo' (1, 4) 'foo' () None None + False False ['foo'] +'(a|ab)(c|bcd)(d*)' (0, 4) 'abcd' ('a', 'bcd', '') 3 None + True True [('a', 'bcd', '')] +'(a*)*' (0, 0) '' ('',) 1 None + True False ['', ''] +'(a?)*' (0, 0) '' ('',) 1 None + True False ['', ''] +'(a*)+' (0, 2) 'aa' ('',) 1 None + True False ['', '', ''] +'(?:ab)+?c' (0, 5) 'ababc' () None None + True True ['ababc'] +'a*?b' (0, 4) 'aaab' () None None + True True ['aaab'] +'a{2,4}?' (0, 2) 'aa' () None None + True True ['aa', 'aa'] +'foo(?=bar)' (0, 3) 'foo' () None None + True False ['foo'] +'foo(?!bar)' (0, 3) 'foo' () None None + True False ['foo'] +'(?<=\\$)\\d+' (6, 8) '42' () None None + False False ['42'] +'(?a+)b' (0, 4) 'aaab' () None None + True True ['aaab'] +'a++b' (0, 4) 'aaab' () None None + True True ['aaab'] +'a++a' None + False False [] +'(\\w)(\\w)?\\2' (0, 3) 'abb' ('a', 'b') 2 None + True True [('a', 'b')] +'(a)?(?(1)b|c)' (0, 2) 'ab' ('a',) 1 None + True True ['a'] +'(a)?(?(1)b|c)' (0, 1) 'c' (None,) None None + True True [''] +'[\\d\\s]+' (0, 5) '1 2 3' () None None + True False ['1 2 3'] +'[\\w.-]+@[\\w.-]+' (6, 17) 'a.b-c@x.org' () None None + False False ['a.b-c@x.org'] +'\\x41é\\101' (0, 3) 'AéA' () None None + True True ['AéA'] +'[\\]]+' (0, 2) ']]' () None None + True False [']]'] +'[]a]+' (0, 2) 'a]' () None None + True False ['a]'] +'[-a]+' (0, 3) '-a-' () None None + True True ['-a-'] +'[a-]+' (0, 3) '-a-' () None None + True True ['-a-'] +'\\.\\*\\+' (0, 3) '.*+' () None None + True True ['.*+'] +'café' None + False False [] +'(?i)café' (0, 4) 'CAFÉ' () None None + True True ['CAFÉ'] +'(?i)[a-z]+' (0, 5) 'HeLLo' () None None + True True ['HeLLo'] +'\\w+' (0, 5) 'naïve' () None None + True False ['naïve', 'café'] +'(?a)\\w+' (0, 2) 'na' () None None + True False ['na', 've'] +'(?x) a b # comment\\n c' (0, 2) 'ab' () None None + True False ['ab'] +'(a)|(b)' (0, 1) 'b' (None, 'b') 2 None + True True [('', 'b')] +'((a)(b))' (0, 2) 'ab' ('ab', 'a', 'b') 1 None + True True [('ab', 'a', 'b')] +'\\Aab\\Z' (0, 2) 'ab' () None None + True True ['ab'] +'$' (2, 2) '' () None None + False False ['', ''] +'x*' (0, 0) '' () None None + True False ['', '', '', ''] +'(?i:a)b' (0, 2) 'Ab' () None None + True True ['Ab'] +'(?i:a)b' None + False False [] +'(?-i:a)b' None + False False [] +'\\s+$' (3, 6) ' \t' () None None + False False [' \t'] +-a-b--d- world hello +a<111>b<222><222> ('bba', 2) +a2 b44 c666 -a-b-c- +True axcb +['Words', 'words', 'words', ''] ['Words', ', ', 'words', ', ', 'words', '.', ''] +['Words', 'words, words.'] ['', 'a', '', 'b', 'c', ''] ['0', '3', '9'] +[('a', '1'), ('b', ''), ('c', '3')] ['o', 't'] ['', '', ''] +[(0, 2), (3, 5), (7, 9)] ['', 'aa', '', ''] +re.compile('(?P\\w+)=(?P\\w*)') (?P\w+)=(?P\w*) 2 {'key': 1, 'val': 2} {'key': 'k', 'val': 'v'} ('k', 'v') v 5 4 +(5, 6) v:k 0 10 x k=v; z= True ((3, 6), (3, 4), (5, 6)) + None None +re.compile('a', re.IGNORECASE|re.MULTILINE) re.compile('a', re.DOTALL) re.IGNORECASE re.IGNORECASE|re.MULTILINE re.compile(b'a+') re.compile('x', re.VERBOSE|re.ASCII) +a\.b\*c\?d\[e\]f\(g\)\ h\-i_j b'a\\.b' True +b'123' [b'a', b'b', b'c'] b'aXc' +('a', 'b') ('a', '-') {} +last None +error 'a**' multiple repeat at position 2 | multiple repeat 2 1 3 +error '*a' nothing to repeat at position 0 | nothing to repeat 0 1 1 +error '(ab' missing ), unterminated subpattern at position 0 | missing ), unterminated subpattern 0 1 1 +error 'ab)' unbalanced parenthesis at position 2 | unbalanced parenthesis 2 1 3 +error '[ab' unterminated character set at position 0 | unterminated character set 0 1 1 +error '\\q' bad escape \q at position 0 | bad escape \q 0 1 1 +error '(?P<1>a)' bad character in group name '1' at position 4 | bad character in group name '1' 4 1 5 +error '(?P=x)' unknown group name 'x' at position 4 | unknown group name 'x' 4 1 5 +error '(?<=a+)b' look-behind requires fixed-width pattern | look-behind requires fixed-width pattern None None None +error 'a{3,2}' min repeat greater than max repeat at position 2 | min repeat greater than max repeat 2 1 3 +error '(?z)' unknown extension ?z at position 1 | unknown extension ?z 1 1 2 +error '\\1(a)' invalid group reference 1 at position 1 | invalid group reference 1 1 1 2 +error 'x(?i)y' global flags not at the start of the expression at position 1 | global flags not at the start of the expression 1 1 2 +error '(?Pa)(?Pb)' redefinition of group name 'n' as group 2; was group 1 at position 12 | redefinition of group name 'n' as group 2; was group 1 12 1 13 +error '[z-a]' bad character range z-a at position 1 | bad character range z-a 1 1 2 +error 'abc\n(' missing ), unterminated subpattern at position 4 (line 2, column 1) | missing ), unterminated subpattern 4 2 1 +error '(?' unexpected end of pattern at position 2 | unexpected end of pattern 2 1 3 +error '\\x4' incomplete escape \x4 at position 0 | incomplete escape \x4 0 1 1 +TypeError cannot use a string pattern on a bytes-like object +error invalid group reference 2 at position 1 +error bad escape \q at position 0 +IndexError no such group +True PatternError True +(0, 6001) 3000 6001 +6001 True diff --git a/maven/python-transpiler/src/test/resources/conformance/re_.py b/maven/python-transpiler/src/test/resources/conformance/re_.py new file mode 100644 index 00000000000..f64c04d42eb --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/re_.py @@ -0,0 +1,72 @@ +import re + +tests = [ + (r"a+", "caaab"), (r"(a)(b)?", "ac"), (r"(?P\w+)\s(?P=word)", "hello hello world"), + (r"\d{2,3}", "a12345"), (r"\d{,2}x", "123x"), (r"x{2}", "xxx"), (r"a{,}", "aaa"), (r"[a-c]+", "xxabcab"), + (r"[^a-c]+", "abcxyzab"), (r"(?i)hello", "HeLLo"), (r"^abc$", "abc\n"), (r"(?m)^b$", "a\nb\nc"), + (r"a.c", "a\nc"), (r"(?s)a.c", "a\nc"), (r"\bfoo\b", "a foo b"), (r"\Bfoo", "afoo"), + (r"(a|ab)(c|bcd)(d*)", "abcd"), (r"(a*)*", "b"), (r"(a?)*", "b"), (r"(a*)+", "aab"), + (r"(?:ab)+?c", "ababc"), (r"a*?b", "aaab"), (r"a{2,4}?", "aaaa"), (r"foo(?=bar)", "foobar"), + (r"foo(?!bar)", "foobaz foobar"), (r"(?<=\$)\d+", "cost $42"), (r"(?a+)b", "aaab"), (r"a++b", "aaab"), (r"a++a", "aaaa"), (r"(\w)(\w)?\2", "abb"), + (r"(a)?(?(1)b|c)", "ab"), (r"(a)?(?(1)b|c)", "c"), (r"[\d\s]+", "1 2 3x"), (r"[\w.-]+@[\w.-]+", "mail: a.b-c@x.org!"), + (r"\x41é\101", "AéA"), (r"[\]]+", "]]a"), (r"[]a]+", "a]b"), (r"[-a]+", "-a-"), (r"[a-]+", "-a-"), + (r"\.\*\+", ".*+"), (r"café", "CAFÉ"), (r"(?i)café", "CAFÉ"), (r"(?i)[a-z]+", "HeLLo"), + (r"\w+", "naïve café"), (r"(?a)\w+", "naïve"), (r"(?x) a b # comment\n c", "abc"), + (r"(a)|(b)", "b"), (r"((a)(b))", "ab"), (r"\Aab\Z", "ab"), (r"$", "ab\n"), (r"x*", "aaa"), + (r"(?i:a)b", "Ab"), (r"(?i:a)b", "AB"), (r"(?-i:a)b", "aB"), (r"\s+$", "abc \t"), +] +for pat, s in tests: + m = re.search(pat, s) + if m: + print(repr(pat), m.span(), repr(m.group()), m.groups(), m.lastindex, m.lastgroup) + else: + print(repr(pat), None) + print(" ", re.match(pat, s) is not None, re.fullmatch(pat, s) is not None, re.findall(pat, s)) + +print(re.sub(r"x*", "-", "abxd"), re.sub(r"(\w+) (\w+)", r"\2 \1", "hello world")) +print(re.sub(r"(?P\d)", r"<\g\g<1>\g<0>>", "a1b22"), re.subn(r"a", "b", "aaa", count=2)) +print(re.sub(r"\d+", lambda m: str(int(m.group()) * 2), "a1 b22 c333"), re.sub("", "-", "abc")) +print(re.sub(r"a", r"\n\t\\", "a") == "\n\t\\", re.sub("b", "x", "abcb", count=1)) +print(re.split(r"\W+", "Words, words, words."), re.split(r"(\W+)", "Words, words, words.")) +print(re.split(r"\W+", "Words, words, words.", maxsplit=1), re.split(r"x*", "axbc"), re.split("[a-f]", "0a3B9", flags=re.I)) +print(re.findall(r"(\w)(\d)?", "a1b c3"), re.findall(r"\b\w", "one two"), re.findall("", "ab")) +print([m.span() for m in re.finditer(r"\w+", "ab cd ef")], [m.group() for m in re.finditer(r"a*", "baac")]) +p = re.compile(r"(?P\w+)=(?P\w*)") +m = p.search("x k=v; z=") +print(p, p.pattern, p.groups, p.groupindex, m, m.groupdict(), m.group("key", 2), m["val"], m.start(2), m.end("key")) +print(m.span("val"), m.expand(r"\g:\1"), m.pos, m.endpos, m.string, m.re is p, m.regs) +print(p.match("a=b", 0), p.search("zz a=b", 2), p.search("a=b c=d", 1, 5), p.fullmatch("a=b"), p.fullmatch("a=b c")) +print(re.compile("a", re.I | re.M), re.compile("a", re.S), re.I, re.I | re.M, re.compile(b"a+"), re.compile("x", re.X | re.A)) +print(re.escape("a.b*c?d[e]f(g) h-i_j"), re.escape(b"a.b"), re.compile("a") == re.compile("a")) +print(re.search(rb"\d+", b"ab123").group(), re.findall(rb"[a-c]", b"abcd"), re.sub(rb"b", b"X", b"abc")) +print(re.match(r"(a)(b)", "ab").groups(), re.match(r"(a)(x)?", "a").groups("-"), re.match(r"(a)(x)?", "a").groupdict()) +print(re.search(r"(?P\w+) (?P\w+)", "Jane Doe").lastgroup, re.purge()) +bads = [r"a**", r"*a", r"(ab", r"ab)", r"[ab", r"\q", r"(?P<1>a)", r"(?P=x)", r"(?<=a+)b", r"a{3,2}", r"(?z)", + r"\1(a)", "x(?i)y", r"(?Pa)(?Pb)", r"[z-a]", "abc\n(", r"(?", r"\x4"] +for b in bads: + try: + re.compile(b) + print("compiled", repr(b)) + except re.error as e: + print("error", repr(b), e, "|", e.msg, e.pos, e.lineno, e.colno) +try: + re.search("a", b"a") +except TypeError as e: + print("TypeError", e) +try: + re.sub("(a)", r"\2", "a") +except re.error as e: + print("error", e) +try: + re.sub("a", r"\q", "a") +except re.error as e: + print("error", e) +try: + re.match("(a)", "a").group(5) +except IndexError as e: + print("IndexError", e) +print(isinstance(re.error("x"), Exception), re.error.__name__, re.PatternError is re.error) +big = "ab" * 3000 + "c" +print(re.search(r"(ab)*c", big).span(), len(re.findall(r"a.", big)), re.match(r".*c", big).end()) +print(re.match(r"(?:a|b)*?c", big).end(), bool(re.fullmatch(r"(?:ab)+c", big))) diff --git a/maven/python-transpiler/src/test/resources/conformance/scopes.out b/maven/python-transpiler/src/test/resources/conformance/scopes.out new file mode 100644 index 00000000000..fe1b15fdbb2 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/scopes.out @@ -0,0 +1,14 @@ +10 11 11 +[10, 11, 12] [12, 12, 12] +outer +10 5 None +unbound: cannot access local variable 'local_var' where it is not associated with a value +265252859812191058636308480000000 600 18446744073709551615 -9223372036854775808 919788 33333333333333333333 +370370367037037036703703703670 11 1180591620717411303424 0x400000000000000000 4 +9223372036854775808 -9223372036854775809 9223372037000250000 (-4, 1) 1 +9 + wrapper +2 +2 1 +main module +[1, 'unbound', 2, 'v', 'e cleared', 'no z', 3] [1, 'unbound', 'no y', 'v', 'e cleared', 'no z', 3] diff --git a/maven/python-transpiler/src/test/resources/conformance/scopes.py b/maven/python-transpiler/src/test/resources/conformance/scopes.py new file mode 100644 index 00000000000..552bb8a6b6a --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/scopes.py @@ -0,0 +1,119 @@ +# Name resolution, globals, closures in loops, recursion and big ints. +G = 10 +def read_global(): + return G +def write_global(): + global G + G += 1 + return G +print(read_global(), write_global(), G) + +def make_adders(): + return [lambda x, i=i: x + i for i in range(3)], [lambda x: x + i for i in range(3)] +early, late = make_adders() +print([f(10) for f in early], [f(10) for f in late]) + +def outer(): + v = "outer" + def mid(): + def inner(): + return v + return inner() + return mid() +print(outer()) + +class K: + attr = 5 + doubled = attr * 2 + items = [attr + i for i in range(2)] if False else None + def get(self): + return self.attr +print(K.doubled, K().get(), K.items) + +def unbound(): + try: + print(local_var) + except UnboundLocalError as e: + print("unbound:", e) + local_var = 1 +unbound() + +def fact(n): + return 1 if n <= 1 else n * fact(n - 1) +print(fact(30), fact(25) // fact(23), 2 ** 64 - 1, -(2 ** 63), (2 ** 64) % 1000007, 10 ** 20 // 3) +print(int("123456789012345678901234567890") * 3, 7 ** 77 % 13, abs(-2 ** 70), hex(2 ** 70), (1 << 100) >> 98) +print(9223372036854775807 + 1, -9223372036854775808 - 1, 3037000500 * 3037000500, divmod(-7, 2), -7 % 2) +def ack(m, n): + if m == 0: + return n + 1 + if n == 0: + return ack(m - 1, 1) + return ack(m - 1, ack(m, n - 1)) +print(ack(2, 3)) + +def deco(fn): + def wrapper(*a, **k): + return "<" + str(fn(*a, **k)) + ">" + return wrapper + +@deco +def hello(name, punct="!"): + return "hi " + name + punct +print(hello("bob"), hello("al", punct="?"), hello.__name__) + +def counter_gen(): + state = {"n": 0} + def step(): + state["n"] += 1 + return state["n"] + return step +st = counter_gen(); st(); print(st()) +x = 1 +def shadow(x=x): + x = x + 1 + return x +print(shadow(), x) +if __name__ == "__main__": + print("main module") + + +def unbound_paths(flag): + out = [] + x = 1 + for i in range(2): + try: + out.append(x) + del x + except UnboundLocalError: + out.append("unbound") + if flag: + y = 2 + try: + out.append(y) + except UnboundLocalError: + out.append("no y") + e = 5 + try: + raise ValueError("v") + except ValueError as e: + out.append(str(e)) + try: + out.append(e) + except UnboundLocalError: + out.append("e cleared") + try: + z = int("q") + except ValueError: + pass + try: + out.append(z) + except UnboundLocalError: + out.append("no z") + while True: + w = 3 + break + out.append(w) + return out + + +print(unbound_paths(True), unbound_paths(False)) diff --git a/maven/python-transpiler/src/test/resources/conformance/statistics_.out b/maven/python-transpiler/src/test/resources/conformance/statistics_.out new file mode 100644 index 00000000000..0b40cc3bbf2 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/statistics_.out @@ -0,0 +1,34 @@ +2.8 1.6071428571428572 5/7 4.25 2.0 +36.000000000000014 48.0 56.0 3.6 +3 4.0 3 5 52.5 3.5 +3 red ['b', 'd', 'f'] [] +1.3720238095238095 1.6666666666666667 67/108 1.25 +1.0810874155219827 0.986893273527251 3.0276503540974917 1.3720238095238095 +[81.0, 86.2, 89.0, 99.4, 102.5, 103.6, 106.0, 109.8, 111.0] +[2.0, 3.0, 4.0] [10, 10] +0.75 -1.0 0.31622776601683794 1.0 +LinearRegression(slope=0.31, intercept=-610.18) LinearRegression(slope=2.0, intercept=0.0) +NormalDist(mu=100.0, sigma=15.0) 0.9772498680518 0.02659615202676218 129.3994597681008 [89.88265374706, 100.0, 110.11734625294] 1.0 0.6538769160102 NormalDist(mu=105.0, sigma=15.0) NormalDist(mu=200.0, sigma=30.0) 225.0 +NormalDist(mu=2.716666666666667, sigma=0.5076087732365021) +StatisticsError mean requires at least one data point +StatisticsError variance requires at least two data points +StatisticsError no median for empty data +StatisticsError no mode for empty data +StatisticsError ('No negative inputs allowed', -2) +sqrt 1.449855761205 -0.775939255547 +exp -2.815278859231 -3.487082142416 +log 0.994792606279 -0.982793723247 +log10 0.432032939545 -0.426821890855 +sin 4.784552064542 -0.331840118511 +cos 0.339295764715 4.679416866449 +tan 0.003205515148 -1.022235159497 +sinh -1.337557189096 -1.830346869722 +cosh -1.477721670135 -1.656735272693 +tanh 1.016334677559 0.09917250556 +asin 0.557728098702 -1.702201269176 +acos 1.013068228093 1.702201269176 +atan 1.348971189281 -0.303034028741 +asinh 1.676147380837 -0.950053155716 +acosh 1.702201269176 -1.013068228093 +atanh 0.188961443694 -1.261140187228 +3.141592653589793 (1.0, 1.5707963267948966) True 2j True True (3+0j) diff --git a/maven/python-transpiler/src/test/resources/conformance/statistics_.py b/maven/python-transpiler/src/test/resources/conformance/statistics_.py new file mode 100644 index 00000000000..33c6587ec33 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/statistics_.py @@ -0,0 +1,32 @@ +# statistics and cmath. +import statistics as st +import cmath +from fractions import Fraction as F + +d = [2.75, 1.75, 1.25, 0.25, 0.5, 1.25, 3.5] +print(st.mean([1, 2, 3, 4, 4]), st.mean(d), st.mean([F(3, 7), F(1, 21), F(5, 3)]), st.fmean([3.5, 4.0, 5.25]), st.fmean([1, 2, 3], weights=[0.25, 0.5, 0.25])) +print(st.geometric_mean([54, 24, 36]), st.harmonic_mean([40, 60]), st.harmonic_mean([40, 60], weights=[5, 30]), st.harmonic_mean([2.5, 3, 10])) +print(st.median([1, 3, 5]), st.median([1, 3, 5, 7]), st.median_low([1, 3, 5, 7]), st.median_high([1, 3, 5, 7]), st.median_grouped([52, 52, 53, 54]), st.median_grouped([1, 3, 3, 5, 7], interval=2)) +print(st.mode([1, 1, 2, 3, 3, 3, 3, 4]), st.mode(["red", "blue", "blue", "red", "green", "red", "red"]), st.multimode("aabbbbccddddeeffffgg"), st.multimode([])) +print(st.variance([2.75, 1.75, 1.25, 0.25, 0.5, 1.25, 3.5]), st.variance([1, 2, 3, 4]), st.variance([F(1, 6), F(1, 2), F(5, 3)]), st.pvariance([0.0, 0.25, 0.25, 1.25, 1.5, 1.75, 2.75, 3.25])) +print(st.stdev([1.5, 2.5, 2.5, 2.75, 3.25, 4.75]), st.pstdev([1.5, 2.5, 2.5, 2.75, 3.25, 4.75]), st.stdev([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]), st.variance(d, st.mean(d))) +print(st.quantiles([105, 129, 87, 86, 111, 111, 89, 81, 108, 92, 110, 100, 75, 105, 103, 109, 76, 119, 99, 91, 103, 129, 106, 101, 84, 111, 74, 87, 86, 103, 103, 106, 86, 111, 75, 87, 102, 121, 111, 88, 89, 101, 106, 95, 103, 107, 101, 81, 109, 104], n=10)) +print(st.quantiles([1, 2, 3, 4, 5], n=4, method="inclusive"), st.quantiles([10], n=3)) +x = [1, 2, 3, 4, 5, 6, 7, 8, 9] +y = [1, 2, 3, 1, 2, 3, 1, 2, 3] +print(st.covariance(x, y), st.correlation(x, x[::-1]), st.correlation(x, y), st.correlation([1, 2, 3, 4], [1, 4, 9, 16], method="ranked")) +print(st.linear_regression([1971, 1975, 1979, 1982, 1983], [1, 2, 3, 4, 5]), st.linear_regression([1, 2, 3], [2, 4, 6], proportional=True)) +nd = st.NormalDist(100, 15) +# erf/erfc are the C library's in CPython, so their last bit is the platform's. +print(nd, round(nd.cdf(130), 13), nd.pdf(100), nd.inv_cdf(0.975), [round(q, 11) for q in nd.quantiles()], nd.zscore(115), round(nd.overlap(st.NormalDist(110, 10)), 13), nd + 5, nd * 2, nd.variance) +print(st.NormalDist.from_samples([2.5, 3.1, 2.1, 2.4, 2.7, 3.5])) +for f, a in [(st.mean, []), (st.variance, [1]), (st.median, []), (st.mode, []), (st.geometric_mean, [1, -2])]: + try: + f(a) + except st.StatisticsError as e: + print(type(e).__name__, e) +z = complex(1.5, -2.25) +for name in ["sqrt", "exp", "log", "log10", "sin", "cos", "tan", "sinh", "cosh", "tanh", "asin", "acos", "atan", "asinh", "acosh", "atanh"]: + r = getattr(cmath, name)(z) + print(name, round(r.real, 12), round(r.imag, 12)) +print(cmath.phase(-1), cmath.polar(1j), cmath.rect(2, cmath.pi / 2).real < 1e-15, cmath.sqrt(-4), cmath.isclose(1 + 1j, 1.0000000001 + 1j), cmath.isinf(cmath.infj), cmath.log(8, 2)) diff --git a/maven/python-transpiler/src/test/resources/conformance/stdlib_collections.out b/maven/python-transpiler/src/test/resources/conformance/stdlib_collections.out new file mode 100644 index 00000000000..adda0dcc210 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/stdlib_collections.out @@ -0,0 +1,31 @@ +deque([0, 2, 3, 4], maxlen=4) 4 0 4 4 +deque([0, 2], maxlen=4) 3 4 [2, 0] False +deque([15, 16, 17, 18, 19]) 2 1 +deque([101, 100, 15, 16, 17, 18, 19]) True +IndexError pop from an empty deque +defaultdict(, {'a': [1, 3], 'b': [2], 'zz': []}) [] ['a', 'b', 'zz'] True +{'m': 1, 'i': 4, 's': 4, 'p': 2} +Counter({'a': 5, 'b': 2, 'r': 2, 'c': 1, 'd': 1}) [('a', 5), ('b', 2)] 0 11 +['a', 'a', 'a', 'a', 'a', 'b'] Counter({'b': 7, 'a': 5, 'r': 2, 'c': 1, 'd': 1, 'z': 1}) Counter({'b': 2, 'r': 2, 'c': 1, 'd': 1, 'z': 1}) Counter({'a': 1}) Counter({'a': 4}) +Counter({'dogs': 8, 'cats': 4}) Counter({'a': 2}) Counter({'b': 2}) +OrderedDict({'z': 3}) ('x', 1) ('y', 2) OrderedDict({'z': 3}) +['w', 'z'] OrderedDict({'a': 0, 'b': 0}) +Point(x=1, y=2) 3 1 {'x': 1, 'y': 2} Point(x=10, y=2) ('x', 'y') Point(x=5, y=6) +1 2 True True 2 +P3(a=1, b=2, c=0) P3(a=1, b=2, c=3) {'c': 0} +TypeError Point.__new__() missing 1 required positional argument: 'y' +matched 2 +1 3 2 ['a', 'b'] True +{'a': 1, 'c': 9} 0 2 +True True True +True False True +True False +3 [0, 1, 4, 9] True 2 [9, 4, 1, 0] 1 +True False +{'b': [3, 4]} {'b': [3]} False (1, [2]) +[1, 2] [1] Box +True False +abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ 0123456789 !"#$%&'()*+,-./:;<=>?@[\]^_`{|}~ +Hello Big World +tim likes pie; $5 x likes ${what}; $5 +KeyError 'what' diff --git a/maven/python-transpiler/src/test/resources/conformance/stdlib_collections.py b/maven/python-transpiler/src/test/resources/conformance/stdlib_collections.py new file mode 100644 index 00000000000..26683a0fafc --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/stdlib_collections.py @@ -0,0 +1,143 @@ +from collections import deque, defaultdict, Counter, OrderedDict, namedtuple, ChainMap +import collections.abc as cabc +import copy +import string + +d = deque([1, 2, 3], maxlen=4) +d.append(4) +d.append(5) +d.appendleft(0) +print(d, len(d), d[0], d[-1], d.maxlen) +d.rotate(1) +print(d, d.pop(), d.popleft(), list(reversed(d)), 3 in d) +d2 = deque() +for i in range(20): + d2.append(i) +for i in range(15): + d2.popleft() +print(d2, d2.index(17), d2.count(18)) +d2.extendleft([100, 101]) +print(d2, d2 == deque([101, 100, 15, 16, 17, 18, 19])) +try: + deque().pop() +except IndexError as e: + print("IndexError", e) + +dd = defaultdict(list) +for k, v in [("a", 1), ("b", 2), ("a", 3)]: + dd[k].append(v) +print(dd, dd["zz"], sorted(dd), isinstance(dd, dict)) +counts = defaultdict(int) +for ch in "mississippi": + counts[ch] += 1 +print(dict(counts)) + +c = Counter("abracadabra") +print(c, c.most_common(2), c["z"], c.total()) +c.update("aaz") +c.subtract({"a": 2}) +print(sorted(c.elements())[:6], c + Counter(b=5), c - Counter(a=100), Counter(a=3) & Counter(a=1, b=2), Counter(a=1) | Counter(a=4)) +print(Counter(cats=4, dogs=8), -Counter(a=-2, b=3), +Counter(a=-1, b=2)) + +od = OrderedDict() +od["x"] = 1 +od["y"] = 2 +od["z"] = 3 +od.move_to_end("x") +print(od, od.popitem(), od.popitem(last=False), od) +od["w"] = 0 +od.move_to_end("w", last=False) +print(list(od.keys()), OrderedDict.fromkeys("ab", 0)) + +Point = namedtuple("Point", "x y") +p = Point(1, y=2) +print(p, p.x + p.y, p[0], p._asdict(), p._replace(x=10), Point._fields, Point._make([5, 6])) +x, y = p +print(x, y, p == (1, 2), isinstance(p, tuple), len(p)) +P3 = namedtuple("P3", ["a", "b", "c"], defaults=[0]) +print(P3(1, 2), P3(1, 2, 3), P3._field_defaults) +try: + Point(1) +except TypeError as e: + print("TypeError", e) + + +match p: + case Point(x=1, y=yy): + print("matched", yy) + +cm = ChainMap({"a": 1}, {"a": 2, "b": 3}) +print(cm["a"], cm["b"], len(cm), sorted(cm), "b" in cm) +cm["c"] = 9 +print(cm.maps[0], cm.new_child({"a": 0})["a"], cm.parents["a"]) + +print(isinstance([], cabc.Sequence), isinstance({}, cabc.Mapping), isinstance(set(), cabc.Set)) +print(isinstance([], cabc.Iterable), isinstance(5, cabc.Iterable), isinstance(len, cabc.Callable)) +print(issubclass(list, cabc.MutableSequence), issubclass(tuple, cabc.MutableSequence)) + + +class Bag(cabc.Sized, cabc.Iterable): + def __init__(self, items): + self.items = list(items) + + def __len__(self): + return len(self.items) + + def __iter__(self): + return iter(self.items) + + +class Ranged(cabc.Sequence): + def __init__(self, n): + self.n = n + + def __len__(self): + return self.n + + def __getitem__(self, i): + if i >= self.n: + raise IndexError(i) + return i * i + + +r = Ranged(4) +print(len(Bag("abc")), list(r), 9 in r, r.index(4), list(reversed(r)), r.count(1)) + + +class Thing: + def __len__(self): + return 0 + + +print(isinstance(Thing(), cabc.Sized), isinstance(Thing(), cabc.Container)) + +nested = {"a": [1, 2, {"b": [3]}], "t": (1, [2])} +shallow = copy.copy(nested) +deep = copy.deepcopy(nested) +nested["a"][2]["b"].append(4) +print(shallow["a"][2], deep["a"][2], shallow is nested, deep["t"]) + + +class Box: + def __init__(self, v): + self.v = v + + +b = Box([1]) +b2 = copy.copy(b) +b3 = copy.deepcopy(b) +b.v.append(2) +print(b2.v, b3.v, type(b3).__name__) +lst = [1] +pair = [lst, lst] +dp = copy.deepcopy(pair) +print(dp[0] is dp[1], dp[0] is lst) + +print(string.ascii_letters, string.digits, string.punctuation) +print(string.capwords("hello big world")) +t = string.Template("$who likes ${what}; $$5") +print(t.substitute(who="tim", what="pie"), t.safe_substitute(who="x")) +try: + t.substitute(who="a") +except KeyError as e: + print("KeyError", e) diff --git a/maven/python-transpiler/src/test/resources/conformance/strings.out b/maven/python-transpiler/src/test/resources/conformance/strings.out new file mode 100644 index 00000000000..20a22d65e1a --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/strings.out @@ -0,0 +1,26 @@ +Hello, World! Hello, World! Hello, World! hello, world! HELLO, WORLD! Hello, World! hELLO, wORLD! +['Hello', 'World!'] ['a', 'b', '', 'c'] ['a', 'b', 'c'] ['a', 'b,c'] ['a,b', 'c'] +x-y-z aBBc 2 3 1 +True True 2 True True True +***abc*** abc | abc 007 -007 +a b ['line1', 'line2', 'line3'] axbxc hello +('hello', ' ', 'world') ('a.b', '.', 'c') bc ab +"it's" 'say "hi"' 'both \' and "' '\n\t\\' '\x00\x7fÿĀ' +1.0 1e+16 1.5e-05 0.1 123456789.12345679 inf -inf +left | right| mid | 003 -3 3.500000e+00 25.000000% 1,234.5 +1100 14 FF A s |7 | aba 1-2 + a|b |00042|ff|10|1.234568e+04|0.0001|% +6 😀 b'w\xc3\xb6rld\xf0\x9f\x98\x80' True ['d', '😀'] +b'bytes\x00\xff' 7 98 255 b'yt' b'bytes\x00\xff!' 627974657300ff b'AB' b'ABC' +8364 True b'\xe2\x82\xac' x +True True True True ['e', 'h', 'l', 'l', 'o'] o +MiSSissippi bcab x +café b'caf\xe9' � b'x' +encode error UnicodeEncodeError +1 -4 1152921504606846977 -230584300921369408 1224995262755759164 16777216 -3550055125485641917 5740354900026072187 -9148893311433065316 -2 True +{(1, 2), (2, 1), (3, 4), (0, 0)} {1099511627776, 100, 7, 1000, 10, -1} {0.5, 1.5, 2.25} {2, 5, 8, 11, 14, 17, 20} +'\u2014a\u4e00\uac01\U0001f600' b'\\N{x}' b'\\N' +'alpha' 'beta' 'gamma' 'delta' 'eps' 22 +a +b +1 3 3 41 diff --git a/maven/python-transpiler/src/test/resources/conformance/strings.py b/maven/python-transpiler/src/test/resources/conformance/strings.py new file mode 100644 index 00000000000..508562742a9 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/strings.py @@ -0,0 +1,55 @@ +# str, bytes and formatting. +s = " Hello, World! " +print(s.strip(), s.lstrip(), s.rstrip(), s.lower(), s.upper(), s.title(), s.swapcase()) +print(s.strip().split(", "), "a,b,,c".split(","), "a b c".split(), "a,b,c".split(",", 1), "a,b,c".rsplit(",", 1)) +print("-".join(["x", "y", "z"]), "abc".replace("b", "BB"), "hello".find("l"), "hello".rfind("l"), "hello".index("e")) +print("hello".startswith("he"), "hello".endswith(("lo", "x")), "hello".count("l"), "42".isdigit(), "ab".isalpha(), " ".isspace()) +print("abc".center(9, "*"), "abc".ljust(5) + "|", "abc".rjust(5), "7".zfill(3), "-7".zfill(4)) +print("a\tb".expandtabs(4), "line1\nline2\r\nline3".splitlines(), "x".join("abc"), "Hello".casefold()) +print("hello world".partition(" "), "a.b.c".rpartition("."), "abc".removeprefix("a"), "abc".removesuffix("c")) +print(repr("it's"), repr('say "hi"'), repr("both ' and \""), repr("\n\t\\"), repr("\x00\x7fÿĀ")) +print(str(1.0), str(1e16), str(1.5e-5), repr(0.1), str(123456789.123456789), str(float("inf")), str(-float("inf"))) +print(f"{'left':<8}|{'right':>8}|{'mid':^8}|", f"{3:03d} {-3:+d} {3.5:e} {0.25:%} {1234.5:,.1f}") +print(f"{12:b} {12:o} {255:X} {65:c} {'s':5}|{7:<4}|", "{0}{1}{0}".format("a", "b"), "{x}-{y}".format(x=1, y=2)) +print("%5s|%-5s|%05d|%x|%o|%e|%g|%%" % ("a", "b", 42, 255, 8, 12345.678, 0.0001)) +w = "wörld\U0001F600" +print(len(w), w[5], w.encode("utf-8"), w.encode("utf-8").decode("utf-8") == w, list(w[-2:])) +b = b"bytes\x00\xff" +print(b, len(b), b[0], b[-1], b[1:3], b + b"!", b.hex(), bytes([65, 66]), b"abc".upper()) +print(ord("€"), chr(0x1F600) == "\U0001F600", "€".encode(), str(b"x", "ascii")) +print("abc" < "abd", "Z" < "a", "a" * 0, "" in "abc", "bc" in "abc", sorted("hello"), max("hello")) +print("Mississippi".replace("ss", "SS", 1), "abcabc".strip("ac"), "{:>{w}}".format("x", w=4)) +print(b"caf\xc3\xa9".decode(), "café".encode("latin-1"), b"\xff".decode("utf-8", "replace"), "xé".encode("ascii", "ignore")) +try: + "é".encode("ascii") +except UnicodeEncodeError as e: + print("encode error", type(e).__name__) +except UnicodeError as e: + print("encode error (base)", type(e).__name__) +print(hash(2**61), hash(-2**63), hash(1.5), hash(-0.1), hash(1e300), hash(5e-324), hash((1, 2)), hash(()), hash((1, (2, 3.5))), hash(-1), hash(1.0) == hash(1) == hash(True)) +print({(1, 2), (3, 4), (0, 0), (2, 1)}, {-1, 10, 100, 1000, 7, 2**40}, {1.5, 2.25, 0.5}, set(range(20, 0, -3))) +print(ascii("\N{EM DASH}\N{latin small letter a}\N{CJK UNIFIED IDEOGRAPH-4E00}\N{HANGUL SYLLABLE GAG}\N{GRINNING FACE}"), b"\N{x}", rb"\N") + + +def words(text): + n = 0 + for w in text.split(): + n += len(w) + print(repr(w), end=" ") + print(n) + for w in "".split(): + print("never") + for w in " ".split(): + print("never") + + +class Splitter: + def split(self): + return ["a", "b"] + + +words(" alpha\tbeta\n\ngamma delta \x1c eps ") +for w in Splitter().split(): + print(w) + +print(len("\ud800"), len("a\U0001F600b"), len("\udc00\ud800x"), len("x" * 40 + "\U0001F600")) diff --git a/maven/python-transpiler/src/test/resources/conformance/struct_.out b/maven/python-transpiler/src/test/resources/conformance/struct_.out new file mode 100644 index 00000000000..e1e6ac8b1ca --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/struct_.out @@ -0,0 +1,34 @@ +4 8 5 16 4 6 +b'\xfe\xff\xff\xff\xff\xff\xff\xff\x00(k\xee\xfb\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff' +b'\xff\xfe\xff\xff\xff\xff\xff\xff\xeek(\x00' b'\x80\xff\x01z' +(-2, 65535, -1, 4000000000, -5, 18446744073709551615) +0.0 0000000000000000 0.0 00000000 0.0 0000 +-0.0 0000000000000080 -0.0 80000000 -0.0 0080 +1.0 000000000000f03f 1.0 3f800000 1.0 003c +-2.5 00000000000004c0 -2.5 c0200000 -2.5 00c1 +0.1 9a9999999999b93f 0.1 3dcccccd 0.10000000149011612 662e +1e+300 9c7500883ce4377e 1e+300 - - - +5e-324 0100000000000000 5e-324 00000000 0.0 0000 +3.4028234663852886e+38 000000e0ffffef47 3.4028234663852886e+38 7f7fffff 3.4028234663852886e+38 - +1.401298464324817e-45 000000000000a036 1.401298464324817e-45 00000001 1.401298464324817e-45 0000 +65504.0 0000000000fcef40 65504.0 477fe000 65504.0 ff7b +6e-08 2b69a4292b1b703e 6e-08 3380d959 5.99999978589949e-08 0100 +inf 000000000000f07f inf - - 007c +0.10000000149011612 0.0999755859375 (nan,) +b'ab\x00\x00\x00' b'ab' b'\x03abc\x00' (b'abc',) (b'xyz',) +b'\x01\x00\x02\x00\x03\x00\x00\x00' b'a\x00\x00\x00\x01\x00\x00\x00' b'\x00\x00\x00\x00\x00\x00\xf0?\x00' +4 <2h b'\x01\x00\x02\x00' (1, 2) [(1, 2), (3, 4)] Struct('<2h') +bytearray(b'\x00\x00\x04\x03\x02\x01\x00\x00') (16909060,) (258,) +'b' format requires -128 <= number <= 127 +'B' format requires 0 <= number <= 255 +'h' format requires -32768 <= number <= 32767 +'I' format requires 0 <= number <= 4294967295 +required argument is not an integer +pack expected 1 items for packing (got 0) +char format requires a bytes object of length 1 +'q' format requires -9223372036854775808 <= number <= 9223372036854775807 +unpack requires a buffer of 4 bytes +unpack requires a buffer of 2 bytes +bad char in struct format +float too large to pack with f format +b'\x00' b'\x01' b'\x07\x00' diff --git a/maven/python-transpiler/src/test/resources/conformance/struct_.py b/maven/python-transpiler/src/test/resources/conformance/struct_.py new file mode 100644 index 00000000000..5e970ee27d6 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/struct_.py @@ -0,0 +1,37 @@ +# struct. +import struct + +print(struct.calcsize("hHiI", -2, 65535, -1, 4000000000), struct.pack("!bB?c", -128, 255, True, b"z")) +print(struct.unpack("f", v).hex() if abs(v) < 3.5e38 or v != v else "-", + struct.unpack(">f", struct.pack(">f", v))[0] if abs(v) < 3.5e38 else "-", struct.pack("H", b"\x00\x01\x02", 1)) +for args in [(" & © © © ¬it; &x � €  | ≂̸ &bogus; +ParseResult(scheme='https', netloc='user:pw@Example.COM:8080', path='/a/b', params='p', query='q=1&r=2', fragment='frag') example.com 8080 user pw https://user:pw@Example.COM:8080/a/b;p?q=1&r=2#frag +SplitResult(scheme='', netloc='net', path='/loc', query='x', fragment='') SplitResult(scheme='mailto', netloc='', path='a@b.c', query='', fragment='') ::1 b'h' +'g' http://a/b/c/g +'./g' http://a/b/c/g +'g/' http://a/b/c/g/ +'/g' http://a/g +'//g' http://g +'?y' http://a/b/c/d;p?y +'g?y' http://a/b/c/g?y +'#s' http://a/b/c/d;p?q#s +'g#s' http://a/b/c/g#s +'..' http://a/b/ +'../g' http://a/b/g +'../..' http://a/ +'../../../g' http://a/g +'g;x' http://a/b/c/g;x +'' http://a/b/c/d;p?q +'.' http://a/b/c/ +'./' http://a/b/c/ +DefragResult(url='http://x/y', fragment='z') http://h/p?a=1#f +a%20b%26c/d%3F%C3%A9 %2Fx%20y a+b%2Bc €%2 a b c b'A%zz' +a=1&b=x+y&c=%C3%A9 k=1&k=2 k=%5B1%2C+2%5D +{'a': ['1', '2'], 'c': ['x y!']} [('a', '1'), ('b', ''), ('c', '')] {'x': ['1'], 'y': ['2']} +16 16 12 True True True False True +True 2 +full +1 2 True +empty +[(1, '1'), (2, '2'), (4, '4'), (5, '5')] +c b 1 +9 True diff --git a/maven/python-transpiler/src/test/resources/conformance/textproc_.py b/maven/python-transpiler/src/test/resources/conformance/textproc_.py new file mode 100644 index 00000000000..0381e9222ca --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/textproc_.py @@ -0,0 +1,82 @@ +# warnings, shlex, html, urllib.parse, secrets, queue. +import warnings, shlex, html, secrets, queue +from urllib.parse import (urlparse, urlsplit, urlunparse, urljoin, urldefrag, quote, quote_plus, unquote, + unquote_plus, urlencode, parse_qs, parse_qsl, unquote_to_bytes) + +with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + warnings.warn("first") + warnings.warn("dep", DeprecationWarning) + warnings.warn(RuntimeWarning("rt")) + print([(type(x.message).__name__, str(x.message), x.category.__name__) for x in w]) +with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("once") + for i in range(3): + warnings.warn("same") + warnings.filterwarnings("ignore", message="skip.*") + warnings.warn("skip me") + warnings.warn("keep me") + print(len(w), [str(x.message) for x in w]) +with warnings.catch_warnings(): + warnings.simplefilter("error") + try: + warnings.warn("boom", UserWarning) + except UserWarning as e: + print("raised", e) +print(warnings.formatwarning("msg", UserWarning, "f.py", 3).strip()) +try: + warnings.simplefilter("bogus") +except ValueError as e: + print(e) + +print(shlex.split('ls -l "my file.txt" \'single q\' a\\ b # c'), shlex.split("a # b", comments=True)) +print(shlex.quote("safe"), shlex.quote("it's here"), shlex.quote(""), shlex.join(["echo", "a b", "c"])) +lex = shlex.shlex("a=1; b='x y' && c", posix=True, punctuation_chars=True) +print(list(lex)) +print(list(shlex.shlex("foo(bar, 'baz')"))) +for bad in ['"unclosed', "trail\\"]: + try: + shlex.split(bad) + except ValueError as e: + print(e) + +print(html.escape("'&'"), html.escape("", quote=False)) +print(html.unescape("<p> &amp; © © © ¬it; &x � €  | ≂̸ &bogus;")) + +u = urlparse("https://user:pw@Example.COM:8080/a/b;p?q=1&r=2#frag") +print(u, u.hostname, u.port, u.username, u.password, u.geturl()) +print(urlsplit("//net/loc?x"), urlsplit("mailto:a@b.c"), urlsplit("http://[::1]:80/x").hostname, urlparse(b"http://h/p").netloc) +for rel in ["g", "./g", "g/", "/g", "//g", "?y", "g?y", "#s", "g#s", "..", "../g", "../..", "../../../g", "g;x", "", ".", "./"]: + print(repr(rel), urljoin("http://a/b/c/d;p?q", rel)) +print(urldefrag("http://x/y#z"), urlunparse(("http", "h", "/p", "", "a=1", "f"))) +print(quote("a b&c/d?é"), quote("/x y", safe=""), quote_plus("a b+c"), unquote("%E2%82%AC%2"), unquote_plus("a+b%20c"), unquote_to_bytes("%41%zz")) +print(urlencode({"a": 1, "b": "x y", "c": "é"}), urlencode([("k", [1, 2])], doseq=True), urlencode({"k": [1, 2]})) +print(parse_qs("a=1&a=2&b=&c=x+y%21"), parse_qsl("a=1&b=&c", keep_blank_values=True), parse_qs("x=1;y=2", separator=";")) + +print(len(secrets.token_bytes(16)), len(secrets.token_hex(8)), len(secrets.token_urlsafe(9)), 0 <= secrets.randbelow(10) < 10, + secrets.choice("ab") in "ab", secrets.compare_digest("abc", "abc"), secrets.compare_digest(b"abc", b"abd"), secrets.randbits(8) < 256) + +q = queue.Queue(maxsize=2) +q.put(1) +q.put(2) +print(q.full(), q.qsize()) +try: + q.put_nowait(3) +except queue.Full: + print("full") +print(q.get(), q.get(), q.empty()) +try: + q.get_nowait() +except queue.Empty: + print("empty") +pq = queue.PriorityQueue() +for x in [5, 1, 4, 2]: + pq.put((x, str(x))) +print([pq.get() for _ in range(4)]) +lq = queue.LifoQueue() +for x in "abc": + lq.put(x) +print(lq.get(), lq.get(), lq.qsize()) +sq = queue.SimpleQueue() +sq.put(9) +print(sq.get(), sq.empty()) diff --git a/maven/python-transpiler/src/test/resources/conformance/textwrap_.out b/maven/python-transpiler/src/test/resources/conformance/textwrap_.out new file mode 100644 index 00000000000..e5a4c7c12d5 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/textwrap_.out @@ -0,0 +1,24 @@ +10 ['The quick', 'brown fox', 'jumps over', 'the lazy', 'dog. A', 'well-known', 'pangram --', 'used by ty', 'pographers', '--', 'contains', 'every', 'letter of', 'the', 'alphabet.', 'Supercalif', 'ragilistic', 'expialidoc', 'ious is', 'long; so', 'is pseudo-', 'pseudo-hyp', 'oparathyro', 'idism.', 'Tabs', 'and', 'newlines', 'too.'] +25 ['The quick brown fox jumps', 'over the lazy dog. A', 'well-known pangram --', 'used by typographers --', 'contains every letter of', 'the alphabet. Supercalif', 'ragilisticexpialidocious', 'is long; so is pseudo-', 'pseudo-', 'hypoparathyroidism.', 'Tabs and newlines too.'] +40 ['The quick brown fox jumps over the lazy', 'dog. A well-known pangram -- used by', 'typographers -- contains every letter of', 'the alphabet.', 'Supercalifragilisticexpialidocious is', 'long; so is pseudo-pseudo-', 'hypoparathyroidism. Tabs and', 'newlines too.'] +79 ['The quick brown fox jumps over the lazy dog. A well-known pangram -- used by', 'typographers -- contains every letter of the alphabet.', 'Supercalifragilisticexpialidocious is long; so is pseudo-pseudo-', 'hypoparathyroidism. Tabs and newlines too.'] +* The quick brown fox jumps + over the lazy dog. A well- + known pangram -- used by + typographers -- contains + every letter of the + alphabet. Supercalifragilis + ticexpialidocious is long; + so is pseudo-pseudo- + hypoparathyroidism. + Tabs and newlines too. +['The quick brown fox', 'jumps over the lazy', 'dog. A well-known', 'pangram -- used by', 'typographers --', 'contains every', 'letter of the', 'alphabet.', 'Supercalifragilisticexpialidocious', 'is long; so is', 'pseudo-pseudo-', 'hypoparathyroidism.', 'Tabs and newlines', 'too.'] +['The quick brown fox', 'jumps over the lazy', 'dog. A well-known', 'pangram -- used by', 'typographers --', 'contains every', 'letter of the', 'alphabet. Supercali', 'fragilisticexpialido', 'cious is long; so is', 'pseudo-pseudo-hypopa', 'rathyroidism.', 'Tabs and newlines', 'too.'] +['The quick brown fox jumps over', 'the lazy dog. A well-known', 'pangram -- used by', 'typographers -- contains every', 'letter of the alphabet. Super', 'califragilisticexpialidocious', 'is long; so is pseudo-pseudo-', 'hypoparathyroidism.', 'Tabs and newlines too.'] +[' leading', 'and', 'trailing'] [' leading', ' '] +['The quick brown fox jumps over', 'the lazy dog. A well-known', 'pangram -- used by [...]'] ['The quick brown fox jumps...'] +Hello world! Hello [...] Hello world +[] ['a', 'b', 'c'] ['a-b-', 'c-d-', 'e-f'] +'\ndef f():\n return 1\n x\n' ' a\n\tb\n' '\na\nb' +'> a\n\n> b\n' '+a\n+\n+b' +invalid width 0 (must be > 0) diff --git a/maven/python-transpiler/src/test/resources/conformance/textwrap_.py b/maven/python-transpiler/src/test/resources/conformance/textwrap_.py new file mode 100644 index 00000000000..72ad3808fea --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/textwrap_.py @@ -0,0 +1,26 @@ +# textwrap. +import textwrap + +text = ("The quick brown fox jumps over the lazy dog. A well-known pangram -- used by typographers -- " + "contains every letter of the alphabet. Supercalifragilisticexpialidocious is long; " + "so is pseudo-pseudo-hypoparathyroidism.\tTabs\tand\nnewlines too.") +for w in [10, 25, 40, 79]: + print(w, textwrap.wrap(text, w)) +print(textwrap.fill(text, 30, initial_indent="* ", subsequent_indent=" ")) +print(textwrap.wrap(text, 20, break_long_words=False)) +print(textwrap.wrap(text, 20, break_on_hyphens=False)) +print(textwrap.wrap(text, 30, fix_sentence_endings=True)) +print(textwrap.wrap(" leading and trailing ", 10), textwrap.wrap(" leading ", 10, drop_whitespace=False)) +print(textwrap.wrap(text, 30, max_lines=3), textwrap.wrap(text, 30, max_lines=1, placeholder="...")) +print(textwrap.shorten("Hello world!", width=12), textwrap.shorten("Hello world!", width=11), textwrap.shorten("Hello world", 50)) +print(textwrap.wrap("", 10), textwrap.wrap("abc", 1), textwrap.wrap("a-b-c-d-e-f", 4)) +print(repr(textwrap.dedent(""" + def f(): + return 1 + x +""")), repr(textwrap.dedent(" a\n\tb\n")), repr(textwrap.dedent(" \n a\n b"))) +print(repr(textwrap.indent("a\n\nb\n", "> ")), repr(textwrap.indent("a\n\nb", "+", lambda l: True))) +try: + textwrap.wrap("x", 0) +except ValueError as e: + print(e) diff --git a/maven/python-transpiler/src/test/resources/conformance/typing_abc.out b/maven/python-transpiler/src/test/resources/conformance/typing_abc.out new file mode 100644 index 00000000000..9a6b7088099 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/typing_abc.out @@ -0,0 +1,36 @@ +int | None int | str typing.List[int] typing.Dict[str, typing.List[int]] typing.Tuple[int, ...] typing.Callable[[int], str] +True (, ) +typing.Literal['a', 'b'] typing.Annotated[int, 'meta'] (, 'm') False +~T True x +2 Stack (typing.Generic[~T],) +Employee(name='ann', id=3) ann#3 ('name', 'id') 9 True +{'title': 'x', 'year': 1} True True +True False +5 UserId +{'name': , 'times': , 'return': } Says hi. +3 +Sq 9 frozenset({'area'}) True +TypeError Can't instantiate abstract class Shape without an implementation for abstract method 'area' +TypeError Can't instantiate abstract class Half without an implementation for abstract method 'area' +True True False +

+body P +

+ + +caught boom +swallowed +after +suppressed + +in stack + +exit b +exit a +using +closed +5 +'hidden\n' + +inside + diff --git a/maven/python-transpiler/src/test/resources/conformance/typing_abc.py b/maven/python-transpiler/src/test/resources/conformance/typing_abc.py new file mode 100644 index 00000000000..ddefb2300ac --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/typing_abc.py @@ -0,0 +1,189 @@ +from typing import (Any, Optional, Union, List, Dict, Tuple, Callable, TypeVar, Generic, Protocol, NamedTuple, + TypedDict, Literal, cast, overload, get_origin, get_args, runtime_checkable, Final, Annotated, + Iterator, Sequence, NewType, ClassVar, get_type_hints, TYPE_CHECKING) +from abc import ABC, abstractmethod, ABCMeta +from contextlib import contextmanager, suppress, ExitStack, closing, nullcontext, redirect_stdout +import io + +print(Optional[int], Union[int, str], List[int], Dict[str, List[int]], Tuple[int, ...], Callable[[int], str]) +print(Union[int, None] == Optional[int], Union[int, int], get_origin(List[int]), get_args(Dict[str, int])) +print(Literal["a", "b"], Annotated[int, "meta"], get_args(Annotated[int, "m"]), TYPE_CHECKING) +T = TypeVar("T") +K = TypeVar("K", bound=str) +print(T, K.__bound__ is str, cast(int, "x")) + + +class Stack(Generic[T]): + def __init__(self): + self.items: List[T] = [] + + def push(self, item: T) -> None: + self.items.append(item) + + def pop(self) -> T: + return self.items.pop() + + +s = Stack[int]() +s.push(1) +s.push(2) +print(s.pop(), type(s).__name__, Stack.__orig_bases__ if hasattr(Stack, "__orig_bases__") else None) + + +class Employee(NamedTuple): + name: str + id: int = 3 + + def badge(self): + return self.name + "#" + str(self.id) + + +e = Employee("ann") +print(e, e.badge(), e._fields, Employee("b", 9).id, isinstance(e, tuple)) + + +class Movie(TypedDict): + title: str + year: int + + +m = Movie(title="x", year=1) +print(m, type(m) is dict, Movie.__required_keys__ == {"title", "year"}) + + +@runtime_checkable +class Closer(Protocol): + def close(self) -> None: + ... + + +class F: + def close(self): + print("closed") + + +print(isinstance(F(), Closer), isinstance(1, Closer)) +UserId = NewType("UserId", int) +print(UserId(5), UserId.__name__) + + +def greet(name: str, times: int = 1) -> str: + """Says hi.""" + return ("hi " + name) * times + + +print(greet.__annotations__, greet.__doc__, get_type_hints(greet)["times"]) + + +@overload +def f(x: int) -> int: + ... + + +@overload +def f(x: str) -> str: + ... + + +def f(x): + return x + + +print(f(3)) + + +class Shape(ABC): + @abstractmethod + def area(self): + ... + + def describe(self): + return type(self).__name__ + " " + str(self.area()) + + +class Sq(Shape): + def __init__(self, s): + self.s = s + + def area(self): + return self.s * self.s + + +print(Sq(3).describe(), Shape.__abstractmethods__, isinstance(Sq(1), Shape)) +try: + Shape() +except TypeError as e: + print("TypeError", e) + + +class Half(Shape): + pass + + +try: + Half() +except TypeError as e: + print("TypeError", e) + + +class Plugin(metaclass=ABCMeta): + pass + + +Plugin.register(int) +print(isinstance(3, Plugin), issubclass(int, Plugin), issubclass(str, Plugin)) + + +@contextmanager +def tag(name): + print("<" + name + ">") + try: + yield name.upper() + finally: + print("") + + +with tag("p") as t: + print("body", t) +try: + with tag("q"): + raise ValueError("boom") +except ValueError as e: + print("caught", e) + + +@contextmanager +def swallow(): + try: + yield + except KeyError: + print("swallowed") + + +with swallow(): + raise KeyError("k") +print("after") +with suppress(ZeroDivisionError): + 1 / 0 +print("suppressed") +with ExitStack() as stack: + for n in ("a", "b"): + stack.callback(print, "exit", n) + stack.enter_context(tag("x")) + print("in stack") +with closing(F()) as fobj: + print("using") +with nullcontext(5) as v: + print(v) +buf = io.StringIO() +with redirect_stdout(buf): + print("hidden") +print(repr(buf.getvalue())) + + +@tag("deco") +def decorated(): + print("inside") + + +decorated() diff --git a/maven/python-transpiler/src/test/resources/conformance/uuid_.out b/maven/python-transpiler/src/test/resources/conformance/uuid_.out new file mode 100644 index 00000000000..223bf5ce8c0 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/uuid_.out @@ -0,0 +1,24 @@ +12345678-1234-5678-1234-567812345678 UUID('12345678-1234-5678-1234-567812345678') 12345678123456781234567812345678 24197857161011715162171839636988778104 urn:uuid:12345678-1234-5678-1234-567812345678 None reserved for NCS compatibility +(305419896, 4660, 22136, 18, 52, 95073701484152) 305419896 4660 4660 95073701484152 b'\x124Vx\x124Vx\x124Vx\x124Vx' b'xV4\x124\x12xV\x124Vx\x124Vx' +True True True True True +6fa459ea-ee8a-3ca4-894e-db77e160355e 886313e1-3b8a-5372-9b90-0c9aee199e5d +53b14841-9895-312b-bb91-82c1f08c9edd 42d5e23b-3a02-5135-85c6-52d1102f1f00 +00000000-0001-8002-8000-000000000003 00000000-0000-0000-0000-000000000000 ffffffff-ffff-ffff-ffff-ffffffffffff None reserved for future definition +uuid1 1 specified in RFC 4122 True 36 True +uuid4 4 specified in RFC 4122 True 36 True +uuid6 6 specified in RFC 4122 True 36 True +uuid7 7 specified in RFC 4122 True 36 True +uuid8 8 specified in RFC 4122 True 36 True +True 2748 7 +00000000-0000-4000-8000-000000000005 True True +1 True True +ValueError badly formed hexadecimal UUID string +ValueError bytes is not a 16-char string +ValueError int is out of range (need a 128-bit value) +ValueError fields is not a 6-tuple +ValueError illegal version number +TypeError one of the hex, bytes, bytes_le, fields, or int arguments must be given +UUID objects are immutable +'k' KeyError +None (None, None, None) +1- 'n'-8-3 [('a', 'b', 'c', None), ('{', None, None, None), ('d}', None, None, None)] diff --git a/maven/python-transpiler/src/test/resources/conformance/uuid_.py b/maven/python-transpiler/src/test/resources/conformance/uuid_.py new file mode 100644 index 00000000000..d3ec01f7683 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/uuid_.py @@ -0,0 +1,36 @@ +# uuid: parsing, fields, name-based versions, random/time versions, and errors. +import uuid +import sys +import string + +u = uuid.UUID("{12345678-1234-5678-1234-567812345678}") +print(u, repr(u), u.hex, u.int, u.urn, u.version, u.variant) +print(u.fields, u.time_low, u.time_mid, u.clock_seq, u.node, u.bytes, u.bytes_le) +print(uuid.UUID(bytes=u.bytes) == u, uuid.UUID(bytes_le=u.bytes_le) == u, uuid.UUID(fields=u.fields) == u, + uuid.UUID(int=u.int) == u, uuid.UUID("urn:uuid:12345678123456781234567812345678") == u) +print(uuid.uuid3(uuid.NAMESPACE_DNS, "python.org"), uuid.uuid5(uuid.NAMESPACE_DNS, "python.org")) +print(uuid.uuid3(uuid.NAMESPACE_URL, b"http://x"), uuid.uuid5(uuid.NAMESPACE_OID, "1.2.3")) +print(uuid.uuid8(1, 2, 3), uuid.NIL, uuid.MAX, uuid.NIL.version, uuid.MAX.variant) +for f in (uuid.uuid1, uuid.uuid4, uuid.uuid6, uuid.uuid7, uuid.uuid8): + a, b = f(), f() + print(f.__name__, a.version, a.variant, a != b, len(str(a)), int(a) == a.int) +s7 = [uuid.uuid7() for _ in range(50)] +print(s7 == sorted(s7), uuid.uuid1(node=0xabc, clock_seq=7).node, uuid.uuid1(clock_seq=7).clock_seq) +print(uuid.UUID(int=5, version=4), sorted([uuid.MAX, uuid.NIL, u])[1] == u, hash(u) == hash(uuid.UUID(str(u)))) +print({u: 1}[uuid.UUID(u.hex)], u < uuid.MAX, uuid.uuid1().node == uuid.getnode()) +for kw in [{"hex": "zz"}, {"bytes": b"ab"}, {"int": -1}, {"fields": (1, 2, 3)}, {"int": 1, "version": 9}, {}]: + try: + uuid.UUID(**kw) + except (ValueError, TypeError) as e: + print(type(e).__name__, e) +try: + u.int = 3 +except TypeError as e: + print(e) +try: + raise KeyError("k") +except KeyError: + print(sys.exception(), sys.exc_info()[0].__name__) +print(sys.exception(), sys.exc_info()) +fm = string.Formatter() +print(fm.format("{0}-{name!r:>6}-{x[1]}-{y.real}", 1, name="n", x=[7, 8], y=3), list(fm.parse("a{b:c}{{d}}"))) diff --git a/maven/python-transpiler/src/test/resources/conformance/weakref_.out b/maven/python-transpiler/src/test/resources/conformance/weakref_.out new file mode 100644 index 00000000000..176c4e83084 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/weakref_.out @@ -0,0 +1,23 @@ +True True ReferenceType True None True True +cannot create weak reference to 'int' object +cannot create weak reference to 'str' object +cannot create weak reference to 'tuple' object +cannot create weak reference to 'list' object +cannot create weak reference to 'dict' object +cannot create weak reference to 'NoneType' object + True True +a hi a Node(a) +a2 +hi a True +2 ['x', 'y'] Node(a) none True [('x', Node(a)), ('y', Node(b))] +Node(b) 1 Node(c) ['q', 'x'] +2 1 True False False [1, 2] 0 +2 True False ['a', 'c'] +True True +finalized c +False None +main done +registered last, runs first +bye second? +bye first! +a finalized at exit diff --git a/maven/python-transpiler/src/test/resources/conformance/weakref_.py b/maven/python-transpiler/src/test/resources/conformance/weakref_.py new file mode 100644 index 00000000000..1e965d87c82 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/weakref_.py @@ -0,0 +1,60 @@ +# weakref and atexit. Referents stay alive throughout: when a dead one is +# cleared is the collector's business, not something a program can rely on. +import weakref, atexit + +class Node: + def __init__(self, name): + self.name = name + def hello(self): + return "hi " + self.name + def __repr__(self): + return "Node(%s)" % self.name + +a, b, c = Node("a"), Node("b"), Node("c") +r = weakref.ref(a) +print(r() is a, r() == a, type(r).__name__, isinstance(r, weakref.ref), r.__callback__, weakref.ref(a) == r, hash(r) == hash(weakref.ref(a))) +for bad in [1, "s", (1,), [1], {}, None]: + try: + weakref.ref(bad) + except TypeError as e: + print(e) +print(weakref.ref(len) if False else "", weakref.ref(Node)() is Node, weakref.ref(print if False else Node.hello)() is Node.hello) +p = weakref.proxy(a) +print(p.name, p.hello(), str(p)) +p.name = "a2" +print(a.name) +a.name = "a" +wm = weakref.WeakMethod(a.hello) +print(wm()(), wm() == a.hello) +wvd = weakref.WeakValueDictionary() +wvd["x"] = a +wvd["y"] = b +print(len(wvd), sorted(wvd.keys()), wvd["x"], wvd.get("z", "none"), "y" in wvd, sorted(wvd.items(), key=lambda kv: kv[0])) +print(wvd.pop("y"), len(wvd), wvd.setdefault("q", c), sorted(wvd)) +wkd = weakref.WeakKeyDictionary() +wkd[a] = 1 +wkd[b] = 2 +print(len(wkd), wkd[a], a in wkd, c in wkd, 5 in wkd, sorted(wkd.values()), wkd.get(c, 0)) +ws = weakref.WeakSet([a, b]) +ws.add(c) +ws.discard(b) +print(len(ws), a in ws, b in ws, sorted(n.name for n in ws)) +f = weakref.finalize(c, print, "finalized", "c") +print(f.alive, f.peek()[1:] == (print, ("finalized", "c"), {})) +f() +print(f.alive, f()) + +def bye(who, punct="!"): + print("bye", who + punct) + +atexit.register(bye, "first") +atexit.register(bye, "second", punct="?") +@atexit.register +def last(): + print("registered last, runs first") +def gone(): + print("never") +atexit.register(gone) +atexit.unregister(gone) +weakref.finalize(a, print, "a finalized at exit") +print("main done") diff --git a/scripts/check-cast-semantics.sh b/scripts/check-cast-semantics.sh index f6c50a53cf4..c7810b8a6a2 100755 --- a/scripts/check-cast-semantics.sh +++ b/scripts/check-cast-semantics.sh @@ -42,6 +42,8 @@ DEFAULT_ROOTS=( "maven/android-compat/target/classes" "maven/swing-compat/target/classes" "maven/javafx-compat/target/classes" + # The Python runtime ships inside every application with Python sources. + "maven/python-runtime/target/classes" ) # Translated too, but built only under the unity-compat Maven profile, which # needs the .NET SDK: maven/unity-compat, the runtime an application with Unity diff --git a/scripts/copyright-header-exclusions.txt b/scripts/copyright-header-exclusions.txt index 3503fb02610..3b53078afc4 100644 --- a/scripts/copyright-header-exclusions.txt +++ b/scripts/copyright-header-exclusions.txt @@ -73,6 +73,7 @@ CodenameOne/src/com/codename1/gaming/physics/box2d/dynamics/World.java | Vendore CodenameOne/src/com/codename1/gaming/physics/box2d/dynamics/joints/FrictionJoint.java | Vendored JBox2D source (Daniel Murphy, BSD 2-Clause) retaining its original notice CodenameOne/src/com/codename1/gaming/physics/box2d/dynamics/joints/Joint.java | Vendored JBox2D source (Daniel Murphy, BSD 2-Clause) retaining its original notice CodenameOne/src/com/codename1/util/MathUtil.java | Port of Sun Microsystems FDLIBM retaining its original freely-distributable notice +maven/python-runtime/src/main/java/com/codename1/python/runtime/Erf.java | Port of Sun Microsystems FDLIBM s_erf.c retaining its original freely-distributable notice # Pulled into the diff-scoped check by the repository-wide @author/@version removal: CodenameOne/src/com/codename1/gaming/physics/box2d/callbacks/ContactFilter.java | Vendored JBox2D source (Daniel Murphy, BSD 2-Clause) retaining its original notice CodenameOne/src/com/codename1/gaming/physics/box2d/callbacks/ContactImpulse.java | Vendored JBox2D source (Daniel Murphy, BSD 2-Clause) retaining its original notice diff --git a/scripts/python-bench/README.md b/scripts/python-bench/README.md new file mode 100644 index 00000000000..52b4b7e62d3 --- /dev/null +++ b/scripts/python-bench/README.md @@ -0,0 +1,60 @@ +# Python vs Codename One benchmark + +The kernels under `kernels/` run twice: by CPython, and transpiled to Java by +the Codename One Python transpiler and run against `codenameone-python-runtime`. + +``` +kernels/*.py one workload each; _harness.py times it and prints the lines below +run_bench.py builds, measures, renders and gates +baseline/ the gate's ratios, seeded from CI (absent until the first fold) +``` + +Each kernel prints `RESULT ` and `BENCH `. +The RESULT must be identical on both sides or the run fails: a faster +program that computes something else is a bug, not a speed-up. BENCH is the +fastest of several rounds timed inside the program, so it excludes start-up; +the harness separately records the whole process's wall time and its peak +resident memory. + +The kernels are our own implementations of the classic algorithms (n-body, +spectral norm, fannkuch, the Richards scheduler) plus a text-processing +workload. They avoid generators and other constructs the transpiler does not +support yet. + +## Running it + +```bash +source tools/env.sh +cd maven +mvn -Dmaven.repo.local=$(pwd)/../.m2-repo -pl core,python-transpiler -am install -DskipTests +JAVA_HOME=$JAVA17_HOME mvn -Dmaven.repo.local=$(pwd)/../.m2-repo -pl python-runtime,python-transpiler install +cd .. +python3 scripts/python-bench/run_bench.py --java-home "$JAVA17_HOME" \ + --runtime-jar maven/python-runtime/target/codenameone-python-runtime-8.0-SNAPSHOT.jar \ + --core-jar .m2-repo/com/codenameone/codenameone-core/8.0-SNAPSHOT/codenameone-core-8.0-SNAPSHOT.jar \ + --transpiler-cp maven/python-transpiler/target/classes +``` + +`--kernel nbody` runs one kernel; `--rounds` sets the rounds per side. + +## The gate + +`--check` compares each kernel's ratio ours/CPython -- compute time, process +wall time, peak memory -- with `baseline/ratios.json` and fails when one grows +past its tolerance. Ratios depend on the machine and the interpreter, so the +baseline comes from the CI runner, never a laptop: download the +`python-bench-report` artifact and run + +```bash +python3 scripts/python-bench/run_bench.py --fold python-bench.json +``` + +Until that file exists `--check` only reports. CPython is pinned in the +workflow (`CPYTHON_VERSION`); moving it is a deliberate re-baseline. + +## What is measured today + +Only the JVM arm, which is what the simulator and the desktop targets run. +Its memory figure is the JVM's own footprint, which dominates a small kernel, +so read it as a ceiling rather than a device number. The ParparVM, Android +and JavaScript arms join as the Python support reaches them. diff --git a/scripts/python-bench/build_parparvm.sh b/scripts/python-bench/build_parparvm.sh new file mode 100755 index 00000000000..b119263bc48 --- /dev/null +++ b/scripts/python-bench/build_parparvm.sh @@ -0,0 +1,75 @@ +#!/bin/bash +# Builds one kernel as a native ParparVM executable, the way an iOS or native +# desktop build lowers it: Python -> Java -> bytecode -> C -> clang -O3. +# +# build_parparvm.sh [maven repo] +# +# Every input comes from the checkout's own Maven repository (the second +# argument, else PYBENCH_M2, else /.m2-repo), never ~/.m2: concurrent checkouts overwrite each other's +# snapshots there. Install them first: +# +# cd maven && mvn -Dmaven.repo.local=$(pwd)/../.m2-repo -pl core,python-transpiler -am install -DskipTests +# JAVA_HOME=$JAVA17_HOME mvn -Dmaven.repo.local=$(pwd)/../.m2-repo -pl python-runtime install -DskipTests +# mvn -Dmaven.repo.local=$(pwd)/../.m2-repo -f ../vm/pom.xml -pl JavaAPI,ByteCodeTranslator -am install -DskipTests +# +# Output: scripts/python-bench/target// (and the translated +# sources beside it). Needs JAVA17_HOME (javac for the generated code, which is +# Java 17) and JDK_8_HOME or JAVA_HOME (the translator), and clang. +set -e +HERE="$(cd "$(dirname "$0")" && pwd)" +REPO="$(cd "$HERE/../.." && pwd)" +K="$1" +[ -n "$K" ] || { echo "usage: build_parparvm.sh [maven repo]" >&2; exit 2; } +M2="${2:-${PYBENCH_M2:-$REPO/.m2-repo}}" +J17="${JAVA17_HOME:?set JAVA17_HOME}" +J8="${JDK_8_HOME:-$JAVA_HOME}" +CC="${CC:-clang}" +art() { echo "$M2/$1/$2/$3/$2-$3.jar"; } +CORE=$(art com/codenameone codenameone-core 8.0-SNAPSHOT) +RUNTIME=$(art com/codenameone codenameone-python-runtime 8.0-SNAPSHOT) +TRANSPILER=$(art com/codenameone codenameone-python-transpiler 8.0-SNAPSHOT) +JAVAAPI=$(art com/codename1/parparvm JavaAPI 1.0-SNAPSHOT) +TRANSLATOR=$(art com/codename1/parparvm ByteCodeTranslator 1.0-SNAPSHOT) +for j in "$CORE" "$RUNTIME" "$TRANSPILER" "$JAVAAPI" "$TRANSLATOR"; do + [ -f "$j" ] || { echo "missing $j -- install it (see the header of this script)" >&2; exit 1; } +done + +W="$HERE/target/$K" +rm -rf "$W" +mkdir -p "$W/src" "$W/gen/com/pybench" "$W/jclasses" "$W/javaapi" "$W/out" +# PY_SOURCES names another directory of programs (the conformance suite). +SRC_DIR="${PY_SOURCES:-$HERE/kernels}" +cp "$SRC_DIR/$K.py" "$W/src/" +[ -f "$HERE/kernels/_harness.py" ] && cp "$HERE/kernels/_harness.py" "$W/src/" +"$J17/bin/java" -cp "$TRANSPILER" com.codename1.python.transpiler.api.Main "$W/src" "$W/gen" "$K" +cat > "$W/gen/com/pybench/PyBench.java" < "$W/translate.log" 2>&1 || { tail -30 "$W/translate.log"; exit 1; } + +# Compiled exactly as vm/selfhost compiles the self-hosted translator and its +# benchmarks, so the flags are the ones a shipped binary gets. +PYTHON=python3 +WINDOWS=0 +native_path() { echo "$1"; } +CN1_SELFHOST_CFLAGS="-flto=thin $CN1_SELFHOST_CFLAGS" +. "$REPO/vm/selfhost/compile-dist.sh" +cn1_compile_dist "$W/out/dist" PyBench "$W/$K" -O3 "$W/cc.log" diff --git a/scripts/python-bench/conformance_parparvm.sh b/scripts/python-bench/conformance_parparvm.sh new file mode 100755 index 00000000000..25722b969f4 --- /dev/null +++ b/scripts/python-bench/conformance_parparvm.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Runs the transpiler's conformance programs as native ParparVM executables +# and compares each output with CPython's recorded one (the .out beside it). +# The JVM run of the same programs is ConformanceTest; this is the device +# half, where a cast that fails is a crash instead of an exception. +# +# conformance_parparvm.sh [program ...] +set -e +HERE="$(cd "$(dirname "$0")" && pwd)" +REPO="$(cd "$HERE/../.." && pwd)" +CONF="$REPO/maven/python-transpiler/src/test/resources/conformance" +progs="$*" +[ -n "$progs" ] || progs=$(cd "$CONF" && ls *.py | sed 's/\.py$//') +failed=0 +for p in $progs; do + if ! PY_SOURCES="$CONF" "$HERE/build_parparvm.sh" "$p" > /dev/null 2>&1; then + echo "BUILD FAILED $p"; failed=1; continue + fi + got="$HERE/target/$p/output.txt" + (cd "$HERE/target/$p/src" && "../$p" > "$got" 2>&1) || true + if diff -q "$CONF/$p.out" "$got" > /dev/null; then + echo "ok $p" + else + echo "DIFF $p"; diff "$CONF/$p.out" "$got" | head -20; failed=1 + fi +done +exit $failed diff --git a/scripts/python-bench/hostfs/com/codename1/io/FileSystemStorage.java b/scripts/python-bench/hostfs/com/codename1/io/FileSystemStorage.java new file mode 100644 index 00000000000..377ef18d8cf --- /dev/null +++ b/scripts/python-bench/hostfs/com/codename1/io/FileSystemStorage.java @@ -0,0 +1,112 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.io; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +/// The benchmark's stand-in for the core class: the members the Python +/// runtime's `Cn1FileSystem` calls, over the host's files. A native +/// benchmark executable is the runtime without the core, whose real +/// `FileSystemStorage` would bring the whole UI and its natives with it. +public final class FileSystemStorage { + private static final FileSystemStorage INSTANCE = new FileSystemStorage(); + private static final String SCHEME = "file://"; + + private FileSystemStorage() { + } + + public static FileSystemStorage getInstance() { + return INSTANCE; + } + + private static File file(String url) { + return new File(url.startsWith(SCHEME) ? url.substring(SCHEME.length()) : url); + } + + /// The directory the executable was started in. + public String getAppHomePath() { + String p = new File(".").getAbsolutePath(); + if (p.endsWith("/.")) { + p = p.substring(0, p.length() - 2); + } + return SCHEME + p + "/"; + } + + public InputStream openInputStream(String url) throws IOException { + return new FileInputStream(file(url)); + } + + public OutputStream openOutputStream(String url) throws IOException { + return new FileOutputStream(file(url)); + } + + /// Appends when `offset` is the file's length, the only way the + /// runtime calls it. + public OutputStream openOutputStream(String url, int offset) throws IOException { + return new FileOutputStream(file(url), offset > 0); + } + + public boolean exists(String url) { + return file(url).exists(); + } + + public boolean isDirectory(String url) { + return file(url).isDirectory(); + } + + public String[] listFiles(String url) throws IOException { + String[] r = file(url).list(); + if (r == null) { + throw new IOException("not a directory: " + url); + } + return r; + } + + public void mkdir(String url) { + file(url).mkdir(); + } + + public void delete(String url) { + file(url).delete(); + } + + /// Renames within the directory, as the core class does. + public void rename(String url, String newName) { + File f = file(url); + f.renameTo(new File(f.getParent(), newName)); + } + + public long getLength(String url) { + return file(url).length(); + } + + public long getLastModified(String url) { + return file(url).lastModified(); + } +} diff --git a/scripts/python-bench/kernels/_harness.py b/scripts/python-bench/kernels/_harness.py new file mode 100644 index 00000000000..3840b559f59 --- /dev/null +++ b/scripts/python-bench/kernels/_harness.py @@ -0,0 +1,35 @@ +# Shared by every kernel: runs `work(size)` `repeat` times and prints the +# result once, then the fastest round (BENCH), the first, cold one (COLD) +# and all rounds together (TOTAL), in the line format the scripts read. +# +# The size is a run-time input (the first argument, else the kernel's own +# default), so no compiler, ahead of time or just in time, can fold the work +# into a constant. Every round must compute the same result; a round a +# compiler skipped or memoized would have to reproduce it exactly anyway. +import sys +import time + + +def measure(name, work, size, repeat): + if len(sys.argv) > 1: + size = int(sys.argv[1]) + best = None + first = None + total = 0.0 + result = None + for _ in range(repeat): + t0 = time.perf_counter() + r = work(size) + dt = time.perf_counter() - t0 + if result is not None and r != result: + raise RuntimeError("rounds disagree: %r != %r" % (r, result)) + result = r + total += dt + if first is None: + first = dt + if best is None or dt < best: + best = dt + print("RESULT", name, result) + print("BENCH", name, int(best * 1000000)) + print("COLD", name, int(first * 1000000)) + print("TOTAL", name, int(total * 1000000)) diff --git a/scripts/python-bench/kernels/dicts_strings.py b/scripts/python-bench/kernels/dicts_strings.py new file mode 100644 index 00000000000..5ee751ea5e7 --- /dev/null +++ b/scripts/python-bench/kernels/dicts_strings.py @@ -0,0 +1,27 @@ +# Text processing: string building, splitting, dict counting and sorting. +from _harness import measure + +WORDS = ["alpha", "beta", "gamma", "delta", "epsilon", "zeta", "eta", "theta", + "iota", "kappa", "lambda", "mu", "nu", "xi", "omicron", "pi"] + + +def work(lines): + parts = [] + seed = 12345 + for i in range(lines): + row = [] + for _ in range(12): + seed = (seed * 1103515245 + 12345) & 0x7fffffff + row.append(WORDS[(seed >> 16) % len(WORDS)]) + parts.append(" ".join(row)) + text = "\n".join(parts) + counts = {} + for line in text.split("\n"): + for w in line.split(): + key = w.upper() if len(w) > 4 else w + counts[key] = counts.get(key, 0) + 1 + top = sorted(counts.items(), key=lambda kv: (-kv[1], kv[0]))[:5] + return ",".join("%s=%d" % kv for kv in top) + " " + str(len(text)) + + +measure("dicts_strings", work, 160000, 5) diff --git a/scripts/python-bench/kernels/fannkuch.py b/scripts/python-bench/kernels/fannkuch.py new file mode 100644 index 00000000000..44b1b70c1c9 --- /dev/null +++ b/scripts/python-bench/kernels/fannkuch.py @@ -0,0 +1,39 @@ +# Pancake flipping over every permutation: list slicing and reversal. +from _harness import measure + + +def work(n): + perm1 = list(range(n)) + count = [0] * n + max_flips = 0 + checksum = 0 + sign = 1 + r = n + while True: + while r != 1: + count[r - 1] = r + r -= 1 + perm = perm1[:] + flips = 0 + k = perm[0] + while k: + perm[:k + 1] = perm[k::-1] + flips += 1 + k = perm[0] + if flips > max_flips: + max_flips = flips + checksum += flips if sign == 1 else -flips + while True: + if r == n: + return "%d %d" % (checksum, max_flips) + p0 = perm1[0] + perm1[:r] = perm1[1:r + 1] + perm1[r] = p0 + count[r] -= 1 + if count[r] > 0: + break + r += 1 + sign = -sign + + +measure("fannkuch", work, 10, 3) diff --git a/scripts/python-bench/kernels/nbody.py b/scripts/python-bench/kernels/nbody.py new file mode 100644 index 00000000000..0261e76dc98 --- /dev/null +++ b/scripts/python-bench/kernels/nbody.py @@ -0,0 +1,87 @@ +# Planetary orbits: float arithmetic on lists, attribute-free inner loop. +from _harness import measure + +PI = 3.14159265358979323 +SOLAR_MASS = 4 * PI * PI +DAYS_PER_YEAR = 365.24 + + +def bodies(): + return [ + [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], SOLAR_MASS], + [[4.84143144246472090e+00, -1.16032004402742839e+00, -1.03622044471123109e-01], + [1.66007664274403694e-03 * DAYS_PER_YEAR, 7.69901118419740425e-03 * DAYS_PER_YEAR, + -6.90460016972063023e-05 * DAYS_PER_YEAR], 9.54791938424326609e-04 * SOLAR_MASS], + [[8.34336671824457987e+00, 4.12479856412430479e+00, -4.03523417114321381e-01], + [-2.76742510726862411e-03 * DAYS_PER_YEAR, 4.99852801234917238e-03 * DAYS_PER_YEAR, + 2.30417297573763929e-05 * DAYS_PER_YEAR], 2.85885980666130812e-04 * SOLAR_MASS], + [[1.28943695621391310e+01, -1.51111514016986312e+01, -2.23307578892655734e-01], + [2.96460137564761618e-03 * DAYS_PER_YEAR, 2.37847173959480950e-03 * DAYS_PER_YEAR, + -2.96589568540237556e-05 * DAYS_PER_YEAR], 4.36624404335156298e-05 * SOLAR_MASS], + [[1.53796971148509165e+01, -2.59193146099879641e+01, 1.79258772950371181e-01], + [2.68067772490389322e-03 * DAYS_PER_YEAR, 1.62824170038242295e-03 * DAYS_PER_YEAR, + -9.51592254519715870e-05 * DAYS_PER_YEAR], 5.15138902046611451e-05 * SOLAR_MASS], + ] + + +def energy(system): + e = 0.0 + n = len(system) + for i in range(n): + (x1, y1, z1), (vx, vy, vz), m1 = system[i] + e += 0.5 * m1 * (vx * vx + vy * vy + vz * vz) + for j in range(i + 1, n): + (x2, y2, z2), _, m2 = system[j] + dx = x1 - x2 + dy = y1 - y2 + dz = z1 - z2 + e -= (m1 * m2) / ((dx * dx + dy * dy + dz * dz) ** 0.5) + return e + + +def offset_momentum(system): + px = py = pz = 0.0 + for _, v, m in system: + px -= v[0] * m + py -= v[1] * m + pz -= v[2] * m + v = system[0][1] + v[0] = px / SOLAR_MASS + v[1] = py / SOLAR_MASS + v[2] = pz / SOLAR_MASS + + +def advance(system, dt, steps): + pairs = [] + for i in range(len(system)): + for j in range(i + 1, len(system)): + pairs.append((system[i], system[j])) + for _ in range(steps): + for (r1, v1, m1), (r2, v2, m2) in pairs: + dx = r1[0] - r2[0] + dy = r1[1] - r2[1] + dz = r1[2] - r2[2] + mag = dt * ((dx * dx + dy * dy + dz * dz) ** -1.5) + b1 = m1 * mag + b2 = m2 * mag + v1[0] -= dx * b2 + v1[1] -= dy * b2 + v1[2] -= dz * b2 + v2[0] += dx * b1 + v2[1] += dy * b1 + v2[2] += dz * b1 + for r, v, _ in system: + r[0] += dt * v[0] + r[1] += dt * v[1] + r[2] += dt * v[2] + + +def work(steps): + system = bodies() + offset_momentum(system) + before = energy(system) + advance(system, 0.01, steps) + return "%.9f %.9f" % (before, energy(system)) + + +measure("nbody", work, 300000, 5) diff --git a/scripts/python-bench/kernels/richards.py b/scripts/python-bench/kernels/richards.py new file mode 100644 index 00000000000..a0529c599e3 --- /dev/null +++ b/scripts/python-bench/kernels/richards.py @@ -0,0 +1,217 @@ +# An operating system task scheduler: classes, attribute access, virtual calls. +from _harness import measure + +IDLE, WORKER, HANDLER_A, HANDLER_B, DEVICE_A, DEVICE_B = range(6) + + +class Packet: + def __init__(self, link, ident, kind): + self.link = link + self.ident = ident + self.kind = kind + self.datum = 0 + self.data = [0] * 4 + + def append_to(self, queue): + self.link = None + if queue is None: + return self + p = queue + while p.link is not None: + p = p.link + p.link = self + return queue + + +class Task: + def __init__(self, scheduler, ident, priority, queue, waiting): + self.scheduler = scheduler + self.ident = ident + self.priority = priority + self.queue = queue + self.waiting = waiting + self.held = False + self.link = scheduler.tasks + scheduler.tasks = self + scheduler.table[ident] = self + + def runnable(self): + return not self.held and not self.waiting + + def take(self): + p = self.queue + self.queue = p.link + self.waiting = self.queue is not None and False + return p + + def add_packet(self, packet, current): + if self.queue is None: + self.queue = packet + self.waiting = False + if self.priority > current.priority: + return self + else: + self.queue = packet.append_to(self.queue) + return current + + +class Idle(Task): + def __init__(self, scheduler, count): + Task.__init__(self, scheduler, IDLE, 0, None, False) + self.control = 1 + self.count = count + + def run(self, packet): + s = self.scheduler + self.count -= 1 + if self.count == 0: + self.held = True + return s.next_of(self) + if self.control & 1 == 0: + self.control >>= 1 + return s.release(DEVICE_A, self) + self.control = (self.control >> 1) ^ 0xD008 + return s.release(DEVICE_B, self) + + +class Worker(Task): + def __init__(self, scheduler, queue): + Task.__init__(self, scheduler, WORKER, 1000, queue, True) + self.destination = HANDLER_A + self.count = 0 + + def run(self, packet): + s = self.scheduler + if packet is None: + self.waiting = True + return s.next_of(self) + self.destination = HANDLER_B if self.destination == HANDLER_A else HANDLER_A + packet.ident = self.destination + packet.datum = 0 + for i in range(4): + self.count += 1 + if self.count > 26: + self.count = 1 + packet.data[i] = 64 + self.count + return s.queue_packet(packet, self) + + +class Handler(Task): + def __init__(self, scheduler, ident, queue): + Task.__init__(self, scheduler, ident, 2000 if ident == HANDLER_A else 3000, queue, True) + self.work = None + self.devices = None + + def run(self, packet): + s = self.scheduler + if packet is not None: + if packet.kind == 1: + self.work = packet.append_to(self.work) + else: + self.devices = packet.append_to(self.devices) + if self.work is not None: + w = self.work + count = w.datum + if count < 4: + if self.devices is not None: + d = self.devices + self.devices = d.link + d.datum = w.data[count] + w.datum = count + 1 + return s.queue_packet(d, self) + else: + self.work = w.link + return s.queue_packet(w, self) + self.waiting = True + return s.next_of(self) + + +class Device(Task): + def __init__(self, scheduler, ident): + Task.__init__(self, scheduler, ident, 4000 if ident == DEVICE_A else 5000, None, True) + self.pending = None + + def run(self, packet): + s = self.scheduler + if packet is None: + if self.pending is None: + self.waiting = True + return s.next_of(self) + p = self.pending + self.pending = None + return s.queue_packet(p, self) + self.pending = packet + s.hold_count += 1 + self.held = True + return s.next_of(self) + + +class Scheduler: + def __init__(self): + self.tasks = None + self.table = [None] * 6 + self.queue_count = 0 + self.hold_count = 0 + + def next_of(self, task): + t = task.link + while t is not None and not t.runnable() and not (t.queue is not None and not t.held): + t = t.link + return t if t is not None else self.first_ready() + + def first_ready(self): + t = self.tasks + while t is not None: + if not t.held and (t.queue is not None or not t.waiting): + return t + t = t.link + return None + + def release(self, ident, current): + t = self.table[ident] + t.held = False + return t if t.priority > current.priority else current + + def queue_packet(self, packet, current): + t = self.table[packet.ident] + self.queue_count += 1 + packet.link = None + packet.ident = current.ident + return t.add_packet(packet, current) + + def run(self): + t = self.tasks + while t is not None: + if t.held or (t.queue is None and t.waiting): + t = t.link + continue + packet = None + if t.queue is not None: + packet = t.queue + t.queue = packet.link + if t.queue is None: + t.waiting = False + t = t.run(packet) + + +def work(count): + s = Scheduler() + Idle(s, count) + wq = Packet(None, 0, 1) + wq = Packet(wq, 0, 1) + Worker(s, wq) + q = Packet(None, DEVICE_A, 0) + q = Packet(q, DEVICE_A, 0) + q = Packet(q, DEVICE_A, 0) + Handler(s, HANDLER_A, q) + q = Packet(None, DEVICE_B, 0) + q = Packet(q, DEVICE_B, 0) + q = Packet(q, DEVICE_B, 0) + Handler(s, HANDLER_B, q) + Device(s, DEVICE_A) + Device(s, DEVICE_B) + s.run() + return "%d %d" % (s.queue_count, s.hold_count) + + +measure("richards", work, 400000, 5) diff --git a/scripts/python-bench/kernels/spectral_norm.py b/scripts/python-bench/kernels/spectral_norm.py new file mode 100644 index 00000000000..216777c6319 --- /dev/null +++ b/scripts/python-bench/kernels/spectral_norm.py @@ -0,0 +1,47 @@ +# Spectral norm of an infinite matrix: integer arithmetic feeding float division. +from _harness import measure + + +def a(i, j): + return 1.0 / (((i + j) * (i + j + 1) >> 1) + i + 1) + + +def mul_av(v, out): + n = len(v) + for i in range(n): + s = 0.0 + for j in range(n): + s += a(i, j) * v[j] + out[i] = s + + +def mul_atv(v, out): + n = len(v) + for i in range(n): + s = 0.0 + for j in range(n): + s += a(j, i) * v[j] + out[i] = s + + +def mul_atav(v, out, tmp): + mul_av(v, tmp) + mul_atv(tmp, out) + + +def work(n): + u = [1.0] * n + v = [0.0] * n + tmp = [0.0] * n + for _ in range(10): + mul_atav(u, v, tmp) + mul_atav(v, u, tmp) + vbv = 0.0 + vv = 0.0 + for i in range(n): + vbv += u[i] * v[i] + vv += v[i] * v[i] + return "%.9f" % ((vbv / vv) ** 0.5) + + +measure("spectral_norm", work, 480, 5) diff --git a/scripts/python-bench/parparvm_table.py b/scripts/python-bench/parparvm_table.py new file mode 100755 index 00000000000..e5113118455 --- /dev/null +++ b/scripts/python-bench/parparvm_table.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +"""Builds every kernel with build_parparvm.sh and compares it with CPython: +compute (best of rounds, timed inside the program), process wall time, peak +resident memory and binary size. Results must match CPython's exactly. + + parparvm_table.py [--rounds 3] [--no-build] [--m2 DIR] [--impl NAME=PATH ...] + [--json out.json] [--check] [kernel ...] + parparvm_table.py --fold + +--check gates the ParparVM/CPython ratios against +baseline/parparvm-ratios.json, with run_bench.py's tolerances; --fold seeds +or moves that baseline from a report CI produced (never from a laptop: the +ratios depend on the machine). + +--impl adds another Python (PyPy, GraalPy) as extra columns: its compute +time and peak RSS, with the same result check. Compute is the best round +inside one process, so a JIT is measured warm; wall time includes its warm-up. +""" +import argparse +import json +import os +import re +import subprocess +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HERE) +import run_bench # noqa: E402 + +BASELINE = os.path.join(HERE, "baseline", "parparvm-ratios.json") + + +def run(cmd, cwd, rounds): + best = None + for _ in range(rounds): + p = subprocess.run(["/usr/bin/time", "-l" if sys.platform == "darwin" else "-v"] + cmd, + cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) + if p.returncode != 0: + raise RuntimeError("%s failed:\n%s" % (cmd, p.stderr[-3000:])) + result = re.search(r"RESULT \S+ (.*)", p.stdout).group(1) + us = int(re.search(r"BENCH \S+ (\d+)", p.stdout).group(1)) + if sys.platform == "darwin": + wall = float(re.search(r"([\d.]+) real", p.stderr).group(1)) + rss = int(re.search(r"(\d+)\s+maximum resident", p.stderr).group(1)) + else: + m = re.search(r"Elapsed \(wall clock\) time.*: (?:(\d+):)?(\d+):([\d.]+)", p.stderr) + wall = int(m.group(1) or 0) * 3600 + int(m.group(2)) * 60 + float(m.group(3)) + rss = int(re.search(r"Maximum resident set size \(kbytes\): (\d+)", p.stderr).group(1)) * 1024 + cold = int(re.search(r"COLD \S+ (\d+)", p.stdout).group(1)) + total = int(re.search(r"TOTAL \S+ (\d+)", p.stdout).group(1)) + r = (us, wall, rss, result, cold, total) + if best is None or us < best[0]: + best = r + return best + + +def main(): + if len(sys.argv) == 3 and sys.argv[1] == "--fold": + with open(sys.argv[2]) as f: + run_bench.write_baseline(json.load(f), BASELINE) + return 0 + p = argparse.ArgumentParser() + p.add_argument("--m2", help="Maven repository build_parparvm.sh reads (default /.m2-repo)") + p.add_argument("--json") + p.add_argument("--check", action="store_true") + p.add_argument("--rounds", type=int, default=3) + p.add_argument("--no-build", action="store_true") + p.add_argument("--impl", action="append", default=[], metavar="NAME=PATH") + p.add_argument("kernel", nargs="*") + a = p.parse_args() + kernels = a.kernel or sorted(f[:-3] for f in os.listdir(os.path.join(HERE, "kernels")) + if f.endswith(".py") and not f.startswith("_")) + impls = [i.split("=", 1) for i in a.impl] + print("| kernel | impl | best | cold | all rounds | process wall | peak RSS | best vs CPython |") + print("|---|---|---|---|---|---|---|---|") + failed = False + report = {"python": subprocess.run([sys.executable, "--version"], stdout=subprocess.PIPE, + universal_newlines=True).stdout.strip(), "kernels": {}} + for k in kernels: + work = os.path.join(HERE, "target", k) + if not a.no_build: + subprocess.run([os.path.join(HERE, "build_parparvm.sh"), k] + ([a.m2] if a.m2 else []), check=True, + stdout=subprocess.DEVNULL) + src = os.path.join(work, "src") + c = run([sys.executable, k + ".py"], src, a.rounds) + n = run([os.path.join(work, k)], src, a.rounds) + same = c[3] == n[3] + failed |= not same + report["kernels"][k] = { + "ratios": {"compute_us": round(n[0] / float(max(1, c[0])), 3), + "wall_ms": round(n[1] / max(0.001, c[1]), 3), + "peak_rss": round(n[2] / float(max(1, c[2])), 3)}, + "sides": {"cpython": {"compute_us": c[0], "wall_ms": int(c[1] * 1000), "peak_rss": c[2]}, + "cn1_parparvm": {"compute_us": n[0], "wall_ms": int(n[1] * 1000), "peak_rss": n[2], + "binary_bytes": os.path.getsize(os.path.join(work, k))}}} + rows = [("CPython", c), ("ParparVM", n)] + for name, path in impls: + rows.append((name, run([path, k + ".py"], src, a.rounds))) + for name, r in rows: + ok = r[3] == c[3] + failed |= not ok + print("| %s | %s%s | %.1f ms | %.1f ms | %.1f ms | %.2fs | %d MB | %.2fx |" % ( + k, name, "" if ok else " **WRONG RESULT**", r[0] / 1000.0, r[4] / 1000.0, r[5] / 1000.0, + r[1], r[2] >> 20, r[0] / float(c[0]))) + sys.stdout.flush() + if a.json: + with open(a.json, "w") as f: + json.dump(report, f, indent=2, sort_keys=True) + if a.check: + if not os.path.exists(BASELINE): + print("no baseline yet (%s): reporting only. Seed it with " + "parparvm_table.py --fold ." % os.path.relpath(BASELINE), file=sys.stderr) + else: + with open(BASELINE) as f: + for line in run_bench.check(report, json.load(f)): + print("REGRESSION " + line, file=sys.stderr) + failed = True + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/python-bench/run_bench.py b/scripts/python-bench/run_bench.py new file mode 100644 index 00000000000..ed7778bfcf4 --- /dev/null +++ b/scripts/python-bench/run_bench.py @@ -0,0 +1,242 @@ +#!/usr/bin/env python3 +"""Python vs Codename One: the same Python kernels run by CPython and run +transpiled to Java. + +Every kernel prints a RESULT line, which must be identical on both sides (a +benchmark that computes something else is not a benchmark), and a BENCH line +with its fastest round in microseconds, timed inside the program so that +start-up is measured separately. The harness times the whole process and reads +its peak resident memory from the operating system. + + run_bench.py --java-home J --runtime-jar R --core-jar C --transpiler-cp T + [--rounds 3] [--json out.json] [--check | --write-baseline] + run_bench.py --fold python-bench.json + +The gate compares the ratio ours/CPython of each metric with +baseline/ratios.json and fails when one grows by more than its tolerance. +""" +import argparse +import json +import os +import shutil +import subprocess +import sys +import tempfile +import time + +HERE = os.path.dirname(os.path.abspath(__file__)) +KERNELS_DIR = os.path.join(HERE, "kernels") +BASELINE = os.path.join(HERE, "baseline", "ratios.json") +GENERATED_PACKAGE = "com.codename1.generated.python" + +LAUNCHER = """ +public class BenchMain { + public static void main(String[] args) { + com.codename1.python.runtime.Py.setArgv("bench", args); + %s.PythonModules.runMain(); + } +} +""" % GENERATED_PACKAGE + + +def kernels(): + return sorted(f[:-3] for f in os.listdir(KERNELS_DIR) + if f.endswith(".py") and not f.startswith("_")) + + +def timed(cmd, cwd=None): + """Runs cmd and answers (stdout, wall seconds).""" + start = time.perf_counter() + proc = subprocess.Popen(cmd, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = proc.communicate() + wall = time.perf_counter() - start + if proc.returncode != 0: + raise RuntimeError("%s failed (%d):\n%s" % (" ".join(cmd), proc.returncode, err.decode("utf-8", "replace"))) + return out.decode("utf-8"), wall + + +def peak_rss(cmd, cwd=None): + """Peak resident set of one run of cmd, measured in a child of its own so + that RUSAGE_CHILDREN holds that run and nothing else.""" + probe = ("import resource,subprocess,sys;" + "subprocess.run(sys.argv[1:],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,check=True);" + "r=resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss;" + "print(r if sys.platform=='darwin' else r*1024)") + out = subprocess.run([sys.executable, "-c", probe] + cmd, cwd=cwd, check=True, + stdout=subprocess.PIPE).stdout + return int(out.strip()) + + +def parse(output): + result = bench = None + for line in output.splitlines(): + if line.startswith("RESULT "): + result = line.split(" ", 2)[2] + elif line.startswith("BENCH "): + bench = int(line.split()[2]) + if result is None or bench is None: + raise RuntimeError("kernel printed no RESULT/BENCH line:\n" + output) + return result, bench + + +def tree_bytes(path): + total = 0 + for root, _, files in os.walk(path): + for f in files: + total += os.path.getsize(os.path.join(root, f)) + return total + + +def build(kernel, work, args): + """Transpiles and compiles one kernel; answers its classes directory.""" + src = os.path.join(work, kernel, "src") + gen = os.path.join(work, kernel, "gen") + cls = os.path.join(work, kernel, "classes") + os.makedirs(src) + for f in (kernel + ".py", "_harness.py"): + shutil.copy(os.path.join(KERNELS_DIR, f), src) + java = os.path.join(args.java_home, "bin", "java") + javac = os.path.join(args.java_home, "bin", "javac") + subprocess.run([java, "-cp", args.transpiler_cp, "com.codename1.python.transpiler.api.Main", + src, gen, kernel], check=True) + with open(os.path.join(gen, "BenchMain.java"), "w") as f: + f.write(LAUNCHER) + sources = [] + for root, _, files in os.walk(gen): + sources += [os.path.join(root, f) for f in files if f.endswith(".java")] + subprocess.run([javac, "-nowarn", "--release", "17", "-cp", + args.runtime_jar + os.pathsep + args.core_jar, "-d", cls] + sources, check=True) + return src, cls + + +def measure(kernel, work, args): + src, cls = build(kernel, work, args) + java = os.path.join(args.java_home, "bin", "java") + ours_cmd = [java, "-cp", os.pathsep.join([cls, args.runtime_jar, args.core_jar]), "BenchMain"] + theirs_cmd = [args.python, os.path.join(src, kernel + ".py")] + sides = {} + for name, cmd in (("cpython", theirs_cmd), ("cn1_jvm", ours_cmd)): + compute, wall, result = [], [], None + for _ in range(args.rounds): + out, seconds = timed(cmd, cwd=src) + r, us = parse(out) + if result is not None and r != result: + raise RuntimeError("%s: %s is not deterministic" % (kernel, name)) + result = r + compute.append(us) + wall.append(seconds) + sides[name] = { + "result": result, + "compute_us": min(compute), + "wall_ms": int(min(wall) * 1000), + "peak_rss": peak_rss(cmd, cwd=src), + } + if sides["cpython"]["result"] != sides["cn1_jvm"]["result"]: + raise RuntimeError("%s computes something else:\n cpython %s\n cn1 %s" + % (kernel, sides["cpython"]["result"], sides["cn1_jvm"]["result"])) + sides["cn1_jvm"]["class_bytes"] = tree_bytes(cls) + ratios = {} + for key in ("compute_us", "wall_ms", "peak_rss"): + ratios[key] = round(sides["cn1_jvm"][key] / float(max(1, sides["cpython"][key])), 3) + return {"sides": sides, "ratios": ratios} + + +def check(report, baseline): + failures = [] + tolerance = baseline.get("tolerance", {}) + for kernel, entry in report["kernels"].items(): + base = baseline.get("ratios", {}).get(kernel) + if base is None: + failures.append("%s has no baseline; run with --write-baseline" % kernel) + continue + for key, ratio in entry["ratios"].items(): + limit = base[key] * (1 + tolerance.get(key, 0.25)) + if ratio > limit: + failures.append("%s %s: ratio %.3f exceeds baseline %.3f (limit %.3f)" + % (kernel, key, ratio, base[key], limit)) + return failures + + +def write_baseline(report, path=BASELINE): + baseline = {"tolerance": {"compute_us": 0.25, "wall_ms": 0.25, "peak_rss": 0.15}, + "ratios": {k: e["ratios"] for k, e in report["kernels"].items()}, + "python": report.get("python")} + if os.path.exists(path): + with open(path) as f: + baseline["tolerance"] = json.load(f).get("tolerance", baseline["tolerance"]) + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w") as f: + json.dump(baseline, f, indent=2, sort_keys=True) + f.write("\n") + + +def render(report): + lines = ["| kernel | compute ours/CPython | wall ours/CPython | RSS ours/CPython | class bytes |", + "|---|---|---|---|---|"] + for kernel, e in sorted(report["kernels"].items()): + r = e["ratios"] + lines.append("| %s | %.2fx | %.2fx | %.2fx | %d |" % ( + kernel, r["compute_us"], r["wall_ms"], r["peak_rss"], e["sides"]["cn1_jvm"]["class_bytes"])) + lines.append("") + lines.append("Runtime jar: %d bytes" % report["runtime_jar_bytes"]) + return "\n".join(lines) + + +def main(): + if len(sys.argv) == 3 and sys.argv[1] == "--fold": + # Seeds or moves the baseline from a report CI produced, without + # measuring anything here. + with open(sys.argv[2]) as f: + write_baseline(json.load(f)) + return 0 + p = argparse.ArgumentParser() + p.add_argument("--java-home", required=True) + p.add_argument("--runtime-jar", required=True) + p.add_argument("--core-jar", required=True) + p.add_argument("--transpiler-cp", required=True) + p.add_argument("--python", default=sys.executable) + p.add_argument("--rounds", type=int, default=3) + p.add_argument("--kernel", action="append") + p.add_argument("--json") + g = p.add_mutually_exclusive_group() + g.add_argument("--check", action="store_true") + g.add_argument("--write-baseline", action="store_true") + args = p.parse_args() + # Kernels run from their own directory, so every path given here must + # survive the change of working directory. + args.runtime_jar = os.path.abspath(args.runtime_jar) + args.core_jar = os.path.abspath(args.core_jar) + args.transpiler_cp = os.pathsep.join(os.path.abspath(x) for x in args.transpiler_cp.split(os.pathsep)) + + work = tempfile.mkdtemp(prefix="python-bench-") + report = {"python": subprocess.run([args.python, "--version"], stdout=subprocess.PIPE, + universal_newlines=True).stdout.strip(), + "runtime_jar_bytes": os.path.getsize(args.runtime_jar), + "kernels": {}} + for k in args.kernel or kernels(): + print("measuring", k, file=sys.stderr) + report["kernels"][k] = measure(k, work, args) + print(render(report)) + if args.json: + with open(args.json, "w") as f: + json.dump(report, f, indent=2, sort_keys=True) + if args.write_baseline: + write_baseline(report) + if args.check and not os.path.exists(BASELINE): + # Ratios depend on the machine and the interpreter, so the baseline + # is seeded from the CI runner's own report, never from a laptop. + print("no baseline yet (%s): reporting only. Seed it with " + "run_bench.py --fold ." + % os.path.relpath(BASELINE), file=sys.stderr) + return 0 + if args.check: + with open(BASELINE) as f: + failures = check(report, json.load(f)) + for line in failures: + print("REGRESSION " + line, file=sys.stderr) + return 1 if failures else 0 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/vm/ByteCodeTranslator/src/cn1_globals.h b/vm/ByteCodeTranslator/src/cn1_globals.h index 3fa65338394..a4e68699604 100644 --- a/vm/ByteCodeTranslator/src/cn1_globals.h +++ b/vm/ByteCodeTranslator/src/cn1_globals.h @@ -3643,7 +3643,11 @@ static inline JAVA_BOOLEAN cn1_array_access_in_bounds(JAVA_OBJECT array, JAVA_IN return array != JAVA_NULL && index >= 0 && index < ((JAVA_ARRAY)array)->length; } -static inline JAVA_BOOLEAN cn1_array_access_validate(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT array, JAVA_INT index) { +/* Out of line: inlined, the two exception constructions made every array + * element helper below too big for clang to inline, and each element access + * was a call (a Python attribute read, one Object[] load, profiled as a third + * of its own cost). `inline` stays so a file that never calls it does not warn. */ +static inline CN1_CLINIT_ATTR JAVA_BOOLEAN cn1_array_access_validate(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT array, JAVA_INT index) { if (array == JAVA_NULL) { throwException(threadStateData, __NEW_java_lang_NullPointerException(threadStateData)); return JAVA_FALSE; diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/BytecodeMethod.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/BytecodeMethod.java index d475d443c7c..7a2f902ee0c 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/BytecodeMethod.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/BytecodeMethod.java @@ -6364,7 +6364,12 @@ boolean optimize() { } } if(frameless) { - // No frame to release -- the operand stack is a method-local array. + // No frame to release -- the operand stack is a method-local array. The + // try-block stack still has to be unwound: a return after several try + // blocks that end on one label runs a single END_TRY, which pops one. + if(hasTryCatch) { + sb.append(" CN1_FRAMELESS_TRY_RETURN();\n"); + } sb.append(" return ").append(retVal).append(";\n"); } else if(hasTryCatch) { sb.append(" releaseForReturnInException(threadStateData, cn1LocalsBeginInThread, methodBlockOffset); return ").append(retVal).append(";\n"); @@ -6429,7 +6434,12 @@ boolean optimize() { } } if(frameless) { - // No frame to release -- the operand stack is a method-local array. + // No frame to release -- the operand stack is a method-local array. The + // try-block stack still has to be unwound: a return after several try + // blocks that end on one label runs a single END_TRY, which pops one. + if(hasTryCatch) { + sb.append(" CN1_FRAMELESS_TRY_RETURN();\n"); + } sb.append(" return ").append(retVal).append(";\n"); } else if(hasTryCatch) { sb.append(" releaseForReturnInException(threadStateData, cn1LocalsBeginInThread, methodBlockOffset); return ").append(retVal).append(";\n"); diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/JavascriptNativeRegistry.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/JavascriptNativeRegistry.java index 9bf8159fcdf..4a3fffeea82 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/JavascriptNativeRegistry.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/JavascriptNativeRegistry.java @@ -157,6 +157,7 @@ enum NativeCategory { "cn1_java_lang_String_charsToBytes_char_1ARRAY_char_1ARRAY_R_byte_1ARRAY", "cn1_java_lang_String_compactBytes_java_lang_String_R_byte_1ARRAY", "cn1_java_lang_String_cn1SubstringFused_int_int_R_java_lang_String", + "cn1_java_lang_String_cn1FusedJoin_java_lang_String_java_lang_Object_1ARRAY_int_int_R_java_lang_String", "cn1_java_lang_String_cn1FusedConcat2_java_lang_String_java_lang_String_R_java_lang_String", "cn1_java_lang_String_cn1FusedConcat3_java_lang_String_java_lang_String_java_lang_String_R_java_lang_String", "cn1_java_lang_String_cn1FusedConcat4_java_lang_String_java_lang_String_java_lang_String_java_lang_String_R_java_lang_String", diff --git a/vm/ByteCodeTranslator/src/javascript/parparvm_runtime.js b/vm/ByteCodeTranslator/src/javascript/parparvm_runtime.js index 833715d51e7..6b9425ec38f 100644 --- a/vm/ByteCodeTranslator/src/javascript/parparvm_runtime.js +++ b/vm/ByteCodeTranslator/src/javascript/parparvm_runtime.js @@ -6417,6 +6417,15 @@ bindNative(["cn1_java_lang_String_releaseNSString_long"], function() { return nu // The Java substring constructor is the portable fallback for this optional C fusion. bindNative(["cn1_java_lang_String_cn1SubstringFused_int_int_R_java_lang_String"], function() { return null; }); function cn1FusedConcatArg(x) { return x == null ? "null" : jvm.toNativeString(x); } +// Reached only for Latin-1 byte[]-backed Strings (see cn1FusedConcatN): unreachable here, bound for reachability. +bindNative(["cn1_java_lang_String_cn1FusedJoin_java_lang_String_java_lang_Object_1ARRAY_int_int_R_java_lang_String"], function(sep, parts, n) { + let out = ""; + for (let i = 0; i < (n | 0); i++) { + if (i > 0) { out += cn1FusedConcatArg(sep); } + out += cn1FusedConcatArg(parts[i]); + } + return createJavaString(out); +}); bindNative(["cn1_java_lang_String_cn1FusedConcat2_java_lang_String_java_lang_String_R_java_lang_String"], function(a, b) { return createJavaString(cn1FusedConcatArg(a) + cn1FusedConcatArg(b)); }); diff --git a/vm/ByteCodeTranslator/src/nativeMethods.m b/vm/ByteCodeTranslator/src/nativeMethods.m index d57c7d5d8f4..a39abf08cab 100644 --- a/vm/ByteCodeTranslator/src/nativeMethods.m +++ b/vm/ByteCodeTranslator/src/nativeMethods.m @@ -1904,6 +1904,47 @@ static JAVA_OBJECT cn1ConcatFallback(CODENAME_ONE_THREAD_STATE, JAVA_ARRAY_BYTE* return so; } +// String.join over Latin-1 Strings. cn1JoinLatin1 has checked every part and +// summed the length (total fits an int), so this only copies bytes. +JAVA_OBJECT java_lang_String_cn1FusedJoin___java_lang_String_java_lang_Object_1ARRAY_int_int_R_java_lang_String(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT sep, JAVA_OBJECT parts, JAVA_INT n, JAVA_INT total) { + JAVA_ARRAY_BYTE* dst; + JAVA_OBJECT so = cn1FusedLatin1Begin(threadStateData, total, &dst); + int fused = so != JAVA_NULL; + if(!fused) { + enteringNativeAllocations(); + JAVA_ARRAY dat = (JAVA_ARRAY)allocArray(threadStateData, total, &class_array1__JAVA_BYTE, sizeof(JAVA_ARRAY_BYTE), 1); + dst = (JAVA_ARRAY_BYTE*) CN1_ARRAY_DATA(dat); + so = __NEW_java_lang_String(threadStateData); + java_lang_String___INIT____(threadStateData, so); + ((struct obj__java_lang_String*)so)->java_lang_String_value = (JAVA_OBJECT)dat; + ((struct obj__java_lang_String*)so)->java_lang_String_count = total; + } + // Pointers are taken only now: the allocation above may run a collection. + JAVA_OBJECT* items = (JAVA_OBJECT*) CN1_ARRAY_DATA((JAVA_ARRAY)parts); + JAVA_ARRAY_BYTE* sp = CN1_SB_PTR(sep); + int sl = CN1_SB_LEN(sep); + int o = 0; + // Short parts (words, separators) are copied by a loop: a memcpy call + // costs more than the bytes it moves. + for(int q = 0 ; q < n ; q++) { + if(q > 0) { + if(sl > 16) { memcpy(dst + o, sp, sl); } else { for(int i = 0 ; i < sl ; i++) dst[o + i] = sp[i]; } + o += sl; + } + JAVA_OBJECT it = items[q]; + int l = CN1_SB_LEN(it); + JAVA_ARRAY_BYTE* ip = CN1_SB_PTR(it); + if(l > 16) { memcpy(dst + o, ip, l); } else { for(int i = 0 ; i < l ; i++) dst[o + i] = ip[i]; } + o += l; + } + if(fused) { + cn1FusedLatin1End(so, total); + } else { + finishedNativeAllocations(); + } + return so; +} + JAVA_OBJECT java_lang_String_cn1FusedConcat2___java_lang_String_java_lang_String_R_java_lang_String(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT a, JAVA_OBJECT b) { JAVA_ARRAY_BYTE* p[2] = { CN1_SB_PTR(a), CN1_SB_PTR(b) }; int l[2] = { CN1_SB_LEN(a), CN1_SB_LEN(b) }; diff --git a/vm/JavaAPI/src/java/lang/String.java b/vm/JavaAPI/src/java/lang/String.java index 79da14c2a8b..b9d99034b15 100644 --- a/vm/JavaAPI/src/java/lang/String.java +++ b/vm/JavaAPI/src/java/lang/String.java @@ -1451,6 +1451,10 @@ public int codePointCount(int beginIndex, int endIndex) { if (beginIndex < 0 || endIndex > count || beginIndex > endIndex) { throw new IndexOutOfBoundsException(); } + if (isLatin1()) { + // No Latin-1 character is a surrogate. + return endIndex - beginIndex; + } return Character.codePointCount(this, beginIndex, endIndex); } @@ -1498,12 +1502,41 @@ private void checkBoundsBeginEnd(int beginIndex, int endIndex) { } } + // A join of Latin-1 Strings, the common case, measured here and built by the + // native in ONE fused allocation with the bytes copied straight in, as + // cn1FusedConcatN does: no StringBuilder, no char[] and no width scan. Null + // when any part is not a Latin-1 String; the general path then runs. + private static String cn1JoinLatin1(CharSequence delimiter, CharSequence[] elements) { + if (!(delimiter instanceof String) || !((String) delimiter).isLatin1()) { + return null; + } + int n = elements.length; + long total = n > 1 ? (long) ((String) delimiter).count * (n - 1) : 0; + for (int i = 0; i < n; i++) { + CharSequence e = elements[i]; + if (!(e instanceof String) || !((String) e).isLatin1()) { + return null; + } + total += ((String) e).count; + } + if (total > Integer.MAX_VALUE) { + throw new OutOfMemoryError(); + } + return cn1FusedJoin((String) delimiter, elements, n, (int) total); + } + + private static native String cn1FusedJoin(String delimiter, Object[] parts, int n, int total); + /// The elements, each as `String.valueOf` writes it, with `delimiter` /// between every two. public static String join(CharSequence delimiter, CharSequence... elements) { if (delimiter == null || elements == null) { throw new NullPointerException(); } + String fused = cn1JoinLatin1(delimiter, elements); + if (fused != null) { + return fused; + } StringBuilder out = new StringBuilder(); for (int i = 0; i < elements.length; i++) { if (i > 0) { diff --git a/vm/JavaAPI/src/java/util/stream/DoublePipeline.java b/vm/JavaAPI/src/java/util/stream/DoublePipeline.java index bc712d6b66c..d2ab0dc04ae 100644 --- a/vm/JavaAPI/src/java/util/stream/DoublePipeline.java +++ b/vm/JavaAPI/src/java/util/stream/DoublePipeline.java @@ -265,7 +265,7 @@ public double[] toArray() { Iterator it = boxed.iterator(); try { while (it.hasNext()) { - all.add(it.next()); + addNext(all, it); } double[] out = new double[all.size()]; for (int i = 0; i < out.length; i++) { @@ -492,6 +492,12 @@ private static double unboxNext(Iterator it) { return it.next().doubleValue(); } + // Outside the caller's try: the element cast javac inserts must not sit + // under a catch, because a failed cast does not throw on ParparVM. + private static void addNext(List all, Iterator it) { + all.add(it.next()); + } + private static double unboxAt(List all, int i) { return all.get(i).doubleValue(); } diff --git a/vm/JavaAPI/src/java/util/stream/IntPipeline.java b/vm/JavaAPI/src/java/util/stream/IntPipeline.java index 80fcf74332f..1719f74ca60 100644 --- a/vm/JavaAPI/src/java/util/stream/IntPipeline.java +++ b/vm/JavaAPI/src/java/util/stream/IntPipeline.java @@ -308,7 +308,7 @@ public int[] toArray() { Iterator it = boxed.iterator(); try { while (it.hasNext()) { - all.add(it.next()); + addNext(all, it); } int[] out = new int[all.size()]; for (int i = 0; i < out.length; i++) { @@ -544,6 +544,12 @@ private static int unboxNext(Iterator it) { return it.next().intValue(); } + // Outside the caller's try: the element cast javac inserts must not sit + // under a catch, because a failed cast does not throw on ParparVM. + private static void addNext(List all, Iterator it) { + all.add(it.next()); + } + private static int unboxAt(List all, int i) { return all.get(i).intValue(); } diff --git a/vm/JavaAPI/src/java/util/stream/LongPipeline.java b/vm/JavaAPI/src/java/util/stream/LongPipeline.java index 9d97d6c64f7..dadcb76e33a 100644 --- a/vm/JavaAPI/src/java/util/stream/LongPipeline.java +++ b/vm/JavaAPI/src/java/util/stream/LongPipeline.java @@ -300,7 +300,7 @@ public long[] toArray() { Iterator it = boxed.iterator(); try { while (it.hasNext()) { - all.add(it.next()); + addNext(all, it); } long[] out = new long[all.size()]; for (int i = 0; i < out.length; i++) { @@ -536,6 +536,12 @@ private static long unboxNext(Iterator it) { return it.next().longValue(); } + // Outside the caller's try: the element cast javac inserts must not sit + // under a catch, because a failed cast does not throw on ParparVM. + private static void addNext(List all, Iterator it) { + all.add(it.next()); + } + private static long unboxAt(List all, int i) { return all.get(i).longValue(); } diff --git a/vm/tests/src/test/java/com/codename1/tools/translator/FramelessTryReturnIntegrationTest.java b/vm/tests/src/test/java/com/codename1/tools/translator/FramelessTryReturnIntegrationTest.java new file mode 100644 index 00000000000..e587a8f9112 --- /dev/null +++ b/vm/tests/src/test/java/com/codename1/tools/translator/FramelessTryReturnIntegrationTest.java @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.tools.translator; + +import org.junit.jupiter.params.ParameterizedTest; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/// A frameless method that returns a value after several try blocks ending on +/// one label. Each such return ran a single END_TRY, which pops one block, and +/// left the others on the thread's try-block stack; a later throw then +/// longjmp'd into a frame that had already returned. The shape is the one +/// javac gives `try { r = f(); return r; } catch (A) ... catch (C) ... finally`. +public class FramelessTryReturnIntegrationTest { + + @ParameterizedTest + @org.junit.jupiter.params.provider.MethodSource("com.codename1.tools.translator.BytecodeInstructionIntegrationTest#provideRuntimeCompilerConfigs") + public void returnAfterSharedTryEndUnwindsTheBlockStack(CompilerHelper.CompilerConfig config) throws Exception { + Parser.cleanup(); + Path sourceDir = Files.createTempDirectory("frameless-try-sources"); + Path classesDir = Files.createTempDirectory("frameless-try-classes"); + Path javaFile = sourceDir.resolve("FramelessTryApp.java"); + Files.write(javaFile, appSource().getBytes(StandardCharsets.UTF_8)); + Path javaApiDir = Files.createTempDirectory("java-api-classes"); + assertTrue(CompilerHelper.isJavaApiCompatible(config), + "JDK " + config.jdkVersion + " must target matching bytecode level for JavaAPI"); + CompilerHelper.compileJavaAPI(javaApiDir, config); + + List compileArgs = new ArrayList<>(); + compileArgs.add("-source"); + compileArgs.add(config.targetVersion); + compileArgs.add("-target"); + compileArgs.add(config.targetVersion); + if (CompilerHelper.useClasspath(config)) { + compileArgs.add("-classpath"); + } else { + compileArgs.add("-Xlint:-options"); + compileArgs.add("-bootclasspath"); + } + compileArgs.add(javaApiDir.toString()); + compileArgs.add("-d"); + compileArgs.add(classesDir.toString()); + compileArgs.add(javaFile.toString()); + assertEquals(0, CompilerHelper.compile(config.jdkHome, compileArgs), "compilation failed with " + config); + CompilerHelper.copyDirectory(javaApiDir, classesDir); + + Path outputDir = Files.createTempDirectory("frameless-try-output"); + CleanTargetIntegrationTest.runTranslator(classesDir, outputDir, "FramelessTryApp"); + Path distDir = outputDir.resolve("dist"); + Path cmakeLists = distDir.resolve("CMakeLists.txt"); + CleanTargetIntegrationTest.replaceLibraryWithExecutableTarget(cmakeLists, "FramelessTryApp-src"); + Path buildDir = distDir.resolve("build"); + Files.createDirectories(buildDir); + CleanTargetIntegrationTest.runCommand(Arrays.asList("cmake", "-S", distDir.toString(), "-B", buildDir.toString(), + "-DCMAKE_C_COMPILER=clang", "-DCMAKE_OBJC_COMPILER=clang"), distDir); + CleanTargetIntegrationTest.runCommand(Arrays.asList("cmake", "--build", buildDir.toString()), distDir); + String out = CleanTargetIntegrationTest.runCommand( + Arrays.asList(buildDir.resolve("FramelessTryApp").toString()), buildDir); + assertTrue(out.contains("caught late"), "the throw after the leaky returns was not caught:\n" + out); + assertTrue(out.contains("calls 6"), out); + } + + private static String appSource() { + return "public class FramelessTryApp {\n" + + " static boolean running;\n" + + " static int calls;\n" + + " static Object make(int i) {\n" + + " if (i < 0) { throw new IllegalStateException(\"late\"); }\n" + + " calls++;\n" + + " return \"v\";\n" + + " }\n" + + " static Object resume(int i) {\n" + + " running = true;\n" + + " try {\n" + + " Object r = make(i);\n" + + " return r;\n" + + " } catch (IllegalStateException e) {\n" + + " throw e;\n" + + " } catch (RuntimeException e) {\n" + + " throw e;\n" + + " } catch (Error e) {\n" + + " throw e;\n" + + " } finally {\n" + + " running = false;\n" + + " }\n" + + " }\n" + + " static void leak() {\n" + + " resume(1);\n" + + " resume(2);\n" + + " resume(3);\n" + + " }\n" + + " public static void main(String[] args) {\n" + + " leak();\n" + + " resume(4);\n" + + " try {\n" + + " for (int k = 0; k < 3; k++) { resume(k == 2 ? -1 : k); }\n" + + " } catch (IllegalStateException e) {\n" + + " System.out.println(\"caught \" + e.getMessage());\n" + + " }\n" + + " System.out.println(\"calls \" + calls);\n" + + " }\n" + + "}\n"; + } +} From f4843120e3878a78a5b9476a67740a1bd4bbd752 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Tue, 27 Oct 2026 17:32:06 +0200 Subject: [PATCH 002/112] Python: Java interop, project import and build integration from java. import resolves at build time against the compile class path; a generated JavaInterop bridge dispatches with instanceof and direct calls (no reflection), converts values both ways, adapts Python callables to functional interfaces and exposes Java collections to len/in/iteration/indexing. cn1:import-python-project and cn1:transpile-python (Maven), the transpilePython task (Gradle), and the archetype's python profile build src/main/python. runMain returns the exit status, honouring SystemExit. Co-Authored-By: Claude Opus 5.5 --- maven/build-engine/pom.xml | 5 + .../codename1/maven/PythonProjectBuilder.java | 213 ++++ .../maven/PythonProjectImporter.java | 346 +++++++ .../codename1/maven/PythonProjectTest.java | 199 ++++ .../archetype-resources/common/pom.xml | 26 + .../resources/archetype-resources/pom.xml | 5 + .../java/com/codename1/gradle/AppSupport.java | 24 + .../gradle/tasks/TranspilePythonTask.java | 80 ++ .../maven/ImportPythonProjectMojo.java | 117 +++ .../com/codename1/maven/PythonPomUpdater.java | 123 +++ .../codename1/maven/TranspilePythonMojo.java | 131 +++ .../codename1/maven/PythonPomUpdaterTest.java | 126 +++ .../codename1/python/runtime/Builtins.java | 12 + .../codename1/python/runtime/JavaBridge.java | 253 +++++ .../java/com/codename1/python/runtime/Py.java | 3 + .../codename1/python/runtime/PyJavaClass.java | 105 ++ .../python/runtime/PyJavaMethod.java | 64 ++ .../python/runtime/PyJavaObject.java | 108 +- .../codename1/python/runtime/StdModules.java | 21 + .../codename1/python/transpiler/api/Main.java | 17 +- .../transpiler/api/PythonTranspiler.java | 41 +- .../transpiler/codegen/ClassPathReader.java | 245 +++++ .../codegen/JavaInteropGenerator.java | 976 ++++++++++++++++++ .../python/transpiler/ConformanceTest.java | 9 +- .../python/transpiler/UnsupportedTest.java | 5 + .../test/resources/conformance/errors_.out | 5 + .../src/test/resources/conformance/errors_.py | 4 + .../resources/conformance/java_interop.out | 11 + .../resources/conformance/java_interop.py | 46 + scripts/python-bench/build_parparvm.sh | 2 +- scripts/python-bench/run_bench.py | 2 +- 31 files changed, 3304 insertions(+), 20 deletions(-) create mode 100644 maven/build-engine/src/main/java/com/codename1/maven/PythonProjectBuilder.java create mode 100644 maven/build-engine/src/main/java/com/codename1/maven/PythonProjectImporter.java create mode 100644 maven/build-engine/src/test/java/com/codename1/maven/PythonProjectTest.java create mode 100644 maven/codenameone-gradle-plugin/src/main/java/com/codename1/gradle/tasks/TranspilePythonTask.java create mode 100644 maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/ImportPythonProjectMojo.java create mode 100644 maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/PythonPomUpdater.java create mode 100644 maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/TranspilePythonMojo.java create mode 100644 maven/codenameone-maven-plugin/src/test/java/com/codename1/maven/PythonPomUpdaterTest.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/JavaBridge.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/PyJavaClass.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/PyJavaMethod.java create mode 100644 maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/ClassPathReader.java create mode 100644 maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/JavaInteropGenerator.java create mode 100644 maven/python-transpiler/src/test/resources/conformance/java_interop.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/java_interop.py diff --git a/maven/build-engine/pom.xml b/maven/build-engine/pom.xml index 77025a84f47..b284c7d69a0 100644 --- a/maven/build-engine/pom.xml +++ b/maven/build-engine/pom.xml @@ -72,6 +72,11 @@ junit-jupiter-engine test + + ${project.groupId} + codenameone-python-transpiler + ${project.version} + ${project.groupId} codenameone-project-model diff --git a/maven/build-engine/src/main/java/com/codename1/maven/PythonProjectBuilder.java b/maven/build-engine/src/main/java/com/codename1/maven/PythonProjectBuilder.java new file mode 100644 index 00000000000..6fab8177886 --- /dev/null +++ b/maven/build-engine/src/main/java/com/codename1/maven/PythonProjectBuilder.java @@ -0,0 +1,213 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.maven; + +import com.codename1.build.Log; +import com.codename1.builders.BuildException; +import com.codename1.python.transpiler.api.PythonSyntaxError; +import com.codename1.python.transpiler.api.PythonTranspiler; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; + +/// Transpiles `src/main/python` to Java source, for the Maven plugin's +/// `transpile-python` goal and the Gradle plugin's `transpilePython` task. +/// +/// Every `.py` file under the source directory is a module named by its path +/// (`pkg/mod.py` is `pkg.mod`). The output directory is owned by this builder: +/// a file is rewritten only when its text changed, so javac recompiles only +/// what did, and a file no module produces any more is deleted. +/// +/// The entry module, run by `PythonModules.runMain()`, is the one given, or +/// `main` when `main.py` exists. An entry of the form `module:function`, as +/// `[project.scripts]` names one, calls that function and exits with what it +/// returns; without either the sources are a library, +/// reached from Java through `PythonModules.registerAll()` and the generated +/// module classes. +public final class PythonProjectBuilder { + /// The runtime every generated class calls. + public static final String RUNTIME_ARTIFACT = "codenameone-python-runtime"; + + /// The module generated for a `module:function` entry point. + static final String ENTRY_MODULE = "__cn1_entry__"; + + private final File sourceDir; + private final File outputDir; + private final String mainModule; + private final Log log; + private java.util.List classPath = java.util.Collections.emptyList(); + + public PythonProjectBuilder(File sourceDir, File outputDir, String mainModule, Log log) { + this.sourceDir = sourceDir; + this.outputDir = outputDir; + this.mainModule = mainModule; + this.log = log; + } + + /// The jars and class directories `from java. import ` + /// resolves against: the project's compile class path. The JDK's own + /// classes are always found. The project's own Java sources are not + /// compiled yet when Python is transpiled, so they cannot be imported. + public PythonProjectBuilder classPath(java.util.List entries) { + this.classPath = entries; + return this; + } + + /// Whether `dir` holds at least one `.py` file. + public static boolean isPythonProject(File dir) { + if (dir == null) { + return false; + } + File[] children = dir.listFiles(); + if (children == null) { + return false; + } + for (File c : children) { + if (c.isDirectory() ? isPythonProject(c) : c.getName().endsWith(".py")) { + return true; + } + } + return false; + } + + /// Transpiles the sources. Returns false when there are none, after + /// emptying the output directory of what an earlier build left there. + public boolean run() throws BuildException { + if (!isPythonProject(sourceDir)) { + sync(new TreeMap()); + return false; + } + Map sources = new TreeMap(); + try { + collect(sourceDir, "", sources); + } catch (IOException e) { + throw new BuildException("Could not read " + sourceDir + ": " + e.getMessage(), e); + } + String entry = mainModule; + if (entry == null && sources.containsKey("main.py")) { + entry = "main"; + } + int colon = entry == null ? -1 : entry.indexOf(':'); + if (colon > 0) { + // A console-script target, `pkg.cli:main`: the function is what + // runs, as the script pip writes for it calls it. + String module = entry.substring(0, colon).trim(); + String function = entry.substring(colon + 1).trim(); + sources.put(ENTRY_MODULE + ".py", "import sys\nfrom " + module + " import " + function + + " as _entry\nsys.exit(_entry())\n"); + entry = ENTRY_MODULE; + } + Map out; + try { + out = PythonTranspiler.transpile(sources, entry, classPath); + } catch (PythonSyntaxError e) { + throw new BuildException("src/main/python/" + e.file + ":" + e.line + ":" + e.col + ": " + e.reason, e); + } catch (IllegalArgumentException e) { + throw new BuildException(e.getMessage(), e); + } + int written = sync(out); + if (written == 0) { + log.info("Python sources are up to date"); + } else { + log.info("Transpiled " + sources.size() + " Python module(s) to " + outputDir + " (" + written + + " file(s) changed)"); + } + return true; + } + + private static void collect(File dir, String prefix, Map into) throws IOException { + File[] children = dir.listFiles(); + if (children == null) { + return; + } + Arrays.sort(children); + for (File c : children) { + if (c.isDirectory()) { + if (!c.getName().startsWith(".") && !"__pycache__".equals(c.getName())) { + collect(c, prefix + c.getName() + "/", into); + } + } else if (c.getName().endsWith(".py")) { + into.put(prefix + c.getName(), new String(Files.readAllBytes(c.toPath()), StandardCharsets.UTF_8)); + } + } + } + + /// Makes the output directory hold exactly `files`, writing only those + /// whose text changed. Returns how many were written or removed. + private int sync(Map files) throws BuildException { + int changed = 0; + try { + for (Map.Entry e : files.entrySet()) { + File f = new File(outputDir, e.getKey()); + byte[] text = e.getValue().getBytes(StandardCharsets.US_ASCII); + if (f.isFile() && Arrays.equals(Files.readAllBytes(f.toPath()), text)) { + continue; + } + File parent = f.getParentFile(); + if (parent != null) { + Files.createDirectories(parent.toPath()); + } + Files.write(f.toPath(), text); + changed++; + } + Set keep = new HashSet(); + for (String k : files.keySet()) { + keep.add(new File(outputDir, k).getAbsolutePath()); + } + List stale = new ArrayList(); + listFiles(outputDir, stale); + for (File f : stale) { + if (!keep.contains(f.getAbsolutePath())) { + Files.delete(f.toPath()); + changed++; + } + } + } catch (IOException e) { + throw new BuildException("Could not write " + outputDir + ": " + e.getMessage(), e); + } + return changed; + } + + private static void listFiles(File dir, List into) { + File[] children = dir.listFiles(); + if (children == null) { + return; + } + for (File c : children) { + if (c.isDirectory()) { + listFiles(c, into); + } else { + into.add(c); + } + } + } +} diff --git a/maven/build-engine/src/main/java/com/codename1/maven/PythonProjectImporter.java b/maven/build-engine/src/main/java/com/codename1/maven/PythonProjectImporter.java new file mode 100644 index 00000000000..94edd3b59be --- /dev/null +++ b/maven/build-engine/src/main/java/com/codename1/maven/PythonProjectImporter.java @@ -0,0 +1,346 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.maven; + +import com.codename1.build.Log; +import com.codename1.builders.BuildException; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.TreeSet; + +/// Copies a Python project's sources into an application's +/// `common/src/main/python`, for `cn1:import-python-project` and the Gradle +/// plugin's `importPythonProject`. +/// +/// The source root is the project's `src` directory when it has the +/// `src/` layout, else the project directory itself. Only `.py` +/// files are copied, never a virtual environment, a cache or a build +/// directory. The entry module is found the way `python -m` and the +/// project's metadata name it, and a Codename One main class that runs it is +/// written. Importing again mirrors the project afresh: a file the project no +/// longer has is removed, through a record of what the last import wrote. +public final class PythonProjectImporter { + /// Directories that are never part of a program's source. + static final List SKIPPED = Collections.unmodifiableList(Arrays.asList( + ".git", ".hg", ".svn", ".venv", "venv", "env", ".env", "__pycache__", ".tox", ".nox", ".mypy_cache", + ".pytest_cache", ".ruff_cache", "build", "dist", "node_modules", ".idea", ".vscode", "site-packages")); + + /// The settings key naming the entry module, read by the build. + public static final String MAIN_MODULE_SETTING = "codename1.python.mainModule"; + + static final String RECORD = ".cn1-python-import"; + + private final Log log; + + public static final class Result { + public int copiedFiles; + public int removedFiles; + /// The module run as `__main__` (or `module:function`), or null for a library. + public String mainModule; + /// Requirements the project declares, which have to be provided as + /// source under `src/main/python` or by the runtime. + public final List requirements = new ArrayList(); + } + + public PythonProjectImporter(Log log) { + this.log = log; + } + + /// @param source the Python project directory + /// @param commonDir the application's common module + /// @param mainPackage `codename1.packageName`, or null to leave the main class alone + /// @param mainClass `codename1.mainName`, or null to leave the main class alone + public Result importProject(File source, File commonDir, String mainPackage, String mainClass) + throws BuildException { + if (source == null || !source.isDirectory()) { + throw new BuildException(source + " is not a directory; name the Python project to import"); + } + File root = sourceRoot(source); + if (!PythonProjectBuilder.isPythonProject(root)) { + throw new BuildException(source + " has no Python sources"); + } + File target = new File(commonDir, "src/main/python"); + Result r = new Result(); + try { + if (target.getCanonicalFile().equals(root.getCanonicalFile())) { + throw new BuildException(source + " is already this application's src/main/python"); + } + Set previous = readRecord(target); + Set imported = new TreeSet(); + copy(root, target, "", imported, r); + for (String path : previous) { + if (!imported.contains(path)) { + File stale = new File(target, path); + if (stale.isFile() && !Files.isSymbolicLink(stale.toPath()) && stale.delete()) { + r.removedFiles++; + } + } + } + writeRecord(target, imported); + String pyproject = read(new File(source, "pyproject.toml")); + r.mainModule = mainModule(root, pyproject); + r.requirements.addAll(requirements(pyproject, read(new File(source, "requirements.txt")))); + if (r.mainModule != null) { + updateSetting(new File(commonDir, "codenameone_settings.properties"), r.mainModule); + if (mainPackage != null && mainClass != null && mainPackage.length() > 0 && mainClass.length() > 0) { + writeEntryPoint(new File(commonDir, "src/main/java"), mainPackage, mainClass); + } + } + } catch (IOException e) { + throw new BuildException("Cannot import " + source + ": " + e.getMessage(), e); + } + return r; + } + + /// `src` in the `src/` layout, else the project itself. + static File sourceRoot(File project) { + File src = new File(project, "src"); + if (src.isDirectory() && PythonProjectBuilder.isPythonProject(src)) { + return src; + } + return project; + } + + /// The entry point: `[project.scripts]`'s first target (`module:function`), else a + /// package's `__main__`, else `main`, `app` or `__main__` at the root. + static String mainModule(File root, String pyproject) { + if (pyproject != null) { + String scripts = section(pyproject, "project.scripts"); + if (scripts != null) { + for (String line : scripts.split("\n")) { + int eq = line.indexOf('='); + if (eq > 0) { + String v = line.substring(eq + 1).trim().replace("\"", "").replace("'", ""); + int colon = v.indexOf(':'); + String mod = colon >= 0 ? v.substring(0, colon).trim() : v; + if (new File(root, mod.replace('.', '/') + ".py").isFile() + || new File(root, mod.replace('.', '/') + "/__init__.py").isFile()) { + return v.trim(); + } + } + } + } + } + File[] children = root.listFiles(); + if (children != null) { + Arrays.sort(children); + for (File c : children) { + if (c.isDirectory() && new File(c, "__main__.py").isFile() && new File(c, "__init__.py").isFile()) { + return c.getName() + ".__main__"; + } + } + } + for (String m : new String[] {"main", "app", "__main__"}) { + if (new File(root, m + ".py").isFile()) { + return m; + } + } + return null; + } + + /// The body of a TOML table, up to the next header, or null. + static String section(String toml, String name) { + String header = "[" + name + "]"; + int at = toml.indexOf(header); + if (at < 0) { + return null; + } + int start = at + header.length(); + int end = toml.indexOf("\n[", start); + return end < 0 ? toml.substring(start) : toml.substring(start, end); + } + + /// The distribution names a project depends on: `[project] dependencies` + /// and `requirements.txt`, without versions, markers or comments. + static List requirements(String pyproject, String requirementsTxt) { + Set names = new TreeSet(); + if (pyproject != null) { + String project = section(pyproject, "project"); + if (project != null) { + int at = project.indexOf("dependencies"); + int open = at < 0 ? -1 : project.indexOf('[', at); + int close = open < 0 ? -1 : project.indexOf(']', open); + if (close > open) { + for (String item : project.substring(open + 1, close).split(",")) { + add(names, item.trim().replace("\"", "").replace("'", "")); + } + } + } + } + if (requirementsTxt != null) { + for (String line : requirementsTxt.split("\n")) { + String l = line.trim(); + if (!l.startsWith("-")) { + add(names, l); + } + } + } + return new ArrayList(names); + } + + private static void add(Set names, String spec) { + int hash = spec.indexOf('#'); + String s = hash >= 0 ? spec.substring(0, hash) : spec; + int end = 0; + while (end < s.length()) { + char c = s.charAt(end); + if (!(Character.isLetterOrDigit(c) || c == '-' || c == '_' || c == '.')) { + break; + } + end++; + } + if (end > 0) { + names.add(s.substring(0, end)); + } + } + + private void copy(File src, File dest, String path, Set imported, Result r) throws IOException { + if (path.length() > 0 && Files.isSymbolicLink(src.toPath())) { + log.warn("Skipped " + path + ": a symbolic link"); + return; + } + if (src.isDirectory()) { + File[] children = src.listFiles(); + if (children == null) { + return; + } + Arrays.sort(children); + for (File f : children) { + String n = f.getName(); + if (SKIPPED.contains(n) || n.endsWith(".egg-info") || n.endsWith(".dist-info")) { + continue; + } + if (f.isDirectory() || n.endsWith(".py")) { + copy(f, new File(dest, n), path.length() == 0 ? n : path + "/" + n, imported, r); + } + } + return; + } + if (Files.isSymbolicLink(dest.toPath()) || dest.isDirectory()) { + throw new IOException(dest + " is in the way of " + path); + } + File parent = dest.getParentFile(); + if (parent != null) { + Files.createDirectories(parent.toPath()); + } + Files.copy(src.toPath(), dest.toPath(), StandardCopyOption.REPLACE_EXISTING); + imported.add(path); + r.copiedFiles++; + } + + private static String read(File f) throws IOException { + return f.isFile() ? new String(Files.readAllBytes(f.toPath()), StandardCharsets.UTF_8) : null; + } + + private static Set readRecord(File target) throws IOException { + Set paths = new TreeSet(); + String text = read(new File(target, RECORD)); + if (text != null) { + for (String line : text.split("\n")) { + String p = line.trim(); + // Never a path that climbs out of the import root. + if (p.length() > 0 && !p.startsWith("/") && !("/" + p + "/").contains("/../")) { + paths.add(p); + } + } + } + return paths; + } + + private static void writeRecord(File target, Set imported) throws IOException { + StringBuilder sb = new StringBuilder(); + for (String p : imported) { + sb.append(p).append('\n'); + } + Files.createDirectories(target.toPath()); + Files.write(new File(target, RECORD).toPath(), sb.toString().getBytes(StandardCharsets.UTF_8)); + } + + /// Sets [#MAIN_MODULE_SETTING], keeping every other line of the file as it is. + static void updateSetting(File settings, String module) throws IOException { + String line = MAIN_MODULE_SETTING + "=" + module; + List lines = new ArrayList(); + boolean found = false; + String text = read(settings); + if (text != null) { + for (String l : text.split("\n", -1)) { + if (l.startsWith(MAIN_MODULE_SETTING + "=")) { + lines.add(line); + found = true; + } else { + lines.add(l); + } + } + if (!lines.isEmpty() && lines.get(lines.size() - 1).isEmpty()) { + lines.remove(lines.size() - 1); + } + } + if (!found) { + lines.add(line); + } + StringBuilder sb = new StringBuilder(); + for (String l : lines) { + sb.append(l).append('\n'); + } + Files.write(settings.toPath(), sb.toString().getBytes(StandardCharsets.ISO_8859_1)); + } + + private void writeEntryPoint(File javaDir, String pkg, String cls) throws IOException { + File f = new File(javaDir, pkg.replace('.', File.separatorChar) + File.separator + cls + ".java"); + Files.createDirectories(f.getParentFile().toPath()); + String src = "package " + pkg + ";\n\n" + + "import com.codename1.system.Lifecycle;\n\n" + + "/// The Codename One entry point: runs the imported Python program's main module.\n" + + "public class " + cls + " extends Lifecycle {\n" + + " @Override\n" + + " public void runApp() {\n" + + " com.codename1.generated.python.PythonModules.runMain();\n" + + " }\n" + + "}\n"; + byte[] bytes = src.getBytes(StandardCharsets.UTF_8); + if (f.isFile()) { + File backup = new File(f.getPath() + ".pre-python-import"); + boolean unchanged = Arrays.equals(bytes, Files.readAllBytes(f.toPath())); + if (backup.exists()) { + if (!unchanged) { + log.warn(f.getName() + " was changed after an earlier import and was left as it is; " + + "delete it and import again to regenerate it"); + return; + } + } else if (!unchanged) { + Files.copy(f.toPath(), backup.toPath()); + log.info("Saved the previous " + f.getName() + " as " + backup.getName()); + } + } + Files.write(f.toPath(), bytes); + } +} diff --git a/maven/build-engine/src/test/java/com/codename1/maven/PythonProjectTest.java b/maven/build-engine/src/test/java/com/codename1/maven/PythonProjectTest.java new file mode 100644 index 00000000000..67b0ee111ab --- /dev/null +++ b/maven/build-engine/src/test/java/com/codename1/maven/PythonProjectTest.java @@ -0,0 +1,199 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.maven; + +import com.codename1.build.SystemStreamLog; +import com.codename1.builders.BuildException; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.Arrays; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/// [PythonProjectImporter] and [PythonProjectBuilder]: what an import copies +/// and records, and what a build writes, keeps and removes. +public class PythonProjectTest { + + @Rule + public TemporaryFolder tmp = new TemporaryFolder(); + + private File project; + private File common; + private File python; + + @Before + public void project() throws Exception { + project = tmp.newFolder("textstats"); + write(new File(project, "pyproject.toml"), "[project]\nname = \"textstats\"\n" + + "dependencies = [\"attrs>=23\", 'python-dateutil ; python_version>\"3\"']\n\n" + + "[project.scripts]\ntextstats = \"textstats.cli:main\"\n"); + write(new File(project, "requirements.txt"), "# pinned\nPyYAML==6.0.1\n-r other.txt\n"); + write(new File(project, "src/textstats/__init__.py"), ""); + write(new File(project, "src/textstats/cli.py"), "def main():\n print('ran')\n return 3\n"); + write(new File(project, "src/textstats/data.txt"), "not code"); + write(new File(project, "src/textstats/__pycache__/cli.cpython-313.pyc"), "x"); + write(new File(project, ".venv/lib/site.py"), "x = 1\n"); + write(new File(project, "tests/test_cli.py"), "x = 1\n"); + common = tmp.newFolder("app", "common"); + python = new File(common, "src/main/python"); + write(new File(common, "codenameone_settings.properties"), "codename1.packageName=com.acme\n"); + } + + @Test + public void importsTheSourceLayoutAndTheConsoleScript() throws Exception { + PythonProjectImporter.Result r = importer().importProject(project, common, "com.acme", "Stats"); + assertEquals("only the .py files of src/", 2, r.copiedFiles); + assertTrue(new File(python, "textstats/cli.py").isFile()); + assertFalse(new File(python, "textstats/data.txt").exists()); + assertFalse(new File(python, "textstats/__pycache__").exists()); + assertFalse(new File(python, "tests").exists()); + assertEquals("textstats.cli:main", r.mainModule); + assertEquals(Arrays.asList("PyYAML", "attrs", "python-dateutil"), r.requirements); + String settings = read(new File(common, "codenameone_settings.properties")); + assertTrue(settings, settings.startsWith("codename1.packageName=com.acme\n")); + assertTrue(settings, settings.contains("codename1.python.mainModule=textstats.cli:main\n")); + String main = read(new File(common, "src/main/java/com/acme/Stats.java")); + assertTrue(main, main.contains("extends Lifecycle")); + assertTrue(main, main.contains("PythonModules.runMain()")); + } + + @Test + public void reimportRemovesWhatTheProjectNoLongerHas() throws Exception { + write(new File(project, "src/textstats/old.py"), "x = 1\n"); + importer().importProject(project, common, null, null); + assertTrue(new File(python, "textstats/old.py").isFile()); + write(new File(python, "mine.py"), "y = 2\n"); + assertTrue(new File(project, "src/textstats/old.py").delete()); + PythonProjectImporter.Result r = importer().importProject(project, common, null, null); + assertEquals(1, r.removedFiles); + assertFalse(new File(python, "textstats/old.py").exists()); + assertTrue("a file the import did not write is kept", new File(python, "mine.py").isFile()); + assertFalse("no main class without a package and class name", + new File(common, "src/main/java").exists()); + } + + @Test + public void flatProjectsFindTheirEntryModule() throws Exception { + File flat = tmp.newFolder("flat"); + write(new File(flat, "app.py"), "print(1)\n"); + assertEquals("app", PythonProjectImporter.mainModule(flat, null)); + write(new File(flat, "tool/__init__.py"), ""); + write(new File(flat, "tool/__main__.py"), "print(2)\n"); + assertEquals("tool.__main__", PythonProjectImporter.mainModule(flat, null)); + File lib = tmp.newFolder("lib"); + write(new File(lib, "util.py"), "x = 1\n"); + assertNull(PythonProjectImporter.mainModule(lib, null)); + } + + @Test + public void notAPythonProject() throws Exception { + try { + importer().importProject(tmp.newFolder("empty"), common, null, null); + fail(); + } catch (BuildException expected) { + assertTrue(expected.getMessage(), expected.getMessage().contains("no Python sources")); + } + } + + @Test + public void buildWritesOnlyWhatChangedAndRemovesWhatIsGone() throws Exception { + File src = tmp.newFolder("py"); + File out = tmp.newFolder("gen"); + write(new File(src, "main.py"), "import helper\nprint(helper.twice(2))\n"); + write(new File(src, "helper.py"), "def twice(x):\n return 2 * x\n"); + assertTrue(builder(src, out, null).run()); + File registry = new File(out, "com/codename1/generated/python/PythonModules.java"); + assertTrue(read(registry), read(registry).contains(".exec(\"__main__\")")); + File helper = javaFor(out, "helper"); + assertTrue(helper.isFile()); + long stamp = helper.lastModified() - 10000; + assertTrue(helper.setLastModified(stamp)); + write(new File(src, "main.py"), "import helper\nprint(helper.twice(3))\n"); + builder(src, out, null).run(); + assertEquals("an unchanged module is not rewritten", stamp, helper.lastModified()); + assertTrue(new File(src, "helper.py").delete()); + write(new File(src, "main.py"), "print(1)\n"); + builder(src, out, null).run(); + assertFalse(helper.exists()); + assertTrue(new File(src, "main.py").delete()); + assertFalse(builder(src, out, null).run()); + assertFalse("the output of a removed project is removed", registry.exists()); + } + + @Test + public void functionEntryPointIsCalled() throws Exception { + File src = tmp.newFolder("py"); + File out = tmp.newFolder("gen"); + write(new File(src, "pkg/__init__.py"), ""); + write(new File(src, "pkg/cli.py"), "def main():\n return 0\n"); + builder(src, out, "pkg.cli:main").run(); + String entry = read(javaFor(out, PythonProjectBuilder.ENTRY_MODULE)); + assertTrue(entry, entry.contains("main")); + assertTrue(read(new File(out, "com/codename1/generated/python/PythonModules.java")).contains("__main__")); + } + + @Test + public void unsupportedCodeNamesTheFileAndLine() throws Exception { + File src = tmp.newFolder("py"); + write(new File(src, "main.py"), "x = 1\ny = eval('x')\n"); + try { + builder(src, tmp.newFolder("gen"), null).run(); + fail(); + } catch (BuildException expected) { + assertTrue(expected.getMessage(), expected.getMessage().startsWith("src/main/python/main.py:2:")); + } + } + + private static File javaFor(File out, String module) { + return new File(out, "com/codename1/generated/python/" + + com.codename1.python.transpiler.api.PythonTranspiler.className(module) + ".java"); + } + + private static PythonProjectImporter importer() { + return new PythonProjectImporter(new SystemStreamLog()); + } + + private static PythonProjectBuilder builder(File src, File out, String main) { + return new PythonProjectBuilder(src, out, main, new SystemStreamLog()); + } + + private static void write(File f, String text) throws IOException { + Files.createDirectories(f.getParentFile().toPath()); + Files.write(f.toPath(), text.getBytes(StandardCharsets.UTF_8)); + } + + private static String read(File f) throws IOException { + return new String(Files.readAllBytes(f.toPath()), StandardCharsets.UTF_8); + } +} diff --git a/maven/cn1app-archetype/src/main/resources/archetype-resources/common/pom.xml b/maven/cn1app-archetype/src/main/resources/archetype-resources/common/pom.xml index 58cb693c8e3..56c8818992b 100644 --- a/maven/cn1app-archetype/src/main/resources/archetype-resources/common/pom.xml +++ b/maven/cn1app-archetype/src/main/resources/archetype-resources/common/pom.xml @@ -143,6 +143,24 @@ + + + python + + + ${basedir}/src/main/python + + + + + com.codenameone + codenameone-python-runtime + + + + transpile-python + generate-sources + + transpile-python + + diff --git a/maven/cn1app-archetype/src/main/resources/archetype-resources/pom.xml b/maven/cn1app-archetype/src/main/resources/archetype-resources/pom.xml index f74281b64f4..50b556ec89d 100644 --- a/maven/cn1app-archetype/src/main/resources/archetype-resources/pom.xml +++ b/maven/cn1app-archetype/src/main/resources/archetype-resources/pom.xml @@ -71,6 +71,11 @@ codenameone-unity-compat ${cn1.version} + + com.codenameone + codenameone-python-runtime + ${cn1.version} + com.codenameone codenameone-swing-compat diff --git a/maven/codenameone-gradle-plugin/src/main/java/com/codename1/gradle/AppSupport.java b/maven/codenameone-gradle-plugin/src/main/java/com/codename1/gradle/AppSupport.java index 220100b96bf..fdcdfc30f17 100644 --- a/maven/codenameone-gradle-plugin/src/main/java/com/codename1/gradle/AppSupport.java +++ b/maven/codenameone-gradle-plugin/src/main/java/com/codename1/gradle/AppSupport.java @@ -153,6 +153,30 @@ static void apply(final Project project, final ProjectLayout layout, final Coden }); main.getJava().srcDir(svg.flatMap(TranscodeSvgTask::getOutputDirectory)); + // Python: src/main/python transpiles to Java the main source set + // compiles, against codenameone-python-runtime, which only a project + // with Python sources gets as a dependency. + final File pythonDir = new File(layout.projectDir(), "src/main/python"); + if (com.codename1.maven.PythonProjectBuilder.isPythonProject(pythonDir)) { + addFramework(project, "implementation", ext.getVersion(), + com.codename1.maven.PythonProjectBuilder.RUNTIME_ARTIFACT); + } + final String pythonMain = AppSettings.read(layout.settingsFile()) + .getProperty(com.codename1.maven.PythonProjectImporter.MAIN_MODULE_SETTING); + TaskProvider python = project.getTasks().register( + "transpilePython", com.codename1.gradle.tasks.TranspilePythonTask.class, t -> { + common(t, project, layout, ext, userProperties); + t.setDescription("Transpiles the Python sources in src/main/python to Java"); + t.getPythonDirectory().set(pythonDir); + t.getSources().from(project.fileTree(pythonDir, f -> f.include("**/*.py"))); + if (pythonMain != null) { + t.getMainModule().set(pythonMain); + } + t.getJavaClassPath().from(project.getConfigurations().getByName("compileClasspath")); + t.getOutputDirectory().set(new File(layout.buildDir(), "generated/sources/cn1-python")); + }); + main.getJava().srcDir(python.flatMap(com.codename1.gradle.tasks.TranspilePythonTask::getOutputDirectory)); + registerGuiSources(project, layout, ext, userProperties, main); // Android compatibility: src/main/android (manifest, res/, assets/, diff --git a/maven/codenameone-gradle-plugin/src/main/java/com/codename1/gradle/tasks/TranspilePythonTask.java b/maven/codenameone-gradle-plugin/src/main/java/com/codename1/gradle/tasks/TranspilePythonTask.java new file mode 100644 index 00000000000..96b81f5cd44 --- /dev/null +++ b/maven/codenameone-gradle-plugin/src/main/java/com/codename1/gradle/tasks/TranspilePythonTask.java @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.gradle.tasks; + +import com.codename1.builders.BuildException; +import com.codename1.maven.PythonProjectBuilder; +import org.gradle.api.GradleException; +import org.gradle.api.file.ConfigurableFileCollection; +import org.gradle.api.file.DirectoryProperty; +import org.gradle.api.provider.Property; +import org.gradle.api.tasks.CacheableTask; +import org.gradle.api.tasks.Classpath; +import org.gradle.api.tasks.Input; +import org.gradle.api.tasks.InputFiles; +import org.gradle.api.tasks.Internal; +import org.gradle.api.tasks.Optional; +import org.gradle.api.tasks.OutputDirectory; +import org.gradle.api.tasks.PathSensitive; +import org.gradle.api.tasks.PathSensitivity; +import org.gradle.api.tasks.TaskAction; + +/// Transpiles `src/main/python` to Java sources, which the main source set +/// compiles. The same [PythonProjectBuilder] as the Maven plugin's +/// `transpile-python` goal. Runs without Python sources too, to empty its +/// output of what an earlier build generated. +@CacheableTask +public abstract class TranspilePythonTask extends Cn1Task { + /// `src/main/python`. + @Internal + public abstract DirectoryProperty getPythonDirectory(); + + /// The `.py` files under it. + @InputFiles + @PathSensitive(PathSensitivity.RELATIVE) + public abstract ConfigurableFileCollection getSources(); + + /// The module run as `__main__`, or unset for `main`/a library. + @Input + @Optional + public abstract Property getMainModule(); + + /// The compile class path `from java. import ` resolves + /// against. + @Classpath + public abstract ConfigurableFileCollection getJavaClassPath(); + + @OutputDirectory + public abstract DirectoryProperty getOutputDirectory(); + + @TaskAction + public void transpile() { + try { + new PythonProjectBuilder(getPythonDirectory().get().getAsFile(), getOutputDirectory().get().getAsFile(), + getMainModule().getOrNull(), log()) + .classPath(new java.util.ArrayList(getJavaClassPath().getFiles())).run(); + } catch (BuildException ex) { + throw new GradleException(ex.getMessage(), ex.getCause() == null ? ex : ex.getCause()); + } + } +} diff --git a/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/ImportPythonProjectMojo.java b/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/ImportPythonProjectMojo.java new file mode 100644 index 00000000000..ba63f51abd6 --- /dev/null +++ b/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/ImportPythonProjectMojo.java @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.maven; + +import org.apache.maven.plugin.MojoExecutionException; +import org.apache.maven.plugin.MojoFailureException; +import org.apache.maven.plugins.annotations.Mojo; +import org.apache.maven.plugins.annotations.Parameter; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; + +/// Imports a Python project into this Codename One application. +/// +/// ``` +/// mvn cn1:import-python-project -Dsource=/path/to/my-python-app +/// ``` +/// +/// Copies the project's `.py` sources into `common/src/main/python`, adds the +/// runtime dependency and the `transpile-python` goal to the common module, +/// records the entry module in `codenameone_settings.properties` and makes the +/// Codename One main class run it. Build and run as usual afterwards. +/// Importing again mirrors the Python project afresh. +@Mojo(name = "import-python-project", requiresProject = true, aggregator = true) +public class ImportPythonProjectMojo extends AbstractCN1Mojo { + + /// The Python project directory. + @Parameter(property = "source") + private File source; + + /// The same, under a name no other plugin reads; it wins when both are given. + @Parameter(property = "cn1.python.import") + private File pythonImport; + + @Override + protected void executeImpl() throws MojoExecutionException, MojoFailureException { + File from = pythonImport != null ? pythonImport : source; + if (from == null) { + throw new MojoFailureException("Name the Python project to import: -Dsource=/path/to/my-python-app"); + } + File common = getCN1ProjectDir(); + if (common == null) { + throw new MojoExecutionException("Run this from a Codename One application project"); + } + if (!PythonProjectBuilder.isPythonProject(PythonProjectImporter.sourceRoot(from))) { + // Before the pom is touched: a mistyped path must change nothing. + throw new MojoFailureException(from + " has no Python sources"); + } + wirePython(common); + String pkg = properties == null ? null : properties.getProperty("codename1.packageName"); + String main = properties == null ? null : properties.getProperty("codename1.mainName"); + PythonProjectImporter.Result r; + try { + r = new PythonProjectImporter(MavenLog.of(getLog())).importProject(from, common, pkg, main); + } catch (com.codename1.builders.BuildException e) { + throw new MojoFailureException(e.getMessage(), e); + } + getLog().info("Imported " + r.copiedFiles + " Python files into " + new File(common, "src/main/python") + + (r.removedFiles == 0 ? "" : "; removed " + r.removedFiles + " the project no longer has")); + getLog().info(r.mainModule == null ? "No entry module was found: the sources are imported as a library" + : "Entry module: " + r.mainModule); + if (!r.requirements.isEmpty()) { + getLog().warn("The project depends on " + r.requirements + ". A pure-Python package builds when its " + + "sources are under src/main/python; see the developer guide for the packages the runtime " + + "provides."); + } + } + + private void wirePython(File common) throws MojoExecutionException, MojoFailureException { + File pomFile = new File(common, "pom.xml"); + String pom; + try { + pom = new String(Files.readAllBytes(pomFile.toPath()), StandardCharsets.UTF_8); + } catch (IOException e) { + throw new MojoExecutionException("Cannot read " + pomFile, e); + } + PythonPomUpdater u = new PythonPomUpdater(pom); + if (!u.manual.isEmpty()) { + StringBuilder sb = new StringBuilder("Nothing was imported: " + pomFile + + " needs Python support wired in by hand. Add:\n"); + for (String m : u.manual) { + sb.append("\n- ").append(m); + } + throw new MojoFailureException(sb.toString()); + } + if (u.changed) { + try { + Files.write(pomFile.toPath(), u.pom.getBytes(StandardCharsets.UTF_8)); + } catch (IOException e) { + throw new MojoExecutionException("Cannot write " + pomFile, e); + } + getLog().info("Added Python support to " + pomFile); + } + } +} diff --git a/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/PythonPomUpdater.java b/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/PythonPomUpdater.java new file mode 100644 index 00000000000..beebf71143e --- /dev/null +++ b/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/PythonPomUpdater.java @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.maven; + +import java.util.ArrayList; +import java.util.List; + +/// Wires Python support into an existing application's common `pom.xml`: +/// the runtime dependency and the `transpile-python` goal. +/// +/// The pom is edited as text, at anchors every archetype-generated common pom +/// has; a pom without them is reported with the snippets to add by hand +/// rather than guessed at. +final class PythonPomUpdater { + + /// Compile scope: the generated classes call the runtime on the device, + /// and it reaches there the way any library does, as a dependency. + static final String PROFILE = " \n" + + " \n" + + " python\n" + + " \n" + + " \n" + + " ${basedir}/src/main/python\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " com.codenameone\n" + + " codenameone-python-runtime\n" + + " ${cn1.version}\n" + + " \n" + + " \n" + + " \n"; + + static final String EXECUTION = " \n" + + " transpile-python\n" + + " generate-sources\n" + + " \n" + + " transpile-python\n" + + " \n" + + " \n"; + + private static final String PLUGIN = "codenameone-maven-plugin"; + + /// The updated pom, or the same text when it is already wired. + final String pom; + /// What could not be added automatically, each with the XML to add. + final List manual = new ArrayList(); + boolean changed; + + /// The pom without comments and `` blocks: a + /// runtime named only there is no dependency of the build. + static String effective(String pom) { + StringBuilder sb = new StringBuilder(); + int at = 0; + while (at < pom.length()) { + int comment = pom.indexOf("" : ""; + int end = pom.indexOf(close, next); + at = end < 0 ? pom.length() : end + close.length(); + } + return sb.toString(); + } + + PythonPomUpdater(String pom) { + String p = pom; + if (effective(p).indexOf("codenameone-python-runtime") < 0) { + int profiles = p.lastIndexOf(""); + if (profiles < 0) { + manual.add("the codenameone-python-runtime dependency, e.g. as this profile:\n" + PROFILE); + } else { + int lineStart = p.lastIndexOf('\n', profiles) + 1; + p = p.substring(0, lineStart) + PROFILE + p.substring(lineStart); + changed = true; + } + } + if (p.indexOf("transpile-python") < 0) { + int plugin = p.indexOf(PLUGIN); + // Only this plugin's own executions block: a later plugin's would + // otherwise receive a goal it does not have. + int pluginEnd = plugin < 0 ? -1 : p.indexOf("", plugin); + int executions = pluginEnd < 0 ? -1 : p.indexOf("", plugin); + if (executions > pluginEnd) { + executions = -1; + } + if (executions < 0) { + manual.add("an execution of transpile-python in the codenameone-maven-plugin:\n" + EXECUTION); + } else { + int at = executions + "".length(); + p = p.substring(0, at) + "\n" + EXECUTION.substring(0, EXECUTION.length() - 1) + p.substring(at); + changed = true; + } + } + this.pom = p; + } +} diff --git a/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/TranspilePythonMojo.java b/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/TranspilePythonMojo.java new file mode 100644 index 00000000000..06fb18645c4 --- /dev/null +++ b/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/TranspilePythonMojo.java @@ -0,0 +1,131 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.maven; + +import org.apache.maven.model.Dependency; +import org.apache.maven.plugin.MojoExecutionException; +import org.apache.maven.plugin.MojoFailureException; +import org.apache.maven.plugins.annotations.LifecyclePhase; +import org.apache.maven.plugins.annotations.Mojo; +import org.apache.maven.plugins.annotations.Parameter; +import org.apache.maven.plugins.annotations.ResolutionScope; + +import java.io.File; + +/// Transpiles the Python sources of an application module, `src/main/python`, +/// to Java under `target/generated-sources/python`, which is added as a source +/// root. From there the program is ordinary Codename One code and every target +/// builds it. A silent no-op for a project without Python sources, apart from +/// removing what an earlier build generated. +/// +/// A construct a translated program cannot run -- `eval`, `exec`, a C +/// extension -- fails the build with the file and line. +@Mojo(name = "transpile-python", defaultPhase = LifecyclePhase.GENERATE_SOURCES, + requiresDependencyResolution = ResolutionScope.COMPILE) +public class TranspilePythonMojo extends AbstractCN1Mojo { + + @Parameter(property = "cn1.python.sourceDir", defaultValue = "${project.basedir}/src/main/python") + private File pythonSourceDir; + + @Parameter(property = "cn1.python.outputDir", defaultValue = "${project.build.directory}/generated-sources/python") + private File pythonOutputDir; + + /// The module run by `PythonModules.runMain()`. Unset, it is + /// `codename1.python.mainModule` from the settings, which + /// `cn1:import-python-project` writes, and then `main` when `main.py` exists. + @Parameter(property = "cn1.python.mainModule") + private String mainModule; + + @Parameter(property = "cn1.python.skip", defaultValue = "false") + private boolean skip; + + @Override + protected void executeImpl() throws MojoExecutionException, MojoFailureException { + if (skip) { + return; + } + String entry = mainModule; + if (entry == null && properties != null) { + entry = properties.getProperty(PythonProjectImporter.MAIN_MODULE_SETTING); + } + PythonProjectBuilder builder = new PythonProjectBuilder(pythonSourceDir, pythonOutputDir, entry, + MavenLog.of(getLog())).classPath(compileClassPath()); + if (PythonProjectBuilder.isPythonProject(pythonSourceDir)) { + checkJdk(); + checkRuntimeDependency(); + } + boolean built; + try { + built = builder.run(); + } catch (com.codename1.builders.BuildException ex) { + throw new MojoFailureException(ex.getMessage(), ex); + } + if (built) { + String path = pythonOutputDir.getAbsolutePath(); + if (!project.getCompileSourceRoots().contains(path)) { + project.addCompileSourceRoot(path); + } + } + } + + private void checkJdk() throws MojoFailureException { + String spec = System.getProperty("java.specification.version", "1.8"); + int major; + try { + major = Integer.parseInt(spec.startsWith("1.") ? spec.substring(2) : spec); + } catch (NumberFormatException e) { + major = 8; + } + if (major < 17) { + throw new MojoFailureException("Python support generates Java 17 source, but this build is running on JDK " + + spec + ". Run Maven with JDK 17 or newer (e.g. from https://adoptium.net) to use src/main/python."); + } + } + + private void checkRuntimeDependency() throws MojoFailureException { + for (Object o : project.getDependencies()) { + if (PythonProjectBuilder.RUNTIME_ARTIFACT.equals(((Dependency) o).getArtifactId())) { + return; + } + } + throw new MojoFailureException("src/main/python contains Python sources but the project does not declare " + + "the Python runtime dependency. Add this to common/pom.xml, or run cn1:import-python-project, " + + "which adds it:\n\n" + + " \n" + + " com.codenameone\n" + + " " + PythonProjectBuilder.RUNTIME_ARTIFACT + "\n" + + " ${cn1.version}\n" + + " \n"); + } + + private java.util.List compileClassPath() { + java.util.List r = new java.util.ArrayList(); + for (Object o : project.getArtifacts()) { + org.apache.maven.artifact.Artifact a = (org.apache.maven.artifact.Artifact) o; + if (a.getFile() != null && !"test".equals(a.getScope())) { + r.add(a.getFile()); + } + } + return r; + } +} diff --git a/maven/codenameone-maven-plugin/src/test/java/com/codename1/maven/PythonPomUpdaterTest.java b/maven/codenameone-maven-plugin/src/test/java/com/codename1/maven/PythonPomUpdaterTest.java new file mode 100644 index 00000000000..9e3b4cc587c --- /dev/null +++ b/maven/codenameone-maven-plugin/src/test/java/com/codename1/maven/PythonPomUpdaterTest.java @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.maven; + +import org.junit.jupiter.api.Test; + +import java.io.File; +import java.nio.file.Files; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/// cn1:import-python-project in a project generated before Python +/// compatibility existed: its common pom has no runtime dependency and no +/// goal, and the imported project would not build. +public class PythonPomUpdaterTest { + + private static String read(String path) throws Exception { + return new String(Files.readAllBytes(new File(path).toPath()), "UTF-8"); + } + + @Test + public void wiresALegacyArchetypePom() throws Exception { + String legacy = read("src/test/resources/android-pom/legacy-common-pom.xml"); + PythonPomUpdater u = new PythonPomUpdater(legacy); + assertTrue(u.manual.isEmpty(), u.manual.toString()); + assertTrue(u.changed); + assertTrue(u.pom.contains("codenameone-python-runtime")); + assertTrue(u.pom.lastIndexOf("") > u.pom.indexOf("python")); + // The goal is one of the Codename One plugin's executions. + int plugin = u.pom.indexOf("codenameone-maven-plugin"); + int goal = u.pom.indexOf("transpile-python"); + assertTrue(goal > plugin && goal < u.pom.indexOf("", plugin)); + // The runtime ships with the application: it must not be `provided`, + // which would compile the game and leave every device without it. + int dep = u.pom.indexOf("codenameone-python-runtime"); + assertFalse(u.pom.substring(dep, u.pom.indexOf("", dep)).contains("")); + // Wired once: a second import changes nothing. + PythonPomUpdater again = new PythonPomUpdater(u.pom); + assertFalse(again.changed); + assertTrue(again.manual.isEmpty()); + assertEquals(u.pom, again.pom); + } + + @Test + public void leavesTheCurrentArchetypeAlone() throws Exception { + String current = read("../cn1app-archetype/src/main/resources/archetype-resources/common/pom.xml"); + PythonPomUpdater u = new PythonPomUpdater(current); + assertFalse(u.changed); + assertTrue(u.manual.isEmpty(), u.manual.toString()); + assertEquals(current, u.pom); + } + + /// A runtime named only in a comment or in dependency management is not + /// a dependency of the build, so the profile still goes in. + @Test + public void aManagedOrCommentedRuntimeIsNotADependency() throws Exception { + String legacy = read("src/test/resources/android-pom/legacy-common-pom.xml"); + String mention = "\n" + + "com.codenameone" + + "codenameone-python-runtime" + + "\n"; + int at = legacy.indexOf(""); + PythonPomUpdater u = new PythonPomUpdater(legacy.substring(0, at) + mention + legacy.substring(at)); + assertTrue(u.pom.contains("python")); + } + + /// Android compatibility wired in first must not confuse this one, nor + /// the other way around: both edit the same two places of the pom. + @Test + public void composesWithTheAndroidWiring() throws Exception { + String legacy = read("src/test/resources/android-pom/legacy-common-pom.xml"); + String both = new PythonPomUpdater(new AndroidPomUpdater(legacy, false).pom).pom; + String reversed = new AndroidPomUpdater(new PythonPomUpdater(legacy).pom, false).pom; + for (String pom : new String[] {both, reversed}) { + assertTrue(pom.contains("transpile-python")); + assertTrue(pom.contains("compile-android-res")); + assertTrue(pom.contains("python")); + assertTrue(pom.contains("android-compat")); + assertFalse(new PythonPomUpdater(pom).changed); + assertFalse(new AndroidPomUpdater(pom, false).changed); + } + } + + @Test + public void neverWiresTheGoalIntoALaterPlugin() { + String pom = "\n" + + " \n" + + " com.codenameone\n" + + " codenameone-maven-plugin\n" + + " \n" + + " \n" + + " maven-antrun-plugin\n" + + " \n" + + " \n" + + " \n" + + "\n"; + PythonPomUpdater u = new PythonPomUpdater(pom); + assertFalse(u.pom.contains("transpile-python")); + // Neither anchor exists: both snippets are handed to the developer. + assertEquals(2, u.manual.size(), u.manual.toString()); + assertTrue(u.manual.get(0).contains("codenameone-python-runtime")); + assertTrue(u.manual.get(1).contains("transpile-python")); + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Builtins.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Builtins.java index 2b69a3f1225..fc60b1b3cb2 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Builtins.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Builtins.java @@ -379,6 +379,18 @@ private static void installExceptions() { Object[] args = exceptionArgs(a[0]); return args.length == 0 ? Py.NONE : args[0]; }), null, null)); + Py.SYSTEM_EXIT.define("code", new PyProperty(PyBuiltin.positional("code", a -> systemExitCode(a[0])), + null, null)); + } + + /// `SystemExit.code`: None without arguments, the argument with one, the + /// tuple of them with more. + static Object systemExitCode(Object exc) { + Object[] args = exceptionArgs(exc); + if (args.length == 0) { + return Py.NONE; + } + return args.length == 1 ? args[0] : new PyTuple(args); } /// OSError(errno, strerror[, filename[, winerror[, filename2]]]) keeps diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/JavaBridge.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/JavaBridge.java new file mode 100644 index 00000000000..96d8b547511 --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/JavaBridge.java @@ -0,0 +1,253 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +/// The Java classes a Python program uses, reached without reflection. +/// +/// The transpiler generates one subclass per program, `JavaInterop`, from +/// the class files on the build's class path. It answers only for the +/// classes the program imports, the classes their methods hand back, and +/// only for the member names the program spells, so nothing it does not use +/// is kept alive. Classes are numbered in that generated class; a +/// [PyJavaClass] carries its number. +/// +/// Every method returns `null` for a member it does not have, and the +/// caller raises the Python error. +public abstract class JavaBridge { + private static JavaBridge current; + + /// Installs the program's bridge; the generated `PythonModules` does. + public static void install(JavaBridge bridge) { + current = bridge; + } + + static JavaBridge get() { + if (current == null) { + throw Py.runtimeError("no Java classes were bridged for this program"); + } + return current; + } + + /// `Cls(*args)`. + public abstract Object construct(int cls, Object[] args); + + /// `Cls.name`: a static field's value, or null. + public abstract Object getStatic(int cls, String name); + + /// Whether `Cls` has a static method `name`. + public abstract boolean hasStatic(int cls, String name); + + /// `Cls.name(*args)` for a static method. + public abstract Object callStatic(int cls, String name, Object[] args); + + /// `obj.name`: a public instance field's value, or null. + public abstract Object getField(Object target, String name); + + /// Whether `obj` has an instance method `name`. + public abstract boolean hasMethod(Object target, String name); + + /// `obj.name(*args)`. + public abstract Object invoke(Object target, String name, Object[] args); + + /// `isinstance(obj, Cls)`. + public abstract boolean isInstance(int cls, Object value); + + /// The Python name of `Cls`: its simple name. + public abstract String className(int cls); + + /// The [PyJavaClass] numbered `cls`, made once. + public final PyJavaClass type(int cls) { + return PyJavaClass.of(this, cls); + } + + // ---- conversions the generated code calls --------------------------- + + /// A Java value as Python sees it: boxed numbers as int and float, a + /// char as a one-character str, null as None, and any other object + /// wrapped as a [PyJavaObject] -- unless it is already a Python value. + public static Object toPy(Object v) { + if (v == null) { + return Py.NONE; + } + if (v instanceof String || v instanceof PyObject || v instanceof Long || v instanceof Double) { + return v; + } + if (v instanceof Integer || v instanceof Short || v instanceof Byte) { + return PyInt.of(((Number) v).longValue()); + } + if (v instanceof Float) { + return Py.ofFloat(((Float) v).doubleValue()); + } + if (v instanceof Boolean) { + return Py.bool(((Boolean) v).booleanValue()); + } + if (v instanceof Character) { + return String.valueOf(((Character) v).charValue()); + } + return new PyJavaObject(v); + } + + /// A Python value handed to a parameter of type `Object`: a wrapped + /// object unwrapped, None as null, an int as an Integer when it fits and + /// a Long when not, a bool as a Boolean; anything else as it is. + public static Object toJava(Object v) { + if (v == Py.NONE) { //NOPMD CompareObjectsWithEquals - identity is meant + return null; + } + if (v instanceof PyJavaObject) { + return ((PyJavaObject) v).value; + } + if (v instanceof PyBool) { + return Boolean.valueOf(v == Py.TRUE); //NOPMD CompareObjectsWithEquals - identity is meant + } + if (v instanceof Long) { + long l = ((Long) v).longValue(); + if (l >= Integer.MIN_VALUE && l <= Integer.MAX_VALUE) { + return Integer.valueOf((int) l); + } + return v; + } + if (v instanceof PyObject) { + return ((PyObject) v).toJava(); + } + return v; + } + + /// A Python list or tuple handed to a `java.util` collection parameter: + /// a new `ArrayList` of its elements, so the Java side's changes are not + /// seen by Python. A Java collection is passed as it is. + public static Object toJavaList(Object v) { + Object j = toJava(v); + if (v instanceof PyList) { + PyList l = (PyList) v; + java.util.ArrayList r = new java.util.ArrayList(l.size()); + for (int i = 0; i < l.size(); i++) { + r.add(toJava(l.get(i))); + } + return r; + } + if (v instanceof PyTuple) { + PyTuple t = (PyTuple) v; + java.util.ArrayList r = new java.util.ArrayList(t.size()); + for (int i = 0; i < t.size(); i++) { + r.add(toJava(t.get(i))); + } + return r; + } + return j; + } + + /// A Python dict handed to a `java.util.Map` parameter: a new + /// `LinkedHashMap` in the dict's order. + public static Object toJavaMap(Object v) { + if (v instanceof PyDict) { + PyDict d = (PyDict) v; + java.util.LinkedHashMap r = new java.util.LinkedHashMap(); + Object[] keys = d.keyArray(); + for (int i = 0; i < keys.length; i++) { + r.put(toJava(keys[i]), toJava(d.getItem(keys[i]))); + } + return r; + } + return toJava(v); + } + + /// Whether `v` can be made a `java.util` list. + public static boolean isSequence(Object v) { + return v instanceof PyList || v instanceof PyTuple; + } + + /// A Java exception thrown into Python code, as a `RuntimeError` naming + /// the Java class and message. + public static PyException javaError(Exception e) { + return Py.runtimeError(e.toString()); + } + + /// Whether `v` can go to a parameter of reference type `t`, whose + /// instance test the generated code made: None always can. + public static boolean isNone(Object v) { + return v == Py.NONE; //NOPMD CompareObjectsWithEquals - identity is meant + } + + /// The Java object `v` wraps, or `v` itself. + public static Object unwrap(Object v) { + return v instanceof PyJavaObject ? ((PyJavaObject) v).value : v; + } + + /// Whether `v` fits an `int`, `long`, `short` or `byte` parameter. + public static boolean isInt(Object v) { + return Py.isInt(v) || v instanceof PyBool; + } + + /// Whether `v` fits a `double` or `float` parameter. + public static boolean isReal(Object v) { + return Py.isInt(v) || Py.isFloat(v) || v instanceof PyBool; + } + + /// Whether `v` fits a `char` parameter: a one-character str. + public static boolean isChar(Object v) { + return v instanceof String && ((String) v).length() == 1; + } + + /// Whether `v` is a Python callable that can stand for a functional + /// interface: not a Java object, not a class. + public static boolean isFunction(Object v) { + return !(v instanceof PyJavaObject) && !(v instanceof PyJavaClass) && !(v instanceof PyType) + && Builtins.isCallable(v); + } + + public static long asLong(Object v) { + return v instanceof PyBool ? (v == Py.TRUE ? 1 : 0) : Py.asLong(v); //NOPMD CompareObjectsWithEquals - identity is meant + } + + public static double asDouble(Object v) { + return v instanceof PyBool ? (v == Py.TRUE ? 1 : 0) : Py.asDouble(v); //NOPMD CompareObjectsWithEquals - identity is meant + } + + public static boolean asBoolean(Object v) { + return Py.isTrue(v); + } + + public static char asChar(Object v) { + return ((String) v).charAt(0); + } + + /// Calls a Python function from a functional interface adapter. + public static Object callPy(Object fn, Object[] args) { + return Py.obj(fn).call(args, Py.NO_KW); + } + + /// The error for a call no overload accepts. + public static PyException noOverload(String owner, String name, Object[] args, String signatures) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < args.length; i++) { + if (i > 0) { + sb.append(", "); + } + sb.append(args[i] instanceof PyJavaObject ? ((PyJavaObject) args[i]).javaClassName() + : Py.typeOf(args[i]).name); + } + return Py.typeError(owner + "." + name + "() got (" + sb + "); it takes " + signatures); + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Py.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Py.java index 8ce47d9b0a4..9be40a13432 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Py.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Py.java @@ -1985,6 +1985,9 @@ public static boolean isinstance(Object o, Object t) { } return false; } + if (t instanceof PyJavaClass) { + return ((PyJavaClass) t).isInstance(o); + } if (!(t instanceof PyType)) { throw typeError("isinstance() arg 2 must be a type, a tuple of types, or a union"); } diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyJavaClass.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyJavaClass.java new file mode 100644 index 00000000000..127afc614be --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyJavaClass.java @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +/// A Java class imported into Python (`from java.com.codename1.ui import +/// Form`): calling it constructs an instance, its attributes are the class's +/// static fields and methods, and `isinstance` tests against it. +public final class PyJavaClass extends PyObject { + private static final PyType TYPE = new PyType("java_class", new PyType[] {Py.OBJECT_TYPE}, null); + + private final JavaBridge bridge; + final int id; + + private PyJavaClass(JavaBridge bridge, int id) { + this.bridge = bridge; + this.id = id; + } + + private static PyJavaClass[] made = new PyJavaClass[0]; + + static PyJavaClass of(JavaBridge bridge, int id) { + if (id >= made.length) { + PyJavaClass[] grown = new PyJavaClass[Math.max(id + 1, made.length * 2)]; + System.arraycopy(made, 0, grown, 0, made.length); + made = grown; + } + PyJavaClass c = made[id]; + if (c == null) { + c = new PyJavaClass(bridge, id); + made[id] = c; + } + return c; + } + + @Override + public PyType type() { + return TYPE; + } + + public String name() { + return bridge.className(id); + } + + @Override + public String repr() { + return ""; + } + + @Override + public Object call(Object[] args, String[] kwNames) { + if (kwNames.length > 0) { + throw Py.typeError(name() + "() takes no keyword arguments"); + } + Object r = bridge.construct(id, args); + if (r == null) { + throw Py.typeError("cannot construct Java class " + name()); + } + return r; + } + + @Override + public Object getAttrOrNull(String name) { + Object v = bridge.getStatic(id, name); + if (v != null) { + return v; + } + if (bridge.hasStatic(id, name)) { + return new PyJavaMethod(bridge, this, null, name); + } + if ("__name__".equals(name) || "__qualname__".equals(name)) { + return name(); + } + return super.getAttrOrNull(name); + } + + boolean isInstance(Object o) { + return o instanceof PyJavaObject && bridge.isInstance(id, ((PyJavaObject) o).value); + } + + @Override + public long hash() { + return id; + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyJavaMethod.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyJavaMethod.java new file mode 100644 index 00000000000..c45b821ca14 --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyJavaMethod.java @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +/// A Java method read as an attribute, static (`Cls.name`) or bound to an +/// object (`obj.name`); calling it picks the overload. +final class PyJavaMethod extends PyObject { + private static final PyType TYPE = new PyType("java_method", new PyType[] {Py.OBJECT_TYPE}, null); + + private final JavaBridge bridge; + private final PyJavaClass owner; + private final Object target; + private final String name; + + PyJavaMethod(JavaBridge bridge, PyJavaClass owner, Object target, String name) { + this.bridge = bridge; + this.owner = owner; + this.target = target; + this.name = name; + } + + @Override + public PyType type() { + return TYPE; + } + + @Override + public Object call(Object[] args, String[] kwNames) { + if (kwNames.length > 0) { + throw Py.typeError(name + "() takes no keyword arguments"); + } + Object r = owner != null ? bridge.callStatic(owner.id, name, args) : bridge.invoke(target, name, args); + if (r == null) { + throw Py.attributeError("Java method '" + name + "' is not available"); + } + return r; + } + + @Override + public String repr() { + return ""; + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyJavaObject.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyJavaObject.java index 9b15a115758..f61ec83761d 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyJavaObject.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyJavaObject.java @@ -23,13 +23,13 @@ */ package com.codename1.python.runtime; -/// A Java object handed to Python code that has no Python counterpart. -/// -/// Python sees it as an opaque value: it can be stored, compared by identity, -/// passed back to Java and printed. Calling its methods by name would need -/// reflection, which ParparVM does not keep; the transpiler instead binds -/// Java calls statically (see the interop contract in the developer guide). +/// A Java object seen from Python. Its attributes are its public methods and +/// fields; `==` and `hash` are the object's own `equals` and `hashCode`, and +/// `str` its `toString`. public final class PyJavaObject extends PyObject { + private static final PyType TYPE = new PyType("java_object", new PyType[] {Py.OBJECT_TYPE}, null); + + /// The Java object. public final Object value; public PyJavaObject(Object value) { @@ -38,7 +38,25 @@ public PyJavaObject(Object value) { @Override public PyType type() { - return Py.OBJECT_TYPE; + return TYPE; + } + + /// The Java class's name, for messages. + public String javaClassName() { + return value.getClass().getName(); + } + + @Override + public Object getAttrOrNull(String name) { + JavaBridge b = JavaBridge.get(); + Object v = b.getField(value, name); + if (v != null) { + return v; + } + if (b.hasMethod(value, name)) { + return new PyJavaMethod(b, null, value, name); + } + return super.getAttrOrNull(name); } @Override @@ -48,7 +66,12 @@ public Object toJava() { @Override public String repr() { - return ""; + return ""; + } + + @Override + public String str() { + return String.valueOf(value); } @Override @@ -56,6 +79,75 @@ public Object eq(Object o) { return o instanceof PyJavaObject ? Py.bool(value.equals(((PyJavaObject) o).value)) : null; } + // A Java collection behaves as its Python counterpart: `len`, `in`, + // iteration and indexing. Elements cross as [JavaBridge#toPy] makes them. + + @Override + public int len() { + if (value instanceof java.util.Collection) { + return ((java.util.Collection) value).size(); + } + if (value instanceof java.util.Map) { + return ((java.util.Map) value).size(); + } + return super.len(); + } + + @Override + public boolean contains(Object x) { + if (value instanceof java.util.Collection) { + return ((java.util.Collection) value).contains(JavaBridge.toJava(x)); + } + if (value instanceof java.util.Map) { + return ((java.util.Map) value).containsKey(JavaBridge.toJava(x)); + } + return super.contains(x); + } + + @Override + public Object iter() { + final java.util.Iterator it; + if (value instanceof java.lang.Iterable) { + it = ((java.lang.Iterable) value).iterator(); + } else if (value instanceof java.util.Map) { + it = ((java.util.Map) value).keySet().iterator(); + } else if (value instanceof java.util.Iterator) { + it = (java.util.Iterator) value; + } else { + return super.iter(); + } + return new PyJavaIter(new PyJavaIter.Source() { + @Override + public Object next() { + return it.hasNext() ? JavaBridge.toPy(it.next()) : null; + } + }); + } + + @Override + public Object getItem(Object key) { + if (value instanceof java.util.List && Py.isInt(key)) { + java.util.List l = (java.util.List) value; + long i = Py.asLong(key); + if (i < 0) { + i += l.size(); + } + if (i < 0 || i >= l.size()) { + throw Py.error(Py.INDEX_ERROR, "list index out of range"); + } + return JavaBridge.toPy(l.get((int) i)); + } + if (value instanceof java.util.Map) { + java.util.Map m = (java.util.Map) value; + Object k = JavaBridge.toJava(key); + if (!m.containsKey(k)) { + throw Py.keyError(key); + } + return JavaBridge.toPy(m.get(k)); + } + return super.getItem(key); + } + @Override public long hash() { return value.hashCode(); diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java index 00c088bb343..1c683f8a6c9 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java @@ -77,6 +77,27 @@ static boolean provide(String name) { /// generated `runMain()` calls this when the main module finishes, /// whether or not it raised; a handler that raises is reported on /// stderr and the rest still run, as in CPython. + /// What a program that ended with `e` exits with, reporting it on stderr + /// as the interpreter does: a `SystemExit` with no code or None is 0, + /// with an int that int, with anything else 1 after printing it; any + /// other exception is printed and gives 1. + public static int exitStatus(PyException e) { + if (e.matches(Py.SYSTEM_EXIT)) { + Object code = Builtins.systemExitCode(e.value); + if (code == Py.NONE) { //NOPMD CompareObjectsWithEquals - identity is meant + return 0; + } + if (Py.isInt(code)) { + return (int) Py.asLong(code); + } + System.err.println(Py.toStr(code)); + return 1; + } + System.err.println("Traceback (most recent call last):"); + System.err.println(e.getMessage()); + return 1; + } + public static void runExitHandlers() { while (!EXIT_CALLS.isEmpty()) { int last = EXIT_CALLS.size() - 1; diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/Main.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/Main.java index 678c9642c4c..401b4b8ad9b 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/Main.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/Main.java @@ -36,7 +36,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -/// Command line: `Main [main module]`. +/// Command line: `Main [-cp ] [main module]`. /// /// Transpiles every `.py` under the source root and writes the Java sources /// under the output root. Exit status is 0 on success, 1 when a source does @@ -47,8 +47,19 @@ private Main() { } public static void main(String[] args) throws IOException { + List classPath = new ArrayList(); + if (args.length >= 2 && "-cp".equals(args[0])) { + for (String e : args[1].split(File.pathSeparator)) { + if (!e.isEmpty()) { + classPath.add(new File(e)); + } + } + String[] rest = new String[args.length - 2]; + System.arraycopy(args, 2, rest, 0, rest.length); + args = rest; + } if (args.length < 2 || args.length > 3) { - System.err.println("usage: Main [main module]"); + System.err.println("usage: Main [-cp ] [main module]"); System.exit(2); return; } @@ -64,7 +75,7 @@ public static void main(String[] args) throws IOException { } Map out; try { - out = PythonTranspiler.transpile(sources, args.length == 3 ? args[2] : null); + out = PythonTranspiler.transpile(sources, args.length == 3 ? args[2] : null, classPath); } catch (PythonSyntaxError e) { System.err.println(e.getMessage()); System.exit(1); diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java index 3611f753efb..d7d9a4a9a13 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java @@ -66,6 +66,16 @@ private PythonTranspiler() { /// @param sources relative `.py` path to source text /// @param mainModule the module run as `__main__`, or null for a library public static Map transpile(Map sources, String mainModule) { + return transpile(sources, mainModule, java.util.Collections.emptyList()); + } + + /// @param sources relative `.py` path to source text + /// @param mainModule the module run as `__main__`, or null for a library + /// @param classPath the jars and class directories a program's + /// `from java. import ` resolves against; the + /// JDK's own classes are always found + public static Map transpile(Map sources, String mainModule, + List classPath) { Map out = new LinkedHashMap(); Map classes = new LinkedHashMap(); Map trees = new LinkedHashMap(); @@ -79,6 +89,15 @@ public static Map transpile(Map sources, String } } boolean closed = Closed.check(bodies); + String interop = null; + if (com.codename1.python.transpiler.codegen.JavaInteropGenerator.usesJava(trees.values())) { + try (com.codename1.python.transpiler.codegen.ClassPathReader cp = + new com.codename1.python.transpiler.codegen.ClassPathReader(classPath)) { + interop = com.codename1.python.transpiler.codegen.JavaInteropGenerator.generate(trees, cp, PACKAGE); + } catch (java.io.IOException e) { + throw new IllegalStateException("cannot read the class path: " + e.getMessage(), e); + } + } Set stdlib = addStdlib(sources, trees); for (Map.Entry e : sources.entrySet()) { String path = e.getKey().replace('\\', '/'); @@ -107,7 +126,10 @@ public static Map transpile(Map sources, String if (mainModule != null && !classes.containsKey(mainModule)) { throw new IllegalArgumentException("main module '" + mainModule + "' is not among the sources"); } - out.put(PACKAGE.replace('.', '/') + "/PythonModules.java", registry(classes, mainModule)); + if (interop != null) { + out.put(PACKAGE.replace('.', '/') + "/JavaInterop.java", interop); + } + out.put(PACKAGE.replace('.', '/') + "/PythonModules.java", registry(classes, mainModule, interop != null)); return out; } @@ -295,7 +317,7 @@ public static String className(String module) { return sb.toString(); } - private static String registry(Map classes, String mainModule) { + private static String registry(Map classes, String mainModule, boolean interop) { StringBuilder sb = new StringBuilder(); sb.append("// Generated by the Codename One Python transpiler.\n"); sb.append("package ").append(PACKAGE).append(";\n\n"); @@ -307,19 +329,30 @@ private static String registry(Map classes, String mainModule) { sb.append(" public static void registerAll() {\n"); sb.append(" if (registered) {\n return;\n }\n"); sb.append(" registered = true;\n"); + if (interop) { + sb.append(" JavaInterop.register();\n"); + } for (String cls : classes.values()) { sb.append(" ").append(cls).append(".register();\n"); } sb.append(" }\n\n"); - sb.append(" /// Runs the entry module as `__main__`.\n"); - sb.append(" public static void runMain() {\n"); + sb.append(" /// Runs the entry module as `__main__` and returns the status the\n"); + sb.append(" /// interpreter would exit with. An exception the program does not\n"); + sb.append(" /// catch is reported on stderr, never thrown: on a device the\n"); + sb.append(" /// application goes on, and only a command line exits with it.\n"); + sb.append(" public static int runMain() {\n"); sb.append(" registerAll();\n"); if (mainModule != null) { sb.append(" try {\n"); sb.append(" ").append(classes.get(mainModule)).append(".exec(\"__main__\");\n"); + sb.append(" return 0;\n"); + sb.append(" } catch (com.codename1.python.runtime.PyException e) {\n"); + sb.append(" return com.codename1.python.runtime.StdModules.exitStatus(e);\n"); sb.append(" } finally {\n"); sb.append(" com.codename1.python.runtime.StdModules.runExitHandlers();\n"); sb.append(" }\n"); + } else { + sb.append(" return 0;\n"); } sb.append(" }\n"); sb.append("}\n"); diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/ClassPathReader.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/ClassPathReader.java new file mode 100644 index 00000000000..44e423bac78 --- /dev/null +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/ClassPathReader.java @@ -0,0 +1,245 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.transpiler.codegen; + +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; + +/// Reads the public shape of compiled classes -- access, supertypes, fields +/// and methods -- from a class path of jars and directories, and from the +/// running JDK for `java.*`. Only what the interop generator needs, and no +/// bytecode library: a class file's constant pool and member tables. +public final class ClassPathReader implements AutoCloseable { + public static final int PUBLIC = 0x1; + public static final int STATIC = 0x8; + public static final int BRIDGE = 0x40; + public static final int VARARGS = 0x80; + public static final int INTERFACE = 0x200; + public static final int ABSTRACT = 0x400; + public static final int SYNTHETIC = 0x1000; + + /// One field or method. + public static final class Member { + public final int access; + public final String name; + public final String desc; + + Member(int access, String name, String desc) { + this.access = access; + this.name = name; + this.desc = desc; + } + + public boolean is(int flag) { + return (access & flag) != 0; + } + } + + /// One class. + public static final class ClassInfo { + public final String name; + public final int access; + public final String superName; + public final List interfaces; + public final List fields; + public final List methods; + + ClassInfo(String name, int access, String superName, List interfaces, List fields, + List methods) { + this.name = name; + this.access = access; + this.superName = superName; + this.interfaces = interfaces; + this.fields = fields; + this.methods = methods; + } + + public boolean is(int flag) { + return (access & flag) != 0; + } + } + + private final List roots = new ArrayList(); + private final Map cache = new HashMap(); + private FileSystem jrt; + + public ClassPathReader(List classPath) throws IOException { + for (File f : classPath) { + if (f.isDirectory()) { + roots.add(f); + } else if (f.isFile()) { + roots.add(new ZipFile(f)); + } + } + } + + /// The class with internal name `name` (`java/util/List`), or null. + public ClassInfo find(String name) { + if (cache.containsKey(name)) { + return cache.get(name); + } + ClassInfo c = null; + try { + byte[] b = bytes(name + ".class"); + if (b != null) { + c = parse(b); + } + } catch (IOException e) { + throw new IllegalStateException("cannot read class " + name + ": " + e.getMessage(), e); + } + cache.put(name, c); + return c; + } + + private byte[] bytes(String path) throws IOException { + for (Object r : roots) { + if (r instanceof File) { + File f = new File((File) r, path); + if (f.isFile()) { + return Files.readAllBytes(f.toPath()); + } + } else { + ZipFile z = (ZipFile) r; + ZipEntry e = z.getEntry(path); + if (e != null) { + try (InputStream in = z.getInputStream(e)) { + return readAll(in); + } + } + } + } + if (path.startsWith("java/") || path.startsWith("javax/")) { + if (jrt == null) { + jrt = FileSystems.getFileSystem(URI.create("jrt:/")); + } + Path p = jrt.getPath("modules", "java.base", path); + if (Files.isRegularFile(p)) { + return Files.readAllBytes(p); + } + } + return null; + } + + private static byte[] readAll(InputStream in) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + byte[] buf = new byte[8192]; + int n; + while ((n = in.read(buf)) > 0) { + out.write(buf, 0, n); + } + return out.toByteArray(); + } + + private static ClassInfo parse(byte[] b) throws IOException { + DataInputStream in = new DataInputStream(new java.io.ByteArrayInputStream(b)); + if (in.readInt() != 0xCAFEBABE) { + throw new IOException("not a class file"); + } + in.readUnsignedShort(); + in.readUnsignedShort(); + int n = in.readUnsignedShort(); + String[] utf = new String[n]; + int[] classIdx = new int[n]; + for (int i = 1; i < n; i++) { + int tag = in.readUnsignedByte(); + switch (tag) { + case 1: + utf[i] = in.readUTF(); + break; + case 7: + classIdx[i] = in.readUnsignedShort(); + break; + case 8: case 16: case 19: case 20: + in.readUnsignedShort(); + break; + case 3: case 4: case 9: case 10: case 11: case 12: case 17: case 18: + in.readInt(); + break; + case 5: case 6: + in.readLong(); + i++; + break; + case 15: + in.readUnsignedByte(); + in.readUnsignedShort(); + break; + default: + throw new IOException("bad constant pool tag " + tag); + } + } + int access = in.readUnsignedShort(); + String name = utf[classIdx[in.readUnsignedShort()]]; + int sup = in.readUnsignedShort(); + String superName = sup == 0 ? null : utf[classIdx[sup]]; + int ni = in.readUnsignedShort(); + List interfaces = new ArrayList(); + for (int i = 0; i < ni; i++) { + interfaces.add(utf[classIdx[in.readUnsignedShort()]]); + } + List fields = members(in, utf); + List methods = members(in, utf); + return new ClassInfo(name, access, superName, interfaces, fields, methods); + } + + private static List members(DataInputStream in, String[] utf) throws IOException { + int n = in.readUnsignedShort(); + List r = new ArrayList(n); + for (int i = 0; i < n; i++) { + int access = in.readUnsignedShort(); + String name = utf[in.readUnsignedShort()]; + String desc = utf[in.readUnsignedShort()]; + int na = in.readUnsignedShort(); + for (int a = 0; a < na; a++) { + in.readUnsignedShort(); + int len = in.readInt(); + in.readFully(new byte[len]); + } + r.add(new Member(access, name, desc)); + } + return r; + } + + @Override + public void close() throws IOException { + for (Object r : roots) { + if (r instanceof ZipFile) { + ((ZipFile) r).close(); + } + } + } +} diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/JavaInteropGenerator.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/JavaInteropGenerator.java new file mode 100644 index 00000000000..da959a2af28 --- /dev/null +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/JavaInteropGenerator.java @@ -0,0 +1,976 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.transpiler.codegen; + +import com.codename1.python.transpiler.api.PythonSyntaxError; +import com.codename1.python.transpiler.ast.Node; +import com.codename1.python.transpiler.codegen.ClassPathReader.ClassInfo; +import com.codename1.python.transpiler.codegen.ClassPathReader.Member; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Comparator; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; + +/// Writes `JavaInterop`, the program's [JavaBridge][jb]: every Java class a +/// program imports with `from java. import `, reached by +/// generated `instanceof` tests and direct calls -- no reflection, which +/// ParparVM does not keep. +/// +/// The bridge is as small as the program: it covers the imported classes, +/// the classes their used members hand back, and of those only the members +/// whose names the program spells as an attribute (`x.add`). Overloads are +/// tried most specific first, by the Python value's type; a Python callable +/// passed where a single-method interface is expected becomes an instance +/// of it. +/// +/// [jb]: com.codename1.python.runtime.JavaBridge +public final class JavaInteropGenerator { + private static final String RT = "com.codename1.python.runtime."; + private static final String JB = RT + "JavaBridge"; + + private final ClassPathReader cp; + /// Imported classes, by internal name, numbered in order. + private final Map imported = new LinkedHashMap(); + /// Python module (`java.com.codename1.ui`) to its imported names. + private final Map> modules = new TreeMap>(); + private final Set names = new HashSet(); + /// Every public class whose instances the bridge dispatches on. + private final Set universe = new LinkedHashSet(); + private final Map adapters = new LinkedHashMap(); + private final StringBuilder adapterCode = new StringBuilder(); + + private JavaInteropGenerator(ClassPathReader cp) { + this.cp = cp; + } + + /// Whether any of `trees` imports from a `java.` package. + public static boolean usesJava(Collection trees) { + for (Node.Module t : trees) { + for (Node n : all(t)) { + if (n instanceof Node.ImportFrom && isJava(((Node.ImportFrom) n).module, ((Node.ImportFrom) n).level)) { + return true; + } + if (n instanceof Node.Import) { + for (Node.Alias a : ((Node.Import) n).names) { + if (isJava(a.name, 0)) { + return true; + } + } + } + } + } + return false; + } + + private static boolean isJava(String module, int level) { + return level == 0 && module != null && module.startsWith("java."); + } + + /// Generates the bridge class source for `trees` (path to tree). + public static String generate(Map trees, ClassPathReader cp, String pkg) { + JavaInteropGenerator g = new JavaInteropGenerator(cp); + g.collect(trees); + g.close(); + return g.emit(pkg); + } + + private void collect(Map trees) { + for (Map.Entry e : trees.entrySet()) { + for (Node n : all(e.getValue())) { + if (n instanceof Node.Attribute) { + names.add(((Node.Attribute) n).attr); + } else if (n instanceof Node.Import) { + for (Node.Alias a : ((Node.Import) n).names) { + if (isJava(a.name, 0)) { + throw new PythonSyntaxError(e.getKey(), n.line, 0, "import the Java classes by name: " + + "from " + a.name + " import "); + } + } + } else if (n instanceof Node.ImportFrom) { + Node.ImportFrom f = (Node.ImportFrom) n; + if (!isJava(f.module, f.level)) { + continue; + } + String pkg = f.module.substring(5).replace('.', '/'); + for (Node.Alias a : f.names) { + if ("*".equals(a.name)) { + throw new PythonSyntaxError(e.getKey(), n.line, 0, + "Java classes must be imported by name, not with *"); + } + String internal = pkg + "/" + a.name.replace('.', '$'); + ClassInfo c = cp.find(internal); + if (c == null) { + throw new PythonSyntaxError(e.getKey(), n.line, 0, "no Java class " + + internal.replace('/', '.') + " on the class path"); + } + if (!c.is(ClassPathReader.PUBLIC)) { + throw new PythonSyntaxError(e.getKey(), n.line, 0, "Java class " + + internal.replace('/', '.') + " is not public"); + } + if (!imported.containsKey(internal)) { + imported.put(internal, imported.size()); + } + modules.computeIfAbsent(f.module, k -> new TreeMap()).put(a.name, internal); + } + } + } + } + for (Map.Entry e : trees.entrySet()) { + for (Node n : all(e.getValue())) { + if (n instanceof Node.ClassDef) { + for (Node.Expr b : ((Node.ClassDef) n).bases) { + if (b instanceof Node.Name && javaName(((Node.Name) b).id)) { + throw new PythonSyntaxError(e.getKey(), n.line, 0, "a Python class cannot extend the " + + "Java class " + ((Node.Name) b).id + "; wrap it, or pass a function where " + + "the Java API takes an interface"); + } + } + } + } + } + } + + private boolean javaName(String id) { + for (Map m : modules.values()) { + if (m.containsKey(id)) { + return true; + } + } + return false; + } + + /// Grows the universe to every class a used member can hand back. + private void close() { + List work = new ArrayList(imported.keySet()); + while (!work.isEmpty()) { + String c = work.remove(work.size() - 1); + if (!universe.add(c)) { + continue; + } + for (String s : supertypes(c)) { + if (isPublic(s)) { + work.add(s); + } + } + for (Member m : visibleMethods(c)) { + if (!names.contains(m.name) && !(imported.containsKey(c) && "".equals(m.name))) { + continue; + } + addType(returnType(m.desc), work); + for (String p : params(m.desc)) { + String sam = samOf(p); + if (sam != null) { + Member sm = samMethod(p); + for (String q : params(sm.desc)) { + addType(q, work); + } + } + } + } + for (Member f : visibleFields(c)) { + if (names.contains(f.name)) { + addType(f.desc, work); + } + } + } + } + + private void addType(String desc, List work) { + if (desc.startsWith("L")) { + String n = desc.substring(1, desc.length() - 1); + if (!universe.contains(n) && isPublic(n)) { + work.add(n); + } + } + } + + private boolean isPublic(String internal) { + ClassInfo c = cp.find(internal); + return c != null && c.is(ClassPathReader.PUBLIC) && !internal.contains("$$"); + } + + private List supertypes(String c) { + List r = new ArrayList(); + ClassInfo i = cp.find(c); + if (i == null) { + return r; + } + if (i.superName != null) { + r.add(i.superName); + } + r.addAll(i.interfaces); + return r; + } + + /// Public methods a reference of type `c` can call that `c` declares + /// itself, or inherits from a non-public superclass (whose methods Java + /// lets a caller reach only through `c`). + private List visibleMethods(String c) { + List r = new ArrayList(); + Set seen = new HashSet(); + String cur = c; + while (cur != null) { + ClassInfo i = cp.find(cur); + if (i == null) { + break; + } + for (Member m : i.methods) { + if (m.is(ClassPathReader.PUBLIC) && !m.is(ClassPathReader.BRIDGE) && !m.is(ClassPathReader.SYNTHETIC) + && !"".equals(m.name) && (cur.equals(c) || !"".equals(m.name)) + && seen.add(m.name + m.desc)) { + r.add(m); + } + } + if (i.superName == null || isPublic(i.superName)) { + break; + } + cur = i.superName; + } + return r; + } + + private List visibleFields(String c) { + List r = new ArrayList(); + String cur = c; + while (cur != null) { + ClassInfo i = cp.find(cur); + if (i == null) { + break; + } + for (Member f : i.fields) { + if (f.is(ClassPathReader.PUBLIC) && !f.is(ClassPathReader.SYNTHETIC)) { + r.add(f); + } + } + if (i.superName == null || isPublic(i.superName)) { + break; + } + cur = i.superName; + } + return r; + } + + // ---- descriptors ----------------------------------------------------- + + static List params(String desc) { + List r = new ArrayList(); + int i = 1; + while (desc.charAt(i) != ')') { + int s = i; + while (desc.charAt(i) == '[') { + i++; + } + if (desc.charAt(i) == 'L') { + i = desc.indexOf(';', i); + } + i++; + r.add(desc.substring(s, i)); + } + return r; + } + + static String returnType(String desc) { + return desc.substring(desc.indexOf(')') + 1); + } + + static String javaType(String d) { + switch (d.charAt(0)) { + case 'I': return "int"; + case 'J': return "long"; + case 'S': return "short"; + case 'B': return "byte"; + case 'C': return "char"; + case 'Z': return "boolean"; + case 'D': return "double"; + case 'F': return "float"; + case 'V': return "void"; + case '[': return javaType(d.substring(1)) + "[]"; + default: return d.substring(1, d.length() - 1).replace('/', '.').replace('$', '.'); + } + } + + /// The single abstract method's interface if `desc` names a functional + /// interface whose method it declares itself, else null. + private String samOf(String desc) { + if (!desc.startsWith("L")) { + return null; + } + String n = desc.substring(1, desc.length() - 1); + return samMethod(desc) != null && isPublic(n) ? n : null; + } + + private Member samMethod(String desc) { + if (!desc.startsWith("L")) { + return null; + } + ClassInfo c = cp.find(desc.substring(1, desc.length() - 1)); + if (c == null || !c.is(ClassPathReader.INTERFACE)) { + return null; + } + Member found = null; + for (Member m : c.methods) { + if (m.is(ClassPathReader.ABSTRACT) && !objectMethod(m)) { + if (found != null) { + return null; + } + found = m; + } + } + if (found == null) { + return null; + } + for (String s : c.interfaces) { + if (hasAbstract(s)) { + return null; + } + } + return found; + } + + private boolean hasAbstract(String iface) { + ClassInfo c = cp.find(iface); + if (c == null) { + return false; + } + for (Member m : c.methods) { + if (m.is(ClassPathReader.ABSTRACT) && !objectMethod(m)) { + return true; + } + } + for (String s : c.interfaces) { + if (hasAbstract(s)) { + return true; + } + } + return false; + } + + private static boolean objectMethod(Member m) { + return ("equals".equals(m.name) && "(Ljava/lang/Object;)Z".equals(m.desc)) + || ("hashCode".equals(m.name) && "()I".equals(m.desc)) + || ("toString".equals(m.name) && "()Ljava/lang/String;".equals(m.desc)); + } + + // ---- argument checks and conversions -------------------------------- + + /// How well a Python value fits a parameter: lower is more specific. + private int rank(String d) { + switch (d.charAt(0)) { + case 'Z': case 'C': + return 0; + case 'I': return 1; + case 'J': return 2; + case 'S': case 'B': return 3; + case 'F': return 5; + case 'D': return 4; + case '[': return 6; + default: + break; + } + if ("Ljava/lang/Object;".equals(d)) { + return 20; + } + if ("Ljava/lang/String;".equals(d)) { + return 4; + } + if (samOf(d) != null) { + return 8; + } + ClassInfo c = cp.find(d.substring(1, d.length() - 1)); + return c != null && c.is(ClassPathReader.INTERFACE) ? 12 : 10 - Math.min(depth(d.substring(1, d.length() - 1)), 5); + } + + private int depth(String c) { + int n = 0; + ClassInfo i = cp.find(c); + while (i != null && i.superName != null) { + n++; + i = cp.find(i.superName); + } + return n; + } + + private String check(String d, String e) { + switch (d.charAt(0)) { + case 'I': case 'J': case 'S': case 'B': + return JB + ".isInt(" + e + ")"; + case 'D': case 'F': + return JB + ".isReal(" + e + ")"; + case 'Z': + return "(" + e + " instanceof " + RT + "PyBool)"; + case 'C': + return JB + ".isChar(" + e + ")"; + default: + break; + } + if (d.startsWith("[")) { + return "(" + JB + ".toJava(" + e + ") instanceof " + javaType(d) + ")"; + } + String t = javaType(d); + switch (d) { + case "Ljava/lang/Object;": + return "true"; + case "Ljava/lang/String;": + case "Ljava/lang/CharSequence;": + return "(" + JB + ".isNone(" + e + ") || " + e + " instanceof String)"; + case "Ljava/lang/Integer;": case "Ljava/lang/Long;": case "Ljava/lang/Short;": case "Ljava/lang/Byte;": + return "(" + JB + ".isNone(" + e + ") || " + JB + ".isInt(" + e + "))"; + case "Ljava/lang/Double;": case "Ljava/lang/Float;": + return "(" + JB + ".isNone(" + e + ") || " + JB + ".isReal(" + e + "))"; + case "Ljava/lang/Boolean;": + return "(" + JB + ".isNone(" + e + ") || " + e + " instanceof " + RT + "PyBool)"; + case "Ljava/lang/Character;": + return "(" + JB + ".isNone(" + e + ") || " + JB + ".isChar(" + e + "))"; + case "Ljava/util/List;": case "Ljava/util/Collection;": case "Ljava/lang/Iterable;": + return "(" + JB + ".isSequence(" + e + ") || " + JB + ".toJava(" + e + ") instanceof " + t + + " || " + JB + ".isNone(" + e + "))"; + case "Ljava/util/Map;": + return "(" + e + " instanceof " + RT + "PyDict || " + JB + ".toJava(" + e + ") instanceof " + t + + " || " + JB + ".isNone(" + e + "))"; + default: + break; + } + String c = "(" + JB + ".isNone(" + e + ") || " + JB + ".toJava(" + e + ") instanceof " + t; + if (samOf(d) != null) { + c += " || " + JB + ".isFunction(" + e + ")"; + } + return c + ")"; + } + + private String convert(String d, String e) { + switch (d.charAt(0)) { + case 'I': return "(int) " + JB + ".asLong(" + e + ")"; + case 'J': return JB + ".asLong(" + e + ")"; + case 'S': return "(short) " + JB + ".asLong(" + e + ")"; + case 'B': return "(byte) " + JB + ".asLong(" + e + ")"; + case 'D': return JB + ".asDouble(" + e + ")"; + case 'F': return "(float) " + JB + ".asDouble(" + e + ")"; + case 'Z': return JB + ".asBoolean(" + e + ")"; + case 'C': return JB + ".asChar(" + e + ")"; + default: + break; + } + String t = javaType(d); + String none = JB + ".isNone(" + e + ") ? null : "; + switch (d) { + case "Ljava/lang/Object;": + return JB + ".toJava(" + e + ")"; + case "Ljava/lang/Integer;": + return "(" + none + "Integer.valueOf((int) " + JB + ".asLong(" + e + ")))"; + case "Ljava/lang/Long;": + return "(" + none + "Long.valueOf(" + JB + ".asLong(" + e + ")))"; + case "Ljava/lang/Short;": + return "(" + none + "Short.valueOf((short) " + JB + ".asLong(" + e + ")))"; + case "Ljava/lang/Byte;": + return "(" + none + "Byte.valueOf((byte) " + JB + ".asLong(" + e + ")))"; + case "Ljava/lang/Double;": + return "(" + none + "Double.valueOf(" + JB + ".asDouble(" + e + ")))"; + case "Ljava/lang/Float;": + return "(" + none + "Float.valueOf((float) " + JB + ".asDouble(" + e + ")))"; + case "Ljava/lang/Boolean;": + return "(" + none + "Boolean.valueOf(" + JB + ".asBoolean(" + e + ")))"; + case "Ljava/lang/Character;": + return "(" + none + "Character.valueOf(" + JB + ".asChar(" + e + ")))"; + case "Ljava/util/List;": case "Ljava/util/Collection;": case "Ljava/lang/Iterable;": + return "((" + t + ") " + JB + ".toJavaList(" + e + "))"; + case "Ljava/util/Map;": + return "((" + t + ") " + JB + ".toJavaMap(" + e + "))"; + default: + break; + } + if (samOf(d) != null) { + return adapter(d) + "(" + e + ")"; + } + return "((" + t + ") " + JB + ".toJava(" + e + "))"; + } + + /// The Java value `e` of type `d` as Python sees it. + private static String toPy(String d, String e) { + switch (d.charAt(0)) { + case 'I': case 'J': case 'S': case 'B': + return RT + "PyInt.of((long) " + e + ")"; + case 'D': case 'F': + return RT + "Py.ofFloat(" + e + ")"; + case 'Z': + return RT + "Py.bool(" + e + ")"; + case 'C': + return "String.valueOf(" + e + ")"; + default: + return JB + ".toPy(" + e + ")"; + } + } + + /// A Python callable's result returned from an adapter as type `d`. + private String fromPy(String d, String e) { + if (d.length() == 1 || "Ljava/lang/Object;".equals(d)) { + return convert(d, e); + } + return "((" + javaType(d) + ") " + JB + ".toJava(" + e + "))"; + } + + /// The method that wraps a Python callable as functional interface `d`. + private String adapter(String d) { + Integer id = adapters.get(d); + if (id != null) { + return "adapt" + id; + } + id = adapters.size(); + adapters.put(d, id); + String t = javaType(d); + Member m = samMethod(d); + List ps = params(m.desc); + String ret = returnType(m.desc); + StringBuilder sb = adapterCode; + sb.append(" @SuppressWarnings({\"rawtypes\", \"unchecked\"})\n"); + sb.append(" static ").append(t).append(" adapt").append(id).append("(final Object fn) {\n"); + sb.append(" Object j = ").append(JB).append(".toJava(fn);\n"); + sb.append(" if (j == null || j instanceof ").append(t).append(") {\n"); + sb.append(" return (").append(t).append(") j;\n }\n"); + sb.append(" return new ").append(t).append("() {\n"); + sb.append(" @Override\n public ").append(javaType(ret)).append(' ').append(m.name).append('('); + StringBuilder args = new StringBuilder(); + for (int i = 0; i < ps.size(); i++) { + if (i > 0) { + sb.append(", "); + args.append(", "); + } + sb.append(javaType(ps.get(i))).append(" p").append(i); + args.append(toPy(ps.get(i), "p" + i)); + } + sb.append(") {\n"); + String call = JB + ".callPy(fn, new Object[] {" + args + "})"; + if ("V".equals(ret)) { + sb.append(" ").append(call).append(";\n"); + } else { + sb.append(" Object r = ").append(call).append(";\n"); + sb.append(" return ").append(fromPy(ret, "r")).append(";\n"); + } + sb.append(" }\n };\n }\n\n"); + return "adapt" + id; + } + + // ---- emission --------------------------------------------------------- + + /// Appends the overload tests of `ms`, called on `recv` (a class name for + /// static calls and `new` for constructors). + private void overloads(StringBuilder sb, List ms, String recv, String name, String ind, boolean ctor) { + List sorted = new ArrayList(ms); + sorted.sort(Comparator.comparingInt((Member m) -> params(m.desc).size()).thenComparingInt(m -> { + int s = 0; + for (String p : params(m.desc)) { + s += rank(p); + } + return s; + })); + for (Member m : sorted) { + List ps = params(m.desc); + emitCall(sb, m, ps, recv, name, ind, ctor, false); + } + for (Member m : sorted) { + if (m.is(ClassPathReader.VARARGS)) { + emitCall(sb, m, params(m.desc), recv, name, ind, ctor, true); + } + } + } + + private void emitCall(StringBuilder sb, Member m, List ps, String recv, String name, String ind, + boolean ctor, boolean spread) { + int n = ps.size(); + StringBuilder cond = new StringBuilder(); + StringBuilder call = new StringBuilder(); + int fixed = spread ? n - 1 : n; + cond.append(spread ? "args.length >= " + fixed : "args.length == " + n); + for (int i = 0; i < fixed; i++) { + String c = check(ps.get(i), "args[" + i + "]"); + if (!"true".equals(c)) { + cond.append(" && ").append(c); + } + if (i > 0) { + call.append(", "); + } + call.append(convert(ps.get(i), "args[" + i + "]")); + } + sb.append(ind).append("if (").append(cond).append(") {\n"); + String in = ind + " "; + if (spread) { + String el = ps.get(n - 1).substring(1); + sb.append(in).append("boolean ok = true;\n"); + sb.append(in).append("for (int i = ").append(fixed).append("; i < args.length; i++) {\n"); + sb.append(in).append(" if (!").append(check(el, "args[i]")).append(") {\n"); + sb.append(in).append(" ok = false;\n").append(in).append(" break;\n"); + sb.append(in).append(" }\n").append(in).append("}\n"); + sb.append(in).append("if (ok) {\n"); + String arr = javaType(el); + String newArr = arr.contains("[") ? "new " + arr.substring(0, arr.indexOf('[')) + "[args.length - " + + fixed + "]" + arr.substring(arr.indexOf('[')) : "new " + arr + "[args.length - " + fixed + "]"; + sb.append(in).append(" ").append(javaType(ps.get(n - 1))).append(" va = ").append(newArr).append(";\n"); + sb.append(in).append(" for (int i = ").append(fixed).append("; i < args.length; i++) {\n"); + sb.append(in).append(" va[i - ").append(fixed).append("] = ").append(convert(el, "args[i]")).append(";\n"); + sb.append(in).append(" }\n"); + if (fixed > 0) { + call.append(", "); + } + call.append("va"); + in = in + " "; + } + String ret = ctor ? "L;" : returnType(m.desc); + String invocation = ctor ? "new " + recv + "(" + call + ")" : recv + "." + name + "(" + call + ")"; + if ("V".equals(ret)) { + sb.append(in).append(invocation).append(";\n"); + sb.append(in).append("return ").append(RT).append("Py.NONE;\n"); + } else { + sb.append(in).append("return ").append(toPy(ret, invocation)).append(";\n"); + } + if (spread) { + sb.append(ind).append(" }\n"); + } + sb.append(ind).append("}\n"); + } + + private static String signatures(List ms, String name) { + Set s = new TreeSet(); + for (Member m : ms) { + StringBuilder b = new StringBuilder(name).append('('); + List ps = params(m.desc); + for (int i = 0; i < ps.size(); i++) { + if (i > 0) { + b.append(", "); + } + String t = javaType(ps.get(i)); + b.append(t.substring(t.lastIndexOf('.') + 1)); + } + s.add(b.append(')').toString()); + } + return String.join(" or ", s); + } + + private static String q(String s) { + return "\"" + s + "\""; + } + + private static void tryOpen(StringBuilder sb) { + sb.append(" try {\n"); + } + + private static void tryClose(StringBuilder sb) { + sb.append(" } catch (").append(RT).append("PyException e) {\n throw e;\n"); + sb.append(" } catch (Exception e) {\n throw ").append(JB).append(".javaError(e);\n }\n"); + } + + private String emit(String pkg) { + List ids = new ArrayList(imported.keySet()); + // Subtypes before their supertypes, so an override is found first + // and an `instanceof` of the wider type does not take it. + List order = new ArrayList(universe); + Map depths = new LinkedHashMap(); + for (String c : order) { + depths.put(c, typeDepth(c, new HashSet())); + } + order.sort(Comparator.comparingInt((String c) -> -depths.get(c)).thenComparing(c -> c)); + + Map>> instanceMethods = new TreeMap>>(); + Map> instanceFields = new TreeMap>(); + for (String c : order) { + for (Member m : visibleMethods(c)) { + if (!m.is(ClassPathReader.STATIC) && !"".equals(m.name) && names.contains(m.name)) { + instanceMethods.computeIfAbsent(m.name, k -> new LinkedHashMap>()) + .computeIfAbsent(c, k -> new ArrayList()).add(m); + } + } + for (Member f : visibleFields(c)) { + if (!f.is(ClassPathReader.STATIC) && names.contains(f.name)) { + instanceFields.computeIfAbsent(f.name, k -> new LinkedHashMap()).putIfAbsent(c, f); + } + } + } + + StringBuilder body = new StringBuilder(); + // construct + body.append(" @Override\n public Object construct(int cls, Object[] args) {\n"); + body.append(" switch (cls) {\n"); + for (String c : ids) { + ClassInfo ci = cp.find(c); + if (!ci.is(ClassPathReader.ABSTRACT) && !ci.is(ClassPathReader.INTERFACE)) { + body.append(" case ").append(imported.get(c)).append(": return new").append(imported.get(c)) + .append("(args);\n"); + } + } + body.append(" default: return null;\n }\n }\n\n"); + for (String c : ids) { + ClassInfo ci = cp.find(c); + if (ci.is(ClassPathReader.ABSTRACT) || ci.is(ClassPathReader.INTERFACE)) { + continue; + } + List ctors = new ArrayList(); + for (Member m : visibleMethods(c)) { + if ("".equals(m.name)) { + ctors.add(m); + } + } + body.append(" @SuppressWarnings({\"rawtypes\", \"unchecked\"})\n"); + body.append(" private static Object new").append(imported.get(c)).append("(Object[] args) {\n"); + tryOpen(body); + overloads(body, ctors, javaType("L" + c + ";"), null, " ", true); + tryClose(body); + body.append(" throw ").append(JB).append(".noOverload(").append(q(simple(c))).append(", \"__init__\", args, ") + .append(q(signatures(ctors, simple(c)))).append(");\n }\n\n"); + } + + // statics + Map>> staticMethods = new TreeMap>>(); + Map> staticFields = new TreeMap>(); + for (String c : ids) { + int id = imported.get(c); + String cur = c; + while (cur != null && cp.find(cur) != null) { + for (Member m : visibleMethods(cur)) { + if (m.is(ClassPathReader.STATIC) && names.contains(m.name)) { + staticMethods.computeIfAbsent(id, k -> new TreeMap>()) + .computeIfAbsent(m.name, k -> new ArrayList()).add(m); + } + } + for (Member f : visibleFields(cur)) { + if (f.is(ClassPathReader.STATIC)) { + staticFields.computeIfAbsent(id, k -> new TreeMap()).putIfAbsent(f.name, f); + } + } + ClassInfo ci = cp.find(cur); + cur = ci.superName; + if (cur != null && !isPublic(cur)) { + break; + } + } + } + body.append(" @Override\n public Object getStatic(int cls, String name) {\n"); + body.append(" switch (cls) {\n"); + for (Map.Entry> e : staticFields.entrySet()) { + String c = javaType("L" + ids.get(e.getKey()) + ";"); + body.append(" case ").append(e.getKey()).append(":\n switch (name) {\n"); + for (Member f : e.getValue().values()) { + body.append(" case ").append(q(f.name)).append(": return ") + .append(toPy(f.desc, c + "." + f.name)).append(";\n"); + } + body.append(" default: return null;\n }\n"); + } + body.append(" default: return null;\n }\n }\n\n"); + body.append(" @Override\n public boolean hasStatic(int cls, String name) {\n"); + body.append(" switch (cls) {\n"); + for (Map.Entry>> e : staticMethods.entrySet()) { + body.append(" case ").append(e.getKey()).append(":\n switch (name) {\n"); + for (String n : e.getValue().keySet()) { + body.append(" case ").append(q(n)).append(":\n"); + } + body.append(" return true;\n default: return false;\n }\n"); + } + body.append(" default: return false;\n }\n }\n\n"); + body.append(" @Override\n public Object callStatic(int cls, String name, Object[] args) {\n"); + int sid = 0; + StringBuilder statics = new StringBuilder(); + body.append(" switch (cls) {\n"); + for (Map.Entry>> e : staticMethods.entrySet()) { + String c = javaType("L" + ids.get(e.getKey()) + ";"); + body.append(" case ").append(e.getKey()).append(":\n switch (name) {\n"); + for (Map.Entry> me : e.getValue().entrySet()) { + String fn = "static" + (sid++); + body.append(" case ").append(q(me.getKey())).append(": return ").append(fn).append("(args);\n"); + statics.append(" @SuppressWarnings({\"rawtypes\", \"unchecked\"})\n"); + statics.append(" private static Object ").append(fn).append("(Object[] args) {\n"); + tryOpen(statics); + overloads(statics, me.getValue(), c, me.getKey(), " ", false); + tryClose(statics); + statics.append(" throw ").append(JB).append(".noOverload(").append(q(simple(ids.get(e.getKey())))) + .append(", ").append(q(me.getKey())).append(", args, ") + .append(q(signatures(me.getValue(), me.getKey()))).append(");\n }\n\n"); + } + body.append(" default: return null;\n }\n"); + } + body.append(" default: return null;\n }\n }\n\n").append(statics); + + // instance members + body.append(" @Override\n public Object getField(Object target, String name) {\n"); + body.append(" switch (name) {\n"); + for (Map.Entry> e : instanceFields.entrySet()) { + body.append(" case ").append(q(e.getKey())).append(":\n"); + for (Map.Entry ce : e.getValue().entrySet()) { + String t = javaType("L" + ce.getKey() + ";"); + body.append(" if (target instanceof ").append(t).append(") {\n"); + body.append(" return ").append(toPy(ce.getValue().desc, "((" + t + ") target)." + e.getKey())) + .append(";\n }\n"); + } + body.append(" return null;\n"); + } + body.append(" default: return null;\n }\n }\n\n"); + body.append(" @Override\n public boolean hasMethod(Object target, String name) {\n"); + body.append(" switch (name) {\n"); + for (Map.Entry>> e : instanceMethods.entrySet()) { + body.append(" case ").append(q(e.getKey())).append(":\n return "); + boolean first = true; + for (String c : e.getValue().keySet()) { + if (!first) { + body.append("\n || "); + } + first = false; + body.append("target instanceof ").append(javaType("L" + c + ";")); + } + body.append(";\n"); + } + body.append(" default: return false;\n }\n }\n\n"); + body.append(" @Override\n public Object invoke(Object target, String name, Object[] args) {\n"); + body.append(" switch (name) {\n"); + int iid = 0; + StringBuilder invokers = new StringBuilder(); + for (Map.Entry>> e : instanceMethods.entrySet()) { + String fn = "invoke" + (iid++); + body.append(" case ").append(q(e.getKey())).append(": return ").append(fn).append("(target, args);\n"); + invokers.append(" @SuppressWarnings({\"rawtypes\", \"unchecked\"})\n"); + invokers.append(" private static Object ").append(fn).append("(Object target, Object[] args) {\n"); + tryOpen(invokers); + List all = new ArrayList(); + for (Map.Entry> ce : e.getValue().entrySet()) { + String t = javaType("L" + ce.getKey() + ";"); + invokers.append(" if (target instanceof ").append(t).append(") {\n"); + overloads(invokers, ce.getValue(), "((" + t + ") target)", e.getKey(), " ", false); + invokers.append(" }\n"); + all.addAll(ce.getValue()); + } + tryClose(invokers); + invokers.append(" throw ").append(JB).append(".noOverload(target.getClass().getName(), ") + .append(q(e.getKey())).append(", args, ").append(q(signatures(all, e.getKey()))).append(");\n }\n\n"); + } + body.append(" default: return null;\n }\n }\n\n").append(invokers); + + body.append(" @Override\n public boolean isInstance(int cls, Object value) {\n"); + body.append(" switch (cls) {\n"); + for (String c : ids) { + body.append(" case ").append(imported.get(c)).append(": return value instanceof ") + .append(javaType("L" + c + ";")).append(";\n"); + } + body.append(" default: return false;\n }\n }\n\n"); + body.append(" @Override\n public String className(int cls) {\n return NAMES[cls];\n }\n\n"); + + StringBuilder sb = new StringBuilder(); + sb.append("// Generated by the Codename One Python transpiler.\n"); + sb.append("package ").append(pkg).append(";\n\n"); + sb.append("/// The Java classes this program imports, bridged without reflection.\n"); + sb.append("@SuppressWarnings({\"rawtypes\", \"unchecked\", \"deprecation\", \"removal\"})\n"); + sb.append("public final class JavaInterop extends ").append(JB).append(" {\n"); + sb.append(" private static final String[] NAMES = {"); + for (int i = 0; i < ids.size(); i++) { + sb.append(i > 0 ? ", " : "").append(q(simple(ids.get(i)))); + } + sb.append("};\n\n"); + sb.append(" /// Installs the bridge and makes each `java.` importable.\n"); + sb.append(" public static void register() {\n"); + sb.append(" JavaInterop b = new JavaInterop();\n"); + sb.append(" ").append(JB).append(".install(b);\n"); + Set pkgs = new TreeSet(); + for (String m : modules.keySet()) { + String p = m; + while (p.contains(".")) { + pkgs.add(p); + p = p.substring(0, p.lastIndexOf('.')); + } + pkgs.add(p); + } + for (String p : pkgs) { + sb.append(" {\n"); + sb.append(" ").append(RT).append("PyModule m = ").append(RT).append("PyModule.begin(") + .append(q(p)).append(", new String[0]);\n"); + Map cls = modules.get(p); + if (cls != null) { + for (Map.Entry e : cls.entrySet()) { + sb.append(" m.setAttr(").append(q(e.getKey())).append(", b.type(") + .append(imported.get(e.getValue())).append("));\n"); + } + } + sb.append(" }\n"); + } + sb.append(" }\n\n"); + sb.append(body).append(adapterCode); + sb.append("}\n"); + return sb.toString(); + } + + private int typeDepth(String c, Set seen) { + if (!seen.add(c)) { + return 0; + } + int d = 0; + for (String s : supertypes(c)) { + if (cp.find(s) != null) { + d = Math.max(d, 1 + typeDepth(s, seen)); + } + } + seen.remove(c); + return d; + } + + private static String simple(String internal) { + String s = internal.substring(internal.lastIndexOf('/') + 1); + return s.substring(s.lastIndexOf('$') + 1); + } + + // ---- tree walk -------------------------------------------------------- + + static List all(Node root) { + List out = new ArrayList(); + walk(root, out, java.util.Collections.newSetFromMap(new java.util.IdentityHashMap())); + return out; + } + + private static void walk(Object o, List out, Set seen) { + if (o instanceof Node) { + if (!seen.add(o)) { + return; + } + out.add((Node) o); + for (Class c = o.getClass(); c != null && c != Object.class; c = c.getSuperclass()) { + for (Field f : c.getDeclaredFields()) { + if (Modifier.isStatic(f.getModifiers()) || f.getType().isPrimitive() || f.getType() == String.class) { + continue; + } + try { + f.setAccessible(true); + walk(f.get(o), out, seen); + } catch (IllegalAccessException e) { + throw new IllegalStateException(e); + } + } + } + } else if (o instanceof Collection) { + for (Object x : (Collection) o) { + walk(x, out, seen); + } + } + } +} diff --git a/maven/python-transpiler/src/test/java/com/codename1/python/transpiler/ConformanceTest.java b/maven/python-transpiler/src/test/java/com/codename1/python/transpiler/ConformanceTest.java index 4cc89b77428..014bb1d370b 100644 --- a/maven/python-transpiler/src/test/java/com/codename1/python/transpiler/ConformanceTest.java +++ b/maven/python-transpiler/src/test/java/com/codename1/python/transpiler/ConformanceTest.java @@ -41,6 +41,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.TreeMap; @@ -98,14 +99,16 @@ private static void run(Path program) throws Exception { Path cls = work.resolve("classes"); Files.createDirectories(cls); List args = new ArrayList(); - for (Map.Entry e : PythonTranspiler.transpile(sources, "main").entrySet()) { + String runtime = location(com.codename1.python.runtime.Py.class); + String core = location(com.codename1.io.Util.class); + // Programs may import Java classes: the JDK's, and the core's. + List classPath = Collections.singletonList(new File(core)); + for (Map.Entry e : PythonTranspiler.transpile(sources, "main", classPath).entrySet()) { Path f = gen.resolve(e.getKey()); Files.createDirectories(f.getParent()); Files.write(f, e.getValue().getBytes(StandardCharsets.US_ASCII)); args.add(f.toString()); } - String runtime = location(com.codename1.python.runtime.Py.class); - String core = location(com.codename1.io.Util.class); args.add(0, "-nowarn"); args.add(1, "--release"); args.add(2, "17"); diff --git a/maven/python-transpiler/src/test/java/com/codename1/python/transpiler/UnsupportedTest.java b/maven/python-transpiler/src/test/java/com/codename1/python/transpiler/UnsupportedTest.java index afeb499f3d7..ce92a074552 100644 --- a/maven/python-transpiler/src/test/java/com/codename1/python/transpiler/UnsupportedTest.java +++ b/maven/python-transpiler/src/test/java/com/codename1/python/transpiler/UnsupportedTest.java @@ -51,6 +51,11 @@ class UnsupportedTest { {"import sys\n\ndef t(*a):\n pass\n\nsys.settrace(t)\n", "6", "sys.settrace"}, {"import sys\nprint(sys._getframe().f_lineno)\n", "2", "sys._getframe"}, {"def f():\n x = 1\n return locals()\n", "3", "locals()"}, + {"x = 1\nfrom java.java.util import NoSuchList\n", "2", "no Java class java.util.NoSuchList"}, + {"import java.java.util\n", "1", "import the Java classes by name"}, + {"from java.java.util import *\n", "1", "imported by name"}, + {"from java.java.util import ArrayList\n\nclass L(ArrayList):\n pass\n", "3", + "cannot extend the Java class ArrayList"}, }; @TestFactory diff --git a/maven/python-transpiler/src/test/resources/conformance/errors_.out b/maven/python-transpiler/src/test/resources/conformance/errors_.out index 02f75b0b508..0d6dae9ff6c 100644 --- a/maven/python-transpiler/src/test/resources/conformance/errors_.out +++ b/maven/python-transpiler/src/test/resources/conformance/errors_.out @@ -70,3 +70,8 @@ ValueError | Unknown format code 'q' for object of type 'int' | None None False ValueError | bytes must be in range(0, 256) | None None False ValueError | value not found in bytearray | None None False AttributeError | 'frozenset' object has no attribute 'add' | None None False +SystemExit () None +SystemExit (None,) None +SystemExit (3,) 3 +SystemExit ('bye',) 'bye' +SystemExit (1, 2) (1, 2) diff --git a/maven/python-transpiler/src/test/resources/conformance/errors_.py b/maven/python-transpiler/src/test/resources/conformance/errors_.py index 91e2cd36b40..51054596066 100644 --- a/maven/python-transpiler/src/test/resources/conformance/errors_.py +++ b/maven/python-transpiler/src/test/resources/conformance/errors_.py @@ -175,3 +175,7 @@ def assert_fail(): show(lambda: bytes([256])) show(lambda: bytearray(b"ab").remove(99)) show(lambda: frozenset([1]).add) + +for args in [(), (None,), (3,), ("bye",), (1, 2)]: + e = SystemExit(*args) + print("SystemExit", args, repr(e.code)) diff --git a/maven/python-transpiler/src/test/resources/conformance/java_interop.out b/maven/python-transpiler/src/test/resources/conformance/java_interop.out new file mode 100644 index 00000000000..762818025e9 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/java_interop.out @@ -0,0 +1,11 @@ +[30] +4 4 5 x 2.5 None +[5, x, 2.5, null] +[9, 5, 3, 1] 9 1 True +a1true2.5z 10 a1true2.5z +43 2147483647 7 2.5 +1 1 2 True +True False False +caught True +type error: java.util.ArrayList.get() got (str); it takes get(Object) or get(int) +3 True True diff --git a/maven/python-transpiler/src/test/resources/conformance/java_interop.py b/maven/python-transpiler/src/test/resources/conformance/java_interop.py new file mode 100644 index 00000000000..e1b52ad5237 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/java_interop.py @@ -0,0 +1,46 @@ +# Java interop: from java. import . CPython cannot import +# java.*, so java_interop.out is written by hand -- do not regenerate it. +from java.java.util import ArrayList, HashMap, Collections +from java.java.lang import StringBuilder, Integer, Math + +class Bag: + def __init__(self): + self.items = [] + def add(self, x): + self.items.append(x * 10) + +b = Bag() +b.add(3) +print(b.items) + +l = ArrayList() +l.add(5) +l.add("x") +l.add(2.5) +l.add(None) +print(l.size(), len(l), l.get(0), l.get(1), l.get(2), l.get(3)) +print(l) +nums = ArrayList() +for v in [5, 3, 9, 1]: + nums.add(v) +Collections.sort(nums, lambda a, b: b - a) +print([x for x in nums], nums[0], nums[-1], 9 in nums) +sb = StringBuilder() +sb.append("a").append(1).append(True).append(2.5).append("z") +print(sb.toString(), sb.length(), str(sb)) +print(Integer.parseInt("42") + 1, Integer.MAX_VALUE, Math.max(3, 7), Math.max(2.5, 1.0)) +m = HashMap() +m.put("k", 1) +m.put("j", [1, 2]) +print(m.get("k"), m["k"], len(m), "k" in m) +print(isinstance(l, ArrayList), isinstance(sb, ArrayList), isinstance(b, ArrayList)) +try: + Integer.parseInt("nope") +except RuntimeError as e: + print("caught", "NumberFormatException" in str(e)) +try: + l.get("bad") +except TypeError as e: + print("type error:", e) +l2 = ArrayList([1, 2, 3]) +print(l2.size(), l2.equals(ArrayList([1, 2, 3])), l2 == ArrayList([1, 2, 3])) diff --git a/scripts/python-bench/build_parparvm.sh b/scripts/python-bench/build_parparvm.sh index b119263bc48..12bf518ac74 100755 --- a/scripts/python-bench/build_parparvm.sh +++ b/scripts/python-bench/build_parparvm.sh @@ -48,7 +48,7 @@ package com.pybench; public class PyBench { public static void main(String[] args) { com.codename1.python.runtime.Py.setArgv("bench", args); - com.codename1.generated.python.PythonModules.runMain(); + System.exit(com.codename1.generated.python.PythonModules.runMain()); } } JAVA diff --git a/scripts/python-bench/run_bench.py b/scripts/python-bench/run_bench.py index ed7778bfcf4..a52095afca2 100644 --- a/scripts/python-bench/run_bench.py +++ b/scripts/python-bench/run_bench.py @@ -33,7 +33,7 @@ public class BenchMain { public static void main(String[] args) { com.codename1.python.runtime.Py.setArgv("bench", args); - %s.PythonModules.runMain(); + System.exit(%s.PythonModules.runMain()); } } """ % GENERATED_PACKAGE From 17511796db5b2bc694a2be4dfd09988cfc5caff7 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Tue, 27 Oct 2026 17:36:33 +0200 Subject: [PATCH 003/112] Python: UI sample, project integration test and java.* against the device library from java.java.* resolves against the class path's own class library (java-runtime, JavaAPI) when it carries one, so an API the device lacks is a build error rather than a bridge that fails to link. python-project-test.sh imports scripts/python-samples/hello-ui (a Codename One UI written in Python), builds it with Maven and Gradle and stages every target; python-project.yml runs it. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/python-project.yml | 109 ++++++++++++++++++ .../codename1/maven/PythonProjectBuilder.java | 5 +- .../integration-tests/python-project-test.sh | 72 ++++++++++++ .../transpiler/api/PythonTranspiler.java | 6 +- .../transpiler/codegen/ClassPathReader.java | 48 +++++++- scripts/python-bench/build_parparvm.sh | 3 +- scripts/python-samples/hello-ui/src/main.py | 35 ++++++ 7 files changed, 267 insertions(+), 11 deletions(-) create mode 100644 .github/workflows/python-project.yml create mode 100755 maven/integration-tests/python-project-test.sh create mode 100644 scripts/python-samples/hello-ui/src/main.py diff --git a/.github/workflows/python-project.yml b/.github/workflows/python-project.yml new file mode 100644 index 00000000000..c8ed9bfe713 --- /dev/null +++ b/.github/workflows/python-project.yml @@ -0,0 +1,109 @@ +name: Python projects + +# Python applications built as Codename One applications: the transpiler and +# runtime (maven/python-transpiler, maven/python-runtime) and the build steps +# that import and transpile a project (build-engine, the Maven and Gradle +# plugins, the archetype's python profile). +# +# unit tests the importer, the incremental builder and the +# pom updater +# python-project-test.sh every sample under scripts/python-samples imported, +# built by Maven and Gradle and staged for iOS, +# Android, JavaScript and the desktop with identical +# uploads from both tools +# +# The transpiler's own conformance tests run in python-bench.yml. +on: + pull_request: + branches: [ master ] + paths: + - 'maven/python-transpiler/**' + - 'maven/python-runtime/**' + - 'maven/build-engine/**' + - 'maven/codenameone-maven-plugin/**' + - 'maven/codenameone-gradle-plugin/**' + - 'maven/cn1app-archetype/**' + - 'maven/pom.xml' + - 'maven/integration-tests/python-project-test.sh' + - 'maven/integration-tests/inc/**' + - 'scripts/python-samples/**' + - '.github/workflows/python-project.yml' + push: + branches: [ master ] + paths: + - 'maven/python-transpiler/**' + - 'maven/python-runtime/**' + - 'maven/build-engine/**' + - 'maven/codenameone-maven-plugin/**' + - 'maven/codenameone-gradle-plugin/**' + - 'maven/cn1app-archetype/**' + - 'maven/pom.xml' + - 'maven/integration-tests/python-project-test.sh' + - 'maven/integration-tests/inc/**' + - 'scripts/python-samples/**' + - '.github/workflows/python-project.yml' + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + python-project: + runs-on: ubuntu-24.04 + timeout-minutes: 60 + env: + CN1_GRADLE_REQUIRED: '1' + steps: + - uses: actions/checkout@v6 + # Installed FIRST so the JDK 8 below is the one JAVA_HOME names. + - name: Set up JDK 17 + uses: actions/setup-java@v5 + with: + java-version: '17' + distribution: 'temurin' + - run: | + echo "GRADLE_JAVA_HOME=$JAVA_HOME" >> $GITHUB_ENV + echo "JAVA17_HOME=$JAVA_HOME" >> $GITHUB_ENV + - name: Set up JDK 8 + uses: actions/setup-java@v5 + with: + java-version: '8' + distribution: 'temurin' + cache: 'maven' + # xvfb for the reactor build's JavaSE port; the application builds below + # compile CSS headless and need no display. + - name: Install xvfb + run: bash scripts/ci/apt-get-install.sh xvfb + - name: Build Codename One Maven artifacts (includes archetypes and both plugins) + env: + RETRY_ONLY_MATCHING: transient + RETRY_ATTEMPTS: '5' + RETRY_DELAY_SECONDS: '60' + run: | + cd maven + bash ../scripts/ci/retry.sh xvfb-run -a mvn install -Plocal-dev-javase -DskipTests -Darchetype.test.skip=true -Dmaven.javadoc.skip=true + # The runtime compiles at release 17, which the JDK 8 reactor skips. + - name: Build the Python runtime on JDK 17 + working-directory: maven + run: JAVA_HOME="$JAVA17_HOME" mvn -B install -DskipTests -Dmaven.javadoc.skip=true -pl python-runtime + - name: Unit tests + working-directory: maven + run: JAVA_HOME="$JAVA17_HOME" mvn -B -pl build-engine,codenameone-maven-plugin test -Dtest='Python*' -Dsurefire.failIfNoSpecifiedTests=false + - name: Refresh archetype catalog + run: | + cd maven + mvn archetype:update-local-catalog -Plocal-dev-javase + mvn archetype:crawl -Plocal-dev-javase + - name: Python samples, imported and built by Maven and Gradle + working-directory: maven/integration-tests + run: bash python-project-test.sh + - name: Upload the test projects' logs + if: failure() + uses: actions/upload-artifact@v7 + with: + name: python-project-logs + path: maven/integration-tests/build/python-project/**/*.log + if-no-files-found: ignore diff --git a/maven/build-engine/src/main/java/com/codename1/maven/PythonProjectBuilder.java b/maven/build-engine/src/main/java/com/codename1/maven/PythonProjectBuilder.java index 6fab8177886..d457835a864 100644 --- a/maven/build-engine/src/main/java/com/codename1/maven/PythonProjectBuilder.java +++ b/maven/build-engine/src/main/java/com/codename1/maven/PythonProjectBuilder.java @@ -74,8 +74,9 @@ public PythonProjectBuilder(File sourceDir, File outputDir, String mainModule, L } /// The jars and class directories `from java. import ` - /// resolves against: the project's compile class path. The JDK's own - /// classes are always found. The project's own Java sources are not + /// resolves against: the project's compile class path, whose + /// `java-runtime` supplies the device's `java.*`. The project's own Java + /// sources are not /// compiled yet when Python is transpiled, so they cannot be imported. public PythonProjectBuilder classPath(java.util.List entries) { this.classPath = entries; diff --git a/maven/integration-tests/python-project-test.sh b/maven/integration-tests/python-project-test.sh new file mode 100755 index 00000000000..2e66c82e1fb --- /dev/null +++ b/maven/integration-tests/python-project-test.sh @@ -0,0 +1,72 @@ +#!/bin/bash +# A Python project (scripts/python-samples/*) is imported into an archetype +# application and built for every target, by Maven and by Gradle. Checks, for +# each sample: +# +# - cn1:import-python-project copies the .py sources into +# common/src/main/python, records the entry module in +# codenameone_settings.properties and writes a main class that runs it; +# - the build transpiled them (the generated module classes, the registry +# and, for a program that imports Java classes, the JavaInterop bridge) +# and compiled them with the application; +# - every target's upload jar (staged, never sent) carries the transpiled +# classes and the Python runtime, so the iOS, Android, JavaScript and +# desktop builders all get them; +# - the Gradle build uploads the same entries as the Maven build. +# +# Needs the reactor installed (mvn install, with -Pflutter-modules for the +# python modules) and a JDK 17+; see inc/gradle.sh. +SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" +set -e +source "$SCRIPTPATH/inc/env.sh" +source "$SCRIPTPATH/inc/gradle.sh" +source "$SCRIPTPATH/inc/compat.sh" +SAMPLES="$SCRIPTPATH/../../scripts/python-samples" + +WORKDIR="$SCRIPTPATH/build/python-project" +rm -rf "$WORKDIR" +mkdir -p "$WORKDIR" + +FAILED=0 + +check_python_upload() { + local jar="$1" + assert_zip_has "$jar" '(^|/)com/codename1/generated/python/PythonModules\.class$' + assert_zip_has "$jar" '(^|/)com/codename1/python/runtime/Py\.class$' +} + +for sample_dir in "$SAMPLES"/*/; do + sample=$(basename "$sample_dir") + echo "== $sample" + W="$WORKDIR/$sample" + mkdir -p "$W" + PKG=com.acme.py + compat_generate_app "$sample" "$W" $PKG PyApp + MAPP="$W/app" + (cd "$MAPP" && mvn_local "com.codenameone:codenameone-maven-plugin:$CN1_VERSION:import-python-project" \ + "-Dcn1.python.import=$sample_dir") > "$W/import.log" 2>&1 \ + || { tail -40 "$W/import.log"; fail "$sample: import-python-project"; } + ls "$MAPP"/common/src/main/python/*.py > /dev/null 2>&1 || fail "$sample: the import copied no Python sources" + grep -q '^codename1.python.mainModule=' "$MAPP/common/codenameone_settings.properties" \ + || fail "$sample: the import recorded no entry module" + grep -q 'PythonModules.runMain()' "$MAPP/common/src/main/java/com/acme/py/PyApp.java" \ + || fail "$sample: the main class does not run the Python program" + + compat_build_common "$sample" "$W" "$MAPP" + classes="$MAPP/common/target/classes" + [ -e "$classes/com/codename1/generated/python/PythonModules.class" ] \ + || fail "$sample: the build produced no PythonModules" + if grep -rqE '^from java\.' "$sample_dir" --include='*.py'; then + [ -e "$classes/com/codename1/generated/python/JavaInterop.class" ] \ + || fail "$sample: a program importing Java classes produced no JavaInterop bridge" + fi + + compat_convert_to_gradle "$sample" "$W" "$MAPP" + GAPP="$W/app-gradle" + ls "$GAPP"/src/main/python/*.py > /dev/null 2>&1 || fail "$sample: the Gradle conversion dropped src/main/python" + + compat_stage_all "$sample" "$W" "$MAPP" "$GAPP" check_python_upload +done + +[ $FAILED -eq 0 ] || exit 1 +echo "python-project-test: OK" diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java index d7d9a4a9a13..d0d906e4b66 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java @@ -72,8 +72,10 @@ public static Map transpile(Map sources, String /// @param sources relative `.py` path to source text /// @param mainModule the module run as `__main__`, or null for a library /// @param classPath the jars and class directories a program's - /// `from java. import ` resolves against; the - /// JDK's own classes are always found + /// `from java. import ` resolves against. When it + /// carries a class library (an application's `java-runtime`), + /// `java.*` resolves there alone, so an API the device lacks is a + /// build error; otherwise `java.*` comes from the running JDK public static Map transpile(Map sources, String mainModule, List classPath) { Map out = new LinkedHashMap(); diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/ClassPathReader.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/ClassPathReader.java index 44e423bac78..ae10e2121ae 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/ClassPathReader.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/ClassPathReader.java @@ -95,8 +95,14 @@ public boolean is(int flag) { } private final List roots = new ArrayList(); + /// Whether the class path carries a class library of its own (an + /// application's `java-runtime`, ParparVM's `JavaAPI`): then `java.*` + /// resolves there alone, and an API the device lacks is reported at the + /// import instead of being bridged from the host JDK. + private final boolean ownClassLibrary; private final Map cache = new HashMap(); private FileSystem jrt; + private ZipFile rtJar; public ClassPathReader(List classPath) throws IOException { for (File f : classPath) { @@ -106,6 +112,7 @@ public ClassPathReader(List classPath) throws IOException { roots.add(new ZipFile(f)); } } + ownClassLibrary = fromRoots("java/lang/Object.class") != null; } /// The class with internal name `name` (`java/util/List`), or null. @@ -127,6 +134,17 @@ public ClassInfo find(String name) { } private byte[] bytes(String path) throws IOException { + byte[] b = fromRoots(path); + if (b != null) { + return b; + } + if (!ownClassLibrary && (path.startsWith("java/") || path.startsWith("javax/"))) { + return jdkClass(path); + } + return null; + } + + private byte[] fromRoots(String path) throws IOException { for (Object r : roots) { if (r instanceof File) { File f = new File((File) r, path); @@ -143,16 +161,31 @@ private byte[] bytes(String path) throws IOException { } } } - if (path.startsWith("java/") || path.startsWith("javax/")) { - if (jrt == null) { + return null; + } + + /// A JDK class: from the module image on JDK 9 and newer, from `rt.jar` + /// on JDK 8. + private byte[] jdkClass(String path) throws IOException { + if (jrt == null && rtJar == null) { + File rt = new File(System.getProperty("java.home"), "lib/rt.jar"); + if (rt.isFile()) { + rtJar = new ZipFile(rt); + } else { jrt = FileSystems.getFileSystem(URI.create("jrt:/")); } - Path p = jrt.getPath("modules", "java.base", path); - if (Files.isRegularFile(p)) { - return Files.readAllBytes(p); + } + if (rtJar != null) { + ZipEntry e = rtJar.getEntry(path); + if (e == null) { + return null; + } + try (InputStream in = rtJar.getInputStream(e)) { + return readAll(in); } } - return null; + Path p = jrt.getPath("modules", "java.base", path); + return Files.isRegularFile(p) ? Files.readAllBytes(p) : null; } private static byte[] readAll(InputStream in) throws IOException { @@ -241,5 +274,8 @@ public void close() throws IOException { ((ZipFile) r).close(); } } + if (rtJar != null) { + rtJar.close(); + } } } diff --git a/scripts/python-bench/build_parparvm.sh b/scripts/python-bench/build_parparvm.sh index 12bf518ac74..c616dbf3828 100755 --- a/scripts/python-bench/build_parparvm.sh +++ b/scripts/python-bench/build_parparvm.sh @@ -41,7 +41,8 @@ mkdir -p "$W/src" "$W/gen/com/pybench" "$W/jclasses" "$W/javaapi" "$W/out" SRC_DIR="${PY_SOURCES:-$HERE/kernels}" cp "$SRC_DIR/$K.py" "$W/src/" [ -f "$HERE/kernels/_harness.py" ] && cp "$HERE/kernels/_harness.py" "$W/src/" -"$J17/bin/java" -cp "$TRANSPILER" com.codename1.python.transpiler.api.Main "$W/src" "$W/gen" "$K" +# Java imports resolve against JavaAPI, the class library the binary links. +"$J17/bin/java" -cp "$TRANSPILER" com.codename1.python.transpiler.api.Main -cp "$JAVAAPI:$CORE" "$W/src" "$W/gen" "$K" cat > "$W/gen/com/pybench/PyBench.java" < Date: Tue, 27 Oct 2026 17:36:59 +0200 Subject: [PATCH 004/112] Python: site package root beside the stdlib Co-Authored-By: Claude Opus 5.5 --- .../python/transpiler/api/PythonTranspiler.java | 10 ++++++++++ .../src/main/resources/com/codename1/python/site/.keep | 0 2 files changed, 10 insertions(+) create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/.keep diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java index d0d906e4b66..099091a1f25 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java @@ -60,6 +60,13 @@ public final class PythonTranspiler { /// those; a module of its own with the same name wins. private static final String STDLIB = "/com/codename1/python/stdlib/"; + /// Where the popular third-party packages live (`requests`, `pydantic`, + /// `attrs`, `yaml`, `dateutil`, `numpy`...): the package's public API + /// written for this runtime, found the same way as the standard library + /// and after it. The real packages build methods with `exec` or call C, + /// neither of which a translated program can do. + private static final String SITE = "/com/codename1/python/site/"; + private PythonTranspiler() { } @@ -283,6 +290,9 @@ private static List> bodiesOf(Node.Stmt s) { private static String resource(String path) { InputStream in = PythonTranspiler.class.getResourceAsStream(STDLIB + path); + if (in == null) { + in = PythonTranspiler.class.getResourceAsStream(SITE + path); + } if (in == null) { return null; } diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/.keep b/maven/python-transpiler/src/main/resources/com/codename1/python/site/.keep new file mode 100644 index 00000000000..e69de29bb2d From b7f26b0b7f4842b97011b5d88a3a9ed8eea39a5e Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Tue, 27 Oct 2026 17:46:48 +0200 Subject: [PATCH 005/112] Python: requests and httpx over the platform HTTP client PyHttp is the client the packages are written over: ConnectionRequest in an application, installable like PyFileSystem. requests (verbs, Session, auth, params/json/form bodies, cookies, exceptions) and httpx (verbs, Client, AsyncClient, URL, Headers, HTTPStatusError) are our own Python over it; http_requests and http_httpx match the real packages' output against the same test server. BaseException.__new__ ignores keyword arguments, as CPython's does, so an exception subclass whose __init__ takes keywords can be raised. Co-Authored-By: Claude Opus 5.5 --- .../codename1/python/runtime/Builtins.java | 9 +- .../com/codename1/python/runtime/Cn1Http.java | 121 ++++ .../com/codename1/python/runtime/PyHttp.java | 80 +++ .../codename1/python/runtime/StdModules.java | 42 ++ .../resources/com/codename1/python/site/.keep | 0 .../com/codename1/python/site/_cn1_http.py | 205 ++++++ .../codename1/python/site/httpx/__init__.py | 673 ++++++++++++++++++ .../python/site/requests/__init__.py | 331 +++++++++ .../python/site/requests/exceptions.py | 98 +++ .../python/site/requests/structures.py | 94 +++ .../python/transpiler/ConformanceTest.java | 35 +- .../python/transpiler/HttpTestServer.java | 183 +++++ .../codename1/python/transpiler/JvmHttp.java | 85 +++ .../test/resources/conformance/errors_.out | 1 + .../src/test/resources/conformance/errors_.py | 13 + .../test/resources/conformance/http_httpx.out | 21 + .../test/resources/conformance/http_httpx.py | 60 ++ .../resources/conformance/http_requests.out | 22 + .../resources/conformance/http_requests.py | 66 ++ 19 files changed, 2136 insertions(+), 3 deletions(-) create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/Cn1Http.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/PyHttp.java delete mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/.keep create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/_cn1_http.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/httpx/__init__.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/requests/__init__.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/requests/exceptions.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/requests/structures.py create mode 100644 maven/python-transpiler/src/test/java/com/codename1/python/transpiler/HttpTestServer.java create mode 100644 maven/python-transpiler/src/test/java/com/codename1/python/transpiler/JvmHttp.java create mode 100644 maven/python-transpiler/src/test/resources/conformance/http_httpx.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/http_httpx.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/http_requests.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/http_requests.py diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Builtins.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Builtins.java index fc60b1b3cb2..86033e358d4 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Builtins.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Builtins.java @@ -336,7 +336,14 @@ private static void installExceptions() { } return Py.NONE; })); - b.define("__new__", new PyStaticMethod(PyBuiltin.positional("__new__", a -> { + // Keyword arguments are accepted and ignored, as CPython's are: they + // are for a subclass's __init__, and args holds the positional ones. + b.define("__new__", new PyStaticMethod(new PyBuiltin("__new__", (all, kw) -> { + Object[] a = all; + if (kw.length > 0) { + a = new Object[all.length - kw.length]; + System.arraycopy(all, 0, a, 0, a.length); + } if (a.length == 0 || !(a[0] instanceof PyType)) { throw Py.notATypeObject("BaseException", a); } diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Cn1Http.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Cn1Http.java new file mode 100644 index 00000000000..6233f9429f0 --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Cn1Http.java @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +import com.codename1.io.ConnectionRequest; +import com.codename1.io.Data; +import com.codename1.io.NetworkManager; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.List; + +/// HTTP through Codename One's `ConnectionRequest`: the platform's own +/// client on every target, with its proxy, certificate and cookie handling. +final class Cn1Http extends PyHttp { + @Override + public Response send(String method, String url, String[] headers, final byte[] body, int timeoutMillis, + boolean followRedirects) throws IOException { + Request r = new Request(); + r.setUrl(url); + r.setHttpMethod(method); + r.setPost(body != null); + r.setFailSilently(true); + r.setReadResponseForErrors(true); + r.setDuplicateSupported(true); + r.setFollowRedirects(followRedirects); + if (timeoutMillis > 0) { + r.setTimeout(timeoutMillis); + r.setReadTimeout(timeoutMillis); + } + for (int i = 0; i + 1 < headers.length; i += 2) { + r.addRequestHeader(headers[i], headers[i + 1]); + } + if (body != null) { + r.setRequestBody(new Data() { + @Override + public void appendTo(OutputStream output) throws IOException { + output.write(body); + } + + @Override + public long getSize() { + return body.length; + } + }); + } + NetworkManager.getInstance().addToQueueAndWait(r); + if (r.failure != null && r.getResponseCode() <= 0) { + throw r.failure instanceof IOException ? (IOException) r.failure + : new IOException(String.valueOf(r.failure.getMessage())); + } + int status = r.getResponseCode(); + if (status <= 0) { + throw new IOException("no response from " + url); + } + String reason = r.getResponseErrorMessage(); + byte[] data = r.getResponseData(); + return new Response(status, reason == null ? "" : reason, url, r.headers.toArray(new String[0]), + data == null ? new byte[0] : data); + } + + /// Keeps what Python needs: every header as received, and the failure + /// instead of the default error dialog. + private static final class Request extends ConnectionRequest { + final List headers = new ArrayList(); + Exception failure; + + @Override + protected void readHeaders(Object connection) throws IOException { + String[] names = getHeaderFieldNames(connection); + if (names == null) { + return; + } + for (String n : names) { + if (n == null) { + continue; + } + String[] values = getHeaders(connection, n); + if (values == null) { + continue; + } + for (String v : values) { + headers.add(n); + headers.add(v); + } + } + } + + @Override + protected void handleException(Exception err) { + failure = err; + } + + @Override + protected void handleErrorResponseCode(int code, String message) { + // The response, body included, is the caller's to read. + } + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyHttp.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyHttp.java new file mode 100644 index 00000000000..41c8f2a439c --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyHttp.java @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +import java.io.IOException; + +/// The HTTP client `requests`, `httpx` and `urllib.request` are written over. +/// +/// An application sends through Codename One's `ConnectionRequest` unless +/// something else is installed: the server's own client on the backend, a +/// stub in a test. A request is synchronous; on the EDT the network manager +/// keeps the UI running while it waits. +public abstract class PyHttp { + private static PyHttp current; + + /// The client in use. + public static PyHttp get() { + if (current == null) { + current = new Cn1Http(); + } + return current; + } + + /// Replaces the client, for a host that is not a Codename One + /// application. + public static void install(PyHttp http) { + current = http; + } + + /// A response, of any status. + public static final class Response { + public final int status; + public final String reason; + /// The URL that answered, after redirects. + public final String url; + /// Header names and values, alternating, in the order received; a + /// header sent twice appears twice. + public final String[] headers; + public final byte[] body; + + public Response(int status, String reason, String url, String[] headers, byte[] body) { + this.status = status; + this.reason = reason; + this.url = url; + this.headers = headers; + this.body = body; + } + } + + /// Sends one request. + /// + /// @param headers names and values, alternating + /// @param body the request body, or null + /// @param timeoutMillis 0 for the client's default + /// @throws IOException when no response arrived (the host could not be + /// reached, the connection failed or timed out) + public abstract Response send(String method, String url, String[] headers, byte[] body, int timeoutMillis, + boolean followRedirects) throws IOException; +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java index 1c683f8a6c9..77a37267180 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java @@ -57,6 +57,10 @@ static boolean provide(String name) { PyModule.registerModule(fileSystem()); return true; } + if ("_cn1http".equals(name)) { + PyModule.registerModule(http()); + return true; + } if ("builtins".equals(name)) { PyModule m = new PyModule("builtins", new String[0]); for (java.util.Map.Entry e : Builtins.NAMES.entrySet()) { @@ -251,6 +255,44 @@ private static PyModule fileSystem() { return m; } + /// `_cn1http`: `request(method, url, headers, body, timeout, redirects)` + /// over [PyHttp], answering `(status, reason, url, headers, body)` with + /// the headers a list of pairs. No response at all raises `OSError`. + private static PyModule http() { + PyModule m = new PyModule("_cn1http", new String[0]); + m.setAttr("request", PyBuiltin.positional("request", a -> { + if (!(a[2] instanceof PyList)) { + throw Py.typeError("headers must be a list of (name, value) pairs"); + } + PyList hs = (PyList) a[2]; + String[] headers = new String[hs.size() * 2]; + for (int i = 0; i < hs.size(); i++) { + Object h = hs.get(i); + if (!(h instanceof PyTuple) || ((PyTuple) h).size() != 2) { + throw Py.typeError("headers must be a list of (name, value) pairs"); + } + PyTuple t = (PyTuple) h; + headers[2 * i] = Py.toStr(t.get(0)); + headers[2 * i + 1] = Py.toStr(t.get(1)); + } + byte[] body = a[3] == Py.NONE ? null : PyBytes.like(a[3]); //NOPMD CompareObjectsWithEquals - identity is meant + int timeout = a[4] == Py.NONE ? 0 : (int) Math.ceil(Py.asDouble(a[4]) * 1000); //NOPMD CompareObjectsWithEquals - identity is meant + PyHttp.Response r; + try { + r = PyHttp.get().send(Py.toStr(a[0]), Py.toStr(a[1]), headers, body, timeout, Py.isTrue(a[5])); + } catch (java.io.IOException e) { + throw Py.error(Py.OS_ERROR, String.valueOf(e.getMessage())); //NOPMD PreserveStackTrace - a Python exception has no Java cause + } + Object[] pairs = new Object[r.headers.length / 2]; + for (int i = 0; i < pairs.length; i++) { + pairs[i] = new PyTuple(new Object[] {Py.str(r.headers[2 * i]), Py.str(r.headers[2 * i + 1])}); + } + return new PyTuple(new Object[] {PyInt.of(r.status), Py.str(r.reason), Py.str(r.url), new PyList(pairs), + new PyBytes(r.body)}); + })); + return m; + } + private interface D1 { double f(double x); } diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/.keep b/maven/python-transpiler/src/main/resources/com/codename1/python/site/.keep deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/_cn1_http.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/_cn1_http.py new file mode 100644 index 00000000000..ef5f00a1311 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/_cn1_http.py @@ -0,0 +1,205 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""What requests and httpx share: headers, bodies, URLs and the transport. + +Requests go through _cn1http, the platform's own HTTP client (Codename +One's ConnectionRequest in an application), so proxies, certificates and +the network indicator behave as they do for the rest of the application. +""" + +import _cn1http +import base64 as _base64 +import json as _json +import time as _time +from urllib.parse import urlencode, urlsplit, urlunsplit + + +class Headers: + """Case-insensitive headers that keep the order and spelling given.""" + + def __init__(self, data=None): + self._items = [] + if data is not None: + self.update(data) + + def update(self, data): + if isinstance(data, Headers): + data = data.multi_items() + elif hasattr(data, "items"): + data = list(data.items()) + for k, v in data: + self[k] = v + + def multi_items(self): + return list(self._items) + + def add(self, key, value): + self._items.append((key, value)) + + def get_list(self, key): + lk = key.lower() + return [v for k, v in self._items if k.lower() == lk] + + def __getitem__(self, key): + values = self.get_list(key) + if not values: + raise KeyError(key) + return ", ".join(values) + + def get(self, key, default=None): + values = self.get_list(key) + return ", ".join(values) if values else default + + def __setitem__(self, key, value): + lk = key.lower() + kept = [] + placed = False + for k, v in self._items: + if k.lower() == lk: + if not placed: + kept.append((key, value)) + placed = True + else: + kept.append((k, v)) + if not placed: + kept.append((key, value)) + self._items = kept + + def __delitem__(self, key): + lk = key.lower() + if not any(k.lower() == lk for k, _ in self._items): + raise KeyError(key) + self._items = [(k, v) for k, v in self._items if k.lower() != lk] + + def __contains__(self, key): + lk = key.lower() + return any(k.lower() == lk for k, _ in self._items) + + def keys(self): + seen = [] + for k, _ in self._items: + if k.lower() not in [s.lower() for s in seen]: + seen.append(k) + return seen + + def values(self): + return [self[k] for k in self.keys()] + + def items(self): + return [(k, self[k]) for k in self.keys()] + + def __iter__(self): + return iter(self.keys()) + + def __len__(self): + return len(self.keys()) + + def copy(self): + return type(self)(self.multi_items()) + + def setdefault(self, key, default=None): + if key not in self: + self[key] = default + return self[key] + + def pop(self, key, *default): + if key in self: + v = self[key] + del self[key] + return v + if default: + return default[0] + raise KeyError(key) + + def lower_items(self): + return [(k.lower(), v) for k, v in self.items()] + + def __eq__(self, other): + if isinstance(other, Headers): + other = dict(other.lower_items()) + elif hasattr(other, "items"): + other = {k.lower(): v for k, v in other.items()} + else: + return NotImplemented + return dict(self.lower_items()) == other + + def __repr__(self): + return repr(dict(self.items())) + + +def with_params(url, params): + """`url` with `params` (a mapping or pairs; a list value repeats, None + drops the key) added to its query.""" + if not params: + return url + if isinstance(params, (str, bytes)): + extra = params.decode("utf-8") if isinstance(params, bytes) else params + else: + pairs = params.items() if hasattr(params, "items") else params + flat = [] + for k, v in pairs: + if v is None: + continue + if isinstance(v, (list, tuple)): + for item in v: + flat.append((k, item)) + else: + flat.append((k, v)) + extra = urlencode(flat) + if not extra: + return url + parts = urlsplit(url) + query = parts.query + "&" + extra if parts.query else extra + return urlunsplit((parts.scheme, parts.netloc, parts.path, query, parts.fragment)) + + +def form_body(data): + pairs = data.items() if hasattr(data, "items") else data + flat = [] + for k, v in pairs: + if isinstance(v, (list, tuple)): + for item in v: + flat.append((k, item)) + elif v is not None: + flat.append((k, v)) + return urlencode(flat).encode("ascii") + + +def basic_auth(user, password): + if isinstance(user, str): + user = user.encode("latin-1") + if isinstance(password, str): + password = password.encode("latin-1") + return "Basic " + _base64.b64encode(user + b":" + password).decode("ascii") + + +def charset(content_type): + """The charset parameter of a Content-Type, or None.""" + if not content_type: + return None + for part in content_type.split(";")[1:]: + k, _, v = part.strip().partition("=") + if k.strip().lower() == "charset": + return v.strip().strip("'\"") or None + return None + + +def send(method, url, headers, body, timeout, follow_redirects): + """Sends a request; answers (status, reason, url, [(name, value)], body, + seconds). Raises OSError when no response arrived.""" + start = _time.monotonic() + status, reason, final_url, pairs, content = _cn1http.request(method, url, headers.multi_items(), body, + timeout, follow_redirects) + return status, reason, final_url, pairs, content, _time.monotonic() - start + + +def json_bytes(obj, compact): + if compact: + return _json.dumps(obj, ensure_ascii=False, separators=(",", ":"), allow_nan=False).encode("utf-8") + return _json.dumps(obj, allow_nan=False).encode("utf-8") + + +def is_timeout(message): + m = (message or "").lower() + return "timed out" in m or "timeout" in m diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/httpx/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/httpx/__init__.py new file mode 100644 index 00000000000..cdaa072b002 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/httpx/__init__.py @@ -0,0 +1,673 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""The httpx API over the platform's HTTP client. + +`httpx.get` and the other verbs, `Client` and `AsyncClient` (base URL, +default headers, params and timeout), `Response`, `Request`, `URL`, +`Headers` and httpx's exception hierarchy. As in httpx, redirects are not +followed unless `follow_redirects=True`. Streaming, HTTP/2, transports and +file uploads are not provided. The asynchronous client runs each request +to completion when it is awaited. +""" + +import datetime as _datetime +import json as _json +from urllib.parse import urljoin as _urljoin, urlsplit as _urlsplit, parse_qsl as _parse_qsl + +import _cn1_http + +__version__ = "0.28.1" + +_REASONS = { + 100: "Continue", 101: "Switching Protocols", 200: "OK", 201: "Created", 202: "Accepted", + 204: "No Content", 206: "Partial Content", 301: "Moved Permanently", 302: "Found", 303: "See Other", + 304: "Not Modified", 307: "Temporary Redirect", 308: "Permanent Redirect", 400: "Bad Request", + 401: "Unauthorized", 403: "Forbidden", 404: "Not Found", 405: "Method Not Allowed", 409: "Conflict", + 410: "Gone", 418: "I'm a teapot", 422: "Unprocessable Entity", 429: "Too Many Requests", + 500: "Internal Server Error", 501: "Not Implemented", 502: "Bad Gateway", 503: "Service Unavailable", + 504: "Gateway Timeout"} + + +class HTTPError(Exception): + def __init__(self, message): + super().__init__(message) + self._request = None + + @property + def request(self): + if self._request is None: + raise RuntimeError("The .request property has not been set.") + return self._request + + @request.setter + def request(self, request): + self._request = request + + +class RequestError(HTTPError): + def __init__(self, message, *, request=None): + super().__init__(message) + self._request = request + + +class TransportError(RequestError): + pass + + +class TimeoutException(TransportError): + pass + + +class ConnectTimeout(TimeoutException): + pass + + +class ReadTimeout(TimeoutException): + pass + + +class WriteTimeout(TimeoutException): + pass + + +class PoolTimeout(TimeoutException): + pass + + +class NetworkError(TransportError): + pass + + +class ConnectError(NetworkError): + pass + + +class ReadError(NetworkError): + pass + + +class ProtocolError(TransportError): + pass + + +class UnsupportedProtocol(TransportError): + pass + + +class DecodingError(RequestError): + pass + + +class TooManyRedirects(RequestError): + pass + + +class HTTPStatusError(HTTPError): + def __init__(self, message, *, request, response): + super().__init__(message) + self.request = request + self.response = response + + +class InvalidURL(Exception): + pass + + +class StreamError(RuntimeError): + pass + + +class codes: + CONTINUE = 100 + OK = 200 + CREATED = 201 + ACCEPTED = 202 + NO_CONTENT = 204 + MOVED_PERMANENTLY = 301 + FOUND = 302 + SEE_OTHER = 303 + NOT_MODIFIED = 304 + TEMPORARY_REDIRECT = 307 + PERMANENT_REDIRECT = 308 + BAD_REQUEST = 400 + UNAUTHORIZED = 401 + FORBIDDEN = 403 + NOT_FOUND = 404 + METHOD_NOT_ALLOWED = 405 + CONFLICT = 409 + IM_A_TEAPOT = 418 + UNPROCESSABLE_ENTITY = 422 + TOO_MANY_REQUESTS = 429 + INTERNAL_SERVER_ERROR = 500 + BAD_GATEWAY = 502 + SERVICE_UNAVAILABLE = 503 + GATEWAY_TIMEOUT = 504 + + @staticmethod + def get_reason_phrase(value): + return _REASONS.get(value, "") + + +class Headers(_cn1_http.Headers): + def __init__(self, headers=None, encoding=None): + super().__init__(headers) + self.encoding = encoding or "ascii" + + def keys(self): + return [k.lower() for k in super().keys()] + + def items(self): + return [(k, self[k]) for k in self.keys()] + + @property + def raw(self): + return [(k.lower().encode("ascii"), v.encode("ascii")) for k, v in self._items] + + def __repr__(self): + return f"Headers({dict(self.items())!r})" + + +class QueryParams: + def __init__(self, pairs=()): + self._pairs = list(pairs) + + def get(self, key, default=None): + for k, v in self._pairs: + if k == key: + return v + return default + + def get_list(self, key): + return [v for k, v in self._pairs if k == key] + + def __getitem__(self, key): + for k, v in self._pairs: + if k == key: + return v + raise KeyError(key) + + def __contains__(self, key): + return any(k == key for k, _ in self._pairs) + + def keys(self): + return list(dict(self._pairs).keys()) + + def items(self): + return list(dict(self._pairs).items()) + + def multi_items(self): + return list(self._pairs) + + def __len__(self): + return len(self.keys()) + + def __iter__(self): + return iter(self.keys()) + + def __str__(self): + from urllib.parse import urlencode + return urlencode(self._pairs) + + def __repr__(self): + return f"QueryParams({str(self)!r})" + + +class URL: + def __init__(self, url="", params=None): + if isinstance(url, URL): + url = str(url) + if params: + url = _cn1_http.with_params(url, params) + self._url = url + self._parts = _urlsplit(url) + + @property + def scheme(self): + return self._parts.scheme + + @property + def host(self): + return self._parts.hostname or "" + + @property + def port(self): + return self._parts.port + + @property + def path(self): + return self._parts.path or "/" + + @property + def query(self): + return self._parts.query.encode("ascii") + + @property + def params(self): + return QueryParams(_parse_qsl(self._parts.query, keep_blank_values=True)) + + @property + def fragment(self): + return self._parts.fragment + + @property + def netloc(self): + return self._parts.netloc.encode("ascii") + + @property + def is_absolute_url(self): + return bool(self._parts.scheme and self._parts.netloc) + + @property + def is_relative_url(self): + return not self.is_absolute_url + + def join(self, url): + return URL(_urljoin(self._url, str(url))) + + def copy_with(self, **kwargs): + p = self._parts + from urllib.parse import urlunsplit + return URL(urlunsplit((kwargs.get("scheme", p.scheme), kwargs.get("netloc", p.netloc), + kwargs.get("path", p.path), kwargs.get("query", p.query), + kwargs.get("fragment", p.fragment)))) + + def __str__(self): + return self._url + + def __repr__(self): + return f"URL({self._url!r})" + + def __eq__(self, other): + if isinstance(other, (URL, str)): + return str(self) == str(other) + return NotImplemented + + def __hash__(self): + return hash(self._url) + + +class Request: + def __init__(self, method, url, *, params=None, headers=None, content=None, data=None, json=None, + files=None, cookies=None, extensions=None): + if files: + raise NotImplementedError("file uploads are not supported") + self.method = method.upper() + self.url = URL(url, params=params) + self.headers = Headers(headers) + body = b"" + if content is not None: + body = content.encode("utf-8") if isinstance(content, str) else bytes(content) + elif data is not None: + body = _cn1_http.form_body(data) + self.headers.setdefault("Content-Type", "application/x-www-form-urlencoded") + elif json is not None: + body = _cn1_http.json_bytes(json, True) + self.headers.setdefault("Content-Type", "application/json") + if cookies: + self.headers["Cookie"] = "; ".join(f"{k}={v}" for k, v in cookies.items()) + self.content = body + self.extensions = extensions or {} + + def read(self): + return self.content + + def __repr__(self): + return f"" + + +class Response: + def __init__(self, status_code, *, headers=None, content=None, text=None, json=None, request=None): + self.status_code = status_code + self.headers = Headers(headers) + if content is None and text is not None: + content = text.encode("utf-8") + self.headers.setdefault("Content-Type", "text/plain; charset=utf-8") + elif content is None and json is not None: + content = _cn1_http.json_bytes(json, True) + self.headers.setdefault("Content-Type", "application/json") + self.content = content if content is not None else b"" + self._request = request + self.http_version = "HTTP/1.1" + self.elapsed = _datetime.timedelta(0) + self.history = [] + self.next_request = None + self.extensions = {} + self.default_encoding = "utf-8" + + @property + def request(self): + if self._request is None: + raise RuntimeError("The request instance has not been set on this response.") + return self._request + + @request.setter + def request(self, value): + self._request = value + + @property + def url(self): + return self.request.url + + @property + def reason_phrase(self): + return codes.get_reason_phrase(self.status_code) + + @property + def encoding(self): + return _cn1_http.charset(self.headers.get("content-type")) or self.default_encoding + + @property + def charset_encoding(self): + return _cn1_http.charset(self.headers.get("content-type")) + + @property + def text(self): + try: + return self.content.decode(self.encoding, "replace") + except LookupError: + return self.content.decode("utf-8", "replace") + + def json(self, **kwargs): + return _json.loads(self.content.decode(self.charset_encoding or "utf-8"), **kwargs) + + def read(self): + return self.content + + async def aread(self): + return self.content + + def iter_bytes(self, chunk_size=None): + size = chunk_size or len(self.content) or 1 + for i in range(0, len(self.content), size): + yield self.content[i:i + size] + + def iter_text(self, chunk_size=None): + t = self.text + size = chunk_size or len(t) or 1 + for i in range(0, len(t), size): + yield t[i:i + size] + + def iter_lines(self): + for line in self.text.splitlines(): + yield line + + @property + def is_informational(self): + return 100 <= self.status_code < 200 + + @property + def is_success(self): + return 200 <= self.status_code < 300 + + @property + def is_redirect(self): + return self.status_code in (301, 302, 303, 307, 308) and "location" in self.headers + + @property + def is_client_error(self): + return 400 <= self.status_code < 500 + + @property + def is_server_error(self): + return 500 <= self.status_code < 600 + + @property + def is_error(self): + return 400 <= self.status_code < 600 + + @property + def has_redirect_location(self): + return self.is_redirect + + def raise_for_status(self): + request = self._request + if request is None: + raise RuntimeError("Cannot call `raise_for_status` as the request instance has not been set on " + "this response.") + if self.is_success: + return self + if self.has_redirect_location: + message = ("{error_type} '{0.status_code} {0.reason_phrase}' for url '{0.url}'\n" + "Redirect location: '{0.headers[location]}'\n" + "For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/" + "{0.status_code}") + else: + message = ("{error_type} '{0.status_code} {0.reason_phrase}' for url '{0.url}'\n" + "For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/" + "{0.status_code}") + kinds = {1: "Informational response", 3: "Redirect response", 4: "Client error", 5: "Server error"} + error_type = kinds.get(self.status_code // 100, "Invalid status code") + raise HTTPStatusError(message.format(self, error_type=error_type), request=request, response=self) + + def close(self): + pass + + async def aclose(self): + pass + + def __repr__(self): + return f"" + + +def _timeout_seconds(timeout): + if timeout is None: + return None + if isinstance(timeout, Timeout): + return timeout.read + if isinstance(timeout, tuple): + return timeout[1] if len(timeout) > 1 else timeout[0] + return timeout + + +class Timeout: + def __init__(self, timeout=5.0, *, connect=None, read=None, write=None, pool=None): + self.connect = connect if connect is not None else timeout + self.read = read if read is not None else timeout + self.write = write if write is not None else timeout + self.pool = pool if pool is not None else timeout + + def __repr__(self): + if self.connect == self.read == self.write == self.pool: + return f"Timeout(timeout={self.connect})" + return (f"Timeout(connect={self.connect}, read={self.read}, write={self.write}, " + f"pool={self.pool})") + + +class BasicAuth: + def __init__(self, username, password): + self._header = _cn1_http.basic_auth(username, password) + + def apply(self, request): + request.headers["Authorization"] = self._header + + +_UNSET = object() + + +def _transport(request, timeout, follow_redirects): + scheme = request.url.scheme + if not scheme: + raise UnsupportedProtocol("Request URL is missing an 'http://' or 'https://' protocol.") + if scheme not in ("http", "https"): + raise UnsupportedProtocol(f"Request URL has an unsupported protocol '{scheme}://'.") + body = request.content if (request.content or request.method in ("POST", "PUT", "PATCH")) else None + try: + status, reason, final_url, pairs, content, seconds = _cn1_http.send( + request.method, str(request.url), request.headers, body, _timeout_seconds(timeout), follow_redirects) + except OSError as e: + msg = str(e) + if _cn1_http.is_timeout(msg): + raise ReadTimeout(msg, request=request) from None + raise ConnectError(msg, request=request) from None + if final_url != str(request.url): + request = Request(request.method, final_url, headers=request.headers.multi_items()) + r = Response(status, headers=pairs, content=content, request=request) + r.elapsed = _datetime.timedelta(seconds=seconds) + return r + + +class Client: + def __init__(self, *, auth=None, params=None, headers=None, cookies=None, timeout=5.0, + follow_redirects=False, base_url="", **kwargs): + self.base_url = URL(base_url) if base_url else URL("") + self.headers = Headers([("Accept", "*/*"), ("User-Agent", "python-httpx/" + __version__)]) + if headers: + self.headers.update(headers) + self.params = QueryParams(list(params.items()) if hasattr(params, "items") else (params or [])) + self.cookies = dict(cookies or {}) + self.timeout = timeout if isinstance(timeout, Timeout) else Timeout(timeout) + self.follow_redirects = follow_redirects + self.auth = auth + self.is_closed = False + + def _merge_url(self, url): + u = URL(url) + if u.is_relative_url and str(self.base_url): + base = str(self.base_url) + if not base.endswith("/"): + base += "/" + return URL(base + str(u).lstrip("/")) + return u + + def build_request(self, method, url, *, content=None, data=None, files=None, json=None, params=None, + headers=None, cookies=None, timeout=_UNSET, extensions=None): + merged = self.params.multi_items() + if params: + merged += list(params.items()) if hasattr(params, "items") else list(params) + h = self.headers.copy() + if headers: + h.update(headers) + jar = dict(self.cookies) + if cookies: + jar.update(cookies) + return Request(method, self._merge_url(url), params=merged, headers=h, content=content, data=data, + json=json, files=files, cookies=jar or None, extensions=extensions) + + def send(self, request, *, auth=_UNSET, follow_redirects=_UNSET, timeout=_UNSET): + a = self.auth if auth is _UNSET else auth + if isinstance(a, tuple): + a = BasicAuth(a[0], a[1]) + if a is not None: + a.apply(request) + follow = self.follow_redirects if follow_redirects is _UNSET else follow_redirects + t = self.timeout if timeout is _UNSET else timeout + r = _transport(request, t, follow) + for value in r.headers.get_list("set-cookie"): + name, _, rest = value.partition("=") + self.cookies[name.strip()] = rest.split(";", 1)[0] + return r + + def request(self, method, url, *, content=None, data=None, files=None, json=None, params=None, + headers=None, cookies=None, auth=_UNSET, follow_redirects=_UNSET, timeout=_UNSET, + extensions=None): + req = self.build_request(method, url, content=content, data=data, files=files, json=json, params=params, + headers=headers, cookies=cookies, extensions=extensions) + return self.send(req, auth=auth, follow_redirects=follow_redirects, timeout=timeout) + + def get(self, url, **kwargs): + return self.request("GET", url, **kwargs) + + def options(self, url, **kwargs): + return self.request("OPTIONS", url, **kwargs) + + def head(self, url, **kwargs): + return self.request("HEAD", url, **kwargs) + + def post(self, url, **kwargs): + return self.request("POST", url, **kwargs) + + def put(self, url, **kwargs): + return self.request("PUT", url, **kwargs) + + def patch(self, url, **kwargs): + return self.request("PATCH", url, **kwargs) + + def delete(self, url, **kwargs): + return self.request("DELETE", url, **kwargs) + + def close(self): + self.is_closed = True + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + + +class AsyncClient(Client): + async def send(self, request, *, auth=_UNSET, follow_redirects=_UNSET, timeout=_UNSET): + return Client.send(self, request, auth=auth, follow_redirects=follow_redirects, timeout=timeout) + + async def request(self, method, url, *, content=None, data=None, files=None, json=None, params=None, + headers=None, cookies=None, auth=_UNSET, follow_redirects=_UNSET, timeout=_UNSET, + extensions=None): + req = self.build_request(method, url, content=content, data=data, files=files, json=json, params=params, + headers=headers, cookies=cookies, extensions=extensions) + return Client.send(self, req, auth=auth, follow_redirects=follow_redirects, timeout=timeout) + + async def get(self, url, **kwargs): + return await self.request("GET", url, **kwargs) + + async def options(self, url, **kwargs): + return await self.request("OPTIONS", url, **kwargs) + + async def head(self, url, **kwargs): + return await self.request("HEAD", url, **kwargs) + + async def post(self, url, **kwargs): + return await self.request("POST", url, **kwargs) + + async def put(self, url, **kwargs): + return await self.request("PUT", url, **kwargs) + + async def patch(self, url, **kwargs): + return await self.request("PATCH", url, **kwargs) + + async def delete(self, url, **kwargs): + return await self.request("DELETE", url, **kwargs) + + async def aclose(self): + self.close() + + async def __aenter__(self): + return self + + async def __aexit__(self, *args): + self.close() + + +def request(method, url, *, params=None, content=None, data=None, files=None, json=None, headers=None, + cookies=None, auth=None, timeout=5.0, follow_redirects=False, **kwargs): + with Client(cookies=cookies, timeout=timeout, follow_redirects=follow_redirects) as c: + return c.request(method, url, content=content, data=data, files=files, json=json, params=params, + headers=headers, auth=auth) + + +def get(url, **kwargs): + return request("GET", url, **kwargs) + + +def options(url, **kwargs): + return request("OPTIONS", url, **kwargs) + + +def head(url, **kwargs): + return request("HEAD", url, **kwargs) + + +def post(url, **kwargs): + return request("POST", url, **kwargs) + + +def put(url, **kwargs): + return request("PUT", url, **kwargs) + + +def patch(url, **kwargs): + return request("PATCH", url, **kwargs) + + +def delete(url, **kwargs): + return request("DELETE", url, **kwargs) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/requests/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/requests/__init__.py new file mode 100644 index 00000000000..31789e3fdef --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/requests/__init__.py @@ -0,0 +1,331 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""HTTP for humans: the requests API over the platform's HTTP client. + +`requests.get(url, params=..., headers=..., json=..., timeout=...)` and the +other verbs, `Session`, `Response` with `.json()`, `.text` and +`.raise_for_status()`, and requests' exception hierarchy. Streaming, file +uploads, proxies and adapters are not provided; cookies are kept per +session. +""" + +import datetime as _datetime +import json as _json + +import _cn1_http +from requests import exceptions +from requests.exceptions import (RequestException, HTTPError, ConnectionError, Timeout, ConnectTimeout, + ReadTimeout, URLRequired, TooManyRedirects, MissingSchema, InvalidSchema, + InvalidURL, InvalidHeader, InvalidJSONError, JSONDecodeError) +from requests.structures import CaseInsensitiveDict + +__version__ = "2.34.2" +__title__ = "requests" + +_DEFAULT_HEADERS = (("User-Agent", "python-requests/" + __version__), ("Accept", "*/*")) + + +class _Codes: + def __init__(self): + table = { + 100: ("continue",), 200: ("ok", "okay", "all_ok", "all_good"), 201: ("created",), + 202: ("accepted",), 204: ("no_content",), 301: ("moved_permanently", "moved"), 302: ("found",), + 304: ("not_modified",), 307: ("temporary_redirect",), 308: ("permanent_redirect",), + 400: ("bad_request", "bad"), 401: ("unauthorized",), 403: ("forbidden",), 404: ("not_found",), + 405: ("method_not_allowed",), 409: ("conflict",), 410: ("gone",), 418: ("im_a_teapot", "teapot"), + 422: ("unprocessable_entity",), 429: ("too_many_requests",), 500: ("internal_server_error",), + 502: ("bad_gateway",), 503: ("service_unavailable", "unavailable"), 504: ("gateway_timeout",)} + for code, names in table.items(): + for n in names: + setattr(self, n, code) + setattr(self, n.upper(), code) + + def __getitem__(self, name): + return getattr(self, name, None) + + +codes = _Codes() + + +class PreparedRequest: + def __init__(self, method, url, headers, body): + self.method = method + self.url = url + self.headers = headers + self.body = body + + def __repr__(self): + return f"" + + +class Response: + def __init__(self): + self.status_code = None + self.reason = None + self.url = None + self.headers = CaseInsensitiveDict() + self._content = b"" + self.encoding = None + self.history = [] + self.request = None + self.elapsed = _datetime.timedelta(0) + self.cookies = {} + + @property + def content(self): + return self._content + + @property + def apparent_encoding(self): + return "utf-8" + + @property + def text(self): + enc = self.encoding or "utf-8" + try: + return self._content.decode(enc, "replace") + except LookupError: + return self._content.decode("utf-8", "replace") + + def json(self, **kwargs): + try: + return _json.loads(self.text, **kwargs) + except _json.JSONDecodeError as e: + raise JSONDecodeError(e.msg, e.doc, e.pos, response=self, request=self.request) from None + + @property + def ok(self): + try: + self.raise_for_status() + except HTTPError: + return False + return True + + @property + def is_redirect(self): + return "location" in self.headers and self.status_code in (301, 302, 303, 307, 308) + + @property + def is_permanent_redirect(self): + return "location" in self.headers and self.status_code in (301, 308) + + @property + def links(self): + return {} + + def raise_for_status(self): + reason = self.reason or "" + msg = "" + if 400 <= self.status_code < 500: + msg = f"{self.status_code} Client Error: {reason} for url: {self.url}" + elif 500 <= self.status_code < 600: + msg = f"{self.status_code} Server Error: {reason} for url: {self.url}" + if msg: + raise HTTPError(msg, response=self) + + def iter_content(self, chunk_size=1, decode_unicode=False): + data = self.text if decode_unicode else self._content + if chunk_size is None: + chunk_size = len(data) or 1 + for i in range(0, len(data), chunk_size): + yield data[i:i + chunk_size] + + def iter_lines(self, chunk_size=512, decode_unicode=False, delimiter=None): + data = self.text if decode_unicode else self._content + lines = data.split(delimiter) if delimiter is not None else data.splitlines() + for line in lines: + yield line + + def close(self): + pass + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + + def __bool__(self): + return self.ok + + def __iter__(self): + return self.iter_content(128) + + def __repr__(self): + return f"" + + +def _encoding_of(content_type): + cs = _cn1_http.charset(content_type) + if cs: + return cs + if content_type is None: + return None + if "text" in content_type: + return "ISO-8859-1" + if "application/json" in content_type: + return "utf-8" + return None + + +class Session: + def __init__(self): + self.headers = CaseInsensitiveDict(dict(_DEFAULT_HEADERS)) + self.auth = None + self.params = {} + self.cookies = {} + self.max_redirects = 30 + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + + def close(self): + pass + + def request(self, method, url, params=None, data=None, headers=None, cookies=None, files=None, auth=None, + timeout=None, allow_redirects=True, proxies=None, hooks=None, stream=None, verify=None, + cert=None, json=None): + if files: + raise NotImplementedError("file uploads are not supported") + if not isinstance(url, str): + url = str(url) + url = url.strip() + if ":" not in url.split("/")[0] or url.startswith(":"): + raise MissingSchema(f"Invalid URL '{url}': No scheme supplied. Perhaps you meant https://{url}?") + scheme = url.split(":", 1)[0].lower() + if scheme not in ("http", "https"): + raise InvalidSchema(f"No connection adapters were found for '{url}'") + merged = dict(self.params) + if params: + if hasattr(params, "items"): + merged.update(params) + full = _cn1_http.with_params(url, merged) + else: + full = _cn1_http.with_params(_cn1_http.with_params(url, merged), params) + else: + full = _cn1_http.with_params(url, merged) + h = _cn1_http.Headers(self.headers.items()) + body = None + if data is not None and data != {} and data != []: + if isinstance(data, str): + body = data.encode("utf-8") + elif isinstance(data, (bytes, bytearray)): + body = bytes(data) + else: + body = _cn1_http.form_body(data) + h.setdefault("Content-Type", "application/x-www-form-urlencoded") + elif json is not None: + body = _cn1_http.json_bytes(json, False) + h.setdefault("Content-Type", "application/json") + if headers: + for k, v in headers.items(): + if v is None: + if k in h: + del h[k] + else: + h[k] = v + jar = dict(self.cookies) + if cookies: + jar.update(cookies) + if jar: + h["Cookie"] = "; ".join(f"{k}={v}" for k, v in jar.items()) + a = auth if auth is not None else self.auth + if a is not None: + if isinstance(a, tuple): + h["Authorization"] = _cn1_http.basic_auth(a[0], a[1]) + elif callable(a): + prepared = PreparedRequest(method.upper(), full, h, body) + prepared = a(prepared) + h = prepared.headers + t = timeout[1] if isinstance(timeout, tuple) else timeout + prepared = PreparedRequest(method.upper(), full, CaseInsensitiveDict(h.items()), body) + try: + status, reason, final_url, pairs, content, seconds = _cn1_http.send(method.upper(), full, h, body, t, + allow_redirects) + except OSError as e: + msg = str(e) + if _cn1_http.is_timeout(msg): + raise ReadTimeout(msg, request=prepared) from None + raise ConnectionError(msg, request=prepared) from None + r = Response() + r.status_code = status + r.reason = reason + r.url = final_url + hs = _cn1_http.Headers(pairs) + r.headers = CaseInsensitiveDict(hs.items()) + r._content = content + r.encoding = _encoding_of(r.headers.get("content-type")) + r.request = prepared + r.elapsed = _datetime.timedelta(seconds=seconds) + for value in hs.get_list("set-cookie"): + name, _, rest = value.partition("=") + v = rest.split(";", 1)[0] + r.cookies[name.strip()] = v + self.cookies[name.strip()] = v + return r + + def get(self, url, **kwargs): + kwargs.setdefault("allow_redirects", True) + return self.request("GET", url, **kwargs) + + def options(self, url, **kwargs): + kwargs.setdefault("allow_redirects", True) + return self.request("OPTIONS", url, **kwargs) + + def head(self, url, **kwargs): + kwargs.setdefault("allow_redirects", False) + return self.request("HEAD", url, **kwargs) + + def post(self, url, data=None, json=None, **kwargs): + return self.request("POST", url, data=data, json=json, **kwargs) + + def put(self, url, data=None, **kwargs): + return self.request("PUT", url, data=data, **kwargs) + + def patch(self, url, data=None, **kwargs): + return self.request("PATCH", url, data=data, **kwargs) + + def delete(self, url, **kwargs): + return self.request("DELETE", url, **kwargs) + + +def session(): + return Session() + + +def request(method, url, **kwargs): + with Session() as s: + return s.request(method, url, **kwargs) + + +def get(url, params=None, **kwargs): + return request("GET", url, params=params, **kwargs) + + +def options(url, **kwargs): + return request("OPTIONS", url, **kwargs) + + +def head(url, **kwargs): + kwargs.setdefault("allow_redirects", False) + return request("HEAD", url, **kwargs) + + +def post(url, data=None, json=None, **kwargs): + return request("POST", url, data=data, json=json, **kwargs) + + +def put(url, data=None, **kwargs): + return request("PUT", url, data=data, **kwargs) + + +def patch(url, data=None, **kwargs): + return request("PATCH", url, data=data, **kwargs) + + +def delete(url, **kwargs): + return request("DELETE", url, **kwargs) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/requests/exceptions.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/requests/exceptions.py new file mode 100644 index 00000000000..5aabe2eae5f --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/requests/exceptions.py @@ -0,0 +1,98 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""The exceptions of requests, in its hierarchy.""" + +import json as _json + + +class RequestException(IOError): + def __init__(self, *args, response=None, request=None): + super().__init__(*args) + self.response = response + self.request = request + if request is None and response is not None and hasattr(response, "request"): + self.request = response.request + + +class InvalidJSONError(RequestException): + pass + + +class JSONDecodeError(InvalidJSONError, _json.JSONDecodeError): + def __init__(self, msg, doc, pos, response=None, request=None): + _json.JSONDecodeError.__init__(self, msg, doc, pos) + self.response = response + self.request = request + self.args = (_json.JSONDecodeError.__str__(self),) + + def __str__(self): + return _json.JSONDecodeError.__str__(self) + + +class HTTPError(RequestException): + pass + + +class ConnectionError(RequestException): + pass + + +class ProxyError(ConnectionError): + pass + + +class SSLError(ConnectionError): + pass + + +class Timeout(RequestException): + pass + + +class ConnectTimeout(ConnectionError, Timeout): + pass + + +class ReadTimeout(Timeout): + pass + + +class URLRequired(RequestException): + pass + + +class TooManyRedirects(RequestException): + pass + + +class MissingSchema(RequestException, ValueError): + pass + + +class InvalidSchema(RequestException, ValueError): + pass + + +class InvalidURL(RequestException, ValueError): + pass + + +class InvalidHeader(RequestException, ValueError): + pass + + +class ChunkedEncodingError(RequestException): + pass + + +class ContentDecodingError(RequestException): + pass + + +class StreamConsumedError(RequestException, TypeError): + pass + + +class RetryError(RequestException): + pass diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/requests/structures.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/requests/structures.py new file mode 100644 index 00000000000..17c2329c810 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/requests/structures.py @@ -0,0 +1,94 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""requests.structures: the case-insensitive dictionary of headers.""" + + +class CaseInsensitiveDict: + def __init__(self, data=None, **kwargs): + self._store = {} + if data is None: + data = {} + self.update(data, **kwargs) + + def update(self, data=(), **kwargs): + items = data.items() if hasattr(data, "items") else data + for k, v in items: + self[k] = v + for k, v in kwargs.items(): + self[k] = v + + def __setitem__(self, key, value): + self._store[key.lower()] = (key, value) + + def __getitem__(self, key): + return self._store[key.lower()][1] + + def __delitem__(self, key): + del self._store[key.lower()] + + def __iter__(self): + return (casedkey for casedkey, _ in self._store.values()) + + def __len__(self): + return len(self._store) + + def __contains__(self, key): + return key.lower() in self._store + + def get(self, key, default=None): + return self[key] if key in self else default + + def keys(self): + return list(self) + + def values(self): + return [v for _, v in self._store.values()] + + def items(self): + return [(k, v) for k, v in self._store.values()] + + def lower_items(self): + return [(k, v[1]) for k, v in self._store.items()] + + def setdefault(self, key, default=None): + if key not in self: + self[key] = default + return self[key] + + def pop(self, key, *default): + if key in self: + v = self[key] + del self[key] + return v + if default: + return default[0] + raise KeyError(key) + + def __eq__(self, other): + if hasattr(other, "items"): + other = CaseInsensitiveDict(other) + else: + return NotImplemented + return dict(self.lower_items()) == dict(other.lower_items()) + + def copy(self): + return CaseInsensitiveDict(self._store.values()) + + def __repr__(self): + return str(dict(self.items())) + + +class LookupDict(dict): + def __init__(self, name=None): + self.name = name + super().__init__() + + def __repr__(self): + return f"" + + def __getitem__(self, key): + return self.__dict__.get(key, None) + + def get(self, key, default=None): + return self.__dict__.get(key, default) diff --git a/maven/python-transpiler/src/test/java/com/codename1/python/transpiler/ConformanceTest.java b/maven/python-transpiler/src/test/java/com/codename1/python/transpiler/ConformanceTest.java index 014bb1d370b..8f819db5947 100644 --- a/maven/python-transpiler/src/test/java/com/codename1/python/transpiler/ConformanceTest.java +++ b/maven/python-transpiler/src/test/java/com/codename1/python/transpiler/ConformanceTest.java @@ -94,6 +94,22 @@ private static void run(Path program) throws Exception { } else { sources.put("main.py", read(program)); } + // An http_ program talks to HttpTestServer, whose address it reads + // from the module _testbase. + HttpTestServer server = stem.startsWith("http_") ? new HttpTestServer(0) : null; + if (server != null) { + sources.put("_testbase.py", "BASE = \"http://127.0.0.1:" + server.port() + "\"\n"); + } + try { + run(stem, sources, server != null); + } finally { + if (server != null) { + server.stop(); + } + } + } + + private static void run(String stem, Map sources, boolean http) throws Exception { Path work = Files.createTempDirectory("py-conf-" + stem); Path gen = work.resolve("gen"); Path cls = work.resolve("classes"); @@ -126,16 +142,30 @@ private static void run(Path program) throws Exception { Path fsRoot = work.resolve("fs"); Files.createDirectories(fsRoot.resolve("home")); ByteArrayOutputStream out = new ByteArrayOutputStream(); + ByteArrayOutputStream err = new ByteArrayOutputStream(); PrintStream saved = System.out; + PrintStream savedErr = System.err; String failure = null; - try (URLClassLoader loader = new URLClassLoader(urls, null)) { + try (URLClassLoader loader = new URLClassLoader(urls, ClassLoader.getPlatformClassLoader())) { System.setOut(new PrintStream(out, true, "UTF-8")); + System.setErr(new PrintStream(err, true, "UTF-8")); try { Class fsType = loader.loadClass("com.codename1.python.runtime.PyFileSystem"); Object fs = loader.loadClass(JvmFileSystem.class.getName()).getConstructor(String.class) .newInstance(fsRoot.toString()); fsType.getMethod("install", fsType).invoke(null, fs); - loader.loadClass(PythonTranspiler.PACKAGE + ".PythonModules").getMethod("runMain").invoke(null); + if (http) { + Class httpType = loader.loadClass("com.codename1.python.runtime.PyHttp"); + httpType.getMethod("install", httpType) + .invoke(null, loader.loadClass(JvmHttp.class.getName()).getConstructor().newInstance()); + } + Object status = loader.loadClass(PythonTranspiler.PACKAGE + ".PythonModules").getMethod("runMain") + .invoke(null); + // runMain reports an uncaught Python exception on stderr and + // answers the status the interpreter would exit with. + if (!Integer.valueOf(0).equals(status)) { + failure = "exit status " + status + ":\n" + err.toString("UTF-8"); + } } catch (InvocationTargetException e) { failure = String.valueOf(e.getCause()); if (!failure.startsWith("com.codename1.python.runtime.PyException")) { @@ -158,6 +188,7 @@ private static void run(Path program) throws Exception { } finally { System.out.flush(); System.setOut(saved); + System.setErr(savedErr); } String expected = read(ROOT.resolve(stem + ".out")); String actual = out.toString("UTF-8"); diff --git a/maven/python-transpiler/src/test/java/com/codename1/python/transpiler/HttpTestServer.java b/maven/python-transpiler/src/test/java/com/codename1/python/transpiler/HttpTestServer.java new file mode 100644 index 00000000000..c9c535e4850 --- /dev/null +++ b/maven/python-transpiler/src/test/java/com/codename1/python/transpiler/HttpTestServer.java @@ -0,0 +1,183 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.transpiler; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +/// The server the `http_*` conformance programs talk to, for both arms: the +/// test starts it in process, and the expected output is made by running +/// the real `requests` and `httpx` against `main(port)`. +/// +/// Routes: `/get` and `/anything` echo the request as JSON (method, query, +/// body, content type and every `X-` header, keys sorted); `/status/` +/// answers `n`; `/redirect` sends 302 to `/get?redirected=1`; +/// `/headers-out` sets response headers; `/basic-auth` wants user:pass; +/// `/bad-json` answers a body that is not JSON. +public final class HttpTestServer { + private final HttpServer server; + + public HttpTestServer(int port) throws IOException { + server = HttpServer.create(new InetSocketAddress("127.0.0.1", port), 0); + server.createContext("/", HttpTestServer::handle); + server.start(); + } + + public int port() { + return server.getAddress().getPort(); + } + + public void stop() { + server.stop(0); + } + + public static void main(String[] args) throws Exception { + HttpTestServer s = new HttpTestServer(args.length > 0 ? Integer.parseInt(args[0]) : 0); + System.out.println(s.port()); + System.out.flush(); + Thread.sleep(Long.MAX_VALUE); + } + + private static void handle(HttpExchange x) throws IOException { + String path = x.getRequestURI().getPath(); + byte[] body = readAll(x.getRequestBody()); + try { + if (path.equals("/get") || path.equals("/anything")) { + send(x, 200, "application/json", echo(x, body)); + } else if (path.startsWith("/status/")) { + int code = Integer.parseInt(path.substring("/status/".length())); + send(x, code, "text/plain", "status " + code); + } else if (path.equals("/redirect")) { + x.getResponseHeaders().add("Location", "/get?redirected=1"); + send(x, 302, "text/plain", ""); + } else if (path.equals("/headers-out")) { + x.getResponseHeaders().add("X-One", "1"); + x.getResponseHeaders().add("X-Two", "two"); + send(x, 200, "text/plain; charset=utf-8", "headers"); + } else if (path.equals("/basic-auth")) { + String auth = x.getRequestHeaders().getFirst("Authorization"); + if ("Basic dXNlcjpwYXNz".equals(auth)) { + send(x, 200, "application/json", "{\"authenticated\": true}"); + } else { + send(x, 401, "application/json", "{\"authenticated\": false}"); + } + } else if (path.equals("/bad-json")) { + send(x, 200, "application/json", "{not json"); + } else { + send(x, 404, "text/plain", "no route " + path); + } + } finally { + x.close(); + } + } + + private static String echo(HttpExchange x, byte[] body) throws IOException { + Map fields = new TreeMap(); + fields.put("method", quote(x.getRequestMethod())); + Map args = new TreeMap(); + String q = x.getRequestURI().getRawQuery(); + if (q != null && !q.isEmpty()) { + for (String kv : q.split("&")) { + int eq = kv.indexOf('='); + String k = URLDecoder.decode(eq < 0 ? kv : kv.substring(0, eq), "UTF-8"); + String v = eq < 0 ? "" : URLDecoder.decode(kv.substring(eq + 1), "UTF-8"); + args.merge(k, v, (a, b) -> a + "," + b); + } + } + fields.put("args", object(args)); + Map headers = new TreeMap(); + for (Map.Entry> e : x.getRequestHeaders().entrySet()) { + String k = e.getKey().toLowerCase(java.util.Locale.ROOT); + if (k.startsWith("x-")) { + headers.put(k, String.join(",", e.getValue())); + } + } + fields.put("headers", object(headers)); + String ct = x.getRequestHeaders().getFirst("Content-Type"); + fields.put("content_type", ct == null ? "null" : quote(ct)); + fields.put("body", quote(new String(body, StandardCharsets.UTF_8))); + return object0(fields); + } + + private static String object(Map m) { + Map q = new TreeMap(); + for (Map.Entry e : m.entrySet()) { + q.put(e.getKey(), quote(e.getValue())); + } + return object0(q); + } + + private static String object0(Map m) { + List parts = new ArrayList(); + for (Map.Entry e : m.entrySet()) { + parts.add(quote(e.getKey()) + ": " + e.getValue()); + } + return "{" + String.join(", ", parts) + "}"; + } + + private static String quote(String s) { + StringBuilder b = new StringBuilder("\""); + for (char c : s.toCharArray()) { + if (c == '"' || c == '\\') { + b.append('\\').append(c); + } else if (c < 0x20 || c > 0x7e) { + b.append(String.format("\\u%04x", (int) c)); + } else { + b.append(c); + } + } + return b.append('"').toString(); + } + + private static void send(HttpExchange x, int code, String type, String text) throws IOException { + byte[] b = text.getBytes(StandardCharsets.UTF_8); + x.getResponseHeaders().add("Content-Type", type); + x.sendResponseHeaders(code, b.length == 0 ? -1 : b.length); + if (b.length > 0) { + try (OutputStream o = x.getResponseBody()) { + o.write(b); + } + } + } + + private static byte[] readAll(InputStream in) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + byte[] buf = new byte[8192]; + for (int n = in.read(buf); n > 0; n = in.read(buf)) { + out.write(buf, 0, n); + } + return out.toByteArray(); + } +} diff --git a/maven/python-transpiler/src/test/java/com/codename1/python/transpiler/JvmHttp.java b/maven/python-transpiler/src/test/java/com/codename1/python/transpiler/JvmHttp.java new file mode 100644 index 00000000000..e22b4629124 --- /dev/null +++ b/maven/python-transpiler/src/test/java/com/codename1/python/transpiler/JvmHttp.java @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.transpiler; + +import com.codename1.python.runtime.PyHttp; +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/// HTTP for the conformance programs, over the JDK's client: what a +/// Codename One application gets from `ConnectionRequest`. +public final class JvmHttp extends PyHttp { + private static final Map REASONS = Map.ofEntries(Map.entry(200, "OK"), + Map.entry(201, "Created"), Map.entry(204, "No Content"), Map.entry(301, "Moved Permanently"), + Map.entry(302, "Found"), Map.entry(400, "Bad Request"), Map.entry(401, "Unauthorized"), + Map.entry(403, "Forbidden"), Map.entry(404, "Not Found"), Map.entry(418, "I'm a Teapot"), + Map.entry(500, "Internal Server Error"), Map.entry(503, "Service Unavailable")); + private static final java.util.Set RESTRICTED = java.util.Set.of("connection", "content-length", + "host", "expect", "upgrade"); + + @Override + public Response send(String method, String url, String[] headers, byte[] body, int timeoutMillis, + boolean followRedirects) throws IOException { + HttpClient client = HttpClient.newBuilder() + .followRedirects(followRedirects ? HttpClient.Redirect.NORMAL : HttpClient.Redirect.NEVER) + .connectTimeout(Duration.ofMillis(timeoutMillis > 0 ? timeoutMillis : 10000)).build(); + HttpRequest.Builder b = HttpRequest.newBuilder(URI.create(url)) + .method(method, body == null ? HttpRequest.BodyPublishers.noBody() + : HttpRequest.BodyPublishers.ofByteArray(body)); + for (int i = 0; i + 1 < headers.length; i += 2) { + if (!RESTRICTED.contains(headers[i].toLowerCase(java.util.Locale.ROOT))) { + b.header(headers[i], headers[i + 1]); + } + } + if (timeoutMillis > 0) { + b.timeout(Duration.ofMillis(timeoutMillis)); + } + HttpResponse r; + try { + r = client.send(b.build(), HttpResponse.BodyHandlers.ofByteArray()); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException("interrupted", e); + } + List hs = new ArrayList(); + for (Map.Entry> e : r.headers().map().entrySet()) { + if (e.getKey().startsWith(":")) { + continue; + } + for (String v : e.getValue()) { + hs.add(e.getKey()); + hs.add(v); + } + } + return new Response(r.statusCode(), REASONS.getOrDefault(r.statusCode(), ""), r.uri().toString(), + hs.toArray(new String[0]), r.body()); + } +} diff --git a/maven/python-transpiler/src/test/resources/conformance/errors_.out b/maven/python-transpiler/src/test/resources/conformance/errors_.out index 0d6dae9ff6c..88f98f6d6e9 100644 --- a/maven/python-transpiler/src/test/resources/conformance/errors_.out +++ b/maven/python-transpiler/src/test/resources/conformance/errors_.out @@ -75,3 +75,4 @@ SystemExit (None,) None SystemExit (3,) 3 SystemExit ('bye',) 'bye' SystemExit (1, 2) (1, 2) +KwError ('msg', 2) 3 diff --git a/maven/python-transpiler/src/test/resources/conformance/errors_.py b/maven/python-transpiler/src/test/resources/conformance/errors_.py index 51054596066..4a94a19454e 100644 --- a/maven/python-transpiler/src/test/resources/conformance/errors_.py +++ b/maven/python-transpiler/src/test/resources/conformance/errors_.py @@ -179,3 +179,16 @@ def assert_fail(): for args in [(), (None,), (3,), ("bye",), (1, 2)]: e = SystemExit(*args) print("SystemExit", args, repr(e.code)) + + +# A subclass whose __init__ takes keywords: BaseException.__new__ ignores them. +class KwError(IOError): + def __init__(self, *args, response=None): + super().__init__(*args) + self.response = response + + +try: + raise KwError("msg", 2, response=3) +except OSError as e: + print(type(e).__name__, e.args, e.response) diff --git a/maven/python-transpiler/src/test/resources/conformance/http_httpx.out b/maven/python-transpiler/src/test/resources/conformance/http_httpx.out new file mode 100644 index 00000000000..2a0d8b4ab9e --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/http_httpx.out @@ -0,0 +1,21 @@ +200 OK True False utf-8 application/json +{'args': {'n': '3', 'q': 'a b'}, 'body': '', 'content_type': None, 'headers': {'x-test': 'yes'}, 'method': 'GET'} /get a b BASE/get?q=a+b&n=3 +HTTP/1.1 GET +{'args': {}, 'body': '{"b":[1,2],"a":null}', 'content_type': 'application/json', 'headers': {}, 'method': 'POST'} +{'args': {}, 'body': 'k=v+w', 'content_type': 'application/x-www-form-urlencoded', 'headers': {}, 'method': 'POST'} +{'args': {}, 'body': 'raw', 'content_type': None, 'headers': {'x-put': '1'}, 'method': 'PUT'} +302 True /get?redirected=1 +200 {'redirected': '1'} +404 True status 404 +Client error '404 Not Found' for url 'BASE/status/404' +For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/404 +404 GET +200 +{'extra': 'e', 'p': '1'} {'x-client': 'c'} +200 {'authenticated': True} +async 200 {'async': 'yes'} +async [1,2,3] +UnsupportedProtocol Request URL is missing an 'http://' or 'https://' protocol. +ConnectError True True +200 404 Timeout(timeout=3.0) +1 2 ['a', 'b'] True diff --git a/maven/python-transpiler/src/test/resources/conformance/http_httpx.py b/maven/python-transpiler/src/test/resources/conformance/http_httpx.py new file mode 100644 index 00000000000..90d1ef8b892 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/http_httpx.py @@ -0,0 +1,60 @@ +# httpx against HttpTestServer (see ConformanceTest); the expected output is +# the real httpx's against the same server. +import asyncio +import httpx +from _testbase import BASE + +r = httpx.get(BASE + "/get", params={"q": "a b", "n": 3}, headers={"X-Test": "yes"}) +print(r.status_code, r.reason_phrase, r.is_success, r.is_error, r.encoding, r.headers["content-type"]) +print(r.json(), r.url.path, r.url.params["q"], str(r.url).replace(BASE, "BASE")) +print(r.http_version, r.request.method, repr(r)) + +r = httpx.post(BASE + "/anything", json={"b": [1, 2], "a": None}) +print(r.json()) +r = httpx.post(BASE + "/anything", data={"k": "v w"}) +print(r.json()) +r = httpx.put(BASE + "/anything", content="raw", headers={"X-Put": "1"}) +print(r.json()) + +r = httpx.get(BASE + "/redirect") +print(r.status_code, r.is_redirect, r.headers["location"]) +r = httpx.get(BASE + "/redirect", follow_redirects=True) +print(r.status_code, r.json()["args"]) + +r = httpx.get(BASE + "/status/404") +print(r.status_code, r.is_client_error, r.text) +try: + r.raise_for_status() +except httpx.HTTPStatusError as e: + print(str(e).replace(BASE, "BASE")) + print(e.response.status_code, e.request.method) +print(httpx.get(BASE + "/get").raise_for_status().status_code) + +with httpx.Client(base_url=BASE, headers={"X-Client": "c"}, params={"p": "1"}) as client: + r = client.get("/get", params={"extra": "e"}) + print(r.json()["args"], r.json()["headers"]) + r = client.get("basic-auth", auth=("user", "pass")) + print(r.status_code, r.json()) + + +async def main(): + async with httpx.AsyncClient(base_url=BASE) as client: + r = await client.get("/get", params={"async": "yes"}) + print("async", r.status_code, r.json()["args"]) + r = await client.post("/anything", json=[1, 2, 3]) + print("async", r.json()["body"]) + +asyncio.run(main()) + +try: + httpx.get("nohost/x") +except httpx.UnsupportedProtocol as e: + print("UnsupportedProtocol", e) +try: + httpx.get("http://127.0.0.1:9/nothing", timeout=2) +except httpx.ConnectError as e: + print("ConnectError", isinstance(e, httpx.RequestError), isinstance(e, httpx.HTTPError)) +print(httpx.codes.OK, httpx.codes.NOT_FOUND, httpx.Timeout(3.0)) +h = httpx.Headers({"A": "1"}) +h["b"] = "2" +print(h["a"], h["B"], list(h.keys()), "a" in h) diff --git a/maven/python-transpiler/src/test/resources/conformance/http_requests.out b/maven/python-transpiler/src/test/resources/conformance/http_requests.out new file mode 100644 index 00000000000..64514b1c289 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/http_requests.out @@ -0,0 +1,22 @@ +200 True OK utf-8 application/json True +{'args': {'many': '1,2', 'n': '3', 'q': 'a b'}, 'body': '', 'content_type': None, 'headers': {'x-test': 'yes'}, 'method': 'GET'} +True GET True +{'args': {}, 'body': '{"b": [1, 2], "a": null, "s": "x"}', 'content_type': 'application/json', 'headers': {}, 'method': 'POST'} +{'args': {}, 'body': 'name=py+thon&tags=a&tags=b', 'content_type': 'application/x-www-form-urlencoded', 'headers': {}, 'method': 'POST'} +{'args': {}, 'body': 'raw text', 'content_type': 'text/plain', 'headers': {'x-a': '1'}, 'method': 'PUT'} +PATCH {'x': '1'} bytes +DELETE True +404 False Not Found status 404 +HTTPError: 404 Client Error: Not Found for url: BASE/status/404 404 +HTTPError 503 Server Error: Service Unavailable for url: BASE/status/503 +200 {'redirected': '1'} BASE/get?redirected=1 +302 /get?redirected=1 True +1 two - headers utf-8 +{'authenticated': True} +401 +JSONDecodeError Expecting property name enclosed in double quotes: line 1 column 2 (char 1) True +{'extra': '2', 'fixed': '1'} {'x-call': 'c', 'x-session': 's1'} +MissingSchema Invalid URL 'nohost': No scheme supplied. Perhaps you meant https://nohost? +InvalidSchema No connection adapters were found for 'ftp://example.com/x' +ConnectionError True +200 404 418 diff --git a/maven/python-transpiler/src/test/resources/conformance/http_requests.py b/maven/python-transpiler/src/test/resources/conformance/http_requests.py new file mode 100644 index 00000000000..240b616a350 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/http_requests.py @@ -0,0 +1,66 @@ +# requests against HttpTestServer (see ConformanceTest); the expected output +# is the real requests' against the same server. +import requests +from _testbase import BASE + +r = requests.get(BASE + "/get", params={"q": "a b", "n": 3, "skip": None, "many": [1, 2]}, + headers={"X-Test": "yes"}) +print(r.status_code, r.ok, r.reason, r.encoding, r.headers["content-type"], "Content-Type" in r.headers) +print(r.json()) +print(r.url == BASE + "/get?q=a+b&n=3&many=1&many=2", r.request.method, bool(r)) + +r = requests.post(BASE + "/anything", json={"b": [1, 2], "a": None, "s": "x"}) +print(r.json()) +r = requests.post(BASE + "/anything", data={"name": "py thon", "tags": ["a", "b"]}) +print(r.json()) +r = requests.put(BASE + "/anything", data="raw text", headers={"Content-Type": "text/plain", "X-A": "1"}) +print(r.json()) +r = requests.patch(BASE + "/anything?x=1", data=b"bytes") +print(r.json()["method"], r.json()["args"], r.json()["body"]) +r = requests.delete(BASE + "/anything") +print(r.json()["method"], r.json()["body"] == "") + +r = requests.get(BASE + "/status/404") +print(r.status_code, r.ok, r.reason, r.text, repr(r)) +try: + r.raise_for_status() +except requests.HTTPError as e: + print("HTTPError:", str(e).replace(BASE, "BASE"), e.response.status_code) +r = requests.get(BASE + "/status/503") +try: + r.raise_for_status() +except requests.exceptions.RequestException as e: + print(type(e).__name__, str(e).replace(BASE, "BASE")) + +r = requests.get(BASE + "/redirect") +print(r.status_code, r.json()["args"], r.url.replace(BASE, "BASE")) +r = requests.get(BASE + "/redirect", allow_redirects=False) +print(r.status_code, r.headers["Location"], r.is_redirect) + +r = requests.get(BASE + "/headers-out") +print(r.headers["x-one"], r.headers.get("X-TWO"), r.headers.get("missing", "-"), r.text, r.encoding) + +print(requests.get(BASE + "/basic-auth", auth=("user", "pass")).json()) +print(requests.get(BASE + "/basic-auth", auth=("user", "nope")).status_code) + +try: + requests.get(BASE + "/bad-json").json() +except requests.JSONDecodeError as e: + print("JSONDecodeError", e, isinstance(e, ValueError)) + +with requests.Session() as s: + s.headers.update({"X-Session": "s1"}) + s.params = {"fixed": "1"} + r = s.get(BASE + "/get", params={"extra": "2"}, headers={"X-Call": "c"}) + print(r.json()["args"], r.json()["headers"]) + +for bad in ["nohost", "ftp://example.com/x"]: + try: + requests.get(bad) + except (requests.exceptions.MissingSchema, requests.exceptions.InvalidSchema) as e: + print(type(e).__name__, e) +try: + requests.get("http://127.0.0.1:9/nothing", timeout=2) +except requests.ConnectionError as e: + print("ConnectionError", isinstance(e, requests.RequestException)) +print(requests.codes.ok, requests.codes.not_found, requests.codes["teapot"]) From abbfa762148e9acfd16279393e810b0065047157 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Tue, 27 Oct 2026 17:53:38 +0200 Subject: [PATCH 006/112] Python: runtime generic aliases and unions, string annotations in get_type_hints Co-Authored-By: Claude Opus 5.5 --- .../java/com/codename1/python/runtime/Py.java | 3 + .../python/runtime/PyGenericAlias.java | 99 ++++++++++ .../com/codename1/python/runtime/PyType.java | 11 +- .../com/codename1/python/runtime/PyUnion.java | 170 ++++++++++++++++++ .../com/codename1/python/stdlib/types.py | 4 +- .../com/codename1/python/stdlib/typing.py | 122 ++++++++++++- .../resources/conformance/typing_alias.out | 17 ++ .../resources/conformance/typing_alias.py | 28 +++ scripts/python-bench/conformance_parparvm.sh | 6 +- 9 files changed, 451 insertions(+), 9 deletions(-) create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/PyGenericAlias.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/PyUnion.java create mode 100644 maven/python-transpiler/src/test/resources/conformance/typing_alias.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/typing_alias.py diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Py.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Py.java index 9be40a13432..a55c2a4d71e 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Py.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Py.java @@ -1988,6 +1988,9 @@ public static boolean isinstance(Object o, Object t) { if (t instanceof PyJavaClass) { return ((PyJavaClass) t).isInstance(o); } + if (t instanceof PyUnion) { + return ((PyUnion) t).instanceCheck(o); + } if (!(t instanceof PyType)) { throw typeError("isinstance() arg 2 must be a type, a tuple of types, or a union"); } diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyGenericAlias.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyGenericAlias.java new file mode 100644 index 00000000000..18e16dc1c01 --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyGenericAlias.java @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +/// `list[int]`: a builtin type subscripted in an annotation. It keeps the +/// origin and the arguments where `typing.get_origin`/`get_args` read them, +/// and calling it builds the origin. +final class PyGenericAlias extends PyObject { + static final PyType TYPE = new PyType("types.GenericAlias", new PyType[] {Py.OBJECT_TYPE}, null); + + final PyType origin; + final PyTuple args; + + PyGenericAlias(PyType origin, Object key) { + this.origin = origin; + this.args = key instanceof PyTuple ? (PyTuple) key : new PyTuple(new Object[] {key}); + } + + @Override + public PyType type() { + return TYPE; + } + + @Override + public Object getAttrOrNull(String attr) { + if ("__origin__".equals(attr)) { + return origin; + } + if ("__args__".equals(attr)) { + return args; + } + if ("__parameters__".equals(attr)) { + return new PyTuple(new Object[0]); + } + return origin.getAttrOrNull(attr); + } + + @Override + public Object call(Object[] a, String[] kwNames) { + return origin.call(a, kwNames); + } + + @Override + public Object or(Object o) { + return PyUnion.of(this, o); + } + + @Override + public Object ror(Object o) { + return PyUnion.of(o, this); + } + + @Override + public Object eq(Object o) { + if (!(o instanceof PyGenericAlias)) { + return null; + } + PyGenericAlias g = (PyGenericAlias) o; + return Py.bool(g.origin == origin && Py.eqBool(g.args, args)); + } + + @Override + public long hash() { + return origin.hash() * 31 + Py.hashOf(args); + } + + @Override + public String repr() { + StringBuilder b = new StringBuilder(PyUnion.typeRepr(origin)).append('['); + for (int i = 0; i < args.items.length; i++) { + if (i > 0) { + b.append(", "); + } + b.append(PyUnion.typeRepr(args.items[i])); + } + return b.append(']').toString(); + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyType.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyType.java index 12afe5913b0..4fae9e7b556 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyType.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyType.java @@ -566,8 +566,7 @@ public Object getItem(Object key) { return Py.obj(f).call(new Object[] {this, key}, Py.NO_KW); } if (factory != null) { - // `list[int]`: a generic alias, which is the type itself here. - return this; + return new PyGenericAlias(this, key); } throw Py.typeError("type '" + name + "' is not subscriptable"); } @@ -915,8 +914,12 @@ public long hash() { @Override public Object or(Object o) { - // `int | None` in annotations evaluates to a union; keep it inert. - return new PyTuple(new Object[] {this, o}); + return PyUnion.of(this, o); + } + + @Override + public Object ror(Object o) { + return PyUnion.of(o, this); } private static PyType[] computeMro(PyType self, PyType[] bases) { diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyUnion.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyUnion.java new file mode 100644 index 00000000000..fb8c9cbcff1 --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyUnion.java @@ -0,0 +1,170 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +import java.util.ArrayList; +import java.util.List; + +/// `int | None`: a union of types, flattened and without duplicates. It +/// answers `isinstance` and carries `__args__` for `typing.get_args`. +final class PyUnion extends PyObject { + static final PyType TYPE = new PyType("typing.Union", new PyType[] {Py.OBJECT_TYPE}, null); + + final PyTuple args; + + private PyUnion(Object[] items) { + this.args = new PyTuple(items); + } + + static Object of(Object a, Object b) { + List out = new ArrayList(); + add(out, a); + add(out, b); + if (out.size() == 1) { + return out.get(0); + } + return new PyUnion(out.toArray()); + } + + private static void add(List out, Object t) { + if (t instanceof PyUnion) { + for (Object o : ((PyUnion) t).args.items) { + add(out, o); + } + return; + } + Object v = t == Py.NONE ? Py.NONE_TYPE : t; + for (Object o : out) { + if (o == v || Py.eqBool(o, v)) { + return; + } + } + out.add(v); + } + + /// How a type reads inside an alias: `int`, `mod.Cls`, `None`. + static String typeRepr(Object t) { + if (t == Py.NONE || t == Py.NONE_TYPE) { + return "None"; + } + if (t == Py.ELLIPSIS) { + return "..."; + } + if (t instanceof PyType) { + PyType p = (PyType) t; + Object q = p.dict.get("__qualname__"); + String qual = q instanceof String ? (String) q : p.name; + Object mod = p.dict.get("__module__"); + if (mod instanceof String && !"builtins".equals(mod)) { + return mod + "." + qual; + } + return qual; + } + return Py.toRepr(t); + } + + boolean instanceCheck(Object o) { + for (Object t : args.items) { + if (Py.isinstance(o, t)) { + return true; + } + } + return false; + } + + @Override + public PyType type() { + return TYPE; + } + + @Override + public Object getAttrOrNull(String attr) { + if ("__args__".equals(attr)) { + return args; + } + if ("__origin__".equals(attr)) { + return TYPE; + } + if ("__parameters__".equals(attr)) { + return new PyTuple(new Object[0]); + } + return super.getAttrOrNull(attr); + } + + @Override + public Object or(Object o) { + return of(this, o); + } + + @Override + public Object ror(Object o) { + return of(o, this); + } + + @Override + public Object eq(Object o) { + if (!(o instanceof PyUnion)) { + return null; + } + Object[] mine = args.items; + Object[] theirs = ((PyUnion) o).args.items; + if (mine.length != theirs.length) { + return Py.FALSE; + } + for (Object t : mine) { + boolean found = false; + for (Object u : theirs) { + if (Py.eqBool(t, u)) { + found = true; + break; + } + } + if (!found) { + return Py.FALSE; + } + } + return Py.TRUE; + } + + @Override + public long hash() { + long h = 0; + for (Object t : args.items) { + h ^= Py.hashOf(t); + } + return h; + } + + @Override + public String repr() { + StringBuilder b = new StringBuilder(); + for (int i = 0; i < args.items.length; i++) { + if (i > 0) { + b.append(" | "); + } + b.append(typeRepr(args.items[i])); + } + return b.toString(); + } +} diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/types.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/types.py index 80eb3e03880..1d5c8ba60b8 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/types.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/types.py @@ -41,8 +41,8 @@ def _m(self): NoneType = type(None) NotImplementedType = type(NotImplemented) EllipsisType = type(...) -GenericAlias = None -UnionType = None +GenericAlias = type(list[int]) +UnionType = type(int | str) class SimpleNamespace: diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/typing.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/typing.py index 6248ae013e1..a4097b279d2 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/typing.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/typing.py @@ -35,6 +35,8 @@ def __init__(self, name, getitem=None): self._getitem = getitem def __repr__(self): + if self._name == "Union": + return "" return "typing." + self._name def __getitem__(self, params): @@ -139,6 +141,13 @@ def _union(form, params): flat.append(i) if len(flat) == 1: return flat[0] + if all(isinstance(i, (type, _RuntimeAlias)) for i in flat): + # Plain types make the runtime's own union, the object `int | None` + # evaluates to, so both spellings compare equal and answer isinstance. + result = flat[0] + for i in flat[1:]: + result = result | i + return result return _UnionGenericAlias(Union, tuple(flat), "Union") @@ -483,7 +492,15 @@ def decorator(cls_or_fn): return decorator +_RuntimeUnion = type(int | str) +_RuntimeAlias = type(list[int]) + + def get_origin(tp): + if isinstance(tp, _RuntimeUnion): + return Union + if isinstance(tp, _RuntimeAlias): + return tp.__origin__ if isinstance(tp, _AnnotatedAlias): return Annotated if isinstance(tp, (_GenericAlias, _SpecialGenericAlias)): @@ -494,6 +511,8 @@ def get_origin(tp): def get_args(tp): + if isinstance(tp, (_RuntimeUnion, _RuntimeAlias)): + return tp.__args__ if isinstance(tp, _AnnotatedAlias): return (tp.__origin__,) + tp.__metadata__ if isinstance(tp, _CallableGenericAlias): @@ -506,13 +525,112 @@ def get_args(tp): return () +class _HintParser: + """Evaluates a string annotation without eval: names, dotted names, + `X[...]`, `|`, `None`, `...` and string literals, which is the grammar + annotations are written in.""" + + def __init__(self, text, ns): + self.text = text + self.pos = 0 + self.ns = ns + + def _ws(self): + while self.pos < len(self.text) and self.text[self.pos] == " ": + self.pos += 1 + + def _peek(self): + self._ws() + return self.text[self.pos] if self.pos < len(self.text) else "" + + def parse(self): + value = self._union() + if self._peek(): + raise NameError("cannot evaluate annotation " + repr(self.text)) + return value + + def _union(self): + value = self._primary() + while self._peek() == "|": + self.pos += 1 + value = Union[value, self._primary()] + return value + + def _primary(self): + c = self._peek() + if c in ("'", '"'): + end = self.text.index(c, self.pos + 1) + inner = self.text[self.pos + 1:end] + self.pos = end + 1 + return _HintParser(inner, self.ns).parse() + if self.text.startswith("...", self.pos): + self.pos += 3 + return ... + if c == "[": + self.pos += 1 + items = self._items("]") + return items + start = self.pos + while self.pos < len(self.text) and (self.text[self.pos].isalnum() or self.text[self.pos] in "_."): + self.pos += 1 + name = self.text[start:self.pos] + if not name: + raise NameError("cannot evaluate annotation " + repr(self.text)) + value = self._lookup(name) + while self._peek() == "[": + self.pos += 1 + items = self._items("]") + value = value[items[0] if len(items) == 1 else tuple(items)] + return value + + def _items(self, close): + items = [] + while self._peek() != close: + items.append(self._union()) + if self._peek() == ",": + self.pos += 1 + self.pos += 1 + return items + + def _lookup(self, name): + parts = name.split(".") + if parts[0] == "None": + return None + if parts[0] in self.ns: + value = self.ns[parts[0]] + else: + import builtins + if not hasattr(builtins, parts[0]): + raise NameError("name '" + parts[0] + "' is not defined") + value = getattr(builtins, parts[0]) + for p in parts[1:]: + value = getattr(value, p) + return value + + +def _resolve_hint(value, ns): + if isinstance(value, str): + return _HintParser(value, ns).parse() + return value + + def get_type_hints(obj, globalns=None, localns=None, include_extras=False): + import sys hints = {} if isinstance(obj, type): for base in reversed(obj.__mro__): - hints.update(base.__dict__.get("__annotations__", {})) + ns = dict(globalns) if globalns is not None else dict(getattr(sys.modules.get(base.__module__), "__dict__", {})) + ns.setdefault(base.__name__, base) + if localns: + ns.update(localns) + for k, v in base.__dict__.get("__annotations__", {}).items(): + hints[k] = _resolve_hint(v, ns) else: - hints.update(getattr(obj, "__annotations__", {})) + ns = dict(globalns) if globalns is not None else dict(getattr(obj, "__globals__", {})) + if localns: + ns.update(localns) + for k, v in getattr(obj, "__annotations__", {}).items(): + hints[k] = _resolve_hint(v, ns) if not include_extras: for k, v in list(hints.items()): if isinstance(v, _AnnotatedAlias): diff --git a/maven/python-transpiler/src/test/resources/conformance/typing_alias.out b/maven/python-transpiler/src/test/resources/conformance/typing_alias.out new file mode 100644 index 00000000000..b6021defe68 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/typing_alias.out @@ -0,0 +1,17 @@ +a None () +b list[int] (,) +c dict[str, float] (, ) +d str | None (, ) +e int | str (, ) +f typing.Literal['x', 'y'] typing.Literal ('x', 'y') +g typing.Annotated[int, 'meta'] typing.Annotated (, 'meta') +h M | None None () +i int | None (, ) +j tuple[int, ...] (, Ellipsis) +k None () +l None () +m set[str] (,) +n typing.Any None () +o list['M'] ('M',) +__main__.M | None +None True M diff --git a/maven/python-transpiler/src/test/resources/conformance/typing_alias.py b/maven/python-transpiler/src/test/resources/conformance/typing_alias.py new file mode 100644 index 00000000000..bc3bb551bdf --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/typing_alias.py @@ -0,0 +1,28 @@ +import typing +from typing import Optional, Union, Literal, Annotated, Any, get_origin, get_args +import datetime, enum + +class Color(enum.Enum): + RED = "red" + +class M: + a: int + b: list[int] + c: dict[str, float] + d: Optional[str] = None + e: Union[int, str] = 1 + f: Literal["x", "y"] = "x" + g: Annotated[int, "meta"] = 0 + h: "M | None" = None + i: int | None = None + j: tuple[int, ...] = () + k: datetime.date = None + l: Color = Color.RED + m: set[str] = set() + n: Any = None + o: list["M"] = [] + +for k, v in M.__annotations__.items(): + print(k, v, get_origin(v), get_args(v)) +print(typing.get_type_hints(M)["h"]) +print(M.__dict__.get("d"), "e" in vars(M), M.__mro__[0].__name__) diff --git a/scripts/python-bench/conformance_parparvm.sh b/scripts/python-bench/conformance_parparvm.sh index 25722b969f4..4684116857e 100755 --- a/scripts/python-bench/conformance_parparvm.sh +++ b/scripts/python-bench/conformance_parparvm.sh @@ -10,7 +10,11 @@ HERE="$(cd "$(dirname "$0")" && pwd)" REPO="$(cd "$HERE/../.." && pwd)" CONF="$REPO/maven/python-transpiler/src/test/resources/conformance" progs="$*" -[ -n "$progs" ] || progs=$(cd "$CONF" && ls *.py | sed 's/\.py$//') +# http_* programs talk to ConformanceTest's server through the host's HTTP +# client, which a bare native executable has none of (ConnectionRequest +# needs the UI port): they run on the JVM only. The packages they test are +# Python, so this binary runs the same code paths everywhere but the socket. +[ -n "$progs" ] || progs=$(cd "$CONF" && ls *.py | sed 's/\.py$//' | grep -v '^http_') failed=0 for p in $progs; do if ! PY_SOURCES="$CONF" "$HERE/build_parparvm.sh" "$p" > /dev/null 2>&1; then From 23bdc60d7fcf8841c8564deb390ab5d900cfc610 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Tue, 27 Oct 2026 17:53:26 +0200 Subject: [PATCH 007/112] Python: PyYAML package Co-Authored-By: Claude Opus 5.5 --- .../codename1/python/site/yaml/__init__.py | 3066 +++++++++++++++++ .../src/test/resources/conformance/yaml_.out | 252 ++ .../src/test/resources/conformance/yaml_.py | 262 ++ .../test/resources/conformance/yaml_dump.out | 676 ++++ .../test/resources/conformance/yaml_dump.py | 203 ++ 5 files changed, 4459 insertions(+) create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/yaml/__init__.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/yaml_.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/yaml_.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/yaml_dump.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/yaml_dump.py diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/yaml/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/yaml/__init__.py new file mode 100644 index 00000000000..7f15d00aa1f --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/yaml/__init__.py @@ -0,0 +1,3066 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""YAML 1.1 loading and dumping with the interface of the PyYAML package. + +This is an independent implementation written from the YAML 1.1 specification +and from the documented behaviour of PyYAML. It covers safe loading, the +document/anchor/merge machinery, the implicit type resolvers, the explicit +core tags, and a dumper whose quoting and layout decisions follow PyYAML. +""" + +import re +import base64 +import datetime + +__version__ = "6.0.3" +__with_libyaml__ = False + +_BLANKZ = "\0 \t\r\n\x85\u2028\u2029" +_HEXDIGITS = "0123456789abcdefABCDEF" + + +class _NS: + pass + + +# ---------------------------------------------------------------- errors + +class Mark: + def __init__(self, name, index, line, column, buffer, pointer): + self.name = name + self.index = index + self.line = line + self.column = column + self.buffer = buffer + self.pointer = pointer + + def get_snippet(self, indent=4, max_length=75): + if self.buffer is None: + return None + buf = self.buffer + head = "" + start = self.pointer + while start > 0 and buf[start - 1] not in "\0\r\n\x85\u2028\u2029": + start -= 1 + if self.pointer - start > max_length / 2 - 1: + head = " ... " + start += 5 + break + tail = "" + end = self.pointer + while end < len(buf) and buf[end] not in "\0\r\n\x85\u2028\u2029": + end += 1 + if end - self.pointer > max_length / 2 - 1: + tail = " ... " + end -= 5 + break + snippet = buf[start:end] + return (" " * indent + head + snippet + tail + "\n" + + " " * (indent + self.pointer - start + len(head)) + "^") + + def __str__(self): + snippet = self.get_snippet() + where = ' in "%s", line %d, column %d' % (self.name, self.line + 1, self.column + 1) + if snippet is not None: + where += ":\n" + snippet + return where + + +class YAMLError(Exception): + pass + + +class MarkedYAMLError(YAMLError): + def __init__(self, context=None, context_mark=None, problem=None, problem_mark=None, note=None): + YAMLError.__init__(self) + self.context = context + self.context_mark = context_mark + self.problem = problem + self.problem_mark = problem_mark + self.note = note + + def __str__(self): + lines = [] + if self.context is not None: + lines.append(self.context) + if (self.context_mark is not None + and (self.problem is None or self.problem_mark is None + or self.context_mark.name != self.problem_mark.name + or self.context_mark.line != self.problem_mark.line + or self.context_mark.column != self.problem_mark.column)): + lines.append(str(self.context_mark)) + if self.problem is not None: + lines.append(self.problem) + if self.problem_mark is not None: + lines.append(str(self.problem_mark)) + if self.note is not None: + lines.append(self.note) + return "\n".join(lines) + + +class ReaderError(YAMLError): + def __init__(self, name, position, character, encoding, reason): + YAMLError.__init__(self) + self.name = name + self.character = character + self.position = position + self.encoding = encoding + self.reason = reason + + def __str__(self): + if isinstance(self.character, bytes): + return ("'%s' codec can't decode byte #x%02x: %s\n in \"%s\", position %d" + % (self.encoding, ord(self.character), self.reason, self.name, self.position)) + return ('unacceptable character #x%04x: %s\n in "%s", position %d' + % (self.character, self.reason, self.name, self.position)) + + +class ScannerError(MarkedYAMLError): + pass + + +class ParserError(MarkedYAMLError): + pass + + +class ComposerError(MarkedYAMLError): + pass + + +class ConstructorError(MarkedYAMLError): + pass + + +class RepresenterError(YAMLError): + pass + + +class EmitterError(YAMLError): + pass + + +class SerializerError(YAMLError): + pass + + +# ----------------------------------------------------------------- nodes + +class Node: + def __init__(self, tag, value, start_mark, end_mark): + self.tag = tag + self.value = value + self.start_mark = start_mark + self.end_mark = end_mark + + def __repr__(self): + return "%s(tag=%r, value=%r)" % (self.__class__.__name__, self.tag, self.value) + + +class ScalarNode(Node): + id = "scalar" + + def __init__(self, tag, value, start_mark=None, end_mark=None, style=None): + Node.__init__(self, tag, value, start_mark, end_mark) + self.style = style + + +class CollectionNode(Node): + def __init__(self, tag, value, start_mark=None, end_mark=None, flow_style=None): + Node.__init__(self, tag, value, start_mark, end_mark) + self.flow_style = flow_style + + +class SequenceNode(CollectionNode): + id = "sequence" + + +class MappingNode(CollectionNode): + id = "mapping" + + +# ------------------------------------------------------------- scanner + +class Token: + def __init__(self, kind, start_mark, end_mark, value=None, plain=False, style=None): + self.id = kind + self.start_mark = start_mark + self.end_mark = end_mark + self.value = value + self.plain = plain + self.style = style + + +class _SimpleKey: + def __init__(self, token_number, required, index, line, column, mark): + self.token_number = token_number + self.required = required + self.index = index + self.line = line + self.column = column + self.mark = mark + + +_ESC_REPL = { + "0": "\0", "a": "\x07", "b": "\x08", "t": "\x09", "\t": "\x09", "n": "\x0a", + "v": "\x0b", "f": "\x0c", "r": "\x0d", "e": "\x1b", " ": " ", '"': '"', + "\\": "\\", "/": "/", "N": "\x85", "_": "\xa0", "L": "\u2028", "P": "\u2029", +} +_ESC_CODES = {"x": 2, "u": 4, "U": 8} + + +class _Scanner: + """Turns the character buffer into the YAML token stream.""" + + def _init_scanner(self, text, name): + self.name = name + self.buffer = text + "\0" + self.pointer = 0 + self.index = 0 + self.line = 0 + self.column = 0 + self.done = False + self.flow_level = 0 + self.tokens = [] + self.tokens_taken = 0 + self.indent = -1 + self.indents = [] + self.allow_simple_key = True + self.possible_simple_keys = {} + mark = self._mark() + self.tokens.append(Token("", mark, mark)) + + # -- character access + def _peek(self, k=0): + i = self.pointer + k + if i < len(self.buffer): + return self.buffer[i] + return "\0" + + def _prefix(self, n=1): + return self.buffer[self.pointer:self.pointer + n] + + def _forward(self, n=1): + while n: + ch = self.buffer[self.pointer] + self.pointer += 1 + self.index += 1 + if ch in "\n\x85\u2028\u2029" or (ch == "\r" and self.buffer[self.pointer] != "\n"): + self.line += 1 + self.column = 0 + elif ch != "\ufeff": + self.column += 1 + n -= 1 + + def _mark(self): + return Mark(self.name, self.index, self.line, self.column, self.buffer, self.pointer) + + # -- token queue + def _check_token(self, *kinds): + while self._need_more_tokens(): + self._fetch_more_tokens() + if self.tokens: + if not kinds: + return True + for k in kinds: + if self.tokens[0].id == k: + return True + return False + + def _peek_token(self): + while self._need_more_tokens(): + self._fetch_more_tokens() + if self.tokens: + return self.tokens[0] + return None + + def _get_token(self): + while self._need_more_tokens(): + self._fetch_more_tokens() + if self.tokens: + self.tokens_taken += 1 + return self.tokens.pop(0) + return None + + def _need_more_tokens(self): + if self.done: + return False + if not self.tokens: + return True + self._stale_keys() + if self._next_key_number() == self.tokens_taken: + return True + return False + + def _next_key_number(self): + best = None + for level in self.possible_simple_keys: + key = self.possible_simple_keys[level] + if best is None or key.token_number < best: + best = key.token_number + return best + + def _stale_keys(self): + for level in list(self.possible_simple_keys): + key = self.possible_simple_keys[level] + if key.line != self.line or self.index - key.index > 1024: + if key.required: + raise ScannerError("while scanning a simple key", key.mark, + "could not find expected ':'", self._mark()) + del self.possible_simple_keys[level] + + def _save_key(self): + required = not self.flow_level and self.indent == self.column + if self.allow_simple_key: + self._remove_key() + number = self.tokens_taken + len(self.tokens) + self.possible_simple_keys[self.flow_level] = _SimpleKey( + number, required, self.index, self.line, self.column, self._mark()) + + def _remove_key(self): + if self.flow_level in self.possible_simple_keys: + key = self.possible_simple_keys[self.flow_level] + if key.required: + raise ScannerError("while scanning a simple key", key.mark, + "could not find expected ':'", self._mark()) + del self.possible_simple_keys[self.flow_level] + + def _unwind(self, column): + if self.flow_level: + return + while self.indent > column: + mark = self._mark() + self.indent = self.indents.pop() + self.tokens.append(Token("", mark, mark)) + + def _add_indent(self, column): + if self.indent < column: + self.indents.append(self.indent) + self.indent = column + return True + return False + + # -- dispatcher + def _fetch_more_tokens(self): + self._scan_to_next_token() + self._stale_keys() + self._unwind(self.column) + ch = self._peek() + if ch == "\0": + return self._fetch_stream_end() + if ch == "%" and self.column == 0: + return self._fetch_directive() + if ch == "-" and self.column == 0 and self._prefix(3) == "---" and self._peek(3) in _BLANKZ: + return self._fetch_document_indicator("") + if ch == "." and self.column == 0 and self._prefix(3) == "..." and self._peek(3) in _BLANKZ: + return self._fetch_document_indicator("") + if ch == "[": + return self._fetch_flow_start("[") + if ch == "{": + return self._fetch_flow_start("{") + if ch == "]": + return self._fetch_flow_end("]") + if ch == "}": + return self._fetch_flow_end("}") + if ch == ",": + return self._fetch_flow_entry() + if ch == "-" and self._peek(1) in _BLANKZ: + return self._fetch_block_entry() + if ch == "?" and (self.flow_level or self._peek(1) in _BLANKZ): + return self._fetch_key() + if ch == ":" and (self.flow_level or self._peek(1) in _BLANKZ): + return self._fetch_value() + if ch == "*": + return self._fetch_anchor("") + if ch == "&": + return self._fetch_anchor("") + if ch == "!": + return self._fetch_tag() + if ch == "|" and not self.flow_level: + return self._fetch_block_scalar("|") + if ch == ">" and not self.flow_level: + return self._fetch_block_scalar(">") + if ch == "'": + return self._fetch_flow_scalar("'") + if ch == '"': + return self._fetch_flow_scalar('"') + if self._check_plain(): + return self._fetch_plain() + raise ScannerError("while scanning for the next token", None, + "found character %r that cannot start any token" % ch, self._mark()) + + def _check_plain(self): + ch = self._peek() + return (ch not in "\0 \t\r\n\x85\u2028\u2029-?:,[]{}#&*!|>'\"%@`" + or (self._peek(1) not in _BLANKZ + and (ch == "-" or (not self.flow_level and ch in "?:")))) + + def _scan_to_next_token(self): + if self.index == 0 and self._peek() == "\ufeff": + self._forward() + found = False + while not found: + while self._peek() == " ": + self._forward() + if self._peek() == "#": + while self._peek() not in "\0\r\n\x85\u2028\u2029": + self._forward() + if self._scan_line_break(): + if not self.flow_level: + self.allow_simple_key = True + else: + found = True + + def _scan_line_break(self): + ch = self._peek() + if ch in "\r\n\x85": + if self._prefix(2) == "\r\n": + self._forward(2) + else: + self._forward() + return "\n" + if ch in "\u2028\u2029": + self._forward() + return ch + return "" + + # -- fetchers + def _fetch_stream_end(self): + self._unwind(-1) + self._remove_key() + self.allow_simple_key = False + self.possible_simple_keys = {} + mark = self._mark() + self.tokens.append(Token("", mark, mark)) + self.done = True + + def _fetch_directive(self): + self._unwind(-1) + self._remove_key() + self.allow_simple_key = False + self.tokens.append(self._scan_directive()) + + def _fetch_document_indicator(self, kind): + self._unwind(-1) + self._remove_key() + self.allow_simple_key = False + start = self._mark() + self._forward(3) + self.tokens.append(Token(kind, start, self._mark())) + + def _fetch_flow_start(self, kind): + self._save_key() + self.flow_level += 1 + self.allow_simple_key = True + start = self._mark() + self._forward() + self.tokens.append(Token(kind, start, self._mark())) + + def _fetch_flow_end(self, kind): + self._remove_key() + if self.flow_level: + self.flow_level -= 1 + self.allow_simple_key = False + start = self._mark() + self._forward() + self.tokens.append(Token(kind, start, self._mark())) + + def _fetch_flow_entry(self): + self.allow_simple_key = True + self._remove_key() + start = self._mark() + self._forward() + self.tokens.append(Token(",", start, self._mark())) + + def _fetch_block_entry(self): + if not self.flow_level: + if not self.allow_simple_key: + raise ScannerError(None, None, "sequence entries are not allowed here", self._mark()) + if self._add_indent(self.column): + mark = self._mark() + self.tokens.append(Token("", mark, mark)) + self.allow_simple_key = True + self._remove_key() + start = self._mark() + self._forward() + self.tokens.append(Token("-", start, self._mark())) + + def _fetch_key(self): + if not self.flow_level: + if not self.allow_simple_key: + raise ScannerError(None, None, "mapping keys are not allowed here", self._mark()) + if self._add_indent(self.column): + mark = self._mark() + self.tokens.append(Token("", mark, mark)) + self.allow_simple_key = not self.flow_level + self._remove_key() + start = self._mark() + self._forward() + self.tokens.append(Token("?", start, self._mark())) + + def _fetch_value(self): + if self.flow_level in self.possible_simple_keys: + key = self.possible_simple_keys[self.flow_level] + del self.possible_simple_keys[self.flow_level] + self.tokens.insert(key.token_number - self.tokens_taken, + Token("?", key.mark, key.mark)) + if not self.flow_level: + if self._add_indent(key.column): + self.tokens.insert(key.token_number - self.tokens_taken, + Token("", key.mark, key.mark)) + self.allow_simple_key = False + else: + if not self.flow_level: + if not self.allow_simple_key: + raise ScannerError(None, None, "mapping values are not allowed here", self._mark()) + if not self.flow_level: + if self._add_indent(self.column): + mark = self._mark() + self.tokens.append(Token("", mark, mark)) + self.allow_simple_key = not self.flow_level + self._remove_key() + start = self._mark() + self._forward() + self.tokens.append(Token(":", start, self._mark())) + + def _fetch_anchor(self, kind): + self._save_key() + self.allow_simple_key = False + self.tokens.append(self._scan_anchor(kind)) + + def _fetch_tag(self): + self._save_key() + self.allow_simple_key = False + self.tokens.append(self._scan_tag()) + + def _fetch_block_scalar(self, style): + self.allow_simple_key = True + self._remove_key() + self.tokens.append(self._scan_block_scalar(style)) + + def _fetch_flow_scalar(self, style): + self._save_key() + self.allow_simple_key = False + self.tokens.append(self._scan_flow_scalar(style)) + + def _fetch_plain(self): + self._save_key() + self.allow_simple_key = False + self.tokens.append(self._scan_plain()) + + # -- scanners + def _scan_directive(self): + start = self._mark() + self._forward() + length = 0 + while self._peek(length).isalnum() or self._peek(length) in "-_": + length += 1 + name = self._prefix(length) + self._forward(length) + if name == "YAML": + while self._peek() == " ": + self._forward() + major = self._scan_version_number(start) + if self._peek() != ".": + raise ScannerError("while scanning a directive", start, + "expected a digit or '.', but found %r" % self._peek(), self._mark()) + self._forward() + minor = self._scan_version_number(start) + value = (major, minor) + end = self._mark() + elif name == "TAG": + while self._peek() == " ": + self._forward() + handle = self._scan_tag_handle("directive", start) + if self._peek() != " ": + raise ScannerError("while scanning a directive", start, + "expected ' ', but found %r" % self._peek(), self._mark()) + while self._peek() == " ": + self._forward() + prefix = self._scan_tag_uri("directive", start) + value = (handle, prefix) + end = self._mark() + else: + end = self._mark() + value = None + while self._peek() not in "\0\r\n\x85\u2028\u2029": + self._forward() + while self._peek() == " ": + self._forward() + if self._peek() == "#": + while self._peek() not in "\0\r\n\x85\u2028\u2029": + self._forward() + if self._peek() not in "\0\r\n\x85\u2028\u2029": + raise ScannerError("while scanning a directive", start, + "expected a comment or a line break, but found %r" % self._peek(), + self._mark()) + self._scan_line_break() + return Token("", start, end, (name, value)) + + def _scan_version_number(self, start): + length = 0 + while "0" <= self._peek(length) <= "9": + length += 1 + if not length: + raise ScannerError("while scanning a directive", start, + "expected a digit, but found %r" % self._peek(), self._mark()) + value = int(self._prefix(length)) + self._forward(length) + return value + + def _scan_anchor(self, kind): + start = self._mark() + what = "alias" if kind == "" else "anchor" + self._forward() + length = 0 + ch = self._peek(length) + while ("0" <= ch <= "9" or "A" <= ch <= "Z" or "a" <= ch <= "z" or ch in "-_"): + length += 1 + ch = self._peek(length) + if not length: + raise ScannerError("while scanning an %s" % what, start, + "expected alphabetic or numeric character, but found %r" % ch, + self._mark()) + value = self._prefix(length) + self._forward(length) + ch = self._peek() + if ch not in "\0 \t\r\n\x85\u2028\u2029?:,]}%@`": + raise ScannerError("while scanning an %s" % what, start, + "expected alphabetic or numeric character, but found %r" % ch, + self._mark()) + return Token(kind, start, self._mark(), value) + + def _scan_tag(self): + start = self._mark() + ch = self._peek(1) + if ch == "<": + handle = None + self._forward(2) + suffix = self._scan_tag_uri("tag", start) + if self._peek() != ">": + raise ScannerError("while parsing a tag", start, + "expected '>', but found %r" % self._peek(), self._mark()) + self._forward() + elif ch in _BLANKZ: + handle = None + suffix = "!" + self._forward() + else: + length = 1 + use_handle = False + while ch not in _BLANKZ: + if ch == "!": + use_handle = True + break + length += 1 + ch = self._peek(length) + if use_handle: + handle = self._scan_tag_handle("tag", start) + else: + handle = "!" + self._forward() + suffix = self._scan_tag_uri("tag", start) + ch = self._peek() + if ch not in "\0 \r\n\x85\u2028\u2029": + raise ScannerError("while scanning a tag", start, + "expected ' ', but found %r" % ch, self._mark()) + return Token("", start, self._mark(), (handle, suffix)) + + def _scan_tag_handle(self, name, start): + ch = self._peek() + if ch != "!": + raise ScannerError("while scanning a %s" % name, start, + "expected '!', but found %r" % ch, self._mark()) + length = 1 + ch = self._peek(length) + if ch != " ": + while ("0" <= ch <= "9" or "A" <= ch <= "Z" or "a" <= ch <= "z" or ch in "-_"): + length += 1 + ch = self._peek(length) + if ch != "!": + self._forward(length) + raise ScannerError("while scanning a %s" % name, start, + "expected '!', but found %r" % ch, self._mark()) + length += 1 + value = self._prefix(length) + self._forward(length) + return value + + def _scan_tag_uri(self, name, start): + chunks = [] + length = 0 + ch = self._peek(length) + while ("0" <= ch <= "9" or "A" <= ch <= "Z" or "a" <= ch <= "z" + or ch in "-;/?:@&=+$,_.!~*'()[]%"): + if ch == "%": + chunks.append(self._prefix(length)) + self._forward(length) + length = 0 + chunks.append(self._scan_uri_escapes(name, start)) + else: + length += 1 + ch = self._peek(length) + if length: + chunks.append(self._prefix(length)) + self._forward(length) + if not chunks: + raise ScannerError("while parsing a %s" % name, start, + "expected URI, but found %r" % ch, self._mark()) + return "".join(chunks) + + def _scan_uri_escapes(self, name, start): + data = [] + mark = self._mark() + while self._peek() == "%": + self._forward() + for k in range(2): + if self._peek(k) not in _HEXDIGITS: + raise ScannerError("while scanning a %s" % name, start, + "expected URI escape sequence of 2 hexadecimal numbers, but found %r" + % self._peek(k), self._mark()) + data.append(int(self._prefix(2), 16)) + self._forward(2) + try: + return bytes(data).decode("utf-8") + except UnicodeDecodeError as exc: + raise ScannerError("while scanning a %s" % name, start, str(exc), mark) + + def _scan_block_scalar(self, style): + folded = style == ">" + chunks = [] + start = self._mark() + self._forward() + chomping, increment = self._scan_block_indicators(start) + self._scan_block_ignored_line(start) + min_indent = self.indent + 1 + if min_indent < 1: + min_indent = 1 + if increment is None: + breaks, max_indent, end = self._scan_block_indentation() + indent = max(min_indent, max_indent) + else: + indent = min_indent + increment - 1 + breaks, end = self._scan_block_breaks(indent) + line_break = "" + while self.column == indent and self._peek() != "\0": + chunks.extend(breaks) + leading_non_space = self._peek() not in " \t" + length = 0 + while self._peek(length) not in "\0\r\n\x85\u2028\u2029": + length += 1 + chunks.append(self._prefix(length)) + self._forward(length) + line_break = self._scan_line_break() + breaks, end = self._scan_block_breaks(indent) + if self.column == indent and self._peek() != "\0": + if folded and line_break == "\n" and leading_non_space and self._peek() not in " \t": + if not breaks: + chunks.append(" ") + else: + chunks.append(line_break) + else: + break + if chomping is not False: + chunks.append(line_break) + if chomping is True: + chunks.extend(breaks) + return Token("", start, end, "".join(chunks), False, style) + + def _scan_block_indicators(self, start): + chomping = None + increment = None + ch = self._peek() + if ch in "+-": + chomping = ch == "+" + self._forward() + ch = self._peek() + if ch in "0123456789": + increment = int(ch) + if increment == 0: + raise ScannerError("while scanning a block scalar", start, + "expected indentation indicator in the range 1-9, but found 0", + self._mark()) + self._forward() + elif ch in "0123456789": + increment = int(ch) + if increment == 0: + raise ScannerError("while scanning a block scalar", start, + "expected indentation indicator in the range 1-9, but found 0", + self._mark()) + self._forward() + ch = self._peek() + if ch in "+-": + chomping = ch == "+" + self._forward() + ch = self._peek() + if ch not in "\0 \r\n\x85\u2028\u2029": + raise ScannerError("while scanning a block scalar", start, + "expected chomping or indentation indicators, but found %r" % ch, + self._mark()) + return chomping, increment + + def _scan_block_ignored_line(self, start): + while self._peek() == " ": + self._forward() + if self._peek() == "#": + while self._peek() not in "\0\r\n\x85\u2028\u2029": + self._forward() + ch = self._peek() + if ch not in "\0\r\n\x85\u2028\u2029": + raise ScannerError("while scanning a block scalar", start, + "expected a comment or a line break, but found %r" % ch, self._mark()) + self._scan_line_break() + + def _scan_block_indentation(self): + chunks = [] + max_indent = 0 + end = self._mark() + while self._peek() in " \r\n\x85\u2028\u2029": + if self._peek() != " ": + chunks.append(self._scan_line_break()) + end = self._mark() + else: + self._forward() + if self.column > max_indent: + max_indent = self.column + return chunks, max_indent, end + + def _scan_block_breaks(self, indent): + chunks = [] + end = self._mark() + while self.column < indent and self._peek() == " ": + self._forward() + while self._peek() in "\r\n\x85\u2028\u2029": + chunks.append(self._scan_line_break()) + end = self._mark() + while self.column < indent and self._peek() == " ": + self._forward() + return chunks, end + + def _scan_flow_scalar(self, style): + double = style == '"' + chunks = [] + start = self._mark() + quote = self._peek() + self._forward() + chunks.extend(self._scan_flow_non_spaces(double, start)) + while self._peek() != quote: + chunks.extend(self._scan_flow_spaces(double, start)) + chunks.extend(self._scan_flow_non_spaces(double, start)) + self._forward() + return Token("", start, self._mark(), "".join(chunks), False, style) + + def _scan_flow_non_spaces(self, double, start): + chunks = [] + while True: + length = 0 + while self._peek(length) not in "'\"\\\0 \t\r\n\x85\u2028\u2029": + length += 1 + if length: + chunks.append(self._prefix(length)) + self._forward(length) + ch = self._peek() + if not double and ch == "'" and self._peek(1) == "'": + chunks.append("'") + self._forward(2) + elif (double and ch == "'") or (not double and ch in '"\\'): + chunks.append(ch) + self._forward() + elif double and ch == "\\": + self._forward() + ch = self._peek() + if ch in _ESC_REPL: + chunks.append(_ESC_REPL[ch]) + self._forward() + elif ch in _ESC_CODES: + length = _ESC_CODES[ch] + self._forward() + for k in range(length): + if self._peek(k) not in _HEXDIGITS: + raise ScannerError( + "while scanning a double-quoted scalar", start, + "expected escape sequence of %d hexadecimal numbers, but found %r" + % (length, self._peek(k)), self._mark()) + code = int(self._prefix(length), 16) + chunks.append(chr(code)) + self._forward(length) + elif ch in "\r\n\x85\u2028\u2029": + self._scan_line_break() + chunks.extend(self._scan_flow_breaks(double, start)) + else: + raise ScannerError("while scanning a double-quoted scalar", start, + "found unknown escape character %r" % ch, self._mark()) + else: + return chunks + + def _scan_flow_spaces(self, double, start): + chunks = [] + length = 0 + while self._peek(length) in " \t": + length += 1 + whitespaces = self._prefix(length) + self._forward(length) + ch = self._peek() + if ch == "\0": + raise ScannerError("while scanning a quoted scalar", start, + "found unexpected end of stream", self._mark()) + elif ch in "\r\n\x85\u2028\u2029": + line_break = self._scan_line_break() + breaks = self._scan_flow_breaks(double, start) + if line_break != "\n": + chunks.append(line_break) + elif not breaks: + chunks.append(" ") + chunks.extend(breaks) + else: + chunks.append(whitespaces) + return chunks + + def _scan_flow_breaks(self, double, start): + chunks = [] + while True: + prefix = self._prefix(3) + if (prefix == "---" or prefix == "...") and self._peek(3) in _BLANKZ: + raise ScannerError("while scanning a quoted scalar", start, + "found unexpected document separator", self._mark()) + while self._peek() in " \t": + self._forward() + if self._peek() in "\r\n\x85\u2028\u2029": + chunks.append(self._scan_line_break()) + else: + return chunks + + def _scan_plain(self): + chunks = [] + start = end = self._mark() + indent = self.indent + 1 + spaces = [] + while True: + length = 0 + if self._peek() == "#": + break + while True: + ch = self._peek(length) + if (ch in _BLANKZ + or (ch == ":" and self._peek(length + 1) in + (_BLANKZ + ",[]{}" if self.flow_level else _BLANKZ)) + or (self.flow_level and ch in ",?[]{}")): + break + length += 1 + if length == 0: + break + self.allow_simple_key = False + chunks.extend(spaces) + chunks.append(self._prefix(length)) + self._forward(length) + end = self._mark() + spaces = self._scan_plain_spaces() + if not spaces or self._peek() == "#" or (not self.flow_level and self.column < indent): + break + return Token("", start, end, "".join(chunks), True) + + def _scan_plain_spaces(self): + chunks = [] + length = 0 + while self._peek(length) in " \t": + length += 1 + whitespaces = self._prefix(length) + self._forward(length) + ch = self._peek() + if ch in "\r\n\x85\u2028\u2029": + line_break = self._scan_line_break() + self.allow_simple_key = True + prefix = self._prefix(3) + if (prefix == "---" or prefix == "...") and self._peek(3) in _BLANKZ: + return [] + breaks = [] + while self._peek() in " \r\n\x85\u2028\u2029": + if self._peek() == " ": + self._forward() + else: + breaks.append(self._scan_line_break()) + prefix = self._prefix(3) + if (prefix == "---" or prefix == "...") and self._peek(3) in _BLANKZ: + return [] + if line_break != "\n": + chunks.append(line_break) + elif not breaks: + chunks.append(" ") + chunks.extend(breaks) + elif whitespaces: + chunks.append(whitespaces) + return chunks + + +# ------------------------------------------------------------ resolvers + +DEFAULT_SCALAR_TAG = "tag:yaml.org,2002:str" +DEFAULT_SEQUENCE_TAG = "tag:yaml.org,2002:seq" +DEFAULT_MAPPING_TAG = "tag:yaml.org,2002:map" + +_RE_BOOL = re.compile(r"^(?:yes|Yes|YES|no|No|NO|true|True|TRUE|false|False|FALSE|on|On|ON|off|Off|OFF)$") +_RE_FLOAT = re.compile( + r"^(?:[-+]?(?:[0-9][0-9_]*)\.[0-9_]*(?:[eE][-+][0-9]+)?" + r"|\.[0-9][0-9_]*(?:[eE][-+][0-9]+)?" + r"|[-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+\.[0-9_]*" + r"|[-+]?\.(?:inf|Inf|INF)" + r"|\.(?:nan|NaN|NAN))$") +_RE_INT = re.compile( + r"^(?:[-+]?0b[0-1_]+" + r"|[-+]?0[0-7_]+" + r"|[-+]?(?:0|[1-9][0-9_]*)" + r"|[-+]?0x[0-9a-fA-F_]+" + r"|[-+]?[1-9][0-9_]*(?::[0-5]?[0-9])+)$") +_RE_MERGE = re.compile(r"^(?:<<)$") +_RE_NULL = re.compile(r"^(?:~|null|Null|NULL|)$") +_RE_TIMESTAMP = re.compile( + r"^(?:[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]" + r"|[0-9][0-9][0-9][0-9]-[0-9][0-9]?-[0-9][0-9]?" + r"(?:[Tt]|[ \t]+)[0-9][0-9]?:[0-9][0-9]:[0-9][0-9]" + r"(?:\.[0-9]*)?(?:[ \t]*(?:Z|[-+][0-9][0-9]?(?::[0-9][0-9])?))?)$") +_RE_TS_PARTS = re.compile( + r"^(?P[0-9][0-9][0-9][0-9])-(?P[0-9][0-9]?)-(?P[0-9][0-9]?)" + r"(?:(?:[Tt]|[ \t]+)(?P[0-9][0-9]?):(?P[0-9][0-9]):(?P[0-9][0-9])" + r"(?:\.(?P[0-9]*))?" + r"(?:[ \t]*(?PZ|(?P[-+])(?P[0-9][0-9]?)(?::(?P[0-9][0-9]))?))?)?$") +_RE_VALUE = re.compile(r"^(?:=)$") + + +def _default_resolvers(): + table = {} + + def add(tag, regexp, first): + for ch in first: + table.setdefault(ch, []).append((tag, regexp)) + + add("tag:yaml.org,2002:bool", _RE_BOOL, list("yYnNtTfFoO")) + add("tag:yaml.org,2002:float", _RE_FLOAT, list("-+0123456789.")) + add("tag:yaml.org,2002:int", _RE_INT, list("-+0123456789")) + add("tag:yaml.org,2002:merge", _RE_MERGE, ["<"]) + add("tag:yaml.org,2002:null", _RE_NULL, ["~", "n", "N", ""]) + add("tag:yaml.org,2002:timestamp", _RE_TIMESTAMP, list("0123456789")) + add("tag:yaml.org,2002:value", _RE_VALUE, ["="]) + return table + + +def _resolve(resolvers, kind, value, implicit): + if kind is ScalarNode and implicit[0]: + if value == "": + found = resolvers.get("", []) + else: + found = resolvers.get(value[0], []) + found = list(found) + list(resolvers.get(None, [])) + for tag, regexp in found: + if regexp.match(value): + return tag + if kind is ScalarNode: + return DEFAULT_SCALAR_TAG + if kind is SequenceNode: + return DEFAULT_SEQUENCE_TAG + return DEFAULT_MAPPING_TAG + + +def _copy_resolvers(table): + out = {} + for k in table: + out[k] = list(table[k]) + return out + + +# --------------------------------------------------------------- loader + +def _is_hashable(key): + if isinstance(key, (list, dict, set, bytearray)): + return False + if isinstance(key, tuple): + for item in key: + if not _is_hashable(item): + return False + return True + + +class _LoaderCore(_Scanner): + yaml_constructors = {} + yaml_multi_constructors = {} + yaml_implicit_resolvers = {} + + def __init__(self, stream): + self._init_reader(stream) + self._init_scanner(self._text, self._name) + self._started = False + self._first = True + self._ended = False + self._pending = False + self._doc_mark = None + self.tag_handles = {} + self.yaml_version = None + self.anchors = {} + self.constructed_objects = {} + self.recursive_objects = {} + self.state_generators = [] + self.deep_construct = False + + # -- reader + def _init_reader(self, stream): + if isinstance(stream, str): + self._name = "" + text = stream + elif isinstance(stream, (bytes, bytearray)): + self._name = "" + text = self._decode(bytes(stream)) + else: + self._name = getattr(stream, "name", "") + data = stream.read() + if isinstance(data, str): + text = data + else: + text = self._decode(bytes(data)) + i = 0 + n = len(text) + while i < n: + o = ord(text[i]) + if not (o == 9 or o == 10 or o == 13 or 0x20 <= o <= 0x7e or o == 0x85 + or 0xa0 <= o <= 0xd7ff or 0xe000 <= o <= 0xfffd or 0x10000 <= o <= 0x10ffff): + raise ReaderError(self._name, i, o, "unicode", "special characters are not allowed") + i += 1 + self._text = text + + def _decode(self, data): + try: + if data.startswith(b"\xff\xfe"): + return data[2:].decode("utf-16-le") + if data.startswith(b"\xfe\xff"): + return data[2:].decode("utf-16-be") + return data.decode("utf-8") + except UnicodeDecodeError as exc: + raise ReaderError(self._name, exc.start, data[exc.start:exc.start + 1], "utf-8", exc.reason) + + def dispose(self): + pass + + # -- parser / composer + def check_node(self): + if self._pending: + return True + if self._has_document(): + self._pending = True + return True + return False + + def get_node(self): + if self.check_node(): + self._pending = False + return self._compose_document() + return None + + def get_single_node(self): + document = None + if self.check_node(): + document = self.get_node() + if self.check_node(): + raise ComposerError("expected a single document in the stream", document.start_mark, + "but found another document", self._doc_mark) + return document + + def _has_document(self): + if self._ended: + return False + if not self._started: + self._get_token() + self._started = True + if self._first: + self._first = False + if not self._check_token("", "", ""): + self.tag_handles = {"!": "!", "!!": "tag:yaml.org,2002:"} + self.yaml_version = None + self._doc_mark = self._peek_token().start_mark + return True + while self._check_token(""): + self._get_token() + if self._check_token(""): + self._get_token() + self._ended = True + return False + token = self._peek_token() + self._doc_mark = token.start_mark + self._process_directives() + if not self._check_token(""): + token = self._peek_token() + raise ParserError(None, None, + "expected '', but found %r" % token.id, token.start_mark) + self._get_token() + return True + + def _process_directives(self): + self.yaml_version = None + self.tag_handles = {} + while self._check_token(""): + token = self._get_token() + if token.value[0] == "YAML": + if self.yaml_version is not None: + raise ParserError(None, None, "found duplicate YAML directive", token.start_mark) + major, minor = token.value[1] + if major != 1: + raise ParserError(None, None, + "found incompatible YAML document (version 1.* is required)", + token.start_mark) + self.yaml_version = token.value[1] + elif token.value[0] == "TAG": + handle, prefix = token.value[1] + if handle in self.tag_handles: + raise ParserError(None, None, "duplicate tag handle %r" % handle, token.start_mark) + self.tag_handles[handle] = prefix + for key in ("!", "!!"): + if key not in self.tag_handles: + self.tag_handles[key] = "!" if key == "!" else "tag:yaml.org,2002:" + + def _compose_document(self): + self.anchors = {} + if self._check_token("", "", "", ""): + token = self._peek_token() + node = self._empty_scalar(token.start_mark, True) + else: + node = self._parse_node(True, False) + if self._check_token(""): + self._get_token() + self.anchors = {} + return node + + def _empty_scalar(self, mark, implicit_plain=True, anchor=None, tag=None): + if tag is None or tag == "!": + tag = _resolve(self.yaml_implicit_resolvers, ScalarNode, "", (True, False)) + node = ScalarNode(tag, "", mark, mark, None) + if anchor is not None: + self._register_anchor(anchor, node, mark) + return node + + def _register_anchor(self, anchor, node, mark): + if anchor in self.anchors: + raise ComposerError("found duplicate anchor %r; first occurrence" % anchor, + self.anchors[anchor].start_mark, "second occurrence", mark) + self.anchors[anchor] = node + + def _parse_node(self, block=False, indentless=False): + if self._check_token(""): + token = self._get_token() + if token.value not in self.anchors: + raise ComposerError(None, None, "found undefined alias %r" % token.value, + token.start_mark) + return self.anchors[token.value] + anchor = None + tag = None + start_mark = end_mark = tag_mark = None + if self._check_token(""): + token = self._get_token() + start_mark = token.start_mark + end_mark = token.end_mark + anchor = token.value + if self._check_token(""): + token = self._get_token() + tag_mark = token.start_mark + end_mark = token.end_mark + tag = token.value + elif self._check_token(""): + token = self._get_token() + start_mark = tag_mark = token.start_mark + end_mark = token.end_mark + tag = token.value + if self._check_token(""): + token = self._get_token() + end_mark = token.end_mark + anchor = token.value + if tag is not None: + handle, suffix = tag + if handle is not None: + if handle not in self.tag_handles: + raise ParserError("while parsing a node", start_mark, + "found undefined tag handle %r" % handle, tag_mark) + tag = self.tag_handles[handle] + suffix + else: + tag = suffix + if start_mark is None: + start_mark = end_mark = self._peek_token().start_mark + implicit = tag is None or tag == "!" + if indentless and self._check_token("-"): + end_mark = self._peek_token().end_mark + node = SequenceNode(self._coll_tag(tag, SequenceNode), [], start_mark, end_mark, False) + if anchor is not None: + self._register_anchor(anchor, node, start_mark) + self._parse_indentless_sequence(node) + return node + if self._check_token(""): + token = self._get_token() + end_mark = token.end_mark + if (token.plain and tag is None) or tag == "!": + imp = (True, False) + elif tag is None: + imp = (False, True) + else: + imp = (False, False) + node_tag = tag + if tag is None or tag == "!": + node_tag = _resolve(self.yaml_implicit_resolvers, ScalarNode, token.value, imp) + node = ScalarNode(node_tag, token.value, start_mark, end_mark, token.style) + if anchor is not None: + self._register_anchor(anchor, node, start_mark) + return node + if self._check_token("["): + end_mark = self._peek_token().end_mark + node = SequenceNode(self._coll_tag(tag, SequenceNode), [], start_mark, end_mark, True) + if anchor is not None: + self._register_anchor(anchor, node, start_mark) + self._parse_flow_sequence(node) + return node + if self._check_token("{"): + end_mark = self._peek_token().end_mark + node = MappingNode(self._coll_tag(tag, MappingNode), [], start_mark, end_mark, True) + if anchor is not None: + self._register_anchor(anchor, node, start_mark) + self._parse_flow_mapping(node) + return node + if block and self._check_token(""): + end_mark = self._peek_token().start_mark + node = SequenceNode(self._coll_tag(tag, SequenceNode), [], start_mark, end_mark, False) + if anchor is not None: + self._register_anchor(anchor, node, start_mark) + self._parse_block_sequence(node) + return node + if block and self._check_token(""): + end_mark = self._peek_token().start_mark + node = MappingNode(self._coll_tag(tag, MappingNode), [], start_mark, end_mark, False) + if anchor is not None: + self._register_anchor(anchor, node, start_mark) + self._parse_block_mapping(node) + return node + if anchor is not None or tag is not None: + return self._empty_scalar(end_mark, True, anchor, tag) + what = "block" if block else "flow" + token = self._peek_token() + raise ParserError("while parsing a %s node" % what, start_mark, + "expected the node content, but found %r" % token.id, token.start_mark) + + def _coll_tag(self, tag, kind): + if tag is None or tag == "!": + return _resolve(self.yaml_implicit_resolvers, kind, None, True) + return tag + + def _parse_block_sequence(self, node): + token = self._get_token() + mark = token.start_mark + while self._check_token("-"): + token = self._get_token() + if not self._check_token("-", ""): + node.value.append(self._parse_node(True, False)) + else: + node.value.append(self._empty_scalar(token.end_mark)) + if not self._check_token(""): + token = self._peek_token() + raise ParserError("while parsing a block collection", mark, + "expected , but found %r" % token.id, token.start_mark) + token = self._get_token() + node.end_mark = token.end_mark + + def _parse_indentless_sequence(self, node): + while self._check_token("-"): + token = self._get_token() + if not self._check_token("-", "?", ":", ""): + node.value.append(self._parse_node(True, False)) + else: + node.value.append(self._empty_scalar(token.end_mark)) + node.end_mark = self._peek_token().start_mark + + def _parse_block_mapping(self, node): + token = self._get_token() + mark = token.start_mark + while self._check_token("?", ":"): + if self._check_token("?"): + token = self._get_token() + if not self._check_token("?", ":", ""): + key = self._parse_node(True, True) + else: + key = self._empty_scalar(token.end_mark) + else: + key = self._empty_scalar(self._peek_token().start_mark) + if self._check_token(":"): + token = self._get_token() + if not self._check_token("?", ":", ""): + value = self._parse_node(True, True) + else: + value = self._empty_scalar(token.end_mark) + else: + value = self._empty_scalar(self._peek_token().start_mark) + node.value.append((key, value)) + if not self._check_token(""): + token = self._peek_token() + raise ParserError("while parsing a block mapping", mark, + "expected , but found %r" % token.id, token.start_mark) + token = self._get_token() + node.end_mark = token.end_mark + + def _parse_flow_sequence(self, node): + token = self._get_token() + mark = token.start_mark + first = True + while not self._check_token("]"): + if not first: + if self._check_token(","): + self._get_token() + else: + token = self._peek_token() + raise ParserError("while parsing a flow sequence", mark, + "expected ',' or ']', but got %r" % token.id, token.start_mark) + first = False + if self._check_token("?"): + token = self._get_token() + pair = MappingNode(DEFAULT_MAPPING_TAG, [], token.start_mark, token.end_mark, True) + if not self._check_token(":", ",", "]"): + key = self._parse_node(False, False) + else: + key = self._empty_scalar(token.end_mark) + if self._check_token(":"): + token = self._get_token() + if not self._check_token(",", "]"): + value = self._parse_node(False, False) + else: + value = self._empty_scalar(token.end_mark) + else: + value = self._empty_scalar(self._peek_token().start_mark) + pair.value.append((key, value)) + node.value.append(pair) + elif not self._check_token("]"): + node.value.append(self._parse_node(False, False)) + token = self._get_token() + node.end_mark = token.end_mark + + def _parse_flow_mapping(self, node): + token = self._get_token() + mark = token.start_mark + first = True + while not self._check_token("}"): + if not first: + if self._check_token(","): + self._get_token() + else: + token = self._peek_token() + raise ParserError("while parsing a flow mapping", mark, + "expected ',' or '}', but got %r" % token.id, token.start_mark) + first = False + if self._check_token("?"): + token = self._get_token() + if not self._check_token(":", ",", "}"): + key = self._parse_node(False, False) + else: + key = self._empty_scalar(token.end_mark) + if self._check_token(":"): + token = self._get_token() + if not self._check_token(",", "}"): + value = self._parse_node(False, False) + else: + value = self._empty_scalar(token.end_mark) + else: + value = self._empty_scalar(self._peek_token().start_mark) + node.value.append((key, value)) + elif not self._check_token("}"): + key = self._parse_node(False, False) + node.value.append((key, self._empty_scalar(self._peek_token().start_mark))) + token = self._get_token() + node.end_mark = token.end_mark + + # -- constructor + def check_data(self): + return self.check_node() + + def get_data(self): + if self.check_node(): + return self.construct_document(self.get_node()) + return None + + def get_single_data(self): + node = self.get_single_node() + if node is not None: + return self.construct_document(node) + return None + + def construct_document(self, node): + data = self.construct_object(node) + while self.state_generators: + gens = self.state_generators + self.state_generators = [] + for gen in gens: + for _ in gen: + pass + self.constructed_objects = {} + self.recursive_objects = {} + self.deep_construct = False + return data + + def construct_object(self, node, deep=False): + key = id(node) + if key in self.constructed_objects: + return self.constructed_objects[key] + if deep: + old_deep = self.deep_construct + self.deep_construct = True + if key in self.recursive_objects: + raise ConstructorError(None, None, "found unconstructable recursive node", node.start_mark) + self.recursive_objects[key] = None + constructor = None + tag_suffix = None + if node.tag in self.yaml_constructors: + constructor = self.yaml_constructors[node.tag] + else: + found = False + for prefix in self.yaml_multi_constructors: + if prefix is not None and node.tag.startswith(prefix): + tag_suffix = node.tag[len(prefix):] + constructor = self.yaml_multi_constructors[prefix] + found = True + break + if not found: + if None in self.yaml_multi_constructors: + tag_suffix = node.tag + constructor = self.yaml_multi_constructors[None] + elif None in self.yaml_constructors: + constructor = self.yaml_constructors[None] + elif isinstance(node, ScalarNode): + constructor = _LoaderCore.construct_scalar + elif isinstance(node, SequenceNode): + constructor = _LoaderCore.construct_sequence + else: + constructor = _LoaderCore.construct_mapping + if tag_suffix is None: + data = constructor(self, node) + else: + data = constructor(self, tag_suffix, node) + if hasattr(data, "__next__"): + gen = data + data = next(gen) + if self.deep_construct: + for _ in gen: + pass + else: + self.state_generators.append(gen) + self.constructed_objects[key] = data + del self.recursive_objects[key] + if deep: + self.deep_construct = old_deep + return data + + def construct_scalar(self, node): + if not isinstance(node, ScalarNode): + raise ConstructorError(None, None, "expected a scalar node, but found %s" % node.id, + node.start_mark) + return node.value + + def construct_sequence(self, node, deep=False): + if not isinstance(node, SequenceNode): + raise ConstructorError(None, None, "expected a sequence node, but found %s" % node.id, + node.start_mark) + return [self.construct_object(child, deep=deep) for child in node.value] + + def flatten_mapping(self, node): + merge = [] + index = 0 + while index < len(node.value): + key_node, value_node = node.value[index] + if key_node.tag == "tag:yaml.org,2002:merge": + del node.value[index] + if isinstance(value_node, MappingNode): + self.flatten_mapping(value_node) + merge.extend(value_node.value) + elif isinstance(value_node, SequenceNode): + submerge = [] + for subnode in value_node.value: + if not isinstance(subnode, MappingNode): + raise ConstructorError("while constructing a mapping", node.start_mark, + "expected a mapping for merging, but found %s" + % subnode.id, subnode.start_mark) + self.flatten_mapping(subnode) + submerge.append(subnode.value) + submerge.reverse() + for value in submerge: + merge.extend(value) + else: + raise ConstructorError("while constructing a mapping", node.start_mark, + "expected a mapping or list of mappings for merging, but found %s" + % value_node.id, value_node.start_mark) + elif key_node.tag == "tag:yaml.org,2002:value": + key_node.tag = "tag:yaml.org,2002:str" + index += 1 + else: + index += 1 + if merge: + node.value = merge + node.value + + def construct_mapping(self, node, deep=False): + if not isinstance(node, MappingNode): + raise ConstructorError(None, None, "expected a mapping node, but found %s" % node.id, + node.start_mark) + self.flatten_mapping(node) + mapping = {} + for key_node, value_node in node.value: + key = self.construct_object(key_node, deep=deep) + if not _is_hashable(key): + raise ConstructorError("while constructing a mapping", node.start_mark, + "found unhashable key", key_node.start_mark) + mapping[key] = self.construct_object(value_node, deep=deep) + return mapping + + def construct_pairs(self, node, deep=False): + if not isinstance(node, MappingNode): + raise ConstructorError(None, None, "expected a mapping node, but found %s" % node.id, + node.start_mark) + pairs = [] + for key_node, value_node in node.value: + pairs.append((self.construct_object(key_node, deep=deep), + self.construct_object(value_node, deep=deep))) + return pairs + + # -- the standard tags + def construct_yaml_null(self, node): + self.construct_scalar(node) + return None + + def construct_yaml_bool(self, node): + value = self.construct_scalar(node) + return value.lower() in ("yes", "true", "on") + + def construct_yaml_int(self, node): + value = self.construct_scalar(node) + value = value.replace("_", "") + sign = 1 + if value[0] == "-": + sign = -1 + if value[0] in "+-": + value = value[1:] + if value == "0": + return 0 + if value.startswith("0b"): + return sign * int(value[2:], 2) + if value.startswith("0x"): + return sign * int(value[2:], 16) + if value[0] == "0": + return sign * int(value, 8) + if ":" in value: + digits = [int(part) for part in value.split(":")] + digits.reverse() + base = 1 + total = 0 + for digit in digits: + total += digit * base + base *= 60 + return sign * total + return sign * int(value) + + def construct_yaml_float(self, node): + value = self.construct_scalar(node) + value = value.replace("_", "").lower() + sign = 1.0 + if value[0] == "-": + sign = -1.0 + if value[0] in "+-": + value = value[1:] + if value == ".inf": + return sign * float("inf") + if value == ".nan": + return float("nan") + if ":" in value: + digits = [float(part) for part in value.split(":")] + digits.reverse() + base = 1 + total = 0.0 + for digit in digits: + total += digit * base + base *= 60 + return sign * total + return sign * float(value) + + def construct_yaml_binary(self, node): + try: + value = self.construct_scalar(node).encode("ascii") + except UnicodeEncodeError as exc: + raise ConstructorError(None, None, + "failed to convert base64 data into ascii: %s" % exc, node.start_mark) + try: + return base64.decodebytes(value) + except Exception as exc: + raise ConstructorError(None, None, "failed to decode base64 data: %s" % exc, + node.start_mark) + + def construct_yaml_timestamp(self, node): + value = self.construct_scalar(node) + match = _RE_TS_PARTS.match(value) + year = int(match.group("year")) + month = int(match.group("month")) + day = int(match.group("day")) + if not match.group("hour"): + return datetime.date(year, month, day) + hour = int(match.group("hour")) + minute = int(match.group("minute")) + second = int(match.group("second")) + fraction = 0 + tzinfo = None + if match.group("fraction"): + frac = match.group("fraction")[:6] + while len(frac) < 6: + frac += "0" + fraction = int(frac) + if match.group("tz_sign"): + tz_hour = int(match.group("tz_hour")) + tz_minute = int(match.group("tz_minute") or 0) + delta = datetime.timedelta(hours=tz_hour, minutes=tz_minute) + if match.group("tz_sign") == "-": + delta = -delta + tzinfo = datetime.timezone(delta) + elif match.group("tz"): + tzinfo = datetime.timezone.utc + return datetime.datetime(year, month, day, hour, minute, second, fraction, tzinfo=tzinfo) + + def construct_yaml_omap(self, node): + omap = [] + yield omap + if not isinstance(node, SequenceNode): + raise ConstructorError("while constructing an ordered map", node.start_mark, + "expected a sequence, but found %s" % node.id, node.start_mark) + for subnode in node.value: + if not isinstance(subnode, MappingNode): + raise ConstructorError("while constructing an ordered map", node.start_mark, + "expected a mapping of length 1, but found %s" % subnode.id, + subnode.start_mark) + if len(subnode.value) != 1: + raise ConstructorError("while constructing an ordered map", node.start_mark, + "expected a single mapping item, but found %d items" + % len(subnode.value), subnode.start_mark) + key_node, value_node = subnode.value[0] + omap.append((self.construct_object(key_node), self.construct_object(value_node))) + + def construct_yaml_pairs(self, node): + pairs = [] + yield pairs + if not isinstance(node, SequenceNode): + raise ConstructorError("while constructing pairs", node.start_mark, + "expected a sequence, but found %s" % node.id, node.start_mark) + for subnode in node.value: + if not isinstance(subnode, MappingNode): + raise ConstructorError("while constructing pairs", node.start_mark, + "expected a mapping of length 1, but found %s" % subnode.id, + subnode.start_mark) + if len(subnode.value) != 1: + raise ConstructorError("while constructing pairs", node.start_mark, + "expected a single mapping item, but found %d items" + % len(subnode.value), subnode.start_mark) + key_node, value_node = subnode.value[0] + pairs.append((self.construct_object(key_node), self.construct_object(value_node))) + + def construct_yaml_set(self, node): + data = set() + yield data + value = self.construct_mapping(node) + data.update(value) + + def construct_yaml_str(self, node): + return self.construct_scalar(node) + + def construct_yaml_seq(self, node): + data = [] + yield data + data.extend(self.construct_sequence(node)) + + def construct_yaml_map(self, node): + data = {} + yield data + value = self.construct_mapping(node) + data.update(value) + + def construct_yaml_tuple(self, node): + return tuple(self.construct_sequence(node)) + + def construct_undefined(self, node): + raise ConstructorError(None, None, + "could not determine a constructor for the tag %r" % node.tag, + node.start_mark) + + @classmethod + def add_constructor(cls, tag, constructor): + table = dict(cls.yaml_constructors) + table[tag] = constructor + cls.yaml_constructors = table + + @classmethod + def add_multi_constructor(cls, tag_prefix, multi_constructor): + table = dict(cls.yaml_multi_constructors) + table[tag_prefix] = multi_constructor + cls.yaml_multi_constructors = table + + @classmethod + def add_implicit_resolver(cls, tag, regexp, first): + table = _copy_resolvers(cls.yaml_implicit_resolvers) + if first is None: + first = [None] + for ch in first: + table.setdefault(ch, []).append((tag, regexp)) + cls.yaml_implicit_resolvers = table + + +def _std_constructors(): + c = _LoaderCore + return { + "tag:yaml.org,2002:null": c.construct_yaml_null, + "tag:yaml.org,2002:bool": c.construct_yaml_bool, + "tag:yaml.org,2002:int": c.construct_yaml_int, + "tag:yaml.org,2002:float": c.construct_yaml_float, + "tag:yaml.org,2002:binary": c.construct_yaml_binary, + "tag:yaml.org,2002:timestamp": c.construct_yaml_timestamp, + "tag:yaml.org,2002:omap": c.construct_yaml_omap, + "tag:yaml.org,2002:pairs": c.construct_yaml_pairs, + "tag:yaml.org,2002:set": c.construct_yaml_set, + "tag:yaml.org,2002:str": c.construct_yaml_str, + "tag:yaml.org,2002:seq": c.construct_yaml_seq, + "tag:yaml.org,2002:map": c.construct_yaml_map, + None: c.construct_undefined, + } + + +class BaseLoader(_LoaderCore): + pass + + +class SafeLoader(_LoaderCore): + pass + + +class FullLoader(_LoaderCore): + pass + + +class UnsafeLoader(_LoaderCore): + pass + + +class Loader(_LoaderCore): + pass + + +BaseLoader.yaml_constructors = {} +BaseLoader.yaml_multi_constructors = {} +BaseLoader.yaml_implicit_resolvers = {} +for _cls in (SafeLoader, FullLoader, UnsafeLoader, Loader): + _cls.yaml_constructors = _std_constructors() + _cls.yaml_multi_constructors = {} + _cls.yaml_implicit_resolvers = _default_resolvers() +for _cls in (FullLoader, UnsafeLoader, Loader): + _cls.yaml_constructors["tag:yaml.org,2002:python/tuple"] = _LoaderCore.construct_yaml_tuple + +CBaseLoader = BaseLoader +CSafeLoader = SafeLoader +CFullLoader = FullLoader +CUnsafeLoader = UnsafeLoader +CLoader = Loader + + +def _run_load(stream, loader_cls): + loader = loader_cls(stream) + try: + return loader.get_single_data() + finally: + loader.dispose() + + +def _run_load_all(stream, loader_cls): + loader = loader_cls(stream) + try: + while loader.check_data(): + yield loader.get_data() + finally: + loader.dispose() + + +def load(stream, Loader): + return _run_load(stream, Loader) + + +def load_all(stream, Loader): + return _run_load_all(stream, Loader) + + +def full_load(stream): + return _run_load(stream, FullLoader) + + +def full_load_all(stream): + return _run_load_all(stream, FullLoader) + + +def safe_load(stream): + return _run_load(stream, SafeLoader) + + +def safe_load_all(stream): + return _run_load_all(stream, SafeLoader) + + +def unsafe_load(stream): + return _run_load(stream, UnsafeLoader) + + +def unsafe_load_all(stream): + return _run_load_all(stream, UnsafeLoader) + + +_LOADER_DEFAULTS = (Loader, FullLoader, UnsafeLoader) + + +def compose(stream, Loader=Loader): + loader = Loader(stream) + try: + return loader.get_single_node() + finally: + loader.dispose() + + +def compose_all(stream, Loader=Loader): + loader = Loader(stream) + try: + while loader.check_node(): + yield loader.get_node() + finally: + loader.dispose() + + +def add_implicit_resolver(tag, regexp, first=None, Loader=None, Dumper=None): + if Dumper is None: + Dumper = _DUMPER_DEFAULT + if Loader is None: + for cls in _LOADER_DEFAULTS: + cls.add_implicit_resolver(tag, regexp, first) + else: + Loader.add_implicit_resolver(tag, regexp, first) + Dumper.add_implicit_resolver(tag, regexp, first) + + +def add_constructor(tag, constructor, Loader=None): + if Loader is None: + for cls in _LOADER_DEFAULTS: + cls.add_constructor(tag, constructor) + else: + Loader.add_constructor(tag, constructor) + + +def add_multi_constructor(tag_prefix, multi_constructor, Loader=None): + if Loader is None: + for cls in _LOADER_DEFAULTS: + cls.add_multi_constructor(tag_prefix, multi_constructor) + else: + Loader.add_multi_constructor(tag_prefix, multi_constructor) + + +# ---------------------------------------------------------------- dumper + +_EMIT_ESC = { + "\0": "0", "\x07": "a", "\x08": "b", "\x09": "t", "\x0a": "n", "\x0b": "v", + "\x0c": "f", "\x0d": "r", "\x1b": "e", '"': '"', "\\": "\\", "\x85": "N", + "\xa0": "_", "\u2028": "L", "\u2029": "P", +} + + +class _Analysis: + def __init__(self, scalar, empty, multiline, allow_flow_plain, allow_block_plain, + allow_single_quoted, allow_double_quoted, allow_block): + self.scalar = scalar + self.empty = empty + self.multiline = multiline + self.allow_flow_plain = allow_flow_plain + self.allow_block_plain = allow_block_plain + self.allow_single_quoted = allow_single_quoted + self.allow_double_quoted = allow_double_quoted + self.allow_block = allow_block + + +class _Buf: + def __init__(self): + self.parts = [] + + def write(self, data): + self.parts.append(data) + + def getvalue(self): + if self.parts and isinstance(self.parts[0], bytes): + return b"".join(self.parts) + return "".join(self.parts) + + +def _node_children(node): + if isinstance(node, SequenceNode): + return list(node.value) + out = [] + for k, v in node.value: + out.append(k) + out.append(v) + return out + + +class _DumperCore: + yaml_representers = {} + yaml_multi_representers = {} + yaml_implicit_resolvers = {} + + def __init__(self, stream, default_style=None, default_flow_style=False, canonical=None, + indent=None, width=None, allow_unicode=None, line_break=None, encoding=None, + explicit_start=None, explicit_end=None, version=None, tags=None, sort_keys=True): + self.stream = stream + self.default_style = default_style + self.default_flow_style = default_flow_style + self.sort_keys = sort_keys + self.canonical = bool(canonical) + self.allow_unicode = bool(allow_unicode) + self.encoding = encoding + self.use_explicit_start = explicit_start + self.use_explicit_end = explicit_end + self.use_version = version + self.best_indent = 2 + if indent and 1 < indent < 10: + self.best_indent = indent + self.best_width = 80 + if width and width > self.best_indent * 2: + self.best_width = width + self.best_line_break = "\n" + if line_break in ("\r", "\n", "\r\n"): + self.best_line_break = line_break + self.tag_prefixes = {"!": "!", "tag:yaml.org,2002:": "!!"} + if tags: + for handle in tags: + self.tag_prefixes[tags[handle]] = handle + self.represented_objects = {} + self.object_keeper = [] + self.alias_key = None + self.anchors = {} + self.serialized_nodes = {} + self.last_anchor_id = 0 + self.indents = [] + self.indent = None + self.flow_level = 0 + self.root_context = False + self.sequence_context = False + self.mapping_context = False + self.simple_key_context = False + self.line = 0 + self.column = 0 + self.whitespace = True + self.indention = True + self.open_ended = False + self._first_doc = True + self._opened = False + self._closed = False + + # ------------------------------------------------------ representer + def open(self): + self._opened = True + + def dispose(self): + pass + + def close(self): + if self._closed: + return + self._closed = True + if self.open_ended: + self.write_indicator("...", True) + self.write_indent() + + def represent(self, data): + node = self.represent_data(data) + self.serialize(node) + self.represented_objects = {} + self.object_keeper = [] + self.alias_key = None + + def ignore_aliases(self, data): + if data is None: + return True + if isinstance(data, tuple) and data == (): + return True + if isinstance(data, (str, bytes, bool, int, float)): + return True + return False + + def represent_data(self, data): + if self.ignore_aliases(data): + self.alias_key = None + else: + self.alias_key = id(data) + if self.alias_key is not None: + if self.alias_key in self.represented_objects: + return self.represented_objects[self.alias_key] + self.object_keeper.append(data) + data_type = type(data) + if data_type in self.yaml_representers: + return self.yaml_representers[data_type](self, data) + for base in data_type.__mro__: + if base in self.yaml_multi_representers: + return self.yaml_multi_representers[base](self, data) + if None in self.yaml_multi_representers: + return self.yaml_multi_representers[None](self, data) + if None in self.yaml_representers: + return self.yaml_representers[None](self, data) + return ScalarNode(None, str(data)) + + def represent_scalar(self, tag, value, style=None): + if style is None: + style = self.default_style + node = ScalarNode(tag, value, None, None, style) + if self.alias_key is not None: + self.represented_objects[self.alias_key] = node + return node + + def represent_sequence(self, tag, sequence, flow_style=None): + value = [] + node = SequenceNode(tag, value, None, None, flow_style) + if self.alias_key is not None: + self.represented_objects[self.alias_key] = node + best_style = True + for item in sequence: + node_item = self.represent_data(item) + if not (isinstance(node_item, ScalarNode) and not node_item.style): + best_style = False + value.append(node_item) + if flow_style is None: + if self.default_flow_style is not None: + node.flow_style = self.default_flow_style + else: + node.flow_style = best_style + return node + + def represent_mapping(self, tag, mapping, flow_style=None): + value = [] + node = MappingNode(tag, value, None, None, flow_style) + if self.alias_key is not None: + self.represented_objects[self.alias_key] = node + best_style = True + if hasattr(mapping, "items"): + items = list(mapping.items()) + if self.sort_keys: + try: + items = sorted(items, key=lambda kv: kv[0]) + except TypeError: + pass + else: + items = list(mapping) + for item_key, item_value in items: + node_key = self.represent_data(item_key) + node_value = self.represent_data(item_value) + if not (isinstance(node_key, ScalarNode) and not node_key.style): + best_style = False + if not (isinstance(node_value, ScalarNode) and not node_value.style): + best_style = False + value.append((node_key, node_value)) + if flow_style is None: + if self.default_flow_style is not None: + node.flow_style = self.default_flow_style + else: + node.flow_style = best_style + return node + + def represent_undefined(self, data): + raise RepresenterError("cannot represent an object", data) + + def represent_none(self, data): + return self.represent_scalar("tag:yaml.org,2002:null", "null") + + def represent_str(self, data): + return self.represent_scalar("tag:yaml.org,2002:str", data) + + def represent_binary(self, data): + return self.represent_scalar("tag:yaml.org,2002:binary", + base64.encodebytes(data).decode("ascii"), style="|") + + def represent_bool(self, data): + return self.represent_scalar("tag:yaml.org,2002:bool", "true" if data else "false") + + def represent_int(self, data): + return self.represent_scalar("tag:yaml.org,2002:int", str(data)) + + def represent_float(self, data): + if data != data: + value = ".nan" + elif data == float("inf"): + value = ".inf" + elif data == -float("inf"): + value = "-.inf" + else: + value = repr(data).lower() + if "." not in value and "e" in value: + value = value.replace("e", ".0e", 1) + return self.represent_scalar("tag:yaml.org,2002:float", value) + + def represent_list(self, data): + return self.represent_sequence("tag:yaml.org,2002:seq", data) + + def represent_tuple(self, data): + return self.represent_sequence("tag:yaml.org,2002:python/tuple", data) + + def represent_dict(self, data): + return self.represent_mapping("tag:yaml.org,2002:map", data) + + def represent_set(self, data): + value = {} + for key in data: + value[key] = None + return self.represent_mapping("tag:yaml.org,2002:set", value) + + def represent_date(self, data): + return self.represent_scalar("tag:yaml.org,2002:timestamp", data.isoformat()) + + def represent_datetime(self, data): + return self.represent_scalar("tag:yaml.org,2002:timestamp", data.isoformat(" ")) + + def represent_yaml_object(self, tag, data, cls, flow_style=None): + state = dict(data.__dict__) + return self.represent_mapping(tag, state, flow_style=flow_style) + + @classmethod + def add_representer(cls, data_type, representer): + table = dict(cls.yaml_representers) + table[data_type] = representer + cls.yaml_representers = table + + @classmethod + def add_multi_representer(cls, data_type, representer): + table = dict(cls.yaml_multi_representers) + table[data_type] = representer + cls.yaml_multi_representers = table + + @classmethod + def add_implicit_resolver(cls, tag, regexp, first): + table = _copy_resolvers(cls.yaml_implicit_resolvers) + if first is None: + first = [None] + for ch in first: + table.setdefault(ch, []).append((tag, regexp)) + cls.yaml_implicit_resolvers = table + + # ------------------------------------------------------- serializer + def serialize(self, node): + self.anchors = {} + self.serialized_nodes = {} + self.last_anchor_id = 0 + self._anchor_node(node) + self._document_start() + self._emit_node(node, True, False, False, False) + self.write_indent() + if self.use_explicit_end: + self.write_indicator("...", True) + self.write_indent() + self.anchors = {} + self.serialized_nodes = {} + self.last_anchor_id = 0 + + def _anchor_node(self, node): + key = id(node) + if key in self.anchors: + if self.anchors[key] is None: + self.last_anchor_id += 1 + self.anchors[key] = "id%03d" % self.last_anchor_id + else: + self.anchors[key] = None + if not isinstance(node, ScalarNode): + for child in _node_children(node): + self._anchor_node(child) + + def _document_start(self): + first = self._first_doc + self._first_doc = False + implicit = first and not self.use_explicit_start and not self.use_version + if self.use_version and self.open_ended: + self.write_indicator("...", True) + self.write_indent() + if self.use_version: + version = self.use_version + text = "%d.%d" % (version[0], version[1]) + self._raw_write("%YAML " + text) + self.write_line_break() + if not implicit: + self.write_indent() + self.write_indicator("---", True) + + def _emit_node(self, node, root, sequence, mapping, simple_key): + self.root_context = root + self.sequence_context = sequence + self.mapping_context = mapping + self.simple_key_context = simple_key + key = id(node) + anchor = self.anchors.get(key) + if key in self.serialized_nodes: + self.write_indicator("*" + anchor, True) + return + self.serialized_nodes[key] = True + if isinstance(node, ScalarNode): + detected = _resolve(self.yaml_implicit_resolvers, ScalarNode, node.value, (True, False)) + default = _resolve(self.yaml_implicit_resolvers, ScalarNode, node.value, (False, True)) + implicit = (node.tag == detected, node.tag == default) + self.analysis = self._analyze_scalar(node.value) + self._process_anchor("&", anchor) + style = self._choose_scalar_style(node, implicit) + self._process_tag_scalar(node, style, implicit) + self.increase_indent(flow=True) + self._process_scalar(node, style) + self.indent = self.indents.pop() + return + if isinstance(node, SequenceNode): + kind = SequenceNode + else: + kind = MappingNode + implicit = node.tag == _resolve(self.yaml_implicit_resolvers, kind, node.value, True) + self._process_anchor("&", anchor) + self._process_tag_collection(node, implicit) + empty = len(node.value) == 0 + if kind is SequenceNode: + if self.flow_level or self.canonical or node.flow_style or empty: + self._flow_sequence(node) + else: + self._block_sequence(node) + else: + if self.flow_level or self.canonical or node.flow_style or empty: + self._flow_mapping(node) + else: + self._block_mapping(node) + + def _process_anchor(self, indicator, anchor): + if anchor is None: + return + self.write_indicator(indicator + anchor, True) + + def _process_tag_scalar(self, node, style, implicit): + tag = node.tag + if (not self.canonical or tag is None) and ( + (style == "" and implicit[0]) or (style != "" and implicit[1])): + return + if implicit[0] and tag is None: + tag = "!" + if not tag: + raise EmitterError("tag is not specified") + self.write_indicator(self._prepare_tag(tag), True) + + def _process_tag_collection(self, node, implicit): + tag = node.tag + if (not self.canonical or tag is None) and implicit: + return + if not tag: + raise EmitterError("tag is not specified") + self.write_indicator(self._prepare_tag(tag), True) + + def _prepare_tag(self, tag): + if not tag: + raise EmitterError("tag must not be empty") + if tag == "!": + return tag + handle = None + suffix = tag + for prefix in sorted(self.tag_prefixes.keys()): + if tag.startswith(prefix) and (prefix == "!" or len(prefix) < len(tag)): + handle = self.tag_prefixes[prefix] + suffix = tag[len(prefix):] + chunks = [] + for ch in suffix: + if ("0" <= ch <= "9" or "A" <= ch <= "Z" or "a" <= ch <= "z" + or ch in "-;/?:@&=+$,_.~*'()[]" or (ch == "!" and handle != "!")): + chunks.append(ch) + else: + for b in ch.encode("utf-8"): + chunks.append("%%%02X" % b) + text = "".join(chunks) + if handle: + return handle + text + return "!<%s>" % text + + # -- collections + def increase_indent(self, flow=False, indentless=False): + self.indents.append(self.indent) + if self.indent is None: + if flow: + self.indent = self.best_indent + else: + self.indent = 0 + elif not indentless: + self.indent += self.best_indent + + def _block_sequence(self, node): + indentless = self.mapping_context and not self.indention + self.increase_indent(flow=False, indentless=indentless) + for item in node.value: + self.write_indent() + self.write_indicator("-", True, indention=True) + self._emit_node(item, False, True, False, False) + self.indent = self.indents.pop() + + def _block_mapping(self, node): + self.increase_indent(flow=False) + for key_node, value_node in node.value: + self.write_indent() + if self._check_simple_key(key_node): + self._emit_node(key_node, False, False, True, True) + self.write_indicator(":", False) + self._emit_node(value_node, False, False, True, False) + else: + self.write_indicator("?", True, indention=True) + self._emit_node(key_node, False, False, True, False) + self.write_indent() + self.write_indicator(":", True, indention=True) + self._emit_node(value_node, False, False, True, False) + self.indent = self.indents.pop() + + def _flow_sequence(self, node): + self.write_indicator("[", True, whitespace=True) + self.increase_indent(flow=True) + self.flow_level += 1 + first = True + for item in node.value: + if not first: + self.write_indicator(",", False) + if self.canonical or self.column > self.best_width: + self.write_indent() + self._emit_node(item, False, True, False, False) + first = False + self.indent = self.indents.pop() + self.flow_level -= 1 + if self.canonical and not first: + self.write_indicator(",", False) + self.write_indent() + self.write_indicator("]", False) + + def _flow_mapping(self, node): + self.write_indicator("{", True, whitespace=True) + self.increase_indent(flow=True) + self.flow_level += 1 + first = True + for key_node, value_node in node.value: + if not first: + self.write_indicator(",", False) + if self.canonical or self.column > self.best_width: + self.write_indent() + if not self.canonical and self._check_simple_key(key_node): + self._emit_node(key_node, False, False, True, True) + self.write_indicator(":", False) + self._emit_node(value_node, False, False, True, False) + else: + self.write_indicator("?", True) + self._emit_node(key_node, False, False, True, False) + if self.canonical or self.column > self.best_width: + self.write_indent() + self.write_indicator(":", True) + self._emit_node(value_node, False, False, True, False) + first = False + self.indent = self.indents.pop() + self.flow_level -= 1 + if self.canonical and not first: + self.write_indicator(",", False) + self.write_indent() + self.write_indicator("}", False) + + def _check_simple_key(self, node): + key = id(node) + length = 0 + anchor = self.anchors.get(key) + if key in self.serialized_nodes: + if anchor is not None: + length += len(anchor) + return length < 128 + if anchor is not None: + length += len(anchor) + if node.tag is not None: + length += len(self._prepare_tag(node.tag)) + if isinstance(node, ScalarNode): + analysis = self._analyze_scalar(node.value) + length += len(analysis.scalar) + return length < 128 and not analysis.empty and not analysis.multiline + return length < 128 and len(node.value) == 0 + + # -- scalars + def _analyze_scalar(self, scalar): + if not scalar: + return _Analysis(scalar, True, False, False, True, True, True, False) + block_indicators = False + flow_indicators = False + line_breaks = False + special_characters = False + leading_space = False + leading_break = False + trailing_space = False + trailing_break = False + break_space = False + space_break = False + preceded_by_whitespace = False + followed_by_whitespace = False + previous_space = False + previous_break = False + if scalar.startswith("---") or scalar.startswith("..."): + block_indicators = True + flow_indicators = True + preceded_by_whitespace = True + n = len(scalar) + followed_by_whitespace = n == 1 or scalar[1] in _BLANKZ + index = 0 + while index < n: + ch = scalar[index] + if index == 0: + if ch in "#,[]{}&*!|>'\"%@`": + flow_indicators = True + block_indicators = True + if ch in "?:": + flow_indicators = True + if followed_by_whitespace: + block_indicators = True + if ch == "-" and followed_by_whitespace: + flow_indicators = True + block_indicators = True + else: + if ch in ",?[]{}": + flow_indicators = True + if ch == ":": + flow_indicators = True + if followed_by_whitespace: + block_indicators = True + if ch == "#" and preceded_by_whitespace: + flow_indicators = True + block_indicators = True + if ch in "\n\x85\u2028\u2029": + line_breaks = True + if not (ch == "\n" or "\x20" <= ch <= "\x7e"): + if ((ch == "\x85" or "\xa0" <= ch <= "\ud7ff" or "\ue000" <= ch <= "\ufffd" + or "\U00010000" <= ch < "\U0010ffff") and ch != "\ufeff"): + if not self.allow_unicode: + special_characters = True + else: + special_characters = True + if ch == " ": + if index == 0: + leading_space = True + if index == n - 1: + trailing_space = True + if previous_break: + break_space = True + previous_space = True + previous_break = False + elif ch in "\n\x85\u2028\u2029": + if index == 0: + leading_break = True + if index == n - 1: + trailing_break = True + if previous_space: + space_break = True + previous_space = False + previous_break = True + else: + previous_space = False + previous_break = False + index += 1 + preceded_by_whitespace = ch in _BLANKZ + followed_by_whitespace = index + 1 >= n or scalar[index + 1] in _BLANKZ + allow_flow_plain = True + allow_block_plain = True + allow_single_quoted = True + allow_double_quoted = True + allow_block = True + if leading_space or leading_break or trailing_space or trailing_break: + allow_flow_plain = False + allow_block_plain = False + if trailing_space: + allow_block = False + if break_space: + allow_flow_plain = False + allow_block_plain = False + allow_single_quoted = False + if space_break or special_characters: + allow_flow_plain = False + allow_block_plain = False + allow_single_quoted = False + allow_block = False + if line_breaks: + allow_flow_plain = False + allow_block_plain = False + if flow_indicators: + allow_flow_plain = False + if block_indicators: + allow_block_plain = False + return _Analysis(scalar, False, line_breaks, allow_flow_plain, allow_block_plain, + allow_single_quoted, allow_double_quoted, allow_block) + + def _choose_scalar_style(self, node, implicit): + analysis = self.analysis + style = node.style + if style == '"' or self.canonical: + return '"' + if not style and implicit[0]: + if (not (self.simple_key_context and (analysis.empty or analysis.multiline)) + and ((self.flow_level and analysis.allow_flow_plain) + or (not self.flow_level and analysis.allow_block_plain))): + return "" + if style and style in "|>": + if (not self.flow_level and not self.simple_key_context and analysis.allow_block): + return style + if not style or style == "'": + if (analysis.allow_single_quoted + and not (self.simple_key_context and analysis.multiline)): + return "'" + return '"' + + def _process_scalar(self, node, style): + analysis = self.analysis + split = not self.simple_key_context + if style == '"': + self._write_double_quoted(analysis.scalar, split) + elif style == "'": + self._write_single_quoted(analysis.scalar, split) + elif style == ">": + self._write_folded(analysis.scalar) + elif style == "|": + self._write_literal(analysis.scalar) + else: + self._write_plain(analysis.scalar, split) + self.analysis = None + + # -- low level output + def _raw_write(self, data): + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + + def write_indicator(self, indicator, need_whitespace, whitespace=False, indention=False): + if self.whitespace or not need_whitespace: + data = indicator + else: + data = " " + indicator + self.whitespace = whitespace + self.indention = self.indention and indention + self.column += len(data) + self.open_ended = False + self._raw_write(data) + + def write_indent(self): + indent = self.indent or 0 + if (not self.indention or self.column > indent + or (self.column == indent and not self.whitespace)): + self.write_line_break() + if self.column < indent: + self.whitespace = True + data = " " * (indent - self.column) + self.column = indent + self._raw_write(data) + + def write_line_break(self, data=None): + if data is None: + data = self.best_line_break + self.whitespace = True + self.indention = True + self.line += 1 + self.column = 0 + self._raw_write(data) + + def _put(self, data): + self.column += len(data) + self._raw_write(data) + + def _write_single_quoted(self, text, split=True): + self.write_indicator("'", True) + spaces = False + breaks = False + start = 0 + end = 0 + n = len(text) + while end <= n: + ch = None + if end < n: + ch = text[end] + if spaces: + if ch is None or ch != " ": + if (start + 1 == end and self.column > self.best_width and split + and start != 0 and end != n): + self.write_indent() + else: + self._put(text[start:end]) + start = end + elif breaks: + if ch is None or ch not in "\n\x85\u2028\u2029": + if text[start] == "\n": + self.write_line_break() + for br in text[start:end]: + if br == "\n": + self.write_line_break() + else: + self.write_line_break(br) + self.write_indent() + start = end + else: + if ch is None or ch in " \n\x85\u2028\u2029" or ch == "'": + if start < end: + self._put(text[start:end]) + start = end + if ch == "'": + self._put("''") + start = end + 1 + if ch is not None: + spaces = ch == " " + breaks = ch in "\n\x85\u2028\u2029" + end += 1 + self.write_indicator("'", False) + + def _write_double_quoted(self, text, split=True): + self.write_indicator('"', True) + start = 0 + end = 0 + n = len(text) + while end <= n: + ch = None + if end < n: + ch = text[end] + if (ch is None or ch in "\"\\\x85\u2028\u2029\ufeff" + or not ("\x20" <= ch <= "\x7e" + or (self.allow_unicode + and ("\xa0" <= ch <= "\ud7ff" or "\ue000" <= ch <= "\ufffd")))): + if start < end: + self._put(text[start:end]) + start = end + if ch is not None: + if ch in _EMIT_ESC: + data = "\\" + _EMIT_ESC[ch] + elif ch <= "\xff": + data = "\\x%02X" % ord(ch) + elif ch <= "\uffff": + data = "\\u%04X" % ord(ch) + else: + data = "\\U%08X" % ord(ch) + self._put(data) + start = end + 1 + if (0 < end < n - 1 and (ch == " " or start >= end) + and self.column + (end - start) > self.best_width and split): + data = text[start:end] + "\\" + if start < end: + start = end + self._put(data) + self.write_indent() + self.whitespace = False + self.indention = False + if text[start] == " ": + self._put("\\") + end += 1 + self.write_indicator('"', False) + + def _block_hints(self, text): + hints = "" + if text: + if text[0] in " \n\x85\u2028\u2029": + hints += str(self.best_indent) + if text[-1] not in "\n\x85\u2028\u2029": + hints += "-" + elif len(text) == 1 or text[-2] in "\n\x85\u2028\u2029": + hints += "+" + return hints + + def _write_folded(self, text): + hints = self._block_hints(text) + self.write_indicator(">" + hints, True) + if hints[-1:] == "+": + self.open_ended = True + self.write_line_break() + leading_space = True + spaces = False + breaks = True + start = 0 + end = 0 + n = len(text) + while end <= n: + ch = None + if end < n: + ch = text[end] + if breaks: + if ch is None or ch not in "\n\x85\u2028\u2029": + if not leading_space and ch is not None and ch != " " and text[start] == "\n": + self.write_line_break() + leading_space = ch == " " + for br in text[start:end]: + if br == "\n": + self.write_line_break() + else: + self.write_line_break(br) + if ch is not None: + self.write_indent() + start = end + elif spaces: + if ch != " ": + if start + 1 == end and self.column > self.best_width: + self.write_indent() + else: + self._put(text[start:end]) + start = end + else: + if ch is None or ch in " \n\x85\u2028\u2029": + self._put(text[start:end]) + if ch is None: + self.write_line_break() + start = end + if ch is not None: + breaks = ch in "\n\x85\u2028\u2029" + spaces = ch == " " + end += 1 + + def _write_literal(self, text): + hints = self._block_hints(text) + self.write_indicator("|" + hints, True) + if hints[-1:] == "+": + self.open_ended = True + self.write_line_break() + breaks = True + start = 0 + end = 0 + n = len(text) + while end <= n: + ch = None + if end < n: + ch = text[end] + if breaks: + if ch is None or ch not in "\n\x85\u2028\u2029": + for br in text[start:end]: + if br == "\n": + self.write_line_break() + else: + self.write_line_break(br) + if ch is not None: + self.write_indent() + start = end + else: + if ch is None or ch in "\n\x85\u2028\u2029": + self._put(text[start:end]) + if ch is None: + self.write_line_break() + start = end + if ch is not None: + breaks = ch in "\n\x85\u2028\u2029" + end += 1 + + def _write_plain(self, text, split=True): + if self.root_context: + self.open_ended = True + if not text: + return + if not self.whitespace: + self._put(" ") + self.whitespace = False + self.indention = False + spaces = False + breaks = False + start = 0 + end = 0 + n = len(text) + while end <= n: + ch = None + if end < n: + ch = text[end] + if spaces: + if ch != " ": + if start + 1 == end and self.column > self.best_width and split: + self.write_indent() + self.whitespace = False + self.indention = False + else: + self._put(text[start:end]) + start = end + elif breaks: + if ch is None or ch not in "\n\x85\u2028\u2029": + if text[start] == "\n": + self.write_line_break() + for br in text[start:end]: + if br == "\n": + self.write_line_break() + else: + self.write_line_break(br) + self.write_indent() + self.whitespace = False + self.indention = False + start = end + else: + if ch is None or ch in " \n\x85\u2028\u2029": + self._put(text[start:end]) + start = end + if ch is not None: + spaces = ch == " " + breaks = ch in "\n\x85\u2028\u2029" + end += 1 + + +def _std_representers(full): + c = _DumperCore + table = { + type(None): c.represent_none, + str: c.represent_str, + bytes: c.represent_binary, + bool: c.represent_bool, + int: c.represent_int, + float: c.represent_float, + list: c.represent_list, + tuple: c.represent_tuple if full else c.represent_list, + dict: c.represent_dict, + set: c.represent_set, + datetime.date: c.represent_date, + datetime.datetime: c.represent_datetime, + None: c.represent_undefined, + } + return table + + +class BaseDumper(_DumperCore): + pass + + +class SafeDumper(_DumperCore): + pass + + +class Dumper(_DumperCore): + pass + + +BaseDumper.yaml_representers = {} +BaseDumper.yaml_multi_representers = {} +BaseDumper.yaml_implicit_resolvers = _default_resolvers() +SafeDumper.yaml_representers = _std_representers(False) +SafeDumper.yaml_multi_representers = {} +SafeDumper.yaml_implicit_resolvers = _default_resolvers() +Dumper.yaml_representers = _std_representers(True) +Dumper.yaml_multi_representers = {} +Dumper.yaml_implicit_resolvers = _default_resolvers() +_DUMPER_DEFAULT = Dumper + +CBaseDumper = BaseDumper +CSafeDumper = SafeDumper +CDumper = Dumper + + +def dump_all(documents, stream=None, Dumper=Dumper, default_style=None, default_flow_style=False, + canonical=None, indent=None, width=None, allow_unicode=None, line_break=None, + encoding=None, explicit_start=None, explicit_end=None, version=None, tags=None, + sort_keys=True): + getvalue = None + if stream is None: + stream = _Buf() + getvalue = stream.getvalue + dumper = Dumper(stream, default_style=default_style, default_flow_style=default_flow_style, + canonical=canonical, indent=indent, width=width, allow_unicode=allow_unicode, + line_break=line_break, encoding=encoding, explicit_start=explicit_start, + explicit_end=explicit_end, version=version, tags=tags, sort_keys=sort_keys) + try: + dumper.open() + for data in documents: + dumper.represent(data) + dumper.close() + finally: + dumper.dispose() + if getvalue is not None: + return getvalue() + return None + + +def dump(data, stream=None, Dumper=Dumper, **kwds): + return dump_all([data], stream, Dumper=Dumper, **kwds) + + +def safe_dump_all(documents, stream=None, **kwds): + return dump_all(documents, stream, Dumper=SafeDumper, **kwds) + + +def safe_dump(data, stream=None, **kwds): + return dump_all([data], stream, Dumper=SafeDumper, **kwds) + + +def add_representer(data_type, representer, Dumper=Dumper): + Dumper.add_representer(data_type, representer) + + +def add_multi_representer(data_type, multi_representer, Dumper=Dumper): + Dumper.add_multi_representer(data_type, multi_representer) + + +def _construct_yaml_object(self, node, cls): + data = cls.__new__(cls) + yield data + state = self.construct_mapping(node, deep=True) + data.__dict__.update(state) + + +_LoaderCore.construct_yaml_object = _construct_yaml_object + + +class YAMLObjectMetaclass(type): + def __init__(cls, name, bases, kwds): + type.__init__(cls, name, bases, kwds) + tag = kwds.get("yaml_tag") + if tag is not None: + loaders = cls.yaml_loader + if not isinstance(loaders, list): + loaders = [loaders] + for loader in loaders: + loader.add_constructor(tag, cls.from_yaml) + cls.yaml_dumper.add_representer(cls, cls.to_yaml) + + +class YAMLObject(metaclass=YAMLObjectMetaclass): + yaml_loader = [Loader, FullLoader, UnsafeLoader] + yaml_dumper = Dumper + yaml_tag = None + yaml_flow_style = None + + @classmethod + def from_yaml(cls, loader, node): + return loader.construct_yaml_object(node, cls) + + @classmethod + def to_yaml(cls, dumper, data): + return dumper.represent_yaml_object(cls.yaml_tag, data, cls, flow_style=cls.yaml_flow_style) + + +def warnings(settings=None): + if settings is None: + return {} + + +# Submodule-style namespaces, so that yaml.constructor.ConstructorError and +# friends resolve as they do in PyYAML. +error = _NS() +error.YAMLError = YAMLError +error.MarkedYAMLError = MarkedYAMLError +error.Mark = Mark +reader = _NS() +reader.ReaderError = ReaderError +scanner = _NS() +scanner.ScannerError = ScannerError +parser = _NS() +parser.ParserError = ParserError +composer = _NS() +composer.ComposerError = ComposerError +constructor = _NS() +constructor.ConstructorError = ConstructorError +constructor.SafeConstructor = SafeLoader +constructor.FullConstructor = FullLoader +constructor.Constructor = Loader +representer = _NS() +representer.RepresenterError = RepresenterError +representer.SafeRepresenter = SafeDumper +representer.Representer = Dumper +emitter = _NS() +emitter.EmitterError = EmitterError +serializer = _NS() +serializer.SerializerError = SerializerError +nodes = _NS() +nodes.Node = Node +nodes.ScalarNode = ScalarNode +nodes.CollectionNode = CollectionNode +nodes.SequenceNode = SequenceNode +nodes.MappingNode = MappingNode +loader = _NS() +loader.Loader = Loader +loader.SafeLoader = SafeLoader +loader.FullLoader = FullLoader +loader.BaseLoader = BaseLoader +loader.UnsafeLoader = UnsafeLoader +dumper = _NS() +dumper.Dumper = Dumper +dumper.SafeDumper = SafeDumper +dumper.BaseDumper = BaseDumper diff --git a/maven/python-transpiler/src/test/resources/conformance/yaml_.out b/maven/python-transpiler/src/test/resources/conformance/yaml_.out new file mode 100644 index 00000000000..d23a04e6f14 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/yaml_.out @@ -0,0 +1,252 @@ +null1 None +null2 None +null3 None +null4 None +t1 True +t2 False +t3 True +t4 False +t5 True +t6 False +i1 42 +i2 -17 +i3 5 +i4 31 +i5 5 +i6 '0o17' +i7 15 +i8 1000000 +i9 90 +f1 3.14 +f2 '-.5e+3' +f3 inf +f4 -inf +f5 685230.15 +f6 10.5 +s1 'hello world' +s2 "single 'quoted' text" +s3 'double "quoted"\t tab A \xe9 \n newline' +s4 '1e3' +s5 '12abc' +s6 '123' +s7 '~' +d1 datetime.date(2001, 12, 14) +d2 datetime.datetime(2001, 12, 14, 21, 59, 43, 100000, tzinfo=datetime.timezone(datetime.timedelta(days=-1, seconds=68400))) +d3 datetime.datetime(2001, 12, 14, 21, 59, 43, 100000) +d4 datetime.datetime(2002, 12, 14, 21, 59, 43, tzinfo=datetime.timezone.utc) +True True +seq ['a', ['b1', 'b2'], {'key': 'value', 'other': [1, 2, 3], 'nested': {'x': 1, 'y': ['a', 'b'], 'z': {'q': 'r'}}}, {'complex': 'value'}, '123', 42, 1.0, True, None, [], {}, ['a', 'b', 'c'], None, [1, [2, [3]]], {'a': 1, 'b': None}] +lit 'line one\nline two\n\nafter blank\n' +fold 'folded text continues here\nnew paragraph\n indented more\nback\n' +strip 'no trailing newline' +keep 'keep trailing\n\n' +clip 'stripped folded' +ind ' two extra\nbase\n' +last 'end' +comments {'a': 1, 'b': 'x # not comment', 'c': 'x#y'} +docs [{'a': 1}, [2, 3], 'plain', 'folded\n'] +empty None +onlycomment None +explicit 'foo' +emptydocs [None, None] +anchors {'base': {'name': 'x', 'size': 1}, 'derived': {'name': 'x', 'size': 2}, 'multi': {'extra': True, 'name': 'x', 'size': 1}, 'list': [1, 2], 'again': [1, 2], 'scalar': 'hello', 'copy': 'hello'} +True +True +True 2 +binary b'\x01\x02\x03\x04\x05' +set ['a', 'b', 'c'] +omap [('a', 1), ('b', 2)] +pairs [('a', 1), ('a', 2)] +str 'true' +seqtag [1, 2] +maptag {'a': 1} +ts datetime.date(2020, 1, 2) +tagdirective '42' +verbatim 5 +multiline {'plain': 'this is a multi line plain scalar', 'quoted': 'folded double', 'single': 'folded\nsingle', 'esc': 'ab', 'flow': ['a', 'b', 'c']} +urlish {'a': 'http://x.y/z?q=1', 'b': ['x:y', 'a: b'], 'c': 'a:b'} +flowkeys {'a': 1, 'b c': 2, 'd': 3, 'e': None} +unicode {'k': 'caf\xe9 \u4e2d\u6587'} +bytesin {'a': 1, 'b': ['x']} +custom {'p': (1, 2)} +custom2 ['ABC', 'X Y'] +multi 'one:1' +subclass 'Xy' +full {'a': 1} +plain Loader {'a': [1, 2]} +unsafe {'a': 1} +fullload [1, 2] +tuple (1, 2) +base {'a': '1', 'b': ['2']} +e1 ScannerError +mapping values are not allowed here + in "", line 1, column 5: + a: b: c + ^ +e2 ParserError +while parsing a flow sequence + in "", line 1, column 4: + a: [1, 2 + ^ +expected ',' or ']', but got '' + in "", line 1, column 9: + a: [1, 2 + ^ +e3 ScannerError +while scanning a quoted scalar + in "", line 1, column 4: + a: 'unterminated + ^ +found unexpected end of stream + in "", line 1, column 17: + a: 'unterminated + ^ +e4 ParserError +while parsing a block collection + in "", line 1, column 1: + - a + ^ +expected , but found '?' + in "", line 2, column 1: + b: c + ^ +e5 ComposerError +found undefined alias 'missing' + in "", line 1, column 4: + a: *missing + ^ +e6 ParserError +while parsing a flow mapping + in "", line 1, column 1: + {a: 1, b: 2 + ^ +expected ',' or '}', but got '' + in "", line 1, column 12: + {a: 1, b: 2 + ^ +e7 ParserError +while parsing a block mapping + in "", line 1, column 1: + a: + ^ +expected , but found '' + in "", line 3, column 2: + c: d + ^ +e8 ConstructorError +could not determine a constructor for the tag '!foo' + in "", line 1, column 4: + a: !foo bar + ^ +e9 ComposerError +expected a single document in the stream + in "", line 1, column 1: + a: 1 + ^ +but found another document + in "", line 2, column 1: + --- + ^ +e10 ParserError +expected '', but found ']' + in "", line 1, column 7: + [1, 2]] + ^ +e11 ScannerError +while scanning a double-quoted scalar + in "", line 1, column 4: + a: "bad \q escape" + ^ +found unknown escape character 'q' + in "", line 1, column 10: + a: "bad \q escape" + ^ +e12 ScannerError +while scanning for the next token +found character '\t' that cannot start any token + in "", line 2, column 1: + indented: x + ^ +e13 ParserError +while parsing a block mapping + in "", line 1, column 1: + ? [1, 2] + ^ +expected , but found '-' + in "", line 3, column 1: + - y + ^ +e14 ComposerError +found duplicate anchor 'x'; first occurrence + in "", line 1, column 4: + a: &x 1 + ^ +second occurrence + in "", line 2, column 4: + b: &x 2 + ^ +e15 ConstructorError +while constructing a mapping + in "", line 1, column 1: + {[1]: 2} + ^ +found unhashable key + in "", line 1, column 2: + {[1]: 2} + ^ +e16 ScannerError +mapping values are not allowed here + in "", line 2, column 4: + c: d + ^ +e18 ScannerError +while scanning for the next token +found character '@' that cannot start any token + in "", line 1, column 4: + x: @foo + ^ +e19 ParserError +while parsing a block mapping + in "", line 1, column 1: + a: 'x'y + ^ +expected , but found '' + in "", line 1, column 7: + a: 'x'y + ^ +e20 ParserError +while parsing a block node +expected the node content, but found ']' + in "", line 2, column 5: + - ] + ^ +e21 ConstructorError +failed to decode base64 data: Invalid base64-encoded string: number of data characters (9) cannot be 1 more than a multiple of 4 + in "", line 1, column 1: + !!binary 'not base64!' + ^ +e22 ParserError +while parsing a block mapping + in "", line 1, column 1: + a: | + ^ +expected , but found '' + in "", line 3, column 2: + y + ^ +e23 ParserError +while parsing a flow node +expected the node content, but found '}' + in "", line 1, column 12: + {a: 1, a: [} + ^ +e24 ParserError +found incompatible YAML document (version 1.* is required) + in "", line 1, column 1: + %YAML 2.0 + ^ +e25 no error +lazy ParserError expected the node content, but found '' 3 0 +ns ParserError while parsing a flow node expected the node content, but found '' +True True +6.0.3 diff --git a/maven/python-transpiler/src/test/resources/conformance/yaml_.py b/maven/python-transpiler/src/test/resources/conformance/yaml_.py new file mode 100644 index 00000000000..754a725c005 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/yaml_.py @@ -0,0 +1,262 @@ +import yaml +import datetime + + +def show(label, value): + print(label, ascii(value)) + + +def err(label, text, loader=yaml.safe_load): + try: + loader(text) + print(label, "no error") + except yaml.YAMLError as e: + print(label, type(e).__name__) + print(str(e)) + + +# ---- scalars and implicit resolution +doc = """ +null1: ~ +null2: null +null3: Null +null4: +t1: yes +t2: No +t3: on +t4: OFF +t5: TRUE +t6: false +i1: 42 +i2: -17 +i3: +5 +i4: 0x1F +i5: 0b101 +i6: 0o17 +i7: 017 +i8: 1_000_000 +i9: 1:30 +f1: 3.14 +f2: -.5e+3 +f3: .inf +f4: -.inf +f5: 6.8523015e+5 +f6: 1_0.5 +s1: hello world +s2: 'single ''quoted'' text' +s3: "double \\"quoted\\"\\t tab \\x41 \\u00e9 \\n newline" +s4: 1e3 +s5: 12abc +s6: "123" +s7: '~' +d1: 2001-12-14 +d2: 2001-12-14t21:59:43.10-05:00 +d3: 2001-12-14 21:59:43.10 +d4: 2002-12-14T21:59:43Z +""" +data = yaml.safe_load(doc) +for k in data: + show(k, data[k]) +print(data["f4"] == float("-inf"), data["f3"] == float("inf")) + +# ---- block collections, flow collections +doc = """ +- a +- - b1 + - b2 +- key: value + other: [1, 2, 3] + nested: {x: 1, y: [a, b], z: {q: r}} +- ? complex + : value +- !!str 123 +- !!int "42" +- !!float 1 +- !!bool "yes" +- !!null "" +- [] +- {} +- [a, b, c] +- +- [1, [2, [3]]] +- {a: 1, b} +""" +show("seq", yaml.safe_load(doc)) + +# ---- block scalars +doc = """ +lit: | + line one + line two + + after blank +fold: > + folded text + continues here + + new paragraph + indented more + back +strip: |- + no trailing newline +keep: |+ + keep trailing + +clip: >- + stripped + folded +ind: |2 + two extra + base +last: end +""" +data = yaml.safe_load(doc) +for k in data: + show(k, data[k]) + +# ---- comments and documents +doc = """# leading comment +a: 1 # trailing +# middle +b: 'x # not comment' +c: x#y +""" +show("comments", yaml.safe_load(doc)) + +docs = list(yaml.safe_load_all("---\na: 1\n---\n- 2\n- 3\n...\n---\nplain\n...\n--- >\n folded\n")) +show("docs", docs) +show("empty", yaml.safe_load("")) +show("onlycomment", yaml.safe_load("# nothing\n")) +show("explicit", yaml.safe_load("--- foo\n...\n")) +show("emptydocs", list(yaml.safe_load_all("---\n---\n"))) + +# ---- anchors, aliases, merge +doc = """ +base: &base + name: x + size: 1 +derived: + <<: *base + size: 2 +multi: + <<: [*base, {extra: true, name: y}] +list: &l [1, 2] +again: *l +scalar: &s hello +copy: *s +""" +data = yaml.safe_load(doc) +show("anchors", data) +print(data["list"] is data["again"]) + +doc = "a: &a [1, 2]\nb: *a\n" +data = yaml.safe_load(doc) +print(data["a"] is data["b"]) +rec = yaml.safe_load("&x [1, *x]") +print(rec[1] is rec, len(rec)) + +# ---- explicit tags +show("binary", yaml.safe_load("!!binary |\n AQID\n BAU=\n")) +show("set", sorted(yaml.safe_load("!!set {a, b, c}"))) +show("omap", yaml.safe_load("!!omap [a: 1, b: 2]")) +show("pairs", yaml.safe_load("!!pairs [a: 1, a: 2]")) +show("str", yaml.safe_load("!!str true")) +show("seqtag", yaml.safe_load("!!seq [1, 2]")) +show("maptag", yaml.safe_load("!!map {a: 1}")) +show("ts", yaml.safe_load("!!timestamp 2020-01-02")) +show("tagdirective", yaml.safe_load("%TAG !e! tag:yaml.org,2002:\n---\n!e!str 42\n")) +show("verbatim", yaml.safe_load("! 5")) + +# ---- multi-line flow / plain / quoted +doc = """ +plain: this is + a multi line + plain scalar +quoted: "folded + double" +single: 'folded + + single' +esc: "a\\ + b" +flow: [a, + b, + c] +""" +show("multiline", yaml.safe_load(doc)) +show("urlish", yaml.safe_load("a: http://x.y/z?q=1\nb: [x:y, 'a: b']\nc: a:b\n")) +show("flowkeys", yaml.safe_load("{a: 1, 'b c': 2, ? d : 3, [1,2]: 4}") if False else yaml.safe_load("{a: 1, 'b c': 2, ? d : 3, e: }")) +show("unicode", yaml.safe_load("k: caf\u00e9 \u4e2d\u6587\n")) +show("bytesin", yaml.safe_load(b"a: 1\nb: [x]\n")) + +# ---- custom constructors +yaml.add_constructor("!pt", lambda loader, node: tuple(loader.construct_sequence(node)), Loader=yaml.SafeLoader) +show("custom", yaml.safe_load("p: !pt [1, 2]")) + + +def upper(loader, node): + return loader.construct_scalar(node).upper() + + +yaml.add_constructor("!up", upper, Loader=yaml.SafeLoader) +show("custom2", yaml.safe_load("[!up abc, !up 'x y']")) + + +def multi(loader, suffix, node): + return suffix + ":" + str(loader.construct_scalar(node)) + + +yaml.add_multi_constructor("!m/", multi, Loader=yaml.SafeLoader) +show("multi", yaml.safe_load("!m/one 1")) + + +class MyLoader(yaml.SafeLoader): + pass + + +MyLoader.add_constructor("!x", lambda l, n: "X" + l.construct_scalar(n)) +show("subclass", yaml.load("!x y", Loader=MyLoader)) +show("full", yaml.load("a: 1", Loader=yaml.FullLoader)) +show("plain Loader", yaml.load("a: [1, 2]", Loader=yaml.Loader)) +show("unsafe", yaml.unsafe_load("a: 1")) +show("fullload", yaml.full_load("[1, 2]")) +show("tuple", yaml.load("!!python/tuple [1, 2]", Loader=yaml.FullLoader)) +show("base", yaml.load("a: 1\nb: [2]", Loader=yaml.BaseLoader)) + +# ---- errors +err("e1", "a: b: c") +err("e2", "a: [1, 2") +err("e3", "a: 'unterminated") +err("e4", "- a\nb: c\n") +err("e5", "a: *missing") +err("e6", "{a: 1, b: 2") +err("e7", "a:\n - b\n c: d") +err("e8", "a: !foo bar") +err("e9", "a: 1\n---\nb: 2\n") +err("e10", "[1, 2]]") +err("e11", "a: \"bad \\q escape\"") +err("e12", "key: value\n\tindented: x\n") +err("e13", "? [1, 2]\n: x\n- y") +err("e14", "a: &x 1\nb: &x 2\n") +err("e15", "{[1]: 2}") +err("e16", "a: b\n c: d\n") +err("e18", "x: @foo") +err("e19", "a: 'x'y") +err("e20", "- - -\n - ]") +err("e21", "!!binary 'not base64!'") +err("e22", "a: |\n x\n y") +err("e23", "{a: 1, a: [}") +err("e24", "%YAML 2.0\n--- a") +err("e25", "- a\n- b\n - c\n") +try: + list(yaml.safe_load_all("a: 1\n---\n[\n")) +except yaml.MarkedYAMLError as e: + print("lazy", type(e).__name__, e.problem, e.problem_mark.line, e.problem_mark.column) +try: + yaml.safe_load("a: [") +except yaml.scanner.ScannerError as e: + print("ns", "ScannerError") +except yaml.parser.ParserError as e: + print("ns", "ParserError", e.context, e.problem) +print(issubclass(yaml.constructor.ConstructorError, yaml.MarkedYAMLError), issubclass(yaml.MarkedYAMLError, yaml.YAMLError)) +print(yaml.__version__) diff --git a/maven/python-transpiler/src/test/resources/conformance/yaml_dump.out b/maven/python-transpiler/src/test/resources/conformance/yaml_dump.out new file mode 100644 index 00000000000..8e464d964bf --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/yaml_dump.out @@ -0,0 +1,676 @@ +== None +null +... +<< +== True +true +... +<< +== False +false +... +<< +== 0 +0 +... +<< +== 42 +42 +... +<< +== -7 +-7 +... +<< +== 100000000000000000000 +100000000000000000000 +... +<< +== 3.14 +3.14 +... +<< +== -0.5 +-0.5 +... +<< +== 1e+16 +1.0e+16 +... +<< +== 1e-07 +1.0e-07 +... +<< +== 1.0 +1.0 +... +<< +== inf +.inf +... +<< +== -inf +-.inf +... +<< +== nan +.nan +... +<< +== 'plain' +plain +... +<< +== '' +'' +<< +== ' lead' +' lead' +<< +== 'trail ' +'trail ' +<< +== 'a: b' +'a: b' +<< +== 'a #b' +'a #b' +<< +== '#c' +'#c' +<< +== '- x' +'- x' +<< +== '-' +'-' +<< +== '?' +'?' +<< +== '? x' +'? x' +<< +== ':' +':' +<< +== 'x:' +'x:' +<< +== 'yes' +'yes' +<< +== 'No' +'No' +<< +== 'on' +'on' +<< +== 'null' +'null' +<< +== '~' +'~' +<< +== 'true' +'true' +<< +== '123' +'123' +<< +== '1.5' +'1.5' +<< +== '0x1f' +'0x1f' +<< +== '1e3' +1e3 +... +<< +== '2001-01-01' +'2001-01-01' +<< +== '1:30' +'1:30' +<< +== '<<' +'<<' +<< +== '=' +'=' +<< +== '@at' +'@at' +<< +== '`tick' +'`tick' +<< +== '%pct' +'%pct' +<< +== '!bang' +'!bang' +<< +== '&' +'&' +<< +== '*star' +'*star' +<< +== '|pipe' +'|pipe' +<< +== '>gt' +'>gt' +<< +== "'q'" +'''q''' +<< +== '"dq"' +'"dq"' +<< +== "a'b" +a'b +... +<< +== 'a"b' +a"b +... +<< +== '[x]' +'[x]' +<< +== '{x}' +'{x}' +<< +== 'a,b' +a,b +... +<< +== 'tab\there' +"tab\there" +<< +== 'nl\nx' +'nl + + x' +<< +== 'two\n\nlines' +'two + + + lines' +<< +== 'end\n' +'end + + ' +<< +== '\nstart' +' + + start' +<< +== 'caf\xe9' +"caf\xE9" +<< +== '\u4e2d\u6587' +"\u4E2D\u6587" +<< +== 'bell\x07' +"bell\a" +<< +== 'nul\x00' +"nul\0" +<< +== '---' +'---' +<< +== '...' +'...' +<< +== '--- x' +'--- x' +<< +== 'a b' +a b +... +<< +== 'a\n b' +"a\n b" +<< +== ' ' +' ' +<< +== 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx' +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +... +<< +== 'word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word ' +'word word word word word word word word word word word word word word word word word + word word word word word word word word word word word word word ' +<< +== '\u2028sep' +"\Lsep" +<< +== '\x85next' +"\Nnext" +<< +== unicode allowed +café 中文 😀 +... +<< +== unicode escaped +"caf\xE9 \u4E2D\u6587 \U0001F600" +<< +== bytes +!!binary | + AAFiaW5hcnkgZGF0YSBoZXJl +<< +== date +2020-02-29 +... +<< +== datetime +2020-02-29 12:30:45 +... +<< +== datetime micro +2020-02-29 12:30:45.123456 +... +<< +== datetime tz +2020-02-29 12:30:45+00:00 +... +<< +== default +'123': numeric key +flag: true +long key kkkkkkkkkkkkkkkkkkkk: v +matrix: +- - 1 + - 2 +- - 3 + - 4 +name: demo +nested: + empty_dict: {} + empty_list: [] + x: 1 + y: + - 1 + - 2 + - z: null +none: null +tags: +- a +- b +- c +text: 'multi + + line' +version: 1.5 +<< +== unsorted +name: demo +version: 1.5 +tags: +- a +- b +- c +nested: + x: 1 + y: + - 1 + - 2 + - z: null + empty_list: [] + empty_dict: {} +matrix: +- - 1 + - 2 +- - 3 + - 4 +flag: true +text: 'multi + + line' +none: null +'123': numeric key +long key kkkkkkkkkkkkkkkkkkkk: v +<< +== flow +{'123': numeric key, flag: true, long key kkkkkkkkkkkkkkkkkkkk: v, matrix: [[1, 2], + [3, 4]], name: demo, nested: {empty_dict: {}, empty_list: [], x: 1, y: [1, 2, + {z: null}]}, none: null, tags: [a, b, c], text: 'multi + + line', version: 1.5} +<< +== flow none +'123': numeric key +flag: true +long key kkkkkkkkkkkkkkkkkkkk: v +matrix: +- [1, 2] +- [3, 4] +name: demo +nested: + empty_dict: {} + empty_list: [] + x: 1 + y: + - 1 + - 2 + - {z: null} +none: null +tags: [a, b, c] +text: 'multi + + line' +version: 1.5 +<< +== indent4 +'123': numeric key +flag: true +long key kkkkkkkkkkkkkkkkkkkk: v +matrix: +- - 1 + - 2 +- - 3 + - 4 +name: demo +nested: + empty_dict: {} + empty_list: [] + x: 1 + y: + - 1 + - 2 + - z: null +none: null +tags: +- a +- b +- c +text: 'multi + + line' +version: 1.5 +<< +== width20 +k: alpha beta gamma delta + epsilon zeta eta theta + iota kappa lambda +<< +== width flow +[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, + 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, + 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, + 33, 34, 35, 36, 37, 38, 39] +<< +== explicit start +--- +- a +- b +- c +<< +== explicit both +--- +a: 1 +... +<< +== version +%YAML 1.1 +--- +a: 1 +<< +== list of dicts +- a: 1 + b: 2 +- c: + - 1 + - 2 +- [] +- - [] +- - {} +<< +== list in list +- - 1 + - - 2 + - 3 +- - - 4 +<< +== tuple +- 1 +- 2 +- - 3 + - 4 +<< +== set +!!set +1: null +2: null +3: null +<< +== set flow +{s: !!set {a: null, b: null}} +<< +== mixed keys +1: 4 +b: 2 +null: 3 +<< +== int keys +1: a +2: b +3: c +<< +== tuple key +? - 1 + - 2 +: t +<< +== complex key +? kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk +: 1 +? 'multi + + line' +: 2 +<< +== dict in flow +[{a: [1, {b: 2}]}] +<< +== long plain +- 'word word word word word word word word word word word word word word word word + word word word word word word word word word word word word word word word word + word word word word word word word word ' +<< +== long quoted +k: 'it''s it''s it''s it''s it''s it''s it''s it''s it''s it''s it''s it''s it''s + it''s it''s it''s it''s it''s it''s it''s it''s it''s it''s it''s it''s it''s it''s + it''s it''s it''s ' +<< +== long dq +k: "tab\t tab\t tab\t tab\t tab\t tab\t tab\t tab\t tab\t tab\t tab\t tab\t tab\t\ + \ tab\t tab\t tab\t tab\t tab\t tab\t tab\t tab\t tab\t tab\t tab\t tab\t tab\t\ + \ tab\t tab\t tab\t tab\t " +<< +== aliases +a: &id001 +- 1 +- 2 +b: *id001 +c: + d: *id001 +<< +== alias order +- &id002 + n: 1 +- &id001 + n: 2 +- *id001 +- *id002 +<< +== recursive +&id001 +- 1 +- *id001 +<< +== date alias +- &id001 2020-01-01 +- *id001 +<< +== no alias +a: +- 1 +- 2 +b: +- 1 +- 2 +<< +== dump +a: !!python/tuple +- 1 +- 2 +b: +- 1 +- 2 +<< +== dump flow +{a: x, b: [1, 2]} +<< +== dump_all +a: 1 +--- +- 1 +- 2 +--- plain +--- 5 +... +<< +== safe_dump_all +--- 1 +--- two +--- +- 3 +<< +== dump_all explicit +--- +a: 1 +... +--- +b: 2 +... +<< +== dump scalar dumper +hello +... +<< +== default_style +"a": "b" +"c": +- !!int "1" +- "d" +<< +== default_style sq +'a': 'b' +'c': +- !!int '1' +- 'd' +<< +== default_style lit +"a": |- + b + c +"d": |- + x +<< +== default_style fold +"a": >- + b c d +"d": >- + x +<< +bytes +b'caf\xc3\xa9\n...\n' +None +k: +- 1 +- 2 +== custom rep +list: +- !point + x: 3 + y: 4 +p: !point + x: 1 + y: 2 +<< +== literal rep +a: | + one + two +b: single +c: |- + x + y +<< +== flow seq rep +t: !seq3 [1, 2, 3] +<< +== indented lists +a: + - 1 + - 2 + - b: + - 3 +c: 1 +<< +RepresenterError ('cannot represent an object', +== yamlobject +!Obj +a: 1 +b: +- 2 +- 3 +<< +Obj 5 text +0 True +0 True +0 True +1 True +1 True +1 True +2 True +2 True +2 True +3 True +3 True +3 True +4 True +4 True +4 True +5 True +5 True +5 True +6 True +6 True +6 True +7 True +7 True +7 True +8 True +8 True +8 True +9 True +9 True +9 True +== tuples dump +t: !!python/tuple +- 1 +- 2 +<< +{'t': (1, 2)} diff --git a/maven/python-transpiler/src/test/resources/conformance/yaml_dump.py b/maven/python-transpiler/src/test/resources/conformance/yaml_dump.py new file mode 100644 index 00000000000..87f102270c9 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/yaml_dump.py @@ -0,0 +1,203 @@ +import yaml +import datetime + + +def out(label, text): + print("== " + label) + print(text, end="") + print("<<") + + +# ---- scalars +for v in [None, True, False, 0, 42, -7, 10 ** 20, 3.14, -0.5, 1e16, 1e-7, 1.0, float("inf"), + float("-inf"), float("nan"), "plain", "", " lead", "trail ", "a: b", "a #b", "#c", + "- x", "-", "?", "? x", ":", "x:", "yes", "No", "on", "null", "~", "true", "123", + "1.5", "0x1f", "1e3", "2001-01-01", "1:30", "<<", "=", "@at", "`tick", "%pct", "!bang", + "&", "*star", "|pipe", ">gt", "'q'", '"dq"', "a'b", 'a"b', "[x]", "{x}", "a,b", + "tab\there", "nl\nx", "two\n\nlines", "end\n", "\nstart", "caf\u00e9", "\u4e2d\u6587", + "bell\x07", "nul\x00", "---", "...", "--- x", "a b", "a\n b", " ", "x" * 100, + "word " * 30, "\u2028sep", "\x85next"]: + out(ascii(v), yaml.safe_dump(v)) +out("unicode allowed", yaml.safe_dump("caf\u00e9 \u4e2d\u6587 \U0001f600", allow_unicode=True)) +out("unicode escaped", yaml.safe_dump("caf\u00e9 \u4e2d\u6587 \U0001f600")) +out("bytes", yaml.safe_dump(b"\x00\x01binary data here")) +out("date", yaml.safe_dump(datetime.date(2020, 2, 29))) +out("datetime", yaml.safe_dump(datetime.datetime(2020, 2, 29, 12, 30, 45))) +out("datetime micro", yaml.safe_dump(datetime.datetime(2020, 2, 29, 12, 30, 45, 123456))) +out("datetime tz", yaml.safe_dump(datetime.datetime(2020, 2, 29, 12, 30, 45, tzinfo=datetime.timezone.utc))) + +# ---- structures +data = { + "name": "demo", + "version": 1.5, + "tags": ["a", "b", "c"], + "nested": {"x": 1, "y": [1, 2, {"z": None}], "empty_list": [], "empty_dict": {}}, + "matrix": [[1, 2], [3, 4]], + "flag": True, + "text": "multi\nline", + "none": None, + "123": "numeric key", + "long key " + "k" * 20: "v", +} +out("default", yaml.safe_dump(data)) +out("unsorted", yaml.safe_dump(data, sort_keys=False)) +out("flow", yaml.safe_dump(data, default_flow_style=True)) +out("flow none", yaml.safe_dump(data, default_flow_style=None)) +out("indent4", yaml.safe_dump(data, indent=4)) +out("width20", yaml.safe_dump({"k": "alpha beta gamma delta epsilon zeta eta theta iota kappa lambda"}, width=20)) +out("width flow", yaml.safe_dump(list(range(40)), default_flow_style=True, width=40)) +out("explicit start", yaml.safe_dump(data["tags"], explicit_start=True)) +out("explicit both", yaml.safe_dump({"a": 1}, explicit_start=True, explicit_end=True)) +out("version", yaml.safe_dump({"a": 1}, version=(1, 1))) +out("list of dicts", yaml.safe_dump([{"a": 1, "b": 2}, {"c": [1, 2]}, [], [[]], [{}]])) +out("list in list", yaml.safe_dump([[1, [2, 3]], [[4]]])) +out("tuple", yaml.safe_dump((1, 2, (3, 4)))) +out("set", yaml.safe_dump({3, 1, 2})) +out("set flow", yaml.safe_dump({"s": {"a", "b"}}, default_flow_style=True)) +out("mixed keys", yaml.safe_dump({1: "a", "b": 2, None: 3, True: 4})) +out("int keys", yaml.safe_dump({3: "c", 1: "a", 2: "b"})) +out("tuple key", yaml.safe_dump({(1, 2): "t"})) +out("complex key", yaml.safe_dump({"k" * 130: 1, "multi\nline": 2})) +out("dict in flow", yaml.safe_dump([{"a": [1, {"b": 2}]}], default_flow_style=True)) +out("long plain", yaml.safe_dump(["word " * 40])) +out("long quoted", yaml.safe_dump({"k": "it's " * 30})) +out("long dq", yaml.safe_dump({"k": "tab\t " * 30})) + +# ---- aliases +shared = [1, 2] +out("aliases", yaml.safe_dump({"a": shared, "b": shared, "c": {"d": shared}})) +a = {"n": 1} +b = {"n": 2} +out("alias order", yaml.safe_dump([a, b, b, a])) +rec = [1] +rec.append(rec) +out("recursive", yaml.safe_dump(rec)) +d1 = datetime.date(2020, 1, 1) +out("date alias", yaml.safe_dump([d1, d1])) + + +class NoAlias(yaml.SafeDumper): + def ignore_aliases(self, data): + return True + + +out("no alias", yaml.dump({"a": shared, "b": shared}, Dumper=NoAlias)) + +# ---- dump / dump_all / Dumper +out("dump", yaml.dump({"b": [1, 2], "a": (1, 2)})) +out("dump flow", yaml.dump({"b": [1, 2], "a": "x"}, default_flow_style=True)) +out("dump_all", yaml.dump_all([{"a": 1}, [1, 2], "plain", 5])) +out("safe_dump_all", yaml.safe_dump_all([1, "two", [3]], explicit_start=True)) +out("dump_all explicit", yaml.dump_all([{"a": 1}, {"b": 2}], explicit_start=True, explicit_end=True)) +out("dump scalar dumper", yaml.dump("hello", Dumper=yaml.Dumper)) +out("default_style", yaml.safe_dump({"a": "b", "c": [1, "d"]}, default_style='"')) +out("default_style sq", yaml.safe_dump({"a": "b", "c": [1, "d"]}, default_style="'")) +out("default_style lit", yaml.safe_dump({"a": "b\nc", "d": "x"}, default_style="|")) +out("default_style fold", yaml.safe_dump({"a": "b c d", "d": "x"}, default_style=">")) +print(type(yaml.safe_dump("x", encoding="utf-8")).__name__) +print(yaml.safe_dump("caf\u00e9", encoding="utf-8", allow_unicode=True)) + + +class Buf: + def __init__(self): + self.parts = [] + + def write(self, s): + self.parts.append(s) + + +buf = Buf() +print(yaml.safe_dump({"k": [1, 2]}, buf)) +print("".join(buf.parts), end="") + + +# ---- custom representers +class Point: + def __init__(self, x, y): + self.x = x + self.y = y + + +def point_rep(dumper, p): + return dumper.represent_mapping("!point", {"x": p.x, "y": p.y}) + + +yaml.add_representer(Point, point_rep, Dumper=yaml.SafeDumper) +out("custom rep", yaml.safe_dump({"p": Point(1, 2), "list": [Point(3, 4)]})) + + +def str_literal(dumper, s): + style = "|" if "\n" in s else None + return dumper.represent_scalar("tag:yaml.org,2002:str", s, style=style) + + +class LitDumper(yaml.SafeDumper): + pass + + +LitDumper.add_representer(str, str_literal) +out("literal rep", yaml.dump({"a": "one\ntwo\n", "b": "single", "c": "x\ny"}, Dumper=LitDumper)) + + +def seq_rep(dumper, s): + return dumper.represent_sequence("!seq3", list(s), flow_style=True) + + +class Triple(tuple): + pass + + +yaml.SafeDumper.add_representer(Triple, seq_rep) +out("flow seq rep", yaml.safe_dump({"t": Triple((1, 2, 3))})) + + +class IndentDumper(yaml.SafeDumper): + def increase_indent(self, flow=False, indentless=False): + return super().increase_indent(flow, False) + + +out("indented lists", yaml.dump({"a": [1, 2, {"b": [3]}], "c": 1}, Dumper=IndentDumper)) + +try: + yaml.safe_dump(object()) +except yaml.YAMLError as e: + print(type(e).__name__, str(e)[:30]) + + +class Obj(yaml.YAMLObject): + yaml_tag = "!Obj" + yaml_loader = yaml.SafeLoader + yaml_dumper = yaml.SafeDumper + + def __init__(self, a, b): + self.a = a + self.b = b + + +out("yamlobject", yaml.safe_dump(Obj(1, [2, 3]))) +o = yaml.safe_load("!Obj {a: 5, b: text}") +print(type(o).__name__, o.a, o.b) + +# ---- round trips +samples = [ + {"a": 1, "b": [1, 2, 3], "c": {"d": "e"}}, + ["x", "y: z", "123", "", None, True, 1.5, "multi\nline\n", " pad "], + {"k": "caf\u00e9", "date": datetime.date(2001, 1, 2), "dt": datetime.datetime(2001, 1, 2, 3, 4, 5)}, + [[], {}, [[]], [{}], {"a": []}], + "just a string", + 42, + {"nested": {"deep": {"deeper": [1, {"x": [2, 3]}]}}}, + b"\x00\xff bytes", + {"set": {1, 2, 3}}, + ["tab\there", "quote'single", 'quote"double', "back\\slash", "null", "~", "yes"], +] +for i, s in enumerate(samples): + text = yaml.safe_dump(s) + back = yaml.safe_load(text) + print(i, back == s) + text2 = yaml.safe_dump(s, default_flow_style=True, allow_unicode=True) + print(i, yaml.safe_load(text2) == s) + text3 = yaml.safe_dump(s, width=30, indent=4) + print(i, yaml.safe_load(text3) == s) +out("tuples dump", yaml.dump({"t": (1, 2)})) +print(yaml.load(yaml.dump({"t": (1, 2)}), Loader=yaml.FullLoader)) From f2225503076b50c8e1fbc576608f63442c01db18 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Tue, 27 Oct 2026 17:55:40 +0200 Subject: [PATCH 008/112] Python: attrs and dateutil packages Co-Authored-By: Claude Opus 5.5 --- .../codename1/python/site/attr/__init__.py | 879 ++++++++++++++++++ .../codename1/python/site/attr/converters.py | 63 ++ .../codename1/python/site/attr/exceptions.py | 52 ++ .../com/codename1/python/site/attr/filters.py | 34 + .../com/codename1/python/site/attr/setters.py | 33 + .../codename1/python/site/attr/validators.py | 260 ++++++ .../codename1/python/site/attrs/__init__.py | 26 + .../codename1/python/site/attrs/converters.py | 6 + .../codename1/python/site/attrs/exceptions.py | 12 + .../codename1/python/site/attrs/filters.py | 6 + .../codename1/python/site/attrs/setters.py | 6 + .../codename1/python/site/attrs/validators.py | 7 + .../python/site/dateutil/__init__.py | 10 + .../codename1/python/site/dateutil/_common.py | 56 ++ .../codename1/python/site/dateutil/parser.py | 587 ++++++++++++ .../python/site/dateutil/relativedelta.py | 345 +++++++ .../codename1/python/site/dateutil/rrule.py | 610 ++++++++++++ .../com/codename1/python/site/dateutil/tz.py | 280 ++++++ .../test/resources/conformance/attrs_pkg.out | 94 ++ .../test/resources/conformance/attrs_pkg.py | 392 ++++++++ .../test/resources/conformance/dateutil_.out | 206 ++++ .../test/resources/conformance/dateutil_.py | 221 +++++ 22 files changed, 4185 insertions(+) create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/attr/__init__.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/attr/converters.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/attr/exceptions.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/attr/filters.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/attr/setters.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/attr/validators.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/attrs/__init__.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/attrs/converters.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/attrs/exceptions.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/attrs/filters.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/attrs/setters.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/attrs/validators.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/dateutil/__init__.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/dateutil/_common.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/dateutil/parser.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/dateutil/relativedelta.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/dateutil/rrule.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/dateutil/tz.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/attrs_pkg.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/attrs_pkg.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/dateutil_.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/dateutil_.py diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/attr/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/attr/__init__.py new file mode 100644 index 00000000000..6db8e8a2bb6 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/attr/__init__.py @@ -0,0 +1,879 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Classes without boilerplate: the attrs API. + +Written from the documented behaviour of attrs. The generated methods are +closures over the attribute list, since no source text is compiled at run +time. +""" + +from dataclasses import _is_classvar + +from attr import exceptions +from attr.exceptions import (FrozenInstanceError, NotAnAttrsClassError, UnannotatedAttributeError) + +__all__ = ["Attribute", "Factory", "NOTHING", "asdict", "assoc", "astuple", "attr", "attrib", "attributes", "attrs", + "converters", "define", "evolve", "exceptions", "field", "fields", "fields_dict", "filters", "frozen", + "has", "ib", "make_class", "mutable", "resolve_types", "s", "setters", "validate", "validators", + "Converter", "cmp_using"] + + +class _Nothing: + _singleton = None + + def __repr__(self): + return "NOTHING" + + def __str__(self): + return "_Nothing.NOTHING" + + def __bool__(self): + return False + + +NOTHING = _Nothing() + + +class Factory: + """A default computed by calling a function at instantiation.""" + + def __init__(self, factory, takes_self=False): + self.factory = factory + self.takes_self = takes_self + + def __repr__(self): + return "Factory(factory=%r, takes_self=%r)" % (self.factory, self.takes_self) + + def __eq__(self, other): + if other.__class__ is not Factory: + return NotImplemented + return self.factory == other.factory and self.takes_self == other.takes_self + + def __hash__(self): + return hash((self.factory, self.takes_self)) + + +class Converter: + """A converter that also receives the instance and/or the attribute.""" + + def __init__(self, converter, *, takes_self=False, takes_field=False): + self.converter = converter + self.takes_self = takes_self + self.takes_field = takes_field + + def __call__(self, value, instance=None, field=None): + if self.takes_self and self.takes_field: + return self.converter(value, instance, field) + if self.takes_self: + return self.converter(value, instance) + if self.takes_field: + return self.converter(value, field) + return self.converter(value) + + +_counter = [0] + + +def _next_count(): + _counter[0] += 1 + return _counter[0] + + +class Attribute: + """One attribute of an attrs class, as listed by fields().""" + + def __init__(self, name, default, validator, repr, cmp, hash, init, inherited, metadata=None, type=None, + converter=None, kw_only=False, eq=None, eq_key=None, order=None, order_key=None, on_setattr=None, + alias=None): + if eq is None: + eq = cmp if cmp is not None else True + if order is None: + order = eq + self.name = name + self.default = default + self.validator = validator + self.repr = repr + self.eq = eq + self.eq_key = eq_key + self.order = order + self.order_key = order_key + self.hash = hash + self.init = init + self.metadata = {} if metadata is None else metadata + self.type = type + self.converter = converter + self.kw_only = kw_only + self.inherited = inherited + self.on_setattr = on_setattr + self.alias = alias + + @property + def cmp(self): + return self.eq and self.order + + def evolve(self, **changes): + new = Attribute(self.name, self.default, self.validator, self.repr, None, self.hash, self.init, + self.inherited, self.metadata, self.type, self.converter, self.kw_only, self.eq, + self.eq_key, self.order, self.order_key, self.on_setattr, self.alias) + for k, v in changes.items(): + setattr(new, k, v) + return new + + def __repr__(self): + return ("Attribute(name=%r, default=%r, validator=%r, repr=%r, eq=%r, eq_key=%r, order=%r, order_key=%r, " + "hash=%r, init=%r, metadata=mappingproxy(%r), type=%r, converter=%r, kw_only=%r, inherited=%r, " + "on_setattr=%r, alias=%r)" % (self.name, self.default, self.validator, self.repr, self.eq, + self.eq_key, self.order, self.order_key, self.hash, self.init, + self.metadata, self.type, self.converter, self.kw_only, + self.inherited, self.on_setattr, self.alias)) + + def __eq__(self, other): + if not isinstance(other, Attribute): + return NotImplemented + return (self.name == other.name and self.default == other.default and self.validator == other.validator and + self.repr == other.repr and self.eq == other.eq and self.order == other.order and + self.hash == other.hash and self.init == other.init and self.type == other.type and + self.converter == other.converter and self.kw_only == other.kw_only and + self.inherited == other.inherited and self.alias == other.alias) + + def __hash__(self): + return hash((self.name, self.inherited, self.alias)) + + +class _CountingAttr: + """What attr.ib()/field() return until the class decorator collects it.""" + + def __init__(self, default, validator, repr, eq, eq_key, order, order_key, hash, init, metadata, type, + converter, kw_only, on_setattr, alias): + self.counter = _next_count() + self._default = default + self._validator = validator + self.repr = repr + self.eq = eq + self.eq_key = eq_key + self.order = order + self.order_key = order_key + self.hash = hash + self.init = init + self.metadata = metadata + self.type = type + self.converter = converter + self.kw_only = kw_only + self.on_setattr = on_setattr + self.alias = alias + + def validator(self, meth): + if self._validator is None: + self._validator = meth + else: + self._validator = validators.and_(self._validator, meth) + return meth + + def default(self, meth): + if self._default is not NOTHING: + raise exceptions.DefaultAlreadySetError() + self._default = Factory(meth, takes_self=True) + return meth + + +def _eq_order(cmp, eq, order): + if cmp is not None and (eq is not None or order is not None): + raise ValueError("Don't mix `cmp` with `eq' and `order`.") + if cmp is not None: + return cmp, cmp + if eq is None: + eq = True + if order is None: + order = eq + if eq is False and order is True: + raise ValueError("`order` can only be True if `eq` is True too.") + return eq, order + + +def _key_of(value): + if callable(value): + return True, value + return value, None + + +def attrib(default=NOTHING, validator=None, repr=True, cmp=None, hash=None, init=True, metadata=None, type=None, + converter=None, factory=None, kw_only=False, eq=None, order=None, on_setattr=None, alias=None): + eq, order = _eq_order(cmp, eq, order) + eq_key = None + order_key = None + if callable(eq): + eq_key = eq + eq = True + if order is True or order is None: + order_key = eq_key + if callable(order): + order_key = order + order = True + if hash is not None and hash is not True and hash is not False: + raise TypeError("Invalid value for hash. Must be True, False, or None.") + if factory is not None: + if default is not NOTHING: + raise ValueError("The `default` and `factory` arguments are mutually exclusive.") + if not callable(factory): + raise ValueError("The `factory` argument must be a callable.") + default = Factory(factory) + if isinstance(validator, (list, tuple)): + validator = validators.and_(*validator) + if isinstance(converter, (list, tuple)): + converter = converters.pipe(*converter) + return _CountingAttr(default, validator, repr, eq, eq_key, order, order_key, hash, init, metadata, type, + converter, kw_only, on_setattr, alias) + + +ib = attrib +attr = attrib + + +def field(*, default=NOTHING, validator=None, repr=True, hash=None, init=True, metadata=None, type=None, + converter=None, factory=None, kw_only=False, eq=None, order=None, on_setattr=None, alias=None): + return attrib(default=default, validator=validator, repr=repr, hash=hash, init=init, metadata=metadata, + type=type, converter=converter, factory=factory, kw_only=kw_only, eq=eq, order=order, + on_setattr=on_setattr, alias=alias) + + +class _AttrsTuple(tuple): + """The tuple fields() returns: also reachable by attribute name.""" + + def __getattr__(self, name): + for a in self: + if a.name == name: + return a + raise AttributeError(name) + + +def fields(cls): + if not isinstance(cls, type): + cls = cls.__class__ + attrs = getattr(cls, "__attrs_attrs__", None) + if attrs is None: + raise NotAnAttrsClassError(repr(cls) + " is not an attrs-decorated class.") + return attrs + + +def fields_dict(cls): + if not isinstance(cls, type): + raise TypeError("Passed object must be a class.") + attrs = getattr(cls, "__attrs_attrs__", None) + if attrs is None: + raise NotAnAttrsClassError(repr(cls) + " is not an attrs-decorated class.") + return dict((a.name, a) for a in attrs) + + +def has(cls): + attrs = getattr(cls, "__attrs_attrs__", None) + if attrs is not None: + return True + return False + + +def resolve_types(cls, globalns=None, localns=None, attribs=None, include_extras=True): + return cls + + +def _convert_one(c, value, instance, attrib): + if isinstance(c, Converter): + return c(value, instance, attrib) + return c(value) + + +def _convert_value(instance, attrib, value): + if attrib.converter is None: + return value + return _convert_one(attrib.converter, value, instance, attrib) + + +def _run_validator(instance, attrib, value): + if attrib.validator is not None and not validators.get_disabled(): + attrib.validator(instance, attrib, value) + return value + + +def validate(inst): + if validators.get_disabled(): + return + for a in fields(inst.__class__): + if a.validator is not None: + a.validator(inst, a, getattr(inst, a.name)) + + +def _is_attrs_instance(obj): + return getattr(obj.__class__, "__attrs_attrs__", None) is not None + + +def asdict(inst, recurse=True, filter=None, dict_factory=dict, retain_collection_types=False, + value_serializer=None): + result = dict_factory() + items = [] + for a in fields(inst.__class__): + v = getattr(inst, a.name) + if filter is not None and not filter(a, v): + continue + if value_serializer is not None: + v = value_serializer(inst, a, v) + if recurse is True: + v = _asdict_value(v, filter, dict_factory, retain_collection_types, value_serializer) + items.append((a.name, v)) + if dict_factory is dict: + return dict(items) + for k, v in items: + result[k] = v + return result + + +def _asdict_value(v, filter, dict_factory, retain, serializer): + if _is_attrs_instance(v): + return asdict(v, True, filter, dict_factory, retain, serializer) + if isinstance(v, (tuple, list, set, frozenset)): + cf = v.__class__ if retain is True else list + items = [_asdict_value(i, filter, dict_factory, retain, serializer) for i in v] + try: + return cf(items) + except TypeError: + if not isinstance(v, tuple): + raise + return cf(*items) + if isinstance(v, dict): + df = dict_factory if dict_factory is not dict else v.__class__ + return df((_asdict_value(kk, filter, dict_factory, retain, serializer), + _asdict_value(vv, filter, dict_factory, retain, serializer)) for kk, vv in v.items()) + return v + + +def astuple(inst, recurse=True, filter=None, tuple_factory=tuple, retain_collection_types=False): + rv = [] + for a in fields(inst.__class__): + v = getattr(inst, a.name) + if filter is not None and not filter(a, v): + continue + if recurse is True: + v = _astuple_value(v, filter, tuple_factory, retain_collection_types) + rv.append(v) + return rv if tuple_factory is list else tuple_factory(rv) + + +def _astuple_value(v, filter, tuple_factory, retain): + if _is_attrs_instance(v): + return astuple(v, True, filter, tuple_factory, retain) + if isinstance(v, (tuple, list, set, frozenset)): + cf = v.__class__ if retain is True else list + items = [_astuple_value(i, filter, tuple_factory, retain) for i in v] + try: + return cf(items) + except TypeError: + if not isinstance(v, tuple): + raise + return cf(*items) + if isinstance(v, dict): + df = v.__class__ + return df((_astuple_value(kk, filter, tuple_factory, retain), _astuple_value(vv, filter, tuple_factory, + retain)) for kk, vv in v.items()) + return v + + +def evolve(*args, **changes): + if len(args) != 1: + raise TypeError("evolve() takes 1 positional argument, but %d were given" % len(args)) + inst = args[0] + cls = inst.__class__ + for a in fields(cls): + if not a.init: + continue + name = a.alias + if name not in changes: + changes[name] = getattr(inst, a.name) + return cls(**changes) + + +def assoc(inst, **changes): + new = inst.__class__.__new__(inst.__class__) + names = set(a.name for a in fields(inst.__class__)) + for k in changes: + if k not in names: + raise exceptions.AttrsAttributeNotFoundError( + "{k} is not an attrs attribute on {cl}.".format(k=k, cl=new.__class__)) + for a in fields(inst.__class__): + object.__setattr__(new, a.name, changes.get(a.name, getattr(inst, a.name))) + return new + + +def cmp_using(eq=None, lt=None, le=None, gt=None, ge=None, require_same_type=True, class_name="Comparable"): + body = {} + if eq is not None: + def __eq__(self, other): + if require_same_type and other.__class__ is not self.__class__: + return NotImplemented + return eq(self.value, other.value) + body["__eq__"] = __eq__ + body["__hash__"] = None + for name, fn in (("__lt__", lt), ("__le__", le), ("__gt__", gt), ("__ge__", ge)): + if fn is not None: + body[name] = _cmp_method(fn, require_same_type) + + def __init__(self, value): + self.value = value + body["__init__"] = __init__ + return type(class_name, (object,), body) + + +def _cmp_method(fn, require_same_type): + def method(self, other): + if require_same_type and other.__class__ is not self.__class__: + return NotImplemented + return fn(self.value, other.value) + return method + + +# -------------------------------------------------------------------------------------------------------------- +# Class building + + +def _public_name(name): + return name.lstrip("_") + + +def _attribute_from(name, ca, annotation, kw_only): + default = ca._default + metadata = ca.metadata + a = Attribute(name, default, ca._validator, ca.repr, None, ca.hash, ca.init, False, metadata, + ca.type if ca.type is not None else annotation, ca.converter, + ca.kw_only or kw_only, ca.eq, ca.eq_key, ca.order, ca.order_key, ca.on_setattr, + ca.alias if ca.alias is not None else _public_name(name)) + return a + + +def _collect_own(cls, these, auto_attribs, kw_only): + own = cls.__dict__ + annotations = own.get("__annotations__", {}) + if these is not None: + pairs = [(k, v) for k, v in these.items()] + if not isinstance(these, dict): + pairs = sorted(pairs, key=lambda kv: kv[1].counter) + return [_attribute_from(k, v, annotations.get(k), kw_only) for k, v in pairs], False + if auto_attribs is not False: + names = [] + stray = False + for name, v in own.items(): + if isinstance(v, _CountingAttr) and name not in annotations: + stray = True + if stray and auto_attribs is None: + auto_attribs = False + elif stray: + raise UnannotatedAttributeError( + "The following `attr.ib`s lack a type annotation: " + + ", ".join(k for k, v in own.items() if isinstance(v, _CountingAttr) and k not in annotations) + + ".") + if auto_attribs is None and len(annotations) == 0: + auto_attribs = False + if auto_attribs is not False: + result = [] + for name, ann in annotations.items(): + if _is_classvar(ann) or (isinstance(ann, str) and "ClassVar" in ann): + continue + if name.startswith("__") and not name.endswith("__"): + pass + v = own.get(name, NOTHING) + if isinstance(v, _CountingAttr): + ca = v + else: + ca = attrib(default=v) + result.append(_attribute_from(name, ca, ann, kw_only)) + return result, True + cas = [(k, v) for k, v in own.items() if isinstance(v, _CountingAttr)] + cas = sorted(cas, key=lambda kv: kv[1].counter) + return [_attribute_from(k, v, annotations.get(k), kw_only) for k, v in cas], True + + +def _collect_base(cls, taken, kw_only): + base_attrs = [] + for base in reversed(cls.__mro__[1:-1]): + for a in base.__dict__.get("__attrs_attrs__", ()): + if a.inherited or a.name in taken: + continue + base_attrs.append(a.evolve(inherited=True)) + seen = set() + filtered = [] + for a in reversed(base_attrs): + if a.name in seen: + continue + filtered.insert(0, a) + seen.add(a.name) + if kw_only: + filtered = [a.evolve(kw_only=True) for a in filtered] + return filtered + + +def _and_join(items): + if len(items) == 1: + return items[0] + if len(items) == 2: + return items[0] + " and " + items[1] + return ", ".join(items[:-1]) + ", and " + items[-1] + + +def _initial(attrib, inst): + d = attrib.default + if isinstance(d, Factory): + if d.takes_self: + return d.factory(inst) + return d.factory() + return d + + +def _make_init(cls, attrs, frozen_class, pre_init, post_init): + init_attrs = [a for a in attrs if a.init] + std = [a for a in init_attrs if not a.kw_only] + kwo = [a for a in init_attrs if a.kw_only] + qualname = cls.__qualname__ + ".__init__" + nstd = len(std) + nreq = len([a for a in std if a.default is NOTHING]) + names = [a.alias for a in std + kwo] + + def __init__(self, *args, **kwargs): + if len(args) > nstd: + if nreq == nstd: + takes = "%d positional argument%s" % (nstd + 1, "" if nstd == 0 else "s") + else: + takes = "from %d to %d positional arguments" % (nreq + 1, nstd + 1) + raise TypeError("%s() takes %s but %d were given" % (qualname, takes, len(args) + 1)) + values = {} + for i in range(len(args)): + values[std[i].alias] = args[i] + for k, v in kwargs.items(): + if k not in names: + raise TypeError(qualname + "() got an unexpected keyword argument " + repr(k)) + if k in values: + raise TypeError(qualname + "() got multiple values for argument " + repr(k)) + values[k] = v + missing = [a.alias for a in std if a.alias not in values and a.default is NOTHING] + if missing: + raise TypeError("%s() missing %d required positional argument%s: %s" + % (qualname, len(missing), "" if len(missing) == 1 else "s", + _and_join(["'" + m + "'" for m in missing]))) + missing = [a.alias for a in kwo if a.alias not in values and a.default is NOTHING] + if missing: + raise TypeError("%s() missing %d required keyword-only argument%s: %s" + % (qualname, len(missing), "" if len(missing) == 1 else "s", + _and_join(["'" + m + "'" for m in missing]))) + if pre_init: + self.__attrs_pre_init__() + assigned = [] + for a in attrs: + if a.init and a.alias in values: + v = values[a.alias] + elif a.default is not NOTHING: + v = _initial(a, self) + else: + continue + if a.converter is not None: + v = _convert_one(a.converter, v, self, a) + object.__setattr__(self, a.name, v) + assigned.append(a) + if not validators.get_disabled(): + for a in assigned: + if a.validator is not None: + a.validator(self, a, getattr(self, a.name)) + if post_init: + self.__attrs_post_init__() + __init__.__qualname__ = qualname + return __init__ + + +_active = set() + + +def _make_repr(attrs): + shown = [a for a in attrs if a.repr is not False] + + def __repr__(self): + key = id(self) + if key in _active: + return "..." + _active.add(key) + try: + qn = self.__class__.__qualname__ + qn = qn.rsplit(">.", 1)[-1] + parts = [] + for a in shown: + v = getattr(self, a.name) + parts.append(a.name + "=" + (repr(v) if a.repr is True else a.repr(v))) + return qn + "(" + ", ".join(parts) + ")" + finally: + _active.discard(key) + return __repr__ + + +def _tuple_getter(attrs, flag): + chosen = [] + for a in attrs: + if flag == "eq": + if a.eq: + chosen.append((a.name, a.eq_key)) + elif a.order: + chosen.append((a.name, a.order_key)) + + def as_tuple(self): + out = [] + for name, key in chosen: + v = getattr(self, name) + out.append(v if key is None else key(v)) + return tuple(out) + return as_tuple + + +def _make_eq(as_tuple): + def __eq__(self, other): + if other.__class__ is not self.__class__: + return NotImplemented + return as_tuple(self) == as_tuple(other) + + def __ne__(self, other): + result = __eq__(self, other) + if result is NotImplemented: + return NotImplemented + return not result + return __eq__, __ne__ + + +def _make_order(as_tuple, op): + def compare(self, other): + if other.__class__ is not self.__class__: + return NotImplemented + a = as_tuple(self) + b = as_tuple(other) + if op == "__lt__": + return a < b + if op == "__le__": + return a <= b + if op == "__gt__": + return a > b + return a >= b + compare.__name__ = op + return compare + + +def _make_hash(attrs, cls): + hashed = [] + for a in attrs: + if (a.eq if a.hash is None else a.hash): + hashed.append(a.name) + seed = cls.__qualname__ + + def __hash__(self): + return hash((seed,) + tuple(getattr(self, n) for n in hashed)) + return __hash__ + + +def _hook_for(a, class_hook): + hook = a.on_setattr if a.on_setattr is not None else class_hook + if hook is None or hook is setters.NO_OP: + return None + return hook + + +def _make_setattr(attrs, frozen_class, class_hook, parent_setattr): + by_name = dict((a.name, a) for a in attrs) + hooks = {} + if not frozen_class: + for a in attrs: + h = _hook_for(a, class_hook) + if h is not None: + hooks[a.name] = h + + def __setattr__(self, name, value): + if frozen_class: + raise FrozenInstanceError() + h = hooks.get(name) + if h is not None: + value = h(self, by_name[name], value) + object.__setattr__(self, name, value) + + def __delattr__(self, name): + if frozen_class: + raise FrozenInstanceError() + object.__delattr__(self, name) + return __setattr__, __delattr__, len(hooks) > 0 + + +def _has_own(cls, name): + return name in cls.__dict__ + + +def _wrap(cls, these, repr_, eq, order, hash_, init, frozen_, kw_only, auto_attribs, auto_detect, on_setattr, + define_mode, match_args, field_transformer, unsafe_hash): + if unsafe_hash is not None: + hash_ = unsafe_hash + if auto_detect: + if _has_own(cls, "__repr__"): + repr_ = False + if _has_own(cls, "__init__"): + init = False + has_eq = _has_own(cls, "__eq__") or _has_own(cls, "__ne__") + if has_eq and eq is None: + eq = False + has_order = (_has_own(cls, "__lt__") or _has_own(cls, "__le__") or _has_own(cls, "__gt__") or + _has_own(cls, "__ge__")) + if has_order and order is None: + order = False + if hash_ is None and _has_own(cls, "__hash__"): + hash_ = False + if eq is None: + eq = True + if order is None: + order = eq if not define_mode else False + if order and not eq: + raise ValueError("`order` can only be True if `eq` is True too.") + if isinstance(on_setattr, (list, tuple)): + on_setattr = setters.pipe(*on_setattr) + if frozen_ and on_setattr is not None: + raise ValueError("Frozen classes can't use on_setattr.") + own, _ = _collect_own(cls, these, auto_attribs, kw_only) + taken = set(a.name for a in own) + base = _collect_base(cls, taken, kw_only) + attrs = base + own + if field_transformer is not None: + attrs = list(field_transformer(cls, attrs)) + had_default = False + for a in attrs: + if a.init is not False and a.kw_only is False: + if had_default and a.default is NOTHING: + raise ValueError("No mandatory attributes allowed after an attribute with a default value or " + "factory. Attribute in question: " + repr(a)) + if a.default is not NOTHING: + had_default = True + # Class-level cleanup: the placeholders must not remain as class attributes. + base_names = set(a.name for a in base) + for a in own: + if a.name not in base_names or True: + v = cls.__dict__.get(a.name, NOTHING) + if v is not NOTHING and (isinstance(v, _CountingAttr) or these is None): + delattr(cls, a.name) + cls.__attrs_attrs__ = _AttrsTuple(attrs) + if define_mode and on_setattr is None and not frozen_: + on_setattr = setters.pipe(setters.convert, setters.validate) + if init: + cls.__init__ = _make_init(cls, attrs, frozen_, hasattr(cls, "__attrs_pre_init__"), + hasattr(cls, "__attrs_post_init__")) + if repr_: + cls.__repr__ = _make_repr(attrs) + if eq: + as_tuple = _tuple_getter(attrs, "eq") + cls.__eq__, cls.__ne__ = _make_eq(as_tuple) + if order: + ot = _tuple_getter(attrs, "order") + for op in ("__lt__", "__le__", "__gt__", "__ge__"): + setattr(cls, op, _make_order(ot, op)) + if frozen_: + cls.__setattr__, cls.__delattr__, _ = _make_setattr(attrs, True, None, None) + elif on_setattr is not None or any(a.on_setattr is not None for a in attrs): + any_hooks = False + s, d, any_hooks = _make_setattr(attrs, False, on_setattr, None) + if any_hooks and (define_mode is False or any(a.validator is not None or a.converter is not None or + a.on_setattr is not None for a in attrs)): + cls.__setattr__ = s + if hash_ is True: + cls.__hash__ = _make_hash(attrs, cls) + elif hash_ is None: + if eq and frozen_: + cls.__hash__ = _make_hash(attrs, cls) + elif eq: + cls.__hash__ = None + elif hash_ is False: + pass + if match_args and "__match_args__" not in cls.__dict__: + cls.__match_args__ = tuple(a.alias for a in attrs if a.init and not a.kw_only) + return cls + + +def attrs(maybe_cls=None, these=None, repr_ns=None, repr=None, cmp=None, hash=None, init=None, slots=False, + frozen=False, weakref_slot=True, str=False, auto_attribs=False, kw_only=False, cache_hash=False, + auto_exc=False, eq=None, order=None, auto_detect=False, collect_by_mro=False, getstate_setstate=None, + on_setattr=None, field_transformer=None, match_args=True, unsafe_hash=None): + eq, order = _eq_order_class(cmp, eq, order) + + def wrap(cls): + return _wrap(cls, these, True if repr is None else repr, eq, order, hash, True if init is None else init, + frozen, kw_only, auto_attribs, auto_detect, on_setattr, False, match_args, field_transformer, + unsafe_hash) + if maybe_cls is None: + return wrap + return wrap(maybe_cls) + + +def _eq_order_class(cmp, eq, order): + if cmp is not None and (eq is not None or order is not None): + raise ValueError("Don't mix `cmp` with `eq' and `order`.") + if cmp is not None: + return cmp, cmp + return eq, order + + +s = attrs +attributes = attrs + + +def dataclass(maybe_cls=None, *, these=None, repr=True, unsafe_hash=None, hash=None, init=True, slots=False, + frozen=False, weakref_slot=True, str=False, auto_attribs=True, kw_only=False, cache_hash=False, + auto_exc=True, eq=True, order=False, auto_detect=True, getstate_setstate=None, on_setattr=None, + field_transformer=None, match_args=True): + return attrs(maybe_cls, these=these, repr=repr, hash=hash, init=init, slots=slots, frozen=frozen, + auto_attribs=auto_attribs, kw_only=kw_only, eq=eq, order=order, auto_detect=auto_detect, + on_setattr=on_setattr, field_transformer=field_transformer, match_args=match_args, + unsafe_hash=unsafe_hash) + + +def define(maybe_cls=None, *, these=None, repr=None, unsafe_hash=None, hash=None, init=None, slots=True, + frozen=False, weakref_slot=True, str=False, auto_attribs=None, kw_only=False, cache_hash=False, + auto_exc=True, eq=None, order=False, auto_detect=True, getstate_setstate=None, on_setattr=None, + field_transformer=None, match_args=True): + def wrap(cls): + return _wrap(cls, these, True if repr is None else repr, eq, order, hash, True if init is None else init, + frozen, kw_only, auto_attribs, auto_detect, on_setattr, True, match_args, field_transformer, + unsafe_hash) + if maybe_cls is None: + return wrap + return wrap(maybe_cls) + + +mutable = define + + +def frozen(maybe_cls=None, **kwargs): + kwargs["frozen"] = True + return define(maybe_cls, **kwargs) + + +def make_class(name, attrs, bases=(object,), class_body=None, **attributes_arguments): + if isinstance(attrs, dict): + cls_dict = {} + for k, v in attrs.items(): + cls_dict[k] = v + cd = dict((k, _recount(v)) for k, v in cls_dict.items()) + elif isinstance(attrs, (list, tuple)): + cd = {} + for k in attrs: + cd[k] = attrib() + else: + raise TypeError("attrs argument must be a dict or a list.") + if class_body is not None: + cd.update(class_body) + cls = type(name, tuple(bases), cd) + return attrs_decorator(cls, attributes_arguments) + + +def _recount(v): + if isinstance(v, _CountingAttr): + v.counter = _next_count() + return v + + +def attrs_decorator(cls, kwargs): + return globals()["attrs"](cls, **kwargs) + + +from attr import converters, filters, setters, validators # noqa: E402 (they import names defined above) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/attr/converters.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/attr/converters.py new file mode 100644 index 00000000000..48e0ac34e35 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/attr/converters.py @@ -0,0 +1,63 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Converters for attribute values.""" + +from attr import NOTHING, Factory + +__all__ = ["default_if_none", "optional", "pipe", "to_bool"] + + +def optional(converter): + def optional_converter(val): + if val is None: + return None + return converter(val) + return optional_converter + + +def pipe(*converters): + def pipe_converter(val): + for c in converters: + val = c(val) + return val + return pipe_converter + + +def default_if_none(default=NOTHING, factory=None): + if default is NOTHING and factory is None: + raise TypeError("Must pass either 'default' or 'factory'.") + if default is not NOTHING and factory is not None: + raise TypeError("Must pass either 'default' or 'factory' but not both.") + if factory is not None: + default = Factory(factory) + if isinstance(default, Factory): + if default.takes_self: + raise ValueError("'takes_self' is not supported for converters.") + + def default_if_none_converter(val): + if val is not None: + return val + return default.factory() + else: + def default_if_none_converter(val): + if val is not None: + return val + return default + return default_if_none_converter + + +def to_bool(val): + if isinstance(val, str): + v = val.lower() + if v in ("true", "t", "yes", "y", "on", "1"): + return True + if v in ("false", "f", "no", "n", "off", "0"): + return False + elif val is True or val is False: + return val + elif val == 1: + return True + elif val == 0: + return False + raise ValueError("Cannot convert value to bool: " + repr(val)) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/attr/exceptions.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/attr/exceptions.py new file mode 100644 index 00000000000..336e7f9db79 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/attr/exceptions.py @@ -0,0 +1,52 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Exceptions raised by attrs.""" + + +class FrozenError(AttributeError): + """A frozen instance or attribute was modified.""" + msg = "can't set attribute" + + def __init__(self, *args): + super(FrozenError, self).__init__(*(args if args else (self.msg,))) + + +class FrozenInstanceError(FrozenError): + """An instance of a frozen class was modified.""" + + +class FrozenAttributeError(FrozenError): + """A frozen attribute was modified.""" + + +class AttrsAttributeNotFoundError(ValueError): + """An attrs function was asked about an attribute the class lacks.""" + + +class NotAnAttrsClassError(ValueError): + """A non-attrs class was passed where an attrs class is required.""" + + +class DefaultAlreadySetError(RuntimeError): + """A default was set twice on one attribute.""" + + +class UnannotatedAttributeError(RuntimeError): + """An attribute of an auto_attribs class has no annotation.""" + + +class PythonTooOldError(RuntimeError): + """Kept for API compatibility.""" + + +class NotCallableError(TypeError): + """A value that must be callable is not.""" + + def __init__(self, msg, value): + super(NotCallableError, self).__init__(msg, value) + self.msg = msg + self.value = value + + def __str__(self): + return str(self.msg) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/attr/filters.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/attr/filters.py new file mode 100644 index 00000000000..a4c9d7e68e2 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/attr/filters.py @@ -0,0 +1,34 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Filters for asdict and astuple.""" + + +def _split(what): + types = [] + names = [] + attrs = [] + for w in what: + if isinstance(w, type): + types.append(w) + elif isinstance(w, str): + names.append(w) + else: + attrs.append(w) + return tuple(types), names, attrs + + +def include(*what): + types, names, attrs = _split(what) + + def include_(attribute, value): + return value.__class__ in types or attribute in attrs or attribute.name in names + return include_ + + +def exclude(*what): + types, names, attrs = _split(what) + + def exclude_(attribute, value): + return not (value.__class__ in types or attribute in attrs or attribute.name in names) + return exclude_ diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/attr/setters.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/attr/setters.py new file mode 100644 index 00000000000..3c8e2694fe1 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/attr/setters.py @@ -0,0 +1,33 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Hooks run when an attribute of an instance is assigned.""" + +from attr.exceptions import FrozenAttributeError + + +def pipe(*setters): + def wrapped_pipe(instance, attrib, new_value): + rv = new_value + for setter in setters: + rv = setter(instance, attrib, rv) + return rv + return wrapped_pipe + + +def frozen(_, __, ___): + raise FrozenAttributeError() + + +def validate(instance, attrib, new_value): + import attr + return attr._run_validator(instance, attrib, new_value) + + +def convert(instance, attrib, new_value): + import attr + return attr._convert_value(instance, attrib, new_value) + + +def NO_OP(_, __, new_value): + return new_value diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/attr/validators.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/attr/validators.py new file mode 100644 index 00000000000..2defa6a718f --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/attr/validators.py @@ -0,0 +1,260 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Validators run on attribute values after assignment.""" + +import re as _re + +from attr.exceptions import NotCallableError + +__all__ = ["and_", "deep_iterable", "deep_mapping", "disabled", "ge", "get_disabled", "gt", "in_", "instance_of", + "is_callable", "le", "lt", "matches_re", "max_len", "min_len", "not_", "optional", "set_disabled"] + +_config = {"disabled": False} + + +def set_disabled(disabled): + _config["disabled"] = bool(disabled) + + +def get_disabled(): + return _config["disabled"] + + +class disabled: + """Context manager switching validators off inside the block.""" + + def __enter__(self): + self._prev = get_disabled() + set_disabled(True) + return self + + def __exit__(self, *exc): + set_disabled(self._prev) + return False + + +class _InstanceOfValidator: + def __init__(self, type): + self.type = type + + def __call__(self, inst, attr, value): + if not isinstance(value, self.type): + raise TypeError("'%s' must be %r (got %r that is a %r)." % (attr.name, self.type, value, value.__class__), + attr, self.type, value) + + def __repr__(self): + return "" % (self.type,) + + +def instance_of(type): + return _InstanceOfValidator(type) + + +class _OptionalValidator: + def __init__(self, validator): + self.validator = validator + + def __call__(self, inst, attr, value): + if value is None: + return + self.validator(inst, attr, value) + + def __repr__(self): + return "" % (self.validator,) + + +def optional(validator): + if isinstance(validator, (list, tuple)): + return _OptionalValidator(_AndValidator(validator)) + return _OptionalValidator(validator) + + +class _InValidator: + def __init__(self, options): + self.options = options + + def __call__(self, inst, attr, value): + try: + ok = value in self.options + except TypeError: + ok = False + if not ok: + raise ValueError("'%s' must be in %r (got %r)" % (attr.name, self.options, value), attr, self.options, + value) + + def __repr__(self): + return "" % (self.options,) + + +def in_(options): + return _InValidator(options) + + +class _AndValidator: + def __init__(self, validators): + self._validators = tuple(validators) + + def __call__(self, inst, attr, value): + for v in self._validators: + v(inst, attr, value) + + +def and_(*validators): + vals = [] + for v in validators: + if isinstance(v, _AndValidator): + vals.extend(v._validators) + else: + vals.append(v) + return _AndValidator(tuple(vals)) + + +class _IsCallableValidator: + def __call__(self, inst, attr, value): + if not callable(value): + raise NotCallableError("'%s' must be callable (got %r that is a %r)." % (attr.name, value, + value.__class__), value) + + def __repr__(self): + return "" + + +def is_callable(): + return _IsCallableValidator() + + +class _DeepIterable: + def __init__(self, member_validator, iterable_validator): + self.member_validator = member_validator + self.iterable_validator = iterable_validator + + def __call__(self, inst, attr, value): + if self.iterable_validator is not None: + self.iterable_validator(inst, attr, value) + for member in value: + self.member_validator(inst, attr, member) + + +def deep_iterable(member_validator, iterable_validator=None): + if isinstance(member_validator, (list, tuple)): + member_validator = and_(*member_validator) + return _DeepIterable(member_validator, iterable_validator) + + +class _DeepMapping: + def __init__(self, key_validator, value_validator, mapping_validator): + self.key_validator = key_validator + self.value_validator = value_validator + self.mapping_validator = mapping_validator + + def __call__(self, inst, attr, value): + if self.mapping_validator is not None: + self.mapping_validator(inst, attr, value) + for key in value: + if self.key_validator is not None: + self.key_validator(inst, attr, key) + if self.value_validator is not None: + self.value_validator(inst, attr, value[key]) + + +def deep_mapping(key_validator=None, value_validator=None, mapping_validator=None): + return _DeepMapping(key_validator, value_validator, mapping_validator) + + +class _MatchesRe: + def __init__(self, pattern, func): + self.pattern = pattern + self.func = func + + def __call__(self, inst, attr, value): + if not self.func(value): + raise ValueError("'%s' must match regex %r (%r doesn't)" % (attr.name, self.pattern.pattern, + value), attr, self.pattern, value) + + +def matches_re(regex, flags=0, func=None): + if isinstance(regex, str): + regex = _re.compile(regex, flags) + if func is None: + func = regex.match + return _MatchesRe(regex, func) + + +class _Bound: + def __init__(self, bound, op, text): + self.bound = bound + self.op = op + self.text = text + + def __call__(self, inst, attr, value): + if not self.op(value, self.bound): + raise ValueError("'%s' must be %s %s: %s" % (attr.name, self.text, self.bound, value)) + + def __repr__(self): + return "" % (self.text, self.bound) + + +def lt(val): + return _Bound(val, lambda a, b: a < b, "<") + + +def le(val): + return _Bound(val, lambda a, b: a <= b, "<=") + + +def ge(val): + return _Bound(val, lambda a, b: a >= b, ">=") + + +def gt(val): + return _Bound(val, lambda a, b: a > b, ">") + + +class _MaxLen: + def __init__(self, n): + self.n = n + + def __call__(self, inst, attr, value): + if len(value) > self.n: + raise ValueError("Length of '%s' must be <= %d: %d" % (attr.name, self.n, len(value))) + + +class _MinLen: + def __init__(self, n): + self.n = n + + def __call__(self, inst, attr, value): + if len(value) < self.n: + raise ValueError("Length of '%s' must be >= %d: %d" % (attr.name, self.n, len(value))) + + +def max_len(length): + return _MaxLen(length) + + +def min_len(length): + return _MinLen(length) + + +class _Not: + def __init__(self, validator, msg, exc_types): + self.validator = validator + self.msg = msg + self.exc_types = exc_types + + def __call__(self, inst, attr, value): + try: + self.validator(inst, attr, value) + except self.exc_types: + return + msg = self.msg + if msg is None: + msg = "not_ validator child '%r' did not raise a captured error" % (self.validator,) + raise ValueError(msg, attr, self.validator, value, self.exc_types) + + +def not_(validator, *, msg=None, exc_types=(ValueError, TypeError)): + if isinstance(exc_types, type): + exc_types = (exc_types,) + return _Not(validator, msg, tuple(exc_types)) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/attrs/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/attrs/__init__.py new file mode 100644 index 00000000000..0b251ed9ce8 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/attrs/__init__.py @@ -0,0 +1,26 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""The attrs namespace: the modern names for the attr API.""" + +from attr import (NOTHING, Attribute, Converter, Factory, cmp_using, define, evolve, field, fields, + fields_dict, frozen, has, make_class, mutable, resolve_types, validate) +import attr as _attr +from attr import converters, exceptions, filters, setters, validators +from attr import attrib as _attrib + + +def asdict(inst, *, recurse=True, filter=None, value_serializer=None): + """Like attr.asdict, but collection types are always retained.""" + return _attr.asdict(inst, recurse=recurse, filter=filter, retain_collection_types=True, + value_serializer=value_serializer) + + +def astuple(inst, *, recurse=True, filter=None): + """Like attr.astuple, but collection types are always retained.""" + return _attr.astuple(inst, recurse=recurse, filter=filter, retain_collection_types=True) + + +__all__ = ["NOTHING", "Attribute", "Converter", "Factory", "asdict", "astuple", "cmp_using", "converters", "define", + "evolve", "exceptions", "field", "fields", "fields_dict", "filters", "frozen", "has", "make_class", + "mutable", "resolve_types", "setters", "validate", "validators"] diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/attrs/converters.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/attrs/converters.py new file mode 100644 index 00000000000..92f982b65f5 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/attrs/converters.py @@ -0,0 +1,6 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""The attrs converters under the attrs package name.""" + +from attr.converters import default_if_none, optional, pipe, to_bool diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/attrs/exceptions.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/attrs/exceptions.py new file mode 100644 index 00000000000..97b2cefc2ec --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/attrs/exceptions.py @@ -0,0 +1,12 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""The attrs exceptions under the attrs package name.""" + +from attr.exceptions import (AttrsAttributeNotFoundError, DefaultAlreadySetError, FrozenAttributeError, + FrozenError, FrozenInstanceError, NotAnAttrsClassError, NotCallableError, + PythonTooOldError, UnannotatedAttributeError) + +__all__ = ["AttrsAttributeNotFoundError", "DefaultAlreadySetError", "FrozenAttributeError", "FrozenError", + "FrozenInstanceError", "NotAnAttrsClassError", "NotCallableError", "PythonTooOldError", + "UnannotatedAttributeError"] diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/attrs/filters.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/attrs/filters.py new file mode 100644 index 00000000000..f3da658d1eb --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/attrs/filters.py @@ -0,0 +1,6 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""The attrs filters under the attrs package name.""" + +from attr.filters import exclude, include diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/attrs/setters.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/attrs/setters.py new file mode 100644 index 00000000000..c9b984273e5 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/attrs/setters.py @@ -0,0 +1,6 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""The attrs setters under the attrs package name.""" + +from attr.setters import NO_OP, convert, frozen, pipe, validate diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/attrs/validators.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/attrs/validators.py new file mode 100644 index 00000000000..fc63543bb9f --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/attrs/validators.py @@ -0,0 +1,7 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""The attrs validators under the attrs package name.""" + +from attr.validators import (and_, deep_iterable, deep_mapping, disabled, ge, get_disabled, gt, in_, instance_of, + is_callable, le, lt, matches_re, max_len, min_len, not_, optional, set_disabled) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/dateutil/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/dateutil/__init__.py new file mode 100644 index 00000000000..b3091055bff --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/dateutil/__init__.py @@ -0,0 +1,10 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Extensions to the standard datetime module: relativedelta, parser, rrule and tz. + +Written from the documented behaviour of python-dateutil. Import the +submodules explicitly, as in ``from dateutil import parser``. +""" + +__version__ = "3.0.0" diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/dateutil/_common.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/dateutil/_common.py new file mode 100644 index 00000000000..7eb06694816 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/dateutil/_common.py @@ -0,0 +1,56 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Helpers shared by the dateutil modules: weekday constants and calendar arithmetic.""" + +_NAMES = ("MO", "TU", "WE", "TH", "FR", "SA", "SU") + + +class weekday: + """A day of the week, optionally with an occurrence count: MO(+1), FR(-1).""" + + def __init__(self, weekday, n=None): + self.weekday = weekday + self.n = n + + def __call__(self, n): + if n == self.n: + return self + return self.__class__(self.weekday, n) + + def __eq__(self, other): + if not isinstance(other, weekday): + return False + return self.weekday == other.weekday and self.n == other.n + + def __ne__(self, other): + return not self.__eq__(other) + + def __hash__(self): + return hash((self.weekday, self.n)) + + def __repr__(self): + s = _NAMES[self.weekday] + if not self.n: + return s + return "%s(%+d)" % (s, self.n) + + +MO, TU, WE, TH, FR, SA, SU = [weekday(i) for i in range(7)] + +_MONTH_LENGTHS = (31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31) + + +def isleap(year): + return year % 4 == 0 and (year % 100 != 0 or year % 400 == 0) + + +def monthrange(year, month): + """Return the weekday of the first day and the number of days of a month.""" + import datetime + if month < 1 or month > 12: + raise ValueError("bad month number " + str(month) + "; must be 1-12") + days = _MONTH_LENGTHS[month - 1] + if month == 2 and isleap(year): + days = 29 + return datetime.date(year, month, 1).weekday(), days diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/dateutil/parser.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/dateutil/parser.py new file mode 100644 index 00000000000..40518be30ec --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/dateutil/parser.py @@ -0,0 +1,587 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Parsing of date and time strings: a forgiving parse() and a strict isoparse().""" + +import datetime +import re as _re + +from dateutil import tz as _tz +from dateutil._common import monthrange +from dateutil.relativedelta import relativedelta + +__all__ = ["parse", "parser", "parserinfo", "isoparse", "isoparser", "ParserError", "UnknownTimezoneWarning"] + + +class ParserError(ValueError): + """A string could not be turned into a date.""" + + def __str__(self): + try: + return self.args[0] % self.args[1:] + except (TypeError, IndexError): + return super(ParserError, self).__str__() + + def __repr__(self): + return "%s(%s)" % (self.__class__.__name__, ", ".join(repr(a) for a in self.args)) + + +class UnknownTimezoneWarning(RuntimeWarning): + """A time zone name could not be resolved.""" + + +class parserinfo: + """Accepted for API compatibility; the default English tables are always used.""" + + def __init__(self, dayfirst=False, yearfirst=False): + self.dayfirst = dayfirst + self.yearfirst = yearfirst + + +_MONTHS = (("jan", "january"), ("feb", "february"), ("mar", "march"), ("apr", "april"), ("may", "may"), + ("jun", "june"), ("jul", "july"), ("aug", "august"), ("sep", "september"), ("oct", "october"), + ("nov", "november"), ("dec", "december")) +_WEEKDAYS = (("mon", "monday"), ("tue", "tuesday"), ("wed", "wednesday"), ("thu", "thursday"), ("fri", "friday"), + ("sat", "saturday"), ("sun", "sunday")) +_MONTH_INDEX = {} +for _i in range(12): + _MONTH_INDEX[_MONTHS[_i][0]] = _i + 1 + _MONTH_INDEX[_MONTHS[_i][1]] = _i + 1 +_MONTH_INDEX["sept"] = 9 +_WEEKDAY_INDEX = {} +for _i in range(7): + _WEEKDAY_INDEX[_WEEKDAYS[_i][0]] = _i + _WEEKDAY_INDEX[_WEEKDAYS[_i][1]] = _i +_AM = ("am", "a") +_PM = ("pm", "p") +_UTC = ("utc", "gmt", "z") +_JUMP = (" ", ".", ",", ";", "-", "/", "'", "at", "on", "and", "ad", "m", "t", "of", "st", "nd", "rd", "th") +_TOKEN = _re.compile(r"\d+|[A-Za-z]+|\s+|.") + + +class _Result: + def __init__(self): + self.hour = None + self.minute = None + self.second = None + self.microsecond = None + self.ampm = None + self.weekday = None + self.tzname = None + self.tzoffset = None + self.any = False + + +def _tokens(s): + out = [] + for t in _TOKEN.findall(s): + if t.isspace(): + out.append(" ") + else: + out.append(t) + return out + + +def _is_num(t): + return t.isdigit() + + +def _is_alpha(t): + return t.isalpha() + + +def _two_digit_year(y): + if y >= 100: + return y + now = datetime.datetime.now().year + century = now // 100 * 100 + y += century + if y >= now + 50: + y -= 100 + elif y < now - 50: + y += 100 + return y + + +def _resolve(ymd, dayfirst, yearfirst): + """Decide which of up to three numbers or month names is the year, month and day.""" + y = None + m = None + d = None + n = len(ymd) + if n == 0: + return y, m, d + mi = -1 + for i in range(n): + if ymd[i][2]: + mi = i + if mi >= 0: + m = ymd[mi][0] + rest = [ymd[i] for i in range(n) if i != mi] + if len(rest) == 0: + pass + elif len(rest) == 1: + v = rest[0] + if v[1] > 2 or v[0] > 31: + y = v[0] + else: + d = v[0] + elif len(rest) == 2: + a = rest[0] + b = rest[1] + if mi == 0: + if yearfirst and not (a[1] <= 2 and a[0] <= 31 and b[1] > 2): + y = a[0] + d = b[0] + elif a[1] > 2 or a[0] > 31: + y = a[0] + d = b[0] + else: + d = a[0] + y = b[0] + else: + if a[1] > 2 or a[0] > 31: + y = a[0] + d = b[0] + else: + d = a[0] + y = b[0] + else: + raise ValueError("too many numbers") + return y, m, d + if n == 1: + v = ymd[0] + if v[1] > 2 or v[0] > 31: + y = v[0] + else: + d = v[0] + return y, m, d + if n == 2: + a = ymd[0] + b = ymd[1] + if a[1] > 2 or a[0] > 31: + y = a[0] + m = b[0] + elif b[1] > 2 or b[0] > 31: + m = a[0] + y = b[0] + elif a[0] > 12: + d = a[0] + m = b[0] + elif b[0] > 12: + m = a[0] + d = b[0] + elif dayfirst: + d = a[0] + m = b[0] + else: + m = a[0] + d = b[0] + return y, m, d + if n > 3: + raise ValueError("too many numbers") + a = ymd[0] + b = ymd[1] + c = ymd[2] + if a[1] > 2 or a[0] > 31: + y = a[0] + if dayfirst and not yearfirst and b[0] <= 31 and c[0] <= 12: + d = b[0] + m = c[0] + else: + m = b[0] + d = c[0] + elif yearfirst and a[0] <= 99 and b[0] <= 12: + y = a[0] + if dayfirst: + d = b[0] + m = c[0] + else: + m = b[0] + d = c[0] + elif dayfirst or (a[0] > 12 and b[0] <= 12): + d = a[0] + m = b[0] + y = c[0] + else: + m = a[0] + d = b[0] + y = c[0] + if b[0] > 12 and a[0] <= 12: + pass + return y, m, d + + +class parser: + """A parser object; parse() the module function uses a shared instance.""" + + def __init__(self, info=None): + self.info = info if info is not None else parserinfo() + + def parse(self, timestr, default=None, ignoretz=False, tzinfos=None, **kwargs): + dayfirst = kwargs.get("dayfirst", None) + yearfirst = kwargs.get("yearfirst", None) + fuzzy = kwargs.get("fuzzy", False) + if dayfirst is None: + dayfirst = self.info.dayfirst + if yearfirst is None: + yearfirst = self.info.yearfirst + if not isinstance(timestr, str): + raise TypeError("Parser must be called with a string") + if default is None: + default = datetime.datetime.now().replace(hour=0, minute=0, second=0, microsecond=0) + res, ymd = self._parse(timestr, dayfirst, yearfirst, fuzzy) + if res is None: + raise ParserError("Unknown string format: %s", timestr) + if not res.any and not ymd: + raise ParserError("String does not contain a date: %s", timestr) + try: + ret = self._build(res, ymd, default, dayfirst, yearfirst) + except ValueError as e: + raise ParserError(str(e) + ": %s", timestr) + except OverflowError as e: + raise ParserError(str(e) + ": %s", timestr) + if not ignoretz: + ret = self._attach_tz(ret, res, tzinfos) + return ret + + def _parse(self, timestr, dayfirst, yearfirst, fuzzy): + toks = _tokens(timestr) + n = len(toks) + res = _Result() + ymd = [] + i = 0 + while i < n: + t = toks[i] + if _is_num(t): + i = self._number(toks, i, res, ymd) + if i < 0: + return None, None + continue + if _is_alpha(t): + low = t.lower() + if low in _MONTH_INDEX: + ymd.append([_MONTH_INDEX[low], 0, True]) + res.any = True + elif low in _WEEKDAY_INDEX: + res.weekday = _WEEKDAY_INDEX[low] + res.any = True + elif low in _AM or low in _PM: + if res.hour is None: + if fuzzy: + i += 1 + continue + return None, None + if low in _PM: + if res.hour < 12: + res.hour += 12 + elif res.hour == 12: + res.hour = 0 + res.ampm = low + elif low in _UTC: + res.tzname = t.upper() if len(t) > 1 else "UTC" + if low == "z": + res.tzname = "Z" + res.tzoffset = 0 + res.any = True + elif low in _JUMP: + pass + elif res.hour is not None and res.tzname is None and t.isupper() and 3 <= len(t) <= 5: + res.tzname = t + res.any = True + elif fuzzy: + pass + else: + return None, None + i += 1 + continue + if (t == "+" or t == "-") and res.hour is not None and res.tzoffset is None and i + 1 < n and \ + _is_num(toks[i + 1]): + j = self._offset(toks, i, res) + if j < 0: + return None, None + i = j + continue + if t in _JUMP or fuzzy: + i += 1 + continue + return None, None + return res, ymd + + def _offset(self, toks, i, res): + n = len(toks) + sign = -1 if toks[i] == "-" else 1 + digits = toks[i + 1] + j = i + 2 + if len(digits) == 4: + hh = int(digits[:2]) + mm = int(digits[2:]) + elif len(digits) <= 2: + hh = int(digits) + mm = 0 + if j + 1 < n and toks[j] == ":" and _is_num(toks[j + 1]) and len(toks[j + 1]) == 2: + mm = int(toks[j + 1]) + j += 2 + else: + return -1 + res.tzoffset = sign * (hh * 3600 + mm * 60) + res.any = True + return j + + def _number(self, toks, i, res, ymd): + n = len(toks) + t = toks[i] + L = len(t) + res.any = True + if i + 1 < n and toks[i + 1] == ":" and res.hour is None and L <= 2: + return self._time(toks, i, res) + # hour followed by am/pm + j = i + 1 + if j < n and toks[j] == " ": + j += 1 + if j < n and L <= 2 and res.hour is None and _is_alpha(toks[j]) and toks[j].lower() in ("am", "pm", "a", "p"): + res.hour = int(t) + res.minute = 0 + return j + # date with separators + if i + 2 < n and toks[i + 1] in ("-", "/", ".") and _is_num(toks[i + 2]) and res.hour is None: + sep = toks[i + 1] + vals = [t] + j = i + 1 + while len(vals) < 3 and j + 1 < n and toks[j] == sep and _is_num(toks[j + 1]): + vals.append(toks[j + 1]) + j += 2 + for v in vals: + ymd.append([int(v), len(v), False]) + return j + if L == 8: + ymd.append([int(t[:4]), 4, False]) + ymd.append([int(t[4:6]), 2, False]) + ymd.append([int(t[6:]), 2, False]) + return i + 1 + if L in (12, 14): + ymd.append([int(t[:4]), 4, False]) + ymd.append([int(t[4:6]), 2, False]) + ymd.append([int(t[6:8]), 2, False]) + res.hour = int(t[8:10]) + res.minute = int(t[10:12]) + if L == 14: + res.second = int(t[12:]) + return i + 1 + if L == 6 and not ymd: + ymd.append([int(t[:2]), 2, False]) + ymd.append([int(t[2:4]), 2, False]) + ymd.append([int(t[4:]), 2, False]) + return i + 1 + if L == 6 or (L == 4 and ymd and res.hour is None and i > 0 and toks[i - 1] == "T"): + if res.hour is None: + res.hour = int(t[:2]) + res.minute = int(t[2:4]) + if L == 6: + res.second = int(t[4:]) + return i + 1 + return -1 + if L > 4 and L != 6: + return -1 + ymd.append([int(t), L, False]) + return i + 1 + + def _time(self, toks, i, res): + n = len(toks) + res.hour = int(toks[i]) + i += 2 + if i >= n or not _is_num(toks[i]): + return -1 + res.minute = int(toks[i]) + i += 1 + if i + 1 < n and toks[i] == ":" and _is_num(toks[i + 1]): + res.second = int(toks[i + 1]) + i += 2 + if i + 1 < n and (toks[i] == "." or toks[i] == ",") and _is_num(toks[i + 1]): + res.microsecond = int((toks[i + 1] + "000000")[:6]) + i += 2 + return i + + def _build(self, res, ymd, default, dayfirst, yearfirst): + y, m, d = _resolve(ymd, dayfirst, yearfirst) + if y is not None: + y = _two_digit_year(y) + year = y if y is not None else default.year + month = m if m is not None else default.month + if d is not None: + day = d + else: + day = min(default.day, monthrange(year, month)[1]) + repl = {"year": year, "month": month, "day": day} + if res.hour is not None: + repl["hour"] = res.hour + if res.minute is not None: + repl["minute"] = res.minute + if res.second is not None: + repl["second"] = res.second + if res.microsecond is not None: + repl["microsecond"] = res.microsecond + if res.hour is not None and res.minute is None: + repl["minute"] = 0 + ret = default.replace(**repl) + if res.weekday is not None and d is None: + ret = ret + relativedelta(weekday=res.weekday) + return ret + + def _attach_tz(self, ret, res, tzinfos): + if res.tzname is None and res.tzoffset is None: + return ret + tzname = res.tzname + offset = res.tzoffset + if tzinfos is not None and tzname is not None: + tzdata = None + if callable(tzinfos): + tzdata = tzinfos(tzname, offset) + elif tzname in tzinfos: + tzdata = tzinfos[tzname] + if tzdata is not None: + if isinstance(tzdata, datetime.tzinfo): + return ret.replace(tzinfo=tzdata) + if isinstance(tzdata, int): + return ret.replace(tzinfo=_tz.tzoffset(tzname, tzdata)) + if isinstance(tzdata, str): + z = _tz.gettz(tzdata) + if z is not None: + return ret.replace(tzinfo=z) + raise ValueError("Offset must be tzinfo subclass, tz string, or int offset.") + if tzname in ("UTC", "GMT", "Z") or offset == 0: + return ret.replace(tzinfo=_tz.UTC) + if offset is not None: + return ret.replace(tzinfo=_tz.tzoffset(tzname, offset)) + return ret + + +DEFAULTPARSER = parser() + + +def parse(timestr, parserinfo=None, **kwargs): + if parserinfo is not None: + return parser(parserinfo).parse(timestr, **kwargs) + return DEFAULTPARSER.parse(timestr, **kwargs) + + +# -------------------------------------------------------------------------------------------------------------- +# ISO 8601 + +_ISO_TZ = _re.compile(r"^(.*?)(Z|[+-]\d{2}(?::?\d{2})?)$") + + +class isoparser: + """A strict ISO 8601 parser.""" + + def __init__(self, sep=None): + if sep is not None: + if len(sep) != 1 or sep.isdigit(): + raise ValueError("Separator must be a single, non-numeric ASCII character") + self._sep = sep + + def isoparse(self, dt_str): + if not isinstance(dt_str, str): + raise TypeError("ISO string must be a string") + date_str, time_str = self._split(dt_str) + components = self._date_components(date_str) + if time_str is None: + return datetime.datetime(components[0], components[1], components[2]) + hh, mm, ss, us, tzinfo, next_day = self._time_components(time_str) + dt = datetime.datetime(components[0], components[1], components[2], hh, mm, ss, us, tzinfo) + if next_day: + dt += datetime.timedelta(days=1) + return dt + + def parse_isodate(self, datestr): + c = self._date_components(datestr) + return datetime.date(c[0], c[1], c[2]) + + def parse_isotime(self, timestr): + hh, mm, ss, us, tzinfo, next_day = self._time_components(timestr) + if next_day: + hh = 0 + return datetime.time(hh, mm, ss, us, tzinfo) + + def _split(self, s): + # The date part ends before the first separator character. + m = _re.match(r"^(\d{4}-?W\d{2}(?:-?\d)?|\d{4}-\d{2}-\d{2}|\d{8}|\d{4}-\d{2}|\d{4})(.*)$", s) + if m is None: + raise ValueError("ISO string too short") + date_str = m.group(1) + rest = m.group(2) + if rest == "": + return date_str, None + if self._sep is not None and rest[0] != self._sep: + raise ValueError("Invalid character found when parsing ISO string") + if len(rest) < 2: + raise ValueError("Incomplete time component") + return date_str, rest[1:] + + def _date_components(self, s): + m = _re.match(r"^(\d{4})-?W(\d{2})(?:-?(\d))?$", s) + if m is not None: + year = int(m.group(1)) + week = int(m.group(2)) + dow = int(m.group(3)) if m.group(3) else 1 + if week < 1 or week > 53: + raise ValueError("Invalid week: " + str(week)) + if dow < 1 or dow > 7: + raise ValueError("Invalid weekday: " + str(dow) + " (must be 1-7)") + jan4 = datetime.date(year, 1, 4) + week1_monday = jan4 - datetime.timedelta(days=jan4.weekday()) + d = week1_monday + datetime.timedelta(days=(week - 1) * 7 + dow - 1) + return d.year, d.month, d.day + m = _re.match(r"^(\d{4})(?:-(\d{2})(?:-(\d{2}))?)?$", s) + if m is None: + m = _re.match(r"^(\d{4})(\d{2})(\d{2})$", s) + if m is None: + raise ValueError("Invalid ISO date: " + s) + year = int(m.group(1)) + month = int(m.group(2)) if m.group(2) else 1 + day = int(m.group(3)) if m.group(3) else 1 + return year, month, day + + def _time_components(self, s): + tzinfo = None + m = _ISO_TZ.match(s) + body = s + if m is not None: + body = m.group(1) + z = m.group(2) + if z == "Z": + tzinfo = _tz.UTC + else: + sign = -1 if z[0] == "-" else 1 + digits = z[1:].replace(":", "") + hh = int(digits[:2]) + mm = int(digits[2:]) if len(digits) > 2 else 0 + if hh > 23 or mm > 59: + raise ValueError("Invalid time zone offset") + off = sign * (hh * 3600 + mm * 60) + tzinfo = _tz.UTC if off == 0 else _tz.tzoffset(None, off) + mm_ = _re.match(r"^(\d{2})(?::(\d{2})(?::(\d{2})(?:[.,](\d+))?)?)?$", body) + if mm_ is None: + mm_ = _re.match(r"^(\d{2})(?:(\d{2})(?:(\d{2})(?:[.,](\d+))?)?)?$", body) + if mm_ is None: + raise ValueError("Invalid ISO time: " + s) + hh = int(mm_.group(1)) + mi = int(mm_.group(2)) if mm_.group(2) else 0 + ss = int(mm_.group(3)) if mm_.group(3) else 0 + us = int((mm_.group(4) + "000000")[:6]) if mm_.group(4) else 0 + next_day = False + if hh == 24: + if mi != 0 or ss != 0 or us != 0: + raise ValueError("Hour may only be 24 at 24:00:00.000") + hh = 0 + next_day = True + if hh > 23 or mi > 59 or ss > 59: + raise ValueError("Invalid time component") + return hh, mi, ss, us, tzinfo, next_day + + +DEFAULT_ISOPARSER = isoparser() + + +def isoparse(dt_str): + return DEFAULT_ISOPARSER.isoparse(dt_str) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/dateutil/relativedelta.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/dateutil/relativedelta.py new file mode 100644 index 00000000000..30fe2a17710 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/dateutil/relativedelta.py @@ -0,0 +1,345 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""relativedelta: calendar-aware differences and offsets between dates.""" + +import datetime + +from dateutil._common import weekday, isleap, monthrange, MO, TU, WE, TH, FR, SA, SU + +__all__ = ["relativedelta", "MO", "TU", "WE", "TH", "FR", "SA", "SU"] + + +def _sign(x): + return -1 if x < 0 else 1 + + +class relativedelta: + """A difference expressed in calendar units, with optional absolute replacements.""" + + def __init__(self, dt1=None, dt2=None, years=0, months=0, days=0, leapdays=0, weeks=0, hours=0, minutes=0, + seconds=0, microseconds=0, year=None, month=None, day=None, weekday=None, yearday=None, + nlyearday=None, hour=None, minute=None, second=None, microsecond=None): + if dt1 and dt2: + if not isinstance(dt1, datetime.date) or not isinstance(dt2, datetime.date): + raise TypeError("relativedelta only diffs datetime/date") + if not isinstance(dt1, datetime.datetime): + dt1 = datetime.datetime.fromordinal(dt1.toordinal()) + if not isinstance(dt2, datetime.datetime): + dt2 = datetime.datetime.fromordinal(dt2.toordinal()) + self.years = 0 + self.months = 0 + self.days = 0 + self.leapdays = 0 + self.hours = 0 + self.minutes = 0 + self.seconds = 0 + self.microseconds = 0 + self.year = None + self.month = None + self.day = None + self.weekday = None + self.hour = None + self.minute = None + self.second = None + self.microsecond = None + self._has_time = 0 + months = (dt1.year - dt2.year) * 12 + (dt1.month - dt2.month) + self._set_months(months) + dtm = self.__radd__(dt2) + if dt1 < dt2: + increment = 1 + while dt1 > dtm: + months += increment + self._set_months(months) + dtm = self.__radd__(dt2) + else: + increment = -1 + while dt1 < dtm: + months += increment + self._set_months(months) + dtm = self.__radd__(dt2) + delta = dt1 - dtm + self.seconds = delta.seconds + delta.days * 86400 + self.microseconds = delta.microseconds + else: + if isinstance(dt1, datetime.timedelta) or isinstance(dt2, datetime.timedelta): + raise TypeError("relativedelta does not take timedelta positionally") + if any(x is not None and x != int(x) for x in (years, months)): + raise ValueError("Non-integer years and months are ambiguous and not currently supported.") + self.years = int(years) + self.months = int(months) + self.days = days + weeks * 7 + self.leapdays = leapdays + self.hours = hours + self.minutes = minutes + self.seconds = seconds + self.microseconds = microseconds + self.year = year + self.month = month + self.day = day + self.hour = hour + self.minute = minute + self.second = second + self.microsecond = microsecond + if isinstance(weekday, int): + weekday = _weekday_class(weekday) + self.weekday = weekday + yday = 0 + if nlyearday: + yday = nlyearday + elif yearday: + yday = yearday + if yearday > 59: + self.leapdays = -1 + if yday: + ydayidx = [31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334, 366] + found = False + for idx in range(12): + if yday <= ydayidx[idx]: + self.month = idx + 1 + if idx == 0: + self.day = yday + else: + self.day = yday - ydayidx[idx - 1] + found = True + break + if not found: + raise ValueError("invalid year day (%d)" % yday) + self._fix() + + def _fix(self): + if abs(self.microseconds) > 999999: + s = _sign(self.microseconds) + div, mod = divmod(self.microseconds * s, 1000000) + self.microseconds = mod * s + self.seconds += div * s + if abs(self.seconds) > 59: + s = _sign(self.seconds) + div, mod = divmod(self.seconds * s, 60) + self.seconds = mod * s + self.minutes += div * s + if abs(self.minutes) > 59: + s = _sign(self.minutes) + div, mod = divmod(self.minutes * s, 60) + self.minutes = mod * s + self.hours += div * s + if abs(self.hours) > 23: + s = _sign(self.hours) + div, mod = divmod(self.hours * s, 24) + self.hours = mod * s + self.days += div * s + if abs(self.months) > 11: + s = _sign(self.months) + div, mod = divmod(self.months * s, 12) + self.months = mod * s + self.years += div * s + if (self.hours or self.minutes or self.seconds or self.microseconds or self.hour is not None or + self.minute is not None or self.second is not None or self.microsecond is not None): + self._has_time = 1 + else: + self._has_time = 0 + + @property + def weeks(self): + return int(self.days / 7.0) + + @weeks.setter + def weeks(self, value): + self.days = self.days - (self.weeks * 7) + value * 7 + + def _set_months(self, months): + self.months = months + if abs(self.months) > 11: + s = _sign(self.months) + div, mod = divmod(self.months * s, 12) + self.months = mod * s + self.years = div * s + else: + self.years = 0 + + def normalized(self): + days = int(self.days) + hours_f = round(self.hours + 24 * (self.days - days), 11) + hours = int(hours_f) + minutes_f = round(self.minutes + 60 * (hours_f - hours), 10) + minutes = int(minutes_f) + seconds_f = round(self.seconds + 60 * (minutes_f - minutes), 8) + seconds = int(seconds_f) + microseconds = round(self.microseconds + 1e6 * (seconds_f - seconds)) + return self.__class__(years=self.years, months=self.months, days=days, hours=hours, minutes=minutes, + seconds=seconds, microseconds=microseconds, leapdays=self.leapdays, year=self.year, + month=self.month, day=self.day, weekday=self.weekday, hour=self.hour, + minute=self.minute, second=self.second, microsecond=self.microsecond) + + def __add__(self, other): + if isinstance(other, relativedelta): + return self.__class__(years=other.years + self.years, months=other.months + self.months, + days=other.days + self.days, hours=other.hours + self.hours, + minutes=other.minutes + self.minutes, seconds=other.seconds + self.seconds, + microseconds=other.microseconds + self.microseconds, + leapdays=other.leapdays or self.leapdays, + year=(other.year if other.year is not None else self.year), + month=(other.month if other.month is not None else self.month), + day=(other.day if other.day is not None else self.day), + weekday=(other.weekday if other.weekday is not None else self.weekday), + hour=(other.hour if other.hour is not None else self.hour), + minute=(other.minute if other.minute is not None else self.minute), + second=(other.second if other.second is not None else self.second), + microsecond=(other.microsecond if other.microsecond is not None + else self.microsecond)) + if isinstance(other, datetime.timedelta): + return self.__class__(years=self.years, months=self.months, days=self.days + other.days, + hours=self.hours, minutes=self.minutes, seconds=self.seconds + other.seconds, + microseconds=self.microseconds + other.microseconds, leapdays=self.leapdays, + year=self.year, month=self.month, day=self.day, weekday=self.weekday, + hour=self.hour, minute=self.minute, second=self.second, + microsecond=self.microsecond) + if not isinstance(other, datetime.date): + return NotImplemented + if self._has_time and not isinstance(other, datetime.datetime): + other = datetime.datetime.fromordinal(other.toordinal()) + year = (self.year or other.year) + self.years + month = self.month or other.month + if self.months: + month += self.months + if month > 12: + year += 1 + month -= 12 + elif month < 1: + year -= 1 + month += 12 + day = min(monthrange(year, month)[1], self.day or other.day) + repl = {"year": year, "month": month, "day": day} + for attr in ("hour", "minute", "second", "microsecond"): + value = getattr(self, attr) + if value is not None: + repl[attr] = value + days = self.days + if self.leapdays and month > 2 and isleap(year): + days += self.leapdays + ret = other.replace(**repl) + datetime.timedelta(days=days, hours=self.hours, minutes=self.minutes, + seconds=self.seconds, microseconds=self.microseconds) + if self.weekday: + wd = self.weekday.weekday + nth = self.weekday.n or 1 + jumpdays = (abs(nth) - 1) * 7 + if nth > 0: + jumpdays += (7 - ret.weekday() + wd) % 7 + else: + jumpdays += (ret.weekday() - wd) % 7 + jumpdays *= -1 + ret += datetime.timedelta(days=jumpdays) + return ret + + def __radd__(self, other): + return self.__add__(other) + + def __rsub__(self, other): + return self.__neg__().__radd__(other) + + def __sub__(self, other): + if not isinstance(other, relativedelta): + return NotImplemented + return self.__class__(years=self.years - other.years, months=self.months - other.months, + days=self.days - other.days, hours=self.hours - other.hours, + minutes=self.minutes - other.minutes, seconds=self.seconds - other.seconds, + microseconds=self.microseconds - other.microseconds, + leapdays=self.leapdays or other.leapdays, + year=(self.year if self.year is not None else other.year), + month=(self.month if self.month is not None else other.month), + day=(self.day if self.day is not None else other.day), + weekday=(self.weekday if self.weekday is not None else other.weekday), + hour=(self.hour if self.hour is not None else other.hour), + minute=(self.minute if self.minute is not None else other.minute), + second=(self.second if self.second is not None else other.second), + microsecond=(self.microsecond if self.microsecond is not None + else other.microsecond)) + + def __abs__(self): + return self.__class__(years=abs(self.years), months=abs(self.months), days=abs(self.days), + hours=abs(self.hours), minutes=abs(self.minutes), seconds=abs(self.seconds), + microseconds=abs(self.microseconds), leapdays=self.leapdays, year=self.year, + month=self.month, day=self.day, weekday=self.weekday, hour=self.hour, + minute=self.minute, second=self.second, microsecond=self.microsecond) + + def __neg__(self): + return self.__class__(years=-self.years, months=-self.months, days=-self.days, hours=-self.hours, + minutes=-self.minutes, seconds=-self.seconds, microseconds=-self.microseconds, + leapdays=self.leapdays, year=self.year, month=self.month, day=self.day, + weekday=self.weekday, hour=self.hour, minute=self.minute, second=self.second, + microsecond=self.microsecond) + + def __bool__(self): + return not (not self.years and not self.months and not self.days and not self.hours and + not self.minutes and not self.seconds and not self.microseconds and not self.leapdays and + self.year is None and self.month is None and self.day is None and self.weekday is None and + self.hour is None and self.minute is None and self.second is None and + self.microsecond is None) + + def __mul__(self, other): + try: + f = float(other) + except (TypeError, ValueError): + return NotImplemented + return self.__class__(years=int(self.years * f), months=int(self.months * f), days=int(self.days * f), + hours=int(self.hours * f), minutes=int(self.minutes * f), + seconds=int(self.seconds * f), microseconds=int(self.microseconds * f), + leapdays=self.leapdays, year=self.year, month=self.month, day=self.day, + weekday=self.weekday, hour=self.hour, minute=self.minute, second=self.second, + microsecond=self.microsecond) + + def __rmul__(self, other): + return self.__mul__(other) + + def __truediv__(self, other): + try: + return self.__mul__(1 / float(other)) + except TypeError: + return NotImplemented + + def __eq__(self, other): + if not isinstance(other, relativedelta): + return NotImplemented + if self.weekday or other.weekday: + if not self.weekday or not other.weekday: + return False + if self.weekday.weekday != other.weekday.weekday: + return False + n1 = self.weekday.n + n2 = other.weekday.n + if n1 != n2 and not ((not n1 or n1 == 1) and (not n2 or n2 == 1)): + return False + return (self.years == other.years and self.months == other.months and self.days == other.days and + self.hours == other.hours and self.minutes == other.minutes and + self.seconds == other.seconds and self.microseconds == other.microseconds and + self.leapdays == other.leapdays and self.year == other.year and self.month == other.month and + self.day == other.day and self.hour == other.hour and self.minute == other.minute and + self.second == other.second and self.microsecond == other.microsecond) + + def __ne__(self, other): + r = self.__eq__(other) + if r is NotImplemented: + return r + return not r + + def __hash__(self): + return hash((self.weekday, self.years, self.months, self.days, self.hours, self.minutes, self.seconds, + self.microseconds, self.leapdays, self.year, self.month, self.day, self.hour, self.minute, + self.second, self.microsecond)) + + def __repr__(self): + parts = [] + for attr in ("years", "months", "days", "leapdays", "hours", "minutes", "seconds", "microseconds"): + value = getattr(self, attr) + if value: + parts.append("%s=%+g" % (attr, value)) + for attr in ("year", "month", "day", "weekday", "hour", "minute", "second", "microsecond"): + value = getattr(self, attr) + if value is not None: + parts.append("%s=%s" % (attr, repr(value))) + return "%s(%s)" % (self.__class__.__name__, ", ".join(parts)) + + +def _weekday_class(n): + return weekday(n) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/dateutil/rrule.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/dateutil/rrule.py new file mode 100644 index 00000000000..1b4c1c3830a --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/dateutil/rrule.py @@ -0,0 +1,610 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Recurrence rules (RFC 5545): rrule, rruleset and rrulestr. + +Not implemented: byeaster, byweekno with a week start other than Monday, and +the parsing of time zone identifiers inside rrulestr input. +""" + +import datetime + +from dateutil import tz as _tz +from dateutil._common import weekday, monthrange, isleap, MO, TU, WE, TH, FR, SA, SU + +__all__ = ["rrule", "rruleset", "rrulestr", "YEARLY", "MONTHLY", "WEEKLY", "DAILY", "HOURLY", "MINUTELY", + "SECONDLY", "MO", "TU", "WE", "TH", "FR", "SA", "SU"] + +YEARLY, MONTHLY, WEEKLY, DAILY, HOURLY, MINUTELY, SECONDLY = range(7) +_FREQ_NAMES = ("YEARLY", "MONTHLY", "WEEKLY", "DAILY", "HOURLY", "MINUTELY", "SECONDLY") +_ONE_DAY = datetime.timedelta(days=1) + + +def _as_list(value): + if value is None: + return None + if isinstance(value, (list, tuple, set, frozenset)): + return list(value) + return [value] + + +def _as_weekdays(value): + out = [] + for w in _as_list(value): + if isinstance(w, int): + out.append(weekday(w)) + else: + out.append(w) + return out + + +class rrulebase: + """Shared behaviour of rrule and rruleset: iteration, indexing, before/after/between.""" + + def __iter__(self): + return self._iter() + + def __getitem__(self, item): + if isinstance(item, slice): + out = [] + start = item.start + stop = item.stop + step = item.step if item.step is not None else 1 + if (start is not None and start < 0) or (stop is not None and stop < 0) or step < 0: + return list(self._iter())[item] + idx = 0 + if start is None: + start = 0 + for v in self._iter(): + if stop is not None and idx >= stop: + break + if idx >= start and (idx - start) % step == 0: + out.append(v) + idx += 1 + return out + if item < 0: + return list(self._iter())[item] + idx = 0 + for v in self._iter(): + if idx == item: + return v + idx += 1 + raise IndexError("list index out of range") + + def __contains__(self, item): + for v in self._iter(): + if v == item: + return True + if v > item: + break + return False + + def count(self): + total = 0 + for _ in self._iter(): + total += 1 + return total + + def before(self, dt, inc=False): + last = None + for i in self._iter(): + if i > dt or (not inc and i == dt): + break + last = i + return last + + def after(self, dt, inc=False): + for i in self._iter(): + if i > dt or (inc and i == dt): + return i + return None + + def xafter(self, dt, count=None, inc=False): + n = 0 + for i in self._iter(): + if i > dt or (inc and i == dt): + if count is not None and n >= count: + return + n += 1 + yield i + + def between(self, after, before, inc=False, count=1): + out = [] + for i in self._iter(): + if i > before or (not inc and i == before): + break + if i > after or (inc and i == after): + out.append(i) + return out + + +class rrule(rrulebase): + """A recurrence rule.""" + + def __init__(self, freq, dtstart=None, interval=1, wkst=None, count=None, until=None, bysetpos=None, + bymonth=None, bymonthday=None, byyearday=None, byeaster=None, byweekno=None, byweekday=None, + byhour=None, byminute=None, bysecond=None, cache=False): + if dtstart is None: + dtstart = datetime.datetime.now().replace(microsecond=0) + elif not isinstance(dtstart, datetime.datetime): + dtstart = datetime.datetime.fromordinal(dtstart.toordinal()) + else: + dtstart = dtstart.replace(microsecond=0) + self._dtstart = dtstart + self._tzinfo = dtstart.tzinfo + self._freq = freq + self._interval = interval + self._count = count + if until is not None and not isinstance(until, datetime.datetime): + until = datetime.datetime.fromordinal(until.toordinal()) + if self._count is not None and until is not None: + until = None if False else until + self._until = until + if wkst is None: + self._wkst = 0 + elif isinstance(wkst, int): + self._wkst = wkst + else: + self._wkst = wkst.weekday + if interval < 1: + raise ValueError("interval must be positive") + self._original_rule = {} + self._bysetpos = _as_list(bysetpos) + if bysetpos is not None: + for p in self._bysetpos: + if p == 0 or not -366 <= p <= 366: + raise ValueError("bysetpos must be between 1 and 366, or between -366 and -1") + if byeaster is not None: + raise ValueError("byeaster is not supported") + if byweekno is None and byyearday is None and bymonthday is None and byweekday is None: + if freq == YEARLY: + if bymonth is None: + bymonth = dtstart.month + self._original_rule["bymonth"] = None + bymonthday = dtstart.day + self._original_rule["bymonthday"] = None + elif freq == MONTHLY: + bymonthday = dtstart.day + self._original_rule["bymonthday"] = None + elif freq == WEEKLY: + byweekday = dtstart.weekday() + self._original_rule["byweekday"] = None + self._bymonth = _as_list(bymonth) + self._bymonthday = _as_list(bymonthday) + self._byyearday = _as_list(byyearday) + self._byweekno = _as_list(byweekno) + if byweekday is None: + self._byweekday = None + self._bynweekday = None + else: + plain = [] + nth = [] + for w in _as_weekdays(byweekday): + if w.n is None or w.n == 0 or freq > MONTHLY: + plain.append(w.weekday) + else: + nth.append((w.weekday, w.n)) + self._byweekday = plain if plain else None + self._bynweekday = nth if nth else None + self._byhour = None if byhour is None else sorted(set(_as_list(byhour))) + self._byminute = None if byminute is None else sorted(set(_as_list(byminute))) + self._bysecond = None if bysecond is None else sorted(set(_as_list(bysecond))) + for key, val in (("bymonth", self._bymonth), ("bymonthday", self._bymonthday), + ("byyearday", self._byyearday), ("byweekno", self._byweekno)): + if key not in self._original_rule: + self._original_rule[key] = val + if "byweekday" not in self._original_rule: + self._original_rule["byweekday"] = _as_weekdays(byweekday) if byweekday is not None else None + self._original_rule["bysetpos"] = self._bysetpos + self._original_rule["byhour"] = self._byhour if byhour is not None else None + self._original_rule["byminute"] = self._byminute if byminute is not None else None + self._original_rule["bysecond"] = self._bysecond if bysecond is not None else None + self._kwargs = dict(freq=freq, dtstart=dtstart, interval=interval, wkst=wkst, count=count, until=until, + bysetpos=bysetpos, bymonth=bymonth, bymonthday=bymonthday, byyearday=byyearday, + byweekno=byweekno, byweekday=byweekday, byhour=byhour, byminute=byminute, + bysecond=bysecond) + + def replace(self, **kwargs): + new = dict(self._kwargs) + for k in ("bymonth", "bymonthday", "byweekday"): + if self._original_rule.get(k, 1) is None and new.get(k) is not None: + new[k] = None + new.update(kwargs) + return rrule(**new) + + # ---- day selection + + def _day_matches(self, d, year_len, month_len, yday): + if self._bymonth is not None and d.month not in self._bymonth: + return False + if self._bymonthday is not None: + ok = False + for md in self._bymonthday: + if md == d.day or (md < 0 and md == d.day - month_len - 1): + ok = True + if not ok: + return False + if self._byyearday is not None: + ok = False + for yd in self._byyearday: + if yd == yday or (yd < 0 and yd == yday - year_len - 1): + ok = True + if not ok: + return False + if self._byweekno is not None: + wk = d.isocalendar()[1] + last = datetime.date(d.isocalendar()[0], 12, 28).isocalendar()[1] + ok = False + for w in self._byweekno: + if w == wk or (w < 0 and w == wk - last - 1): + ok = True + if not ok: + return False + if self._byweekday is not None or self._bynweekday is not None: + wd = d.weekday() + ok = self._byweekday is not None and wd in self._byweekday + if not ok and self._bynweekday is not None: + month_scope = self._freq == MONTHLY or (self._freq == YEARLY and self._bymonth is not None) + if month_scope: + k = d.day - 1 + total = month_len + else: + k = yday - 1 + total = year_len + for w, n in self._bynweekday: + if w != wd: + continue + if n > 0 and k // 7 + 1 == n: + ok = True + elif n < 0 and (total - 1 - k) // 7 + 1 == -n: + ok = True + if not ok: + return False + return True + + def _days_of(self, first, last): + """Dates from first to last inclusive that pass the date filters.""" + out = [] + d = first + cur_year = -1 + year_len = 365 + month_len = 30 + cur_month = -1 + while d <= last: + if d.year != cur_year: + cur_year = d.year + year_len = 366 if isleap(cur_year) else 365 + if d.month != cur_month or True: + month_len = monthrange(d.year, d.month)[1] + yday = (d - datetime.date(d.year, 1, 1)).days + 1 + if self._day_matches(d, year_len, month_len, yday): + out.append(d) + if d == last: + break + d = d + _ONE_DAY + return out + + def _times(self, hour, minute): + dt = self._dtstart + freq = self._freq + if freq >= HOURLY: + if self._byhour is not None and freq == HOURLY and hour not in self._byhour: + return [] + if freq >= MINUTELY and self._byminute is not None and freq == MINUTELY and \ + minute not in self._byminute: + return [] + out = [] + if freq < HOURLY: + hours = self._byhour if self._byhour is not None else [dt.hour] + minutes = self._byminute if self._byminute is not None else [dt.minute] + seconds = self._bysecond if self._bysecond is not None else [dt.second] + for h in hours: + for m in minutes: + for s in seconds: + out.append((h, m, s)) + elif freq == HOURLY: + minutes = self._byminute if self._byminute is not None else [dt.minute] + seconds = self._bysecond if self._bysecond is not None else [dt.second] + for m in minutes: + for s in seconds: + out.append((hour, m, s)) + elif freq == MINUTELY: + seconds = self._bysecond if self._bysecond is not None else [dt.second] + for s in seconds: + out.append((hour, minute, s)) + return out + + def _iter(self): + dt = self._dtstart + freq = self._freq + interval = self._interval + count = self._count + until = self._until + tzinfo = self._tzinfo + if until is not None and tzinfo is not None and until.tzinfo is None: + until = until.replace(tzinfo=_tz.UTC) + produced = 0 + year = dt.year + month = dt.month + cursor = dt + week_start = dt.date() - datetime.timedelta(days=(dt.weekday() - self._wkst) % 7) + day = dt.date() + while True: + if year > 9999: + return + # The period's dates and times. + if freq == YEARLY: + days = self._days_of(datetime.date(year, 1, 1), datetime.date(year, 12, 31)) + times = self._times(0, 0) + elif freq == MONTHLY: + days = self._days_of(datetime.date(year, month, 1), + datetime.date(year, month, monthrange(year, month)[1])) + times = self._times(0, 0) + elif freq == WEEKLY: + days = self._days_of(week_start, week_start + datetime.timedelta(days=6)) + times = self._times(0, 0) + elif freq == DAILY: + days = self._days_of(day, day) + times = self._times(0, 0) + else: + days = self._days_of(cursor.date(), cursor.date()) + if freq == HOURLY: + times = self._times(cursor.hour, 0) + elif freq == MINUTELY: + times = self._times(cursor.hour, cursor.minute) + else: + times = [] + ok = ((self._byhour is None or cursor.hour in self._byhour) and + (self._byminute is None or cursor.minute in self._byminute) and + (self._bysecond is None or cursor.second in self._bysecond)) + if ok: + times = [(cursor.hour, cursor.minute, cursor.second)] + if freq == MINUTELY and self._byhour is not None and cursor.hour not in self._byhour: + times = [] + if freq == HOURLY and self._byhour is not None and cursor.hour not in self._byhour: + times = [] + if freq == SECONDLY and not days: + times = [] + cands = [] + for d in days: + for h, mi, s in times: + cands.append(datetime.datetime(d.year, d.month, d.day, h, mi, s, 0, tzinfo)) + cands.sort() + if self._bysetpos is not None and cands: + picked = [] + total = len(cands) + for p in self._bysetpos: + idx = p - 1 if p > 0 else total + p + if 0 <= idx < total and cands[idx] not in picked: + picked.append(cands[idx]) + picked.sort() + cands = picked + for c in cands: + if c < dt: + continue + if until is not None and c > until: + return + produced += 1 + yield c + if count is not None and produced >= count: + return + # Next period. + try: + if freq == YEARLY: + year += interval + elif freq == MONTHLY: + total = year * 12 + month - 1 + interval + year = total // 12 + month = total % 12 + 1 + elif freq == WEEKLY: + week_start = week_start + datetime.timedelta(days=7 * interval) + year = week_start.year + elif freq == DAILY: + day = day + datetime.timedelta(days=interval) + year = day.year + elif freq == HOURLY: + cursor = cursor + datetime.timedelta(hours=interval) + year = cursor.year + elif freq == MINUTELY: + cursor = cursor + datetime.timedelta(minutes=interval) + year = cursor.year + else: + cursor = cursor + datetime.timedelta(seconds=interval) + year = cursor.year + except OverflowError: + return + + def __str__(self): + output = [] + h, m, s = [None] * 3 + if self._dtstart: + output.append(self._dtstart.strftime("DTSTART:%Y%m%dT%H%M%S")) + parts = ["FREQ=" + _FREQ_NAMES[self._freq]] + if self._interval != 1: + parts.append("INTERVAL=" + str(self._interval)) + if self._wkst: + parts.append("WKST=" + repr(weekday(self._wkst))) + if self._count is not None: + parts.append("COUNT=" + str(self._count)) + if self._until: + parts.append(self._until.strftime("UNTIL=%Y%m%dT%H%M%S")) + orig = self._original_rule + names = (("bysetpos", "BYSETPOS"), ("bymonth", "BYMONTH"), ("bymonthday", "BYMONTHDAY"), + ("byyearday", "BYYEARDAY"), ("byweekno", "BYWEEKNO"), ("byweekday", "BYDAY"), + ("byhour", "BYHOUR"), ("byminute", "BYMINUTE"), ("bysecond", "BYSECOND")) + for key, label in names: + value = orig.get(key) + if value is None: + continue + if key == "byweekday": + parts.append(label + "=" + ",".join(repr(w) for w in value)) + else: + parts.append(label + "=" + ",".join(str(v) for v in value)) + output.append("RRULE:" + ";".join(parts)) + return "\n".join(output) + + +def _merge(iterators): + heads = [] + for it in iterators: + try: + heads.append([next(it), it]) + except StopIteration: + pass + while heads: + mi = 0 + for i in range(1, len(heads)): + if heads[i][0] < heads[mi][0]: + mi = i + v = heads[mi][0] + yield v + try: + heads[mi][0] = next(heads[mi][1]) + except StopIteration: + del heads[mi] + + +class rruleset(rrulebase): + """A set of rules and dates, with exclusions.""" + + def __init__(self, cache=False): + self._rrule = [] + self._rdate = [] + self._exrule = [] + self._exdate = [] + + def rrule(self, rrule): + self._rrule.append(rrule) + + def rdate(self, rdate): + self._rdate.append(rdate) + + def exrule(self, exrule): + self._exrule.append(exrule) + + def exdate(self, exdate): + self._exdate.append(exdate) + + def _iter(self): + inc = [iter(sorted(self._rdate))] + [iter(r) for r in self._rrule] + exc = [iter(sorted(self._exdate))] + [iter(r) for r in self._exrule] + ex = _merge(exc) + exh = next(ex, None) + last = None + for d in _merge(inc): + if last is not None and d == last: + continue + last = d + while exh is not None and exh < d: + exh = next(ex, None) + if exh is not None and exh == d: + continue + yield d + + +_WEEKDAY_CODES = {"MO": 0, "TU": 1, "WE": 2, "TH": 3, "FR": 4, "SA": 5, "SU": 6} + + +def _parse_dt(text, tzinfo=None): + text = text.strip() + z = text.endswith("Z") + if z: + text = text[:-1] + if "T" in text: + d, t = text.split("T") + t = (t + "000000")[:6] + v = datetime.datetime(int(d[:4]), int(d[4:6]), int(d[6:8]), int(t[:2]), int(t[2:4]), int(t[4:6])) + else: + v = datetime.datetime(int(text[:4]), int(text[4:6]), int(text[6:8])) + if z: + v = v.replace(tzinfo=_tz.UTC) + elif tzinfo is not None: + v = v.replace(tzinfo=tzinfo) + return v + + +def _rule_kwargs(body, tzinfo): + kw = {} + for part in body.split(";"): + if not part: + continue + name, value = part.split("=", 1) + name = name.upper() + if name == "FREQ": + kw["freq"] = _FREQ_NAMES.index(value.upper()) + elif name == "INTERVAL": + kw["interval"] = int(value) + elif name == "COUNT": + kw["count"] = int(value) + elif name == "UNTIL": + kw["until"] = _parse_dt(value) + elif name == "WKST": + kw["wkst"] = _WEEKDAY_CODES[value.upper()] + elif name == "BYDAY" or name == "BYWEEKDAY": + days = [] + for item in value.split(","): + code = item[-2:].upper() + n = item[:-2] + days.append(weekday(_WEEKDAY_CODES[code], int(n) if n else None)) + kw["byweekday"] = days + else: + key = name.lower() + if key in ("bymonth", "bymonthday", "byyearday", "byweekno", "byhour", "byminute", "bysecond", + "bysetpos"): + kw[key] = [int(v) for v in value.split(",")] + else: + raise ValueError("unknown parameter name: " + name) + return kw + + +def rrulestr(s, **kwargs): + """Parse an RFC 5545 text into an rrule, or an rruleset when it holds more than one rule.""" + dtstart = kwargs.get("dtstart") + tzinfo = kwargs.get("tzinfo") + forceset = kwargs.get("forceset", False) + unfold = kwargs.get("unfold", False) + lines = [ln.strip() for ln in s.replace("\r", "").split("\n") if ln.strip()] + if unfold: + merged = [] + for ln in s.replace("\r", "").split("\n"): + if ln.startswith(" ") and merged: + merged[-1] += ln[1:] + elif ln.strip(): + merged.append(ln.strip()) + lines = merged + rules = [] + exrules = [] + rdates = [] + exdates = [] + for ln in lines: + if ":" not in ln: + rules.append(_rule_kwargs(ln, tzinfo)) + continue + name, value = ln.split(":", 1) + name = name.split(";")[0].upper() + if name == "DTSTART": + dtstart = _parse_dt(value, tzinfo) + elif name == "RRULE": + rules.append(_rule_kwargs(value, tzinfo)) + elif name == "EXRULE": + exrules.append(_rule_kwargs(value, tzinfo)) + elif name == "RDATE": + for v in value.split(","): + rdates.append(_parse_dt(v, tzinfo)) + elif name == "EXDATE": + for v in value.split(","): + exdates.append(_parse_dt(v, tzinfo)) + else: + raise ValueError("unsupported property: " + name) + if (not forceset and len(rules) == 1 and not exrules and not rdates and not exdates): + return rrule(dtstart=dtstart, **rules[0]) + rset = rruleset() + for kw in rules: + rset.rrule(rrule(dtstart=dtstart, **kw)) + for kw in exrules: + rset.exrule(rrule(dtstart=dtstart, **kw)) + for d in rdates: + rset.rdate(d) + for d in exdates: + rset.exdate(d) + return rset diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/dateutil/tz.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/dateutil/tz.py new file mode 100644 index 00000000000..28bf65f86c9 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/dateutil/tz.py @@ -0,0 +1,280 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Time zones: UTC, fixed offsets, the local zone and simple UTC+N strings. + +Zones that need the tz database (named regions with daylight saving rules) are +not available; gettz answers None for them. +""" + +import datetime +import re as _re +import time as _time + +__all__ = ["tzutc", "tzoffset", "tzlocal", "tzstr", "gettz", "UTC", "datetime_exists", "datetime_ambiguous", + "resolve_imaginary"] + +ZERO = datetime.timedelta(0) + + +class tzutc(datetime.tzinfo): + """The UTC zone; every instance is the same object.""" + _instance = None + + def __new__(cls): + if cls._instance is None: + cls._instance = datetime.tzinfo.__new__(cls) + return cls._instance + + def utcoffset(self, dt): + return ZERO + + def dst(self, dt): + return ZERO + + def tzname(self, dt): + return "UTC" + + def is_ambiguous(self, dt): + return False + + def fromutc(self, dt): + return dt + + def __eq__(self, other): + if isinstance(other, tzutc): + return True + if isinstance(other, tzoffset): + return other._offset == ZERO + return NotImplemented + + def __ne__(self, other): + r = self.__eq__(other) + if r is NotImplemented: + return r + return not r + + def __hash__(self): + return hash(ZERO) + + def __repr__(self): + return "tzutc()" + + +UTC = tzutc() + + +class tzoffset(datetime.tzinfo): + """A zone with a fixed offset from UTC, given in seconds or as a timedelta.""" + _cache = {} + + def __new__(cls, name, offset): + if isinstance(offset, datetime.timedelta): + secs = int(offset.total_seconds()) + else: + secs = int(offset) + key = (name, secs) + inst = cls._cache.get(key) + if inst is None: + inst = datetime.tzinfo.__new__(cls) + inst._name = name + inst._offset = datetime.timedelta(seconds=secs) + cls._cache[key] = inst + return inst + + def __init__(self, name, offset): + pass + + def utcoffset(self, dt): + return self._offset + + def dst(self, dt): + return ZERO + + def tzname(self, dt): + return self._name + + def is_ambiguous(self, dt): + return False + + def fromutc(self, dt): + return dt + self._offset + + def __eq__(self, other): + if isinstance(other, tzoffset): + return self._offset == other._offset + if isinstance(other, tzutc): + return self._offset == ZERO + return NotImplemented + + def __ne__(self, other): + r = self.__eq__(other) + if r is NotImplemented: + return r + return not r + + def __hash__(self): + return hash(self._offset) + + def __repr__(self): + return "%s(%s, %d)" % (self.__class__.__name__, repr(self._name), int(self._offset.total_seconds())) + + +class tzlocal(datetime.tzinfo): + """The zone of the machine, from the time module's view of it.""" + + def __init__(self): + std = -getattr(_time, "timezone", 0) + self._std_offset = datetime.timedelta(seconds=std) + if getattr(_time, "daylight", 0): + self._dst_offset = datetime.timedelta(seconds=-getattr(_time, "altzone", -std)) + else: + self._dst_offset = self._std_offset + self._dst_saved = self._dst_offset - self._std_offset + self._hasdst = bool(self._dst_saved) + names = getattr(_time, "tzname", ("UTC", "UTC")) + self._tznames = names + + def utcoffset(self, dt): + if dt is None and self._hasdst: + return None + if self._isdst(dt): + return self._dst_offset + return self._std_offset + + def dst(self, dt): + if dt is None and self._hasdst: + return None + if self._isdst(dt): + return self._dst_saved + return ZERO + + def tzname(self, dt): + return self._tznames[1 if self._isdst(dt) else 0] + + def is_ambiguous(self, dt): + return False + + def _isdst(self, dt): + if not self._hasdst or dt is None: + return False + try: + stamp = (dt.replace(tzinfo=None) - datetime.datetime(1970, 1, 1)).total_seconds() - \ + self._std_offset.total_seconds() + return bool(_time.localtime(stamp).tm_isdst > 0) + except (OverflowError, ValueError, OSError, AttributeError): + return False + + def __eq__(self, other): + if isinstance(other, tzlocal): + return self._std_offset == other._std_offset and self._dst_offset == other._dst_offset + if isinstance(other, tzutc): + return not self._hasdst and self._std_offset == ZERO + if isinstance(other, tzoffset): + return not self._hasdst and self._std_offset == other._offset + return NotImplemented + + def __ne__(self, other): + r = self.__eq__(other) + if r is NotImplemented: + return r + return not r + + def __hash__(self): + return hash((self._std_offset, self._dst_offset)) + + def __repr__(self): + return "tzlocal()" + + +_TZSTR = _re.compile(r"^([A-Za-z]{3,})?([+-])(\d{1,2})(?::?(\d{2}))?$") + + +class tzstr(datetime.tzinfo): + """A zone from a string such as UTC+3 or GMT-05:30 (without daylight saving).""" + + def __init__(self, s): + m = _TZSTR.match(s) + if m is None: + raise ValueError("unknown string format") + self._s = s + self._std_abbr = m.group(1) or "" + secs = int(m.group(3)) * 3600 + int(m.group(4) or 0) * 60 + if m.group(2) == "-": + secs = -secs + self._offset = datetime.timedelta(seconds=secs) + + def utcoffset(self, dt): + return self._offset + + def dst(self, dt): + return ZERO + + def tzname(self, dt): + return self._std_abbr + + def is_ambiguous(self, dt): + return False + + def fromutc(self, dt): + return dt + self._offset + + def __eq__(self, other): + if not isinstance(other, tzstr): + return NotImplemented + return self._s == other._s + + def __ne__(self, other): + r = self.__eq__(other) + if r is NotImplemented: + return r + return not r + + def __hash__(self): + return hash(self._s) + + def __repr__(self): + return "%s(%s)" % (self.__class__.__name__, repr(self._s)) + + +def gettz(name=None): + """Return a zone for a name, or None when it is not known.""" + if not name: + return tzlocal() + if name in ("GMT", "UTC"): + return UTC + if name in getattr(_time, "tzname", ()): + return tzlocal() + if _TZSTR.match(name): + return tzstr(name) + return None + + +def datetime_exists(dt, tz=None): + if tz is None: + if dt.tzinfo is None: + raise ValueError("Datetime is naive and no time zone provided.") + tz = dt.tzinfo + dt = dt.replace(tzinfo=None) + dt_rt = dt.replace(tzinfo=tz).astimezone(UTC).astimezone(tz) + dt_rt = dt_rt.replace(tzinfo=None) + return dt == dt_rt + + +def datetime_ambiguous(dt, tz=None): + if tz is None: + if dt.tzinfo is None: + raise ValueError("Ambiguous naive datetime and no time zone provided.") + tz = dt.tzinfo + is_ambiguous = getattr(tz, "is_ambiguous", None) + if is_ambiguous is not None: + return is_ambiguous(dt) + return False + + +def resolve_imaginary(dt): + if dt.tzinfo is not None and not datetime_exists(dt): + curr_offset = (dt + datetime.timedelta(hours=24)).utcoffset() + old_offset = (dt - datetime.timedelta(hours=24)).utcoffset() + dt += curr_offset - old_offset + return dt diff --git a/maven/python-transpiler/src/test/resources/conformance/attrs_pkg.out b/maven/python-transpiler/src/test/resources/conformance/attrs_pkg.out new file mode 100644 index 00000000000..7e9f35c2ce8 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/attrs_pkg.out @@ -0,0 +1,94 @@ +Point(x=1, y=2) Point(x=3, y=0) Point(x=4, y=5) +True True False +True +lt raised TypeError '<' not supported between instances of 'Point' and 'Point' +missing raised TypeError Point.__init__() missing 1 required positional argument: 'x' +extra raised TypeError Point.__init__() takes from 2 to 3 positional arguments but 4 were given +badkw raised TypeError Point.__init__() got an unexpected keyword argument 'z' +True False +['x', 'y'] x 0 +{'x': 1, 'y': 2} (1, 2) +Point(x=10, y=2) +('x', 'y') +True True [Ver(major=1, minor=0), Ver(major=1, minor=5), Ver(major=2, minor=0)] +True False +Frozen(name='a', tags=(1, 2)) True True +frozen set raised FrozenInstanceError can't set attribute +FrozenInstanceError caught True +attr.exceptions caught +Frozen(name='z', tags=(1, 2)) Frozen(name='a', tags=(1, 2)) +WithFactory(items=[1], d={}, n=3, twice=6) WithFactory(items=[], d={}, n=5, twice=10) [] +Factory(factory=, takes_self=False) +SelfFactory(a=1, b=2) SelfFactory(a=1, b=7) +Validated(x=1, name='n', opt=None, kind='a', pos=1) +v type raised TypeError 'x' must be (got 's' that is a ). +v name raised ValueError 'name' must match regex '^[a-z]+$' ('Bad1' doesn't) +v opt ok -> Validated(x=1, name='n', opt=3, kind='a', pos=1) +v opt bad raised TypeError 'opt' must be (got 'x' that is a ). +v in raised ValueError 'kind' must be in ['a', 'b'] (got 'c') +v pos raised ValueError 'pos' must be >= 0: -1 +v pos2 raised ValueError 'pos' must be < 100: 100 +v dec raised ValueError unlucky x +set invalid raised TypeError 'x' must be (got 'q' that is a ). +set invalid2 raised ValueError unlucky x +4 +validate() raised TypeError 'x' must be (got 'zz' that is a ). +Lists(xs=[1, 2], m={}, s='ab', c=) +deep it raised TypeError 'xs' must be (got 'a' that is a ). +deep it2 raised TypeError 'xs' must be (got (1, 2) that is a ). +deep map raised TypeError 'm' must be (got 'b' that is a ). +min raised ValueError Length of 's' must be >= 1: 0 +max raised ValueError Length of 's' must be <= 3: 4 +callable raised NotCallableError 'c' must be callable (got 5 that is a ). +Converted(n=5, s=None, z=7, b=True, p=3) +Converted(n=1, s='2', z=7, b=False, p=9) +8 int +conv bad raised ValueError invalid literal for int() with base 10: 'x' +to_bool bad raised ValueError Cannot convert value to bool: 'maybe' +Kw(a=1, b=2, c=3) +kw missing raised TypeError Kw.__init__() missing 1 required keyword-only argument: 'c' +kw positional raised TypeError Kw.__init__() takes 2 positional arguments but 4 were given +AllKw(a=1, b=5) +allkw raised TypeError AllKw.__init__() takes 1 positional argument but 2 were given +Post(a=4, b=40) 40 +Priv(_hidden=1, shown=2) Priv(_hidden=5, shown=2) hidden +Opts(a=1, ignored=0, shout=X) True False +Child(a=1, c=3, b=20) ['a', 'c', 'b'] [True, False, False] +False True +mandatory after default raised ValueError No mandatory attributes allowed after an attribute with a default value or factory. Attribute in question: Attribute(name='b', default=NOTHING, validator=None, repr=True, eq=True, eq_key=None, order=True, order_key=None, hash=None, init=True, metadata=mappingproxy({}), type=, converter=None, kw_only=False, inherited=False, on_setattr=None, alias='b') +{'inner': {'v': 1}, 'items': [{'v': 2}, 3], 'mapping': {'k': {'v': 4}}, 'tup': ({'v': 5}, 6)} +{'inner': Inner(v=1), 'items': [Inner(v=2), 3], 'mapping': {'k': Inner(v=4)}, 'tup': (Inner(v=5), 6)} +{'inner': {'v': 1}, 'items': [{'v': 2}, 3], 'mapping': {'k': {'v': 4}}, 'tup': ({'v': 5}, 6)} +((1,), [(2,), 3], {'k': (4,)}, ((5,), 6)) +{'inner': {'v': 1}, 'items': [{'v': 2}, 3], 'mapping': {'k': {'v': 4}}} +{'inner': {'v': 1}, 'mapping': {'k': {'v': 4}}} +{'v': 4} +(1,) (Inner(v=5), 6) +Priv(_hidden=9, shown=2) +evolve bad raised TypeError Point.__init__() got an unexpected keyword argument 'q' +fields bad raised NotAnAttrsClassError is not an attrs-decorated class. +fields inst -> (Attribute(name='x', default=NOTHING, validator=None, repr=True, eq=True, eq_key=None, order=True, order_key=None, hash=None, init=True, metadata=mappingproxy({}), type=, converter=None, kw_only=False, inherited=False, on_setattr=None, alias='x'), Attribute(name='y', default=0, validator=None, repr=True, eq=True, eq_key=None, order=True, order_key=None, hash=None, init=True, metadata=mappingproxy({}), type=, converter=None, kw_only=False, inherited=False, on_setattr=None, alias='y')) +Old(x=1, y=5, z=[]) True True True +['x', 'y', 'z', 'w'] 5 +old validator raised TypeError 'y' must be (got 'no' that is a ). +changes freely unhashable +OldFrozen(a=1, b='b') True +old frozen raised FrozenInstanceError can't set attribute +OldFrozen(a=1, b='c') +{'a': 1, 'b': 'b'} +False True +no order raised TypeError '<' not supported between instances of 'NoOrder' and 'NoOrder' +OrderBy(b=1, a=2) ['b', 'a'] +Decorated(n=2, m=3) +decorated raised ValueError neg +Auto(a=1, b=[], c=3) True +Dyn(a=1, b=2) True +Dyn2(x=1, y=2) Dyn2(x=1, y=5) +custom! True +False +_Nothing.NOTHING NOTHING False +Validated(x='not an int', name='n', opt=None, kind='a', pos=1) +False +5 6 +> +> or None> diff --git a/maven/python-transpiler/src/test/resources/conformance/attrs_pkg.py b/maven/python-transpiler/src/test/resources/conformance/attrs_pkg.py new file mode 100644 index 00000000000..66b65a7f269 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/attrs_pkg.py @@ -0,0 +1,392 @@ +import attr +import attrs +from attrs import define, frozen, field, mutable, asdict, astuple, evolve, fields, has, validators, converters +from attrs import Factory, NOTHING +from attrs.exceptions import FrozenInstanceError + + +def show(label, fn): + try: + print(label, "->", fn()) + except Exception as e: + print(label, "raised", type(e).__name__, e.args[0] if e.args else "") + + +@define +class Point: + x: int + y: int = 0 + + +p = Point(1, 2) +print(p, Point(3), Point(y=5, x=4)) +print(p == Point(1, 2), p != Point(1, 3), p == 5) +print(Point.__hash__ is None) +show("lt", lambda: p < Point(2, 2)) +show("missing", lambda: Point()) +show("extra", lambda: Point(1, 2, 3)) +show("badkw", lambda: Point(1, z=3)) +print(has(Point), has(int)) +print([f.name for f in fields(Point)], fields(Point).x.name, fields(Point).y.default) +print(asdict(p), astuple(p)) +p.x = 10 +print(p) +print(Point.__match_args__) + + +@define(order=True) +class Ver: + major: int + minor: int = 0 + + +print(Ver(1, 2) < Ver(1, 3), Ver(2) > Ver(1, 9), sorted([Ver(2), Ver(1, 5), Ver(1)])) +print(Ver(1) <= Ver(1), Ver(1) >= Ver(2)) + + +@frozen +class Frozen: + name: str + tags: tuple = () + + +f = Frozen("a", (1, 2)) +print(f, hash(f) == hash(Frozen("a", (1, 2))), f == Frozen("a", (1, 2))) +show("frozen set", lambda: setattr(f, "name", "b")) +try: + f.name = "c" +except FrozenInstanceError as e: + print("FrozenInstanceError caught", isinstance(e, AttributeError)) +try: + f.name = "c" +except attr.exceptions.FrozenInstanceError: + print("attr.exceptions caught") +print(evolve(f, name="z"), f) + + +@define +class WithFactory: + items: list = Factory(list) + d: dict = field(factory=dict) + n: int = field(default=3) + twice: int = field(init=False) + + @twice.default + def _twice(self): + return self.n * 2 + + +a = WithFactory() +b = WithFactory(n=5) +a.items.append(1) +print(a, b, b.items) +print(Factory(list)) + + +@define +class SelfFactory: + a: int + b: int = Factory(lambda self: self.a + 1, takes_self=True) + + +print(SelfFactory(1), SelfFactory(1, 7)) + + +@define +class Validated: + x: int = field(validator=validators.instance_of(int)) + name: str = field(default="n", validator=[validators.instance_of(str), validators.matches_re("^[a-z]+$")]) + opt: object = field(default=None, validator=validators.optional(validators.instance_of(int))) + kind: str = field(default="a", validator=validators.in_(["a", "b"])) + pos: int = field(default=1, validator=[validators.ge(0), validators.lt(100)]) + + @x.validator + def _check_x(self, attribute, value): + if value == 13: + raise ValueError("unlucky " + attribute.name) + + +print(Validated(1)) +show("v type", lambda: Validated("s")) +show("v name", lambda: Validated(1, name="Bad1")) +show("v opt ok", lambda: Validated(1, opt=3)) +show("v opt bad", lambda: Validated(1, opt="x")) +show("v in", lambda: Validated(1, kind="c")) +show("v pos", lambda: Validated(1, pos=-1)) +show("v pos2", lambda: Validated(1, pos=100)) +show("v dec", lambda: Validated(13)) +v = Validated(2) +show("set invalid", lambda: setattr(v, "x", "q")) +show("set invalid2", lambda: setattr(v, "x", 13)) +v.x = 4 +print(v.x) +vv = Validated(1) +attrs.validate(vv) +object.__setattr__(vv, "x", "zz") +show("validate()", lambda: attrs.validate(vv)) + + +@define +class Lists: + xs: list = field(validator=validators.deep_iterable(validators.instance_of(int), validators.instance_of(list))) + m: dict = field(default=Factory(dict), + validator=validators.deep_mapping(validators.instance_of(str), validators.instance_of(int))) + s: str = field(default="ab", validator=[validators.min_len(1), validators.max_len(3)]) + c: object = field(default=len, validator=validators.is_callable()) + + +print(Lists([1, 2])) +show("deep it", lambda: Lists([1, "a"])) +show("deep it2", lambda: Lists((1, 2))) +show("deep map", lambda: Lists([1], {"a": "b"})) +show("min", lambda: Lists([1], s="")) +show("max", lambda: Lists([1], s="abcd")) +show("callable", lambda: Lists([1], c=5)) + + +@define +class Converted: + n: int = field(converter=int) + s: str = field(default=None, converter=converters.optional(str)) + z: int = field(default=None, converter=converters.default_if_none(7)) + b: bool = field(default="yes", converter=converters.to_bool) + p: int = field(default="3", converter=converters.pipe(str.strip, int)) + + +c = Converted("5") +print(c) +print(Converted(1, 2, None, "off", " 9 ")) +c.n = "8" +print(c.n, type(c.n).__name__) +show("conv bad", lambda: Converted("x")) +show("to_bool bad", lambda: converters.to_bool("maybe")) + + +@define +class Kw: + a: int + b: int = field(kw_only=True, default=2) + c: int = field(kw_only=True) + + +print(Kw(1, c=3)) +show("kw missing", lambda: Kw(1)) +show("kw positional", lambda: Kw(1, 2, 3)) + + +@define(kw_only=True) +class AllKw: + a: int + b: int = 5 + + +print(AllKw(a=1)) +show("allkw", lambda: AllKw(1)) + + +@define +class Post: + a: int + b: int = field(init=False) + + def __attrs_post_init__(self): + self.b = self.a * 10 + + +print(Post(4), Post(4).b) + + +@define +class Priv: + _hidden: int = 1 + shown: int = 2 + + +print(Priv(), Priv(hidden=5), fields(Priv)._hidden.alias) + + +@define +class Opts: + a: int + secret: str = field(default="s", repr=False) + ignored: int = field(default=0, eq=False) + shout: str = field(default="x", repr=lambda v: v.upper()) + + +print(Opts(1), Opts(1, ignored=3) == Opts(1, ignored=4), Opts(1, secret="q") == Opts(1)) + + +@define +class Base: + a: int + b: int = 2 + + +@define +class Child(Base): + c: int = 3 + b: int = 20 + + +print(Child(1), [f.name for f in fields(Child)], [f.inherited for f in fields(Child)]) +print(Child(1) == Base(1), Child(1) == Child(1)) +show("mandatory after default", lambda: define(type("Bad", (), {"__annotations__": {"a": int, "b": int}, + "a": 1}))) + + +@define +class Inner: + v: int + + +@define +class Outer: + inner: Inner + items: list + mapping: dict + tup: tuple = () + + +o = Outer(Inner(1), [Inner(2), 3], {"k": Inner(4)}, (Inner(5), 6)) +print(asdict(o)) +print(asdict(o, recurse=False)) +print(attr.asdict(o, retain_collection_types=True)) +print(astuple(o)) +print(asdict(o, filter=lambda a, v: a.name != "tup")) +print(asdict(o, filter=attrs.filters.exclude(attrs.fields(Outer).tup, "items"))) +print(asdict(Inner(3), value_serializer=lambda inst, a, v: v + 1)) +print(evolve(o, tup=(1,)).tup, o.tup) +print(evolve(Priv(), hidden=9)) +show("evolve bad", lambda: evolve(Point(1), q=2)) +show("fields bad", lambda: fields(int)) +show("fields inst", lambda: fields(Point(1))) + + +# attr.s / attr.ib style +@attr.s +class Old(object): + x = attr.ib() + y = attr.ib(default=5, validator=attr.validators.instance_of(int)) + z = attr.ib(factory=list) + w = attr.ib(default="w", repr=False) + + +o1 = Old(1) +print(o1, o1 == Old(1), o1 < Old(2), Old(1, 2) > Old(1, 1)) +print([a.name for a in attr.fields(Old)], attr.fields(Old).y.default) +show("old validator", lambda: Old(1, "no")) +o1.x = "changes freely" +print(o1.x, hash(o1) if False else "unhashable" if Old.__hash__ is None else "hashable") + + +@attr.s(frozen=True, auto_attribs=True) +class OldFrozen: + a: int + b: str = "b" + + +of = OldFrozen(1) +print(of, hash(of) == hash(OldFrozen(1))) +show("old frozen", lambda: setattr(of, "a", 2)) +print(attr.evolve(of, b="c")) +print(attr.asdict(of)) + + +@attr.s(eq=False) +class NoEq: + a = attr.ib() + + +print(NoEq(1) == NoEq(1), hash(NoEq(1)) is not None) + + +@attr.s(order=False) +class NoOrder: + a = attr.ib() + + +show("no order", lambda: NoOrder(1) < NoOrder(2)) + + +@attr.s +class OrderBy: + b = attr.ib(default=1) + a = attr.ib(default=2) + + +print(OrderBy(), [f.name for f in attr.fields(OrderBy)]) + + +@attr.s +class Decorated: + n = attr.ib() + m = attr.ib() + + @n.validator + def _vn(self, attribute, value): + if value < 0: + raise ValueError("neg") + + @m.default + def _dm(self): + return self.n + 1 + + +print(Decorated(2)) +show("decorated", lambda: Decorated(-1)) + + +@attr.s(auto_attribs=True) +class Auto: + a: int + b: list = attr.Factory(list) + c: int = attr.ib(default=3) + + +print(Auto(1), Auto(1) == Auto(1, [], 3)) + +Dyn = attr.make_class("Dyn", ["a", "b"]) +print(Dyn(1, 2), Dyn(1, 2) == Dyn(1, 2)) +Dyn2 = attrs.make_class("Dyn2", {"x": attr.ib(default=1), "y": attr.ib(default=2)}, frozen=True) +print(Dyn2(), Dyn2(y=5)) + + +@define +class Custom: + a: int + + def __repr__(self): + return "custom!" + + def __eq__(self, other): + return True + + +print(Custom(1), Custom(1) == Custom(2)) + + +@define(repr=False, eq=False) +class Plain: + a: int + + +print(Plain(1) == Plain(1)) +print(attr.NOTHING, repr(attr.NOTHING), bool(attr.NOTHING)) + +with validators.disabled(): + print(Validated("not an int")) +print(validators.get_disabled()) + + +@define +class SetHooks: + a: int = field(default=1, on_setattr=attrs.setters.NO_OP, converter=int) + b: int = field(default=1, converter=int) + + +sh = SetHooks() +sh.a = "5" +sh.b = "6" +print(sh.a, sh.b) +print(validators.instance_of(int)) +print(repr(validators.optional(validators.instance_of(int)))) diff --git a/maven/python-transpiler/src/test/resources/conformance/dateutil_.out b/maven/python-transpiler/src/test/resources/conformance/dateutil_.out new file mode 100644 index 00000000000..772853c9c79 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/dateutil_.out @@ -0,0 +1,206 @@ +== relativedelta +2024-02-29 10:30:15 +2025-02-26 10:30:15 +2025-02-28 10:30:15 +2023-11-30 10:30:15 +2024-02-01 15:45:15 +2024-02-14 10:30:15 +2024-01-01 10:30:15 +2024-02-29 10:30:15 +2024-01-31 00:00:00 +2024-03-25 +2024-03-15 05:00:00 +2024-02-05 10:30:15 +2024-02-12 10:30:15 +2024-01-26 10:30:15 +2024-03-15 00:00:00 +2024-03-29 00:00:00 +2024-02-29 00:00:00 2023-03-01 00:00:00 +2023-03-01 00:00:00 +2025-02-28 2020-02-29 +2024-06-04 00:00:00 +2024-02-15 00:00:00 +relativedelta(years=+1, months=+2, days=+5) +relativedelta(years=-1, months=-2, days=-5) +relativedelta(months=+1, days=+2, hours=+10, minutes=+57, seconds=+58, microseconds=+496) +relativedelta(years=+24, months=+9, days=+26) +relativedelta(years=+1, months=+2, days=+3, hours=+4) relativedelta(months=+5, days=-10, hours=+1, minutes=+1, seconds=+40) +relativedelta(years=+1, months=+7, days=-7, hours=+5, minutes=+1, seconds=+40) relativedelta(years=+1, months=-3, days=+13, hours=+3, minutes=-1, seconds=-40) relativedelta(years=-1, months=-2, days=-3, hours=-4) relativedelta(days=+3, hours=+2) +relativedelta(years=+2, months=+4, days=+6, hours=+8) relativedelta(years=+2, months=+4, days=+6, hours=+8) relativedelta(months=+1, days=+1, hours=+2) +relativedelta(years=+1, months=+2, days=+42, hours=+3, minutes=+4, seconds=-40) +relativedelta(years=+1, month=4, day=3) relativedelta(day=4) +True True False +False True True +2 21 -1 +relativedelta(days=+1, hours=+12) relativedelta(hours=+1, minutes=+15) +True False +MO FR(-1) SU(+3) False True +frac months raised ValueError +bad yday raised ValueError +weekday0 -> MO +add int raised TypeError +relativedelta(years=-1, months=-3, days=-4) relativedelta(seconds=+1, microseconds=+500000) +True +relativedelta(months=+1) relativedelta(years=+1, days=+1) +2021-02-28 00:00:00 2024-02-29 00:00:00 +2025-02-28 00:00:00 2023-12-01 00:00:00 +== parser +'2024-03-15' -> 2024-03-15T00:00:00 None +'2024-03-15T10:30:00' -> 2024-03-15T10:30:00 None +'2024-03-15 10:30:00.123456' -> 2024-03-15T10:30:00.123456 None +'2024-03-15T10:30:00Z' -> 2024-03-15T10:30:00+00:00 tzutc() +'2024-03-15T10:30:00+02:00' -> 2024-03-15T10:30:00+02:00 tzoffset(None, 7200) +'2024-03-15T10:30:00-0530' -> 2024-03-15T10:30:00-05:30 tzoffset(None, -19800) +'2024-03-15T10:30' -> 2024-03-15T10:30:00 None +'20240315' -> 2024-03-15T00:00:00 None +'20240315T103000' -> 2024-03-15T10:30:00 None +'March 15, 2024' -> 2024-03-15T00:00:00 None +'15 March 2024' -> 2024-03-15T00:00:00 None +'Mar 15 2024 3:45 PM' -> 2024-03-15T15:45:00 None +'3/15/2024' -> 2024-03-15T00:00:00 None +'2024/03/15' -> 2024-03-15T00:00:00 None +'15.03.2024' -> 2024-03-15T00:00:00 None +'Fri, 15 Mar 2024 10:30:00 GMT' -> 2024-03-15T10:30:00+00:00 tzutc() +'10:30' -> 2000-01-01T10:30:00 None +'10:30:45' -> 2000-01-01T10:30:45 None +'3pm' -> 2000-01-01T15:00:00 None +'3 PM' -> 2000-01-01T15:00:00 None +'12 AM' -> 2000-01-01T00:00:00 None +'12:15 pm' -> 2000-01-01T12:15:00 None +'15 Mar' -> 2000-03-15T00:00:00 None +'Mar 2024' -> 2024-03-01T00:00:00 None +'2024' -> 2024-01-01T00:00:00 None +'Friday' -> 2000-01-07T00:00:00 None +'Sat 3rd March 2024' -> 2024-03-03T00:00:00 None +'1st January 2025' -> 2025-01-01T00:00:00 None +'2024-03-15 10:30 PM' -> 2024-03-15T22:30:00 None +'Dec 25 10:00' -> 2000-12-25T10:00:00 None +'2024-03-15T10:30:00 EST' -> 2024-03-15T10:30:00 None +2003-01-02 00:00:00 2003-02-01 00:00:00 2001-02-03 00:00:00 2001-03-02 00:00:00 +2024-02-13 00:00:00 2024-02-13 00:00:00 2024-03-02 00:00:00 +2024-03-15 10:30:00 +1:00:00 -1 day, 20:30:00 +tzoffset('EST', -18000) +tzoffset('PST', -28800) +tzoffset('XYZ', 3600) +skip +2021-01-05 16:00:00 +2024-02-10 05:06:07 +1999-12-31 10:00:07 +1999-02-28 00:00:00 +'not a date' ValueError True +'' ValueError True +'2024-13-45' ValueError True +'25:00' ValueError True +'Feb 30 2024' ValueError True +bad type raised TypeError +Unknown string format: x +500000 250000 +== isoparse +'2024' -> 2024-01-01T00:00:00 None +'2024-03' -> 2024-03-01T00:00:00 None +'2024-03-15' -> 2024-03-15T00:00:00 None +'20240315' -> 2024-03-15T00:00:00 None +'2024-W11-5' -> 2024-03-15T00:00:00 None +'2024W115' -> 2024-03-15T00:00:00 None +'2024-W11' -> 2024-03-11T00:00:00 None +'2024-03-15T10' -> 2024-03-15T10:00:00 None +'2024-03-15T10:30' -> 2024-03-15T10:30:00 None +'2024-03-15T10:30:45' -> 2024-03-15T10:30:45 None +'2024-03-15T10:30:45.123456' -> 2024-03-15T10:30:45.123456 None +'2024-03-15T10:30:45.1234567' -> 2024-03-15T10:30:45.123456 None +'2024-03-15T10:30:45Z' -> 2024-03-15T10:30:45+00:00 tzutc() +'2024-03-15T10:30:45+05:30' -> 2024-03-15T10:30:45+05:30 tzoffset(None, 19800) +'2024-03-15T10:30:45-0800' -> 2024-03-15T10:30:45-08:00 tzoffset(None, -28800) +'2024-03-15T10:30:45+00:00' -> 2024-03-15T10:30:45+00:00 tzutc() +'2024-03-15T103045' -> 2024-03-15T10:30:45 None +'2024-03-15T24:00' -> 2024-03-16T00:00:00 None +'2024-03-15 10:30' -> 2024-03-15T10:30:00 None +'2024-03-15T10:30:45+05' -> 2024-03-15T10:30:45+05:00 tzoffset(None, 18000) +'20240315T103045Z' -> 2024-03-15T10:30:45+00:00 tzutc() +'2024-3-15' ValueError +'2024-03-15T25:00' ValueError +'2024-13-01' ValueError +'hello' ValueError +'2024-03-15T10:30:45+25:00' ValueError +'' ValueError +'2024-W54' ValueError +iso type raised TypeError +== tz +tzutc() tzutc() True True 0:00:00 UTC 0:00:00 +tzoffset('IST', 19800) tzoffset('IST', 19800) 5:30:00 IST True +tzoffset(None, -3600) True True True True +2024-03-15 12:00:00+05:30 2024-03-15T12:00:00+05:30 5:30:00 IST 2024-03-15 06:30:00+00:00 2024-03-15 01:30:00-05:00 +2024-03-15 17:30:00+05:30 2024-03-16 +True -1 day, 18:30:00 +1704067200.0 1704047400.0 +True True tzstr('UTC+3') 3:00:00 -1 day, 18:30:00 None +tzlocal() True +2024-03-15T12:00:00+03:00 +True False +True True +1970-01-02 00:00:00+00:00 1970-01-02 05:30:00+05:30 2024-06-01 12:00:00+05:30 +== rrule +[datetime.datetime(2024, 1, 31, 9, 30), datetime.datetime(2024, 2, 1, 9, 30), datetime.datetime(2024, 2, 2, 9, 30), datetime.datetime(2024, 2, 3, 9, 30)] +[datetime.datetime(2024, 1, 31, 9, 30), datetime.datetime(2024, 2, 3, 9, 30), datetime.datetime(2024, 2, 6, 9, 30), datetime.datetime(2024, 2, 9, 9, 30)] +[datetime.datetime(2024, 1, 31, 9, 30), datetime.datetime(2024, 2, 7, 9, 30), datetime.datetime(2024, 2, 14, 9, 30), datetime.datetime(2024, 2, 21, 9, 30)] +[datetime.datetime(2024, 2, 2, 9, 30), datetime.datetime(2024, 2, 12, 9, 30), datetime.datetime(2024, 2, 16, 9, 30)] +[datetime.datetime(2024, 1, 31, 9, 30), datetime.datetime(2024, 3, 31, 9, 30), datetime.datetime(2024, 5, 31, 9, 30), datetime.datetime(2024, 7, 31, 9, 30), datetime.datetime(2024, 8, 31, 9, 30)] +[datetime.datetime(2024, 1, 31, 9, 30), datetime.datetime(2024, 2, 1, 9, 30), datetime.datetime(2024, 2, 15, 9, 30), datetime.datetime(2024, 2, 29, 9, 30)] +[datetime.datetime(2024, 2, 2, 9, 30), datetime.datetime(2024, 3, 1, 9, 30), datetime.datetime(2024, 4, 5, 9, 30), datetime.datetime(2024, 5, 3, 9, 30)] +[datetime.datetime(2024, 2, 13, 9, 30), datetime.datetime(2024, 2, 26, 9, 30), datetime.datetime(2024, 3, 12, 9, 30), datetime.datetime(2024, 3, 25, 9, 30)] +[datetime.datetime(2024, 1, 31, 9, 30), datetime.datetime(2024, 2, 29, 9, 30), datetime.datetime(2024, 3, 29, 9, 30), datetime.datetime(2024, 4, 30, 9, 30), datetime.datetime(2024, 5, 31, 9, 30), datetime.datetime(2024, 6, 28, 9, 30)] +[datetime.datetime(2024, 3, 9, 9, 30), datetime.datetime(2024, 5, 11, 9, 30), datetime.datetime(2024, 7, 13, 9, 30)] +[datetime.datetime(2024, 1, 31, 9, 30), datetime.datetime(2025, 1, 31, 9, 30), datetime.datetime(2026, 1, 31, 9, 30)] +[datetime.datetime(2024, 2, 29, 0, 0), datetime.datetime(2028, 2, 29, 0, 0), datetime.datetime(2032, 2, 29, 0, 0)] +[datetime.datetime(2024, 2, 10, 9, 30), datetime.datetime(2024, 6, 10, 9, 30), datetime.datetime(2025, 2, 10, 9, 30), datetime.datetime(2025, 6, 10, 9, 30)] +[datetime.datetime(2024, 11, 28, 9, 30), datetime.datetime(2025, 11, 27, 9, 30), datetime.datetime(2026, 11, 26, 9, 30)] +[datetime.datetime(2024, 1, 31, 9, 30), datetime.datetime(2029, 1, 31, 9, 30), datetime.datetime(2034, 1, 31, 9, 30)] +[datetime.datetime(2024, 12, 31, 9, 30), datetime.datetime(2025, 1, 1, 9, 30), datetime.datetime(2025, 12, 31, 9, 30)] +[datetime.datetime(2024, 5, 13, 9, 30), datetime.datetime(2025, 5, 12, 9, 30), datetime.datetime(2026, 5, 11, 9, 30)] +[datetime.datetime(2024, 1, 31, 9, 30), datetime.datetime(2024, 2, 1, 9, 30), datetime.datetime(2024, 2, 2, 9, 30), datetime.datetime(2024, 2, 3, 9, 30)] +[datetime.datetime(2024, 1, 31, 9, 30), datetime.datetime(2024, 2, 1, 9, 30), datetime.datetime(2024, 2, 2, 9, 30)] +[datetime.datetime(2024, 1, 31, 18, 0), datetime.datetime(2024, 1, 31, 18, 30), datetime.datetime(2024, 2, 1, 8, 0)] +[datetime.datetime(2024, 2, 3, 9, 30), datetime.datetime(2024, 2, 4, 9, 30), datetime.datetime(2024, 2, 10, 9, 30), datetime.datetime(2024, 2, 11, 9, 30), datetime.datetime(2024, 2, 17, 9, 30)] +[datetime.datetime(2024, 1, 31, 9, 30), datetime.datetime(2024, 1, 31, 10, 30), datetime.datetime(2024, 1, 31, 11, 30), datetime.datetime(2024, 1, 31, 12, 30)] +[datetime.datetime(2024, 1, 31, 9, 30), datetime.datetime(2024, 1, 31, 16, 30), datetime.datetime(2024, 1, 31, 23, 30), datetime.datetime(2024, 2, 1, 6, 30), datetime.datetime(2024, 2, 1, 13, 30)] +[datetime.datetime(2024, 1, 31, 12, 0), datetime.datetime(2024, 1, 31, 12, 45), datetime.datetime(2024, 1, 31, 13, 0), datetime.datetime(2024, 1, 31, 13, 45)] +[datetime.datetime(2024, 1, 31, 9, 30), datetime.datetime(2024, 1, 31, 9, 50), datetime.datetime(2024, 1, 31, 10, 10)] +[datetime.datetime(2024, 1, 31, 10, 0), datetime.datetime(2024, 1, 31, 10, 15), datetime.datetime(2024, 1, 31, 11, 0), datetime.datetime(2024, 1, 31, 11, 15)] +[datetime.datetime(2024, 1, 31, 9, 30), datetime.datetime(2024, 1, 31, 9, 30, 45), datetime.datetime(2024, 1, 31, 9, 31, 30)] +[datetime.datetime(2024, 1, 31, 9, 30), datetime.datetime(2024, 1, 31, 9, 30, 30), datetime.datetime(2024, 1, 31, 9, 31)] +[datetime.datetime(2024, 1, 31, 9, 30), datetime.datetime(2024, 3, 31, 9, 30), datetime.datetime(2024, 5, 31, 9, 30)] +[datetime.datetime(2024, 2, 4, 9, 30), datetime.datetime(2024, 2, 5, 9, 30), datetime.datetime(2024, 2, 11, 9, 30)] +[datetime.datetime(2024, 2, 5, 9, 30), datetime.datetime(2024, 2, 12, 9, 30), datetime.datetime(2024, 2, 19, 9, 30)] +[datetime.datetime(2024, 5, 1, 0, 0), datetime.datetime(2024, 5, 2, 0, 0)] +2024-01-31 09:30:00 2024-02-09 09:30:00 [datetime.datetime(2024, 2, 2, 9, 30), datetime.datetime(2024, 2, 5, 9, 30), datetime.datetime(2024, 2, 7, 9, 30)] [datetime.datetime(2024, 2, 5, 9, 30), datetime.datetime(2024, 2, 9, 9, 30), datetime.datetime(2024, 2, 14, 9, 30)] +2024-02-12 09:30:00 2024-02-12 09:30:00 2024-02-09 09:30:00 +2024-02-09 09:30:00 2024-02-07 09:30:00 2024-02-09 09:30:00 +None [datetime.datetime(2024, 2, 2, 9, 30), datetime.datetime(2024, 2, 5, 9, 30), datetime.datetime(2024, 2, 7, 9, 30)] [datetime.datetime(2024, 2, 5, 9, 30), datetime.datetime(2024, 2, 7, 9, 30), datetime.datetime(2024, 2, 9, 9, 30)] +True False +10 10 10 +5 +DTSTART:20240131T093000 +RRULE:FREQ=WEEKLY;INTERVAL=2;COUNT=4;BYDAY=MO,FR +DTSTART:20240131T093000 +RRULE:FREQ=MONTHLY;WKST=SU;UNTIL=20250101T000000;BYMONTHDAY=1,-1 +DTSTART:20240131T093000 +RRULE:FREQ=YEARLY;COUNT=2;BYSETPOS=1;BYMONTH=1,2;BYHOUR=9 +[datetime.datetime(2024, 1, 31, 9, 30), datetime.datetime(2024, 2, 1, 9, 30)] +bad setpos raised ValueError +['2024-03-01T08:00:00+01:00', '2024-03-02T08:00:00+01:00'] +[datetime.datetime(9998, 1, 1, 0, 0), datetime.datetime(9999, 1, 1, 0, 0)] +== rruleset +[datetime.datetime(2024, 2, 1, 9, 30), datetime.datetime(2024, 2, 3, 9, 30), datetime.datetime(2024, 2, 4, 9, 30), datetime.datetime(2024, 2, 5, 9, 30), datetime.datetime(2024, 2, 20, 12, 0)] +2024-02-01 09:30:00 2024-02-03 09:30:00 2024-02-05 09:30:00 5 +[datetime.datetime(2024, 2, 5, 9, 30), datetime.datetime(2024, 2, 12, 9, 30), datetime.datetime(2024, 2, 19, 9, 30)] True +[] +== rrulestr +[datetime.datetime(2024, 1, 31, 9, 30), datetime.datetime(2024, 2, 1, 9, 30), datetime.datetime(2024, 2, 2, 9, 30)] +[datetime.datetime(2024, 2, 2, 9, 30), datetime.datetime(2024, 2, 5, 9, 30), datetime.datetime(2024, 2, 9, 9, 30), datetime.datetime(2024, 2, 12, 9, 30)] +[datetime.datetime(2024, 1, 9, 8, 0), datetime.datetime(2024, 2, 13, 8, 0), datetime.datetime(2024, 3, 12, 8, 0)] +[datetime.datetime(2024, 1, 26, 8, 0), datetime.datetime(2024, 3, 29, 8, 0), datetime.datetime(2024, 5, 31, 8, 0)] +[datetime.datetime(2024, 1, 1, 8, 0), datetime.datetime(2024, 1, 2, 8, 0), datetime.datetime(2024, 1, 3, 8, 0), datetime.datetime(2024, 1, 4, 8, 0)] +[datetime.datetime(2024, 1, 1, 8, 0), datetime.datetime(2024, 1, 3, 8, 0), datetime.datetime(2024, 1, 5, 8, 0)] +rrule rruleset +rrulestr bad raised ValueError diff --git a/maven/python-transpiler/src/test/resources/conformance/dateutil_.py b/maven/python-transpiler/src/test/resources/conformance/dateutil_.py new file mode 100644 index 00000000000..279a238396f --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/dateutil_.py @@ -0,0 +1,221 @@ +import datetime +from datetime import date, datetime as dt, timedelta, timezone + +from dateutil import tz +from dateutil.parser import parse, isoparse, ParserError +from dateutil.relativedelta import relativedelta, MO, TU, WE, TH, FR, SA, SU +from dateutil.rrule import (rrule, rruleset, rrulestr, YEARLY, MONTHLY, WEEKLY, DAILY, HOURLY, MINUTELY, + SECONDLY) + + +def show(label, fn): + try: + print(label, "->", fn()) + except Exception as e: + print(label, "raised", type(e).__name__) + + +print("== relativedelta") +base = dt(2024, 1, 31, 10, 30, 15) +print(base + relativedelta(months=1)) +print(base + relativedelta(months=+13, days=-2)) +print(base + relativedelta(years=1, month=2, day=29)) +print(base - relativedelta(months=2)) +print(base + relativedelta(hours=30, minutes=-45)) +print(base + relativedelta(weeks=2)) +print(base + relativedelta(day=1)) +print(base + relativedelta(day=31, months=+1)) +print(base + relativedelta(hour=0, minute=0, second=0)) +print(date(2024, 3, 15) + relativedelta(days=10)) +print(date(2024, 3, 15) + relativedelta(hours=5)) +print(base + relativedelta(weekday=MO)) +print(base + relativedelta(weekday=MO(+2))) +print(base + relativedelta(weekday=FR(-1))) +print(dt(2024, 3, 15) + relativedelta(weekday=FR)) +print(dt(2024, 3, 1) + relativedelta(day=31, weekday=FR(-1))) +print(dt(2024, 1, 1) + relativedelta(yearday=60), dt(2023, 1, 1) + relativedelta(nlyearday=60)) +print(dt(2023, 1, 1) + relativedelta(yearday=60)) +print(date(2024, 2, 29) + relativedelta(years=1), date(2024, 2, 29) + relativedelta(years=-4)) +print(dt(2024, 5, 1) + timedelta(days=3) + relativedelta(months=1)) +print(relativedelta(months=1) + dt(2024, 1, 15)) +print(relativedelta(dt(2024, 3, 15), dt(2023, 1, 10))) +print(relativedelta(dt(2023, 1, 10), dt(2024, 3, 15))) +print(relativedelta(dt(2024, 3, 31, 12, 0, 1, 500), dt(2024, 2, 29, 1, 2, 3, 4))) +print(relativedelta(date(2024, 12, 25), date(2000, 2, 29))) +r1 = relativedelta(years=1, months=2, days=3, hours=4) +r2 = relativedelta(months=5, days=-10, seconds=3700) +print(r1, r2) +print(r1 + r2, r1 - r2, -r1, abs(relativedelta(days=-3, hours=-2))) +print(r1 * 2, 2 * r1, r1 / 2) +print(relativedelta(months=14, days=40, hours=50, minutes=130, seconds=-4000)) +print(relativedelta(day=3, month=4) + relativedelta(years=1), relativedelta(day=3) + relativedelta(day=4)) +print(r1 == relativedelta(years=1, months=2, days=3, hours=4), r1 != r2, r1 == 5) +print(bool(relativedelta()), bool(relativedelta(days=1)), bool(relativedelta(day=1))) +print(relativedelta(days=15).weeks, relativedelta(weeks=3).days, relativedelta(days=-10).weeks) +print(relativedelta(days=1.5).normalized(), relativedelta(hours=1.25).normalized()) +print(relativedelta(weekday=MO(+1)) == relativedelta(weekday=MO), relativedelta(weekday=MO(2)) == relativedelta(weekday=MO)) +print(repr(MO), repr(FR(-1)), repr(SU(+3)), MO == MO(+1), MO(+1) == MO(1)) +show("frac months", lambda: relativedelta(months=1.5)) +show("bad yday", lambda: relativedelta(yearday=400)) +show("weekday0", lambda: MO(0)) +show("add int", lambda: relativedelta(days=1) + 5) +print(relativedelta(years=-1, months=-3, days=-4), relativedelta(microseconds=1500000)) +print(hash(relativedelta(days=1)) == hash(relativedelta(days=1))) +print(relativedelta(dt(2024, 1, 31), dt(2023, 12, 31)), relativedelta(dt(2024, 2, 29), dt(2023, 2, 28))) +print(dt(2020, 2, 29) + relativedelta(years=1), dt(2020, 2, 29) + relativedelta(years=4)) +print(dt(2024, 12, 31) + relativedelta(months=2), dt(2024, 1, 1) - relativedelta(months=1)) + +print("== parser") +D = dt(2000, 1, 1) +for s in ["2024-03-15", "2024-03-15T10:30:00", "2024-03-15 10:30:00.123456", "2024-03-15T10:30:00Z", + "2024-03-15T10:30:00+02:00", "2024-03-15T10:30:00-0530", "2024-03-15T10:30", "20240315", "20240315T103000", + "March 15, 2024", "15 March 2024", "Mar 15 2024 3:45 PM", "3/15/2024", "2024/03/15", "15.03.2024", + "Fri, 15 Mar 2024 10:30:00 GMT", "10:30", "10:30:45", "3pm", "3 PM", "12 AM", "12:15 pm", "15 Mar", + "Mar 2024", "2024", "Friday", "Sat 3rd March 2024", "1st January 2025", "2024-03-15 10:30 PM", + "Dec 25 10:00", "2024-03-15T10:30:00 EST"]: + try: + r = parse(s, default=D) + print(repr(s), "->", r.isoformat(), repr(r.tzinfo)) + except Exception as e: + print(repr(s), "raised", type(e).__name__) +print(parse("01/02/03", default=D), parse("01/02/03", dayfirst=True, default=D), + parse("01/02/03", yearfirst=True, default=D), parse("01/02/03", dayfirst=True, yearfirst=True, default=D)) +print(parse("13/02/2024", default=D), parse("02/13/2024", default=D), parse("2/3/2024", dayfirst=True, default=D)) +print(parse("2024-03-15 10:30 UTC", ignoretz=True, default=D)) +print(parse("10:30 +0100", default=D).utcoffset(), parse("10:30 -03:30", default=D).utcoffset()) +print(repr(parse("10:30 EST", default=D, tzinfos={"EST": -18000}).tzinfo)) +print(repr(parse("10:30 PST", default=D, tzinfos={"PST": tz.tzoffset("PST", -28800)}).tzinfo)) +print(repr(parse("10:30 XYZ", default=D, tzinfos=lambda name, offset: 3600).tzinfo)) +print(parse("Feb 30 2023 foo", default=D, fuzzy=True) if False else "skip") +print(parse("Today is January 5th, 2021 at 4pm", default=D, fuzzy=True)) +print(parse("2024-02-10", default=dt(1999, 12, 31, 5, 6, 7))) +print(parse("10:00", default=dt(1999, 12, 31, 5, 6, 7))) +print(parse("Feb", default=dt(1999, 12, 31))) +for s in ["not a date", "", "2024-13-45", "25:00", "Feb 30 2024"]: + try: + parse(s, default=D) + print(repr(s), "parsed") + except ValueError as e: + print(repr(s), "ValueError", isinstance(e, ParserError)) +show("bad type", lambda: parse(5)) +print(str(ParserError("Unknown string format: %s", "x"))) +print(parse("2024-03-15T10:30:00.5", default=D).microsecond, parse("2024-03-15T10:30:00,25", default=D).microsecond) + +print("== isoparse") +for s in ["2024", "2024-03", "2024-03-15", "20240315", "2024-W11-5", "2024W115", "2024-W11", "2024-03-15T10", + "2024-03-15T10:30", "2024-03-15T10:30:45", "2024-03-15T10:30:45.123456", "2024-03-15T10:30:45.1234567", + "2024-03-15T10:30:45Z", "2024-03-15T10:30:45+05:30", "2024-03-15T10:30:45-0800", "2024-03-15T10:30:45+00:00", + "2024-03-15T103045", "2024-03-15T24:00", "2024-03-15 10:30", "2024-03-15T10:30:45+05", "20240315T103045Z"]: + try: + r = isoparse(s) + print(repr(s), "->", r.isoformat(), repr(r.tzinfo)) + except Exception as e: + print(repr(s), "raised", type(e).__name__) +for s in ["2024-3-15", "2024-03-15T25:00", "2024-13-01", "hello", "2024-03-15T10:30:45+25:00", "", "2024-W54"]: + try: + isoparse(s) + print(repr(s), "parsed") + except ValueError: + print(repr(s), "ValueError") +show("iso type", lambda: isoparse(5)) + +print("== tz") +u = tz.tzutc() +print(u, repr(u), u is tz.tzutc(), u is tz.UTC, u.utcoffset(None), u.tzname(None), u.dst(None)) +o = tz.tzoffset("IST", 19800) +print(o, repr(o), o.utcoffset(None), o.tzname(None), o is tz.tzoffset("IST", timedelta(seconds=19800))) +print(repr(tz.tzoffset(None, -3600)), tz.tzoffset(None, 0) == tz.tzutc(), tz.tzutc() == tz.tzoffset(None, 0), + tz.tzoffset(None, 60) == tz.tzoffset("x", 60), tz.tzoffset(None, 60) != tz.tzoffset(None, 61)) +d1 = dt(2024, 3, 15, 12, 0, tzinfo=o) +print(d1, d1.isoformat(), d1.utcoffset(), d1.tzname(), d1.astimezone(u), d1.astimezone(tz.tzoffset(None, -18000))) +print(dt(2024, 3, 15, 12, 0, tzinfo=u).astimezone(o), dt(2024, 3, 15, 23, 0, tzinfo=u).astimezone(o).date()) +print(d1 == dt(2024, 3, 15, 6, 30, tzinfo=u), d1 - dt(2024, 3, 15, 12, 0, tzinfo=u)) +print(dt(2024, 1, 1, tzinfo=u).timestamp(), dt(2024, 1, 1, tzinfo=o).timestamp()) +print(tz.gettz("UTC") is not None, tz.gettz("GMT") is not None, repr(tz.gettz("UTC+3")), tz.gettz("UTC+3").utcoffset(None), + tz.gettz("GMT-05:30").utcoffset(None), tz.gettz("No/Such_Zone")) +print(repr(tz.tzlocal()), isinstance(tz.tzlocal(), datetime.tzinfo)) +print(dt(2024, 3, 15, 12, tzinfo=tz.gettz("UTC+3")).isoformat()) +print(tz.datetime_exists(dt(2024, 3, 15, 12, tzinfo=o)), tz.datetime_ambiguous(dt(2024, 3, 15, 12, tzinfo=o))) +print(dt(2024, 3, 15, 12, tzinfo=timezone.utc) == dt(2024, 3, 15, 12, tzinfo=u), dt.now(u).tzinfo is u) +print(dt.fromtimestamp(86400, u), dt.fromtimestamp(86400, o), dt(2024, 6, 1, 12, tzinfo=u).replace(tzinfo=o)) + +print("== rrule") +S = dt(2024, 1, 31, 9, 30) +print(list(rrule(DAILY, dtstart=S, count=4))) +print(list(rrule(DAILY, dtstart=S, interval=3, count=4))) +print(list(rrule(WEEKLY, dtstart=S, count=4))) +print(list(rrule(WEEKLY, dtstart=S, interval=2, count=3, byweekday=(MO, FR)))) +print(list(rrule(MONTHLY, dtstart=S, count=5))) +print(list(rrule(MONTHLY, dtstart=S, count=4, bymonthday=(1, 15, -1)))) +print(list(rrule(MONTHLY, dtstart=S, count=4, byweekday=FR(+1)))) +print(list(rrule(MONTHLY, dtstart=S, count=4, byweekday=(MO(-1), TU(2))))) +print(list(rrule(MONTHLY, dtstart=S, count=6, byweekday=(MO, TU, WE, TH, FR), bysetpos=-1))) +print(list(rrule(MONTHLY, dtstart=S, count=3, byweekday=SA(+2), interval=2))) +print(list(rrule(YEARLY, dtstart=S, count=3))) +print(list(rrule(YEARLY, dtstart=dt(2024, 2, 29), count=3))) +print(list(rrule(YEARLY, dtstart=S, count=4, bymonth=(2, 6), bymonthday=10))) +print(list(rrule(YEARLY, dtstart=S, count=3, bymonth=11, byweekday=TH(+4)))) +print(list(rrule(YEARLY, dtstart=S, count=3, interval=5))) +print(list(rrule(YEARLY, dtstart=S, count=3, byyearday=(1, -1)))) +print(list(rrule(YEARLY, dtstart=S, count=3, byweekno=20, byweekday=MO))) +print(list(rrule(DAILY, dtstart=S, until=dt(2024, 2, 3, 9, 30)))) +print(list(rrule(DAILY, dtstart=S, until=dt(2024, 2, 3, 9, 29)))) +print(list(rrule(DAILY, dtstart=S, count=3, byhour=(8, 18), byminute=(0, 30)))) +print(list(rrule(DAILY, dtstart=S, count=5, bymonth=2, byweekday=(SA, SU)))) +print(list(rrule(HOURLY, dtstart=S, count=4))) +print(list(rrule(HOURLY, dtstart=S, count=5, interval=7))) +print(list(rrule(HOURLY, dtstart=S, count=4, byhour=(12, 13), byminute=(0, 45)))) +print(list(rrule(MINUTELY, dtstart=S, count=3, interval=20))) +print(list(rrule(MINUTELY, dtstart=S, count=4, byminute=(0, 15)))) +print(list(rrule(SECONDLY, dtstart=S, count=3, interval=45))) +print(list(rrule(SECONDLY, dtstart=S, count=3, bysecond=(0, 30)))) +print(list(rrule(MONTHLY, dtstart=S, count=3, bymonthday=31))) +print(list(rrule(WEEKLY, dtstart=S, count=3, wkst=SU, byweekday=(SU, MO)))) +print(list(rrule(DAILY, dtstart=S, count=3, byweekday=0))) +print(list(rrule(DAILY, dtstart=date(2024, 5, 1), count=2))) +r = rrule(WEEKLY, dtstart=S, byweekday=(MO, WE, FR)) +print(r[0], r[4], r[-1] if False else "", r[1:4], r[::3] if False else "", r[2:8:2]) +print(r.after(dt(2024, 2, 10)), r.after(dt(2024, 2, 9, 9, 30)), r.after(dt(2024, 2, 9, 9, 30), inc=True)) +print(r.before(dt(2024, 2, 10)), r.before(dt(2024, 2, 9, 9, 30)), r.before(dt(2024, 2, 9, 9, 30), inc=True)) +print(r.before(dt(2024, 1, 1)), r.between(dt(2024, 2, 1), dt(2024, 2, 9)), + r.between(dt(2024, 2, 5, 9, 30), dt(2024, 2, 9, 9, 30), inc=True)) +print(dt(2024, 2, 5, 9, 30) in r, dt(2024, 2, 6, 9, 30) in r) +rc = rrule(DAILY, dtstart=S, count=10, bymonth=(1, 2)) +print(rc.count(), len(list(rc)), rc.count()) +print(rrule(DAILY, dtstart=S, until=dt(2024, 2, 5)).count()) +print(str(rrule(WEEKLY, dtstart=S, count=4, byweekday=(MO, FR), interval=2))) +print(str(rrule(MONTHLY, dtstart=S, until=dt(2025, 1, 1), bymonthday=(1, -1), wkst=SU))) +print(str(rrule(YEARLY, dtstart=S, bymonth=(1, 2), byhour=9, bysetpos=1, count=2))) +print(list(rrule(DAILY, dtstart=S, count=3).replace(count=2))) +show("bad setpos", lambda: rrule(DAILY, bysetpos=0)) +tzd = dt(2024, 3, 1, 8, 0, tzinfo=tz.tzoffset(None, 3600)) +print([x.isoformat() for x in rrule(DAILY, dtstart=tzd, count=2)]) +far = rrule(YEARLY, dtstart=dt(9998, 1, 1)) +print(list(far)) + +print("== rruleset") +rs = rruleset() +rs.rrule(rrule(DAILY, dtstart=S, count=6)) +rs.exdate(dt(2024, 2, 2, 9, 30)) +rs.rdate(dt(2024, 2, 20, 12, 0)) +rs.rdate(dt(2024, 2, 1, 9, 30)) +rs.exrule(rrule(DAILY, dtstart=S, count=1)) +print(list(rs)) +print(rs[0], rs[-1] if False else "", rs.after(dt(2024, 2, 3)), rs.before(dt(2024, 2, 10)), rs.count()) +rs2 = rruleset() +rs2.rrule(rrule(WEEKLY, dtstart=S, byweekday=MO, count=3)) +rs2.rrule(rrule(WEEKLY, dtstart=S, byweekday=(MO, TU), count=3)) +rs2.exrule(rrule(WEEKLY, dtstart=S, byweekday=TU, count=1)) +print(list(rs2), dt(2024, 2, 5, 9, 30) in rs2) +print(list(rruleset())) + +print("== rrulestr") +print(list(rrulestr("FREQ=DAILY;COUNT=3", dtstart=S))) +print(list(rrulestr("RRULE:FREQ=WEEKLY;BYDAY=MO,FR;COUNT=4", dtstart=S))) +print(list(rrulestr("DTSTART:20240101T080000\nRRULE:FREQ=MONTHLY;BYDAY=2TU;COUNT=3"))) +print(list(rrulestr("DTSTART:20240101T080000\nRRULE:FREQ=MONTHLY;BYDAY=-1FR;INTERVAL=2;COUNT=3"))) +print(list(rrulestr("DTSTART:20240101T080000\nRRULE:FREQ=DAILY;UNTIL=20240104T080000"))) +print(list(rrulestr("DTSTART:20240101T080000\nRRULE:FREQ=DAILY;COUNT=5\nEXDATE:20240102T080000,20240104T080000"))) +print(type(rrulestr("FREQ=DAILY;COUNT=2", dtstart=S)).__name__, + type(rrulestr("FREQ=DAILY;COUNT=2", dtstart=S, forceset=True)).__name__) +show("rrulestr bad", lambda: rrulestr("FREQ=DAILY;BOGUS=1", dtstart=S)) From 0eed8b72dbe9d6d707f5c63a2f64a9370020b886 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:09:15 +0200 Subject: [PATCH 009/112] Python: pydantic package Co-Authored-By: Claude Opus 5.5 --- .../python/site/pydantic/__init__.py | 2175 +++++++++++++++++ .../python/site/pydantic/dataclasses.py | 37 + .../test/resources/conformance/pydantic_.out | 162 ++ .../test/resources/conformance/pydantic_.py | 514 ++++ .../conformance/pydantic_dataclass.out | 3 + .../conformance/pydantic_dataclass.py | 20 + 6 files changed, 2911 insertions(+) create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/pydantic/__init__.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/pydantic/dataclasses.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/pydantic_.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/pydantic_.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/pydantic_dataclass.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/pydantic_dataclass.py diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/pydantic/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/pydantic/__init__.py new file mode 100644 index 00000000000..b72213e80d2 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/pydantic/__init__.py @@ -0,0 +1,2175 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Data validation from type annotations: the commonly used pydantic v2 API. + +`BaseModel` reads its fields from class annotations, validates and coerces +input the way pydantic's default lax mode does, and reports failures as a +`ValidationError` whose `errors()` carry pydantic's error types, locations and +messages. Also provided: `Field` constraints, `ConfigDict`, `field_validator`, +`model_validator`, `computed_field`, `field_serializer`, `RootModel`, +`TypeAdapter`, `SecretStr`, the constrained-type aliases and a basic +`model_json_schema`. Written from the public documentation and observed +behaviour; no validation core is compiled, so everything is plain Python. +""" + +import copy as _copy +import datetime as _dt +import decimal as _decimal +import enum as _enum +import json as _json +import re as _re +import sys as _sys +import typing as _typing +import uuid as _uuid + +from typing import Any, Union, Annotated, Literal + +VERSION = "2.14.1" +_ERR_VERSION = "2.14" + +_NoneType = type(None) + + +class _Undefined: + def __repr__(self): + return "PydanticUndefined" + + def __bool__(self): + return False + + +PydanticUndefined = _Undefined() +_BAD = _Undefined() + + +# -------------------------------------------------------------------------- +# errors +# -------------------------------------------------------------------------- + +_MSG = { + "missing": "Field required", + "int_type": "Input should be a valid integer", + "int_parsing": "Input should be a valid integer, unable to parse string as an integer", + "int_from_float": "Input should be a valid integer, got a number with a fractional part", + "float_type": "Input should be a valid number", + "float_parsing": "Input should be a valid number, unable to parse string as a number", + "string_type": "Input should be a valid string", + "bytes_type": "Input should be a valid bytes", + "bool_type": "Input should be a valid boolean", + "bool_parsing": "Input should be a valid boolean, unable to interpret input", + "none_required": "Input should be None", + "list_type": "Input should be a valid list", + "tuple_type": "Input should be a valid tuple", + "set_type": "Input should be a valid set", + "frozen_set_type": "Input should be a valid frozenset", + "dict_type": "Input should be a valid dictionary", + "extra_forbidden": "Extra inputs are not permitted", + "model_type": "Input should be a valid dictionary or instance of %(class_name)s", + "greater_than": "Input should be greater than %(gt)s", + "greater_than_equal": "Input should be greater than or equal to %(ge)s", + "less_than": "Input should be less than %(lt)s", + "less_than_equal": "Input should be less than or equal to %(le)s", + "multiple_of": "Input should be a multiple of %(multiple_of)s", + "string_pattern_mismatch": "String should match pattern '%(pattern)s'", + "frozen_instance": "Instance is frozen", + "datetime_type": "Input should be a valid datetime", + "date_type": "Input should be a valid date", + "time_type": "Input should be a valid time", + "timedelta_type": "Input should be a valid timedelta", + "datetime_parsing": "Input should be a valid datetime, invalid character in year", + "date_from_datetime_parsing": "Input should be a valid date or datetime, invalid character in year", + "time_parsing": "Input should be in a valid time format, invalid character in hour", + "timedelta_parsing": "Input should be a valid timedelta, invalid digit in duration", + "date_from_datetime_inexact": "Datetimes provided to dates should have zero time - e.g. be exact dates", + "decimal_type": "Decimal input should be an integer, float, string or Decimal object", + "decimal_parsing": "Input should be a valid decimal", + "uuid_type": "UUID input should be a string, bytes or UUID object", + "uuid_parsing": "Input should be a valid UUID, invalid character: expected an optional prefix of `urn:uuid:` followed by [0-9a-fA-F-], found `%(ch)s`", + "is_instance_of": "Input should be an instance of %(class)s", + "json_invalid": "Invalid JSON: %(error)s", + "iterable_type": "Input should be iterable", +} + + +class ValidationError(ValueError): + """Raised when validation fails; `errors()` lists every failure.""" + + def __init__(self, title, errors): + ValueError.__init__(self, title) + self.title = title + self._errors = errors + + def error_count(self): + return len(self._errors) + + def errors(self, include_url=True, include_context=True, include_input=True): + out = [] + for e in self._errors: + d = {"type": e["type"], "loc": e["loc"], "msg": e["msg"]} + if include_input: + d["input"] = e["input"] + if include_context and e.get("ctx"): + d["ctx"] = e["ctx"] + if include_url and e.get("url"): + d["url"] = e["url"] + out.append(d) + return out + + def json(self, indent=None): + errs = [] + for e in self.errors(): + d = dict(e) + d["loc"] = list(d["loc"]) + if "ctx" in d: + d["ctx"] = dict((k, v if isinstance(v, (int, float, str, bool)) or v is None else str(v)) + for k, v in d["ctx"].items()) + d["input"] = _to_jsonable(d["input"]) + errs.append(d) + return _jsonify(errs, indent, 0) + + def __str__(self): + n = len(self._errors) + lines = ["%d validation error%s for %s" % (n, "" if n == 1 else "s", self.title)] + for e in self._errors: + loc = ".".join([str(x) for x in e["loc"]]) + if loc: + lines.append(loc) + lines.append(" %s [type=%s, input_value=%s, input_type=%s]" % ( + e["msg"], e["type"], _short_repr(e["input"]), type(e["input"]).__name__)) + if e.get("url"): + lines.append(" For further information visit " + e["url"]) + return "\n".join(lines) + + def __repr__(self): + return str(self) + + +def _short_repr(v): + r = repr(v) + if len(r) > 50: + r = r[:24] + "..." + r[-23:] + return r + + +def _url(typ): + return "https://errors.pydantic.dev/" + _ERR_VERSION + "/v/" + typ + + +class _Ctx: + def __init__(self, strict=False, jsonmode=False, context=None, from_attributes=False): + self.errs = [] + self.strict = strict + self.json = jsonmode + self.context = context + self.from_attributes = from_attributes + self.cfg = {} + self.cls = None + + +def _E(c, loc, typ, inp, ctx=None, msg=None): + if msg is None: + msg = _MSG[typ] + if ctx: + msg = msg % ctx + d = {"type": typ, "loc": tuple(loc), "msg": msg, "input": inp, "url": _url(typ)} + if ctx: + d["ctx"] = ctx + c.errs.append(d) + return _BAD + + +def _plural(n, word): + return "%d %s%s" % (n, word, "" if n == 1 else "s") + + +# -------------------------------------------------------------------------- +# field declarations +# -------------------------------------------------------------------------- + +_CONSTRAINT_KEYS = ("gt", "ge", "lt", "le", "multiple_of", "min_length", "max_length", "pattern") + + +class FieldInfo: + """What `Field(...)` records about one field.""" + + def __init__(self, default=PydanticUndefined, **kw): + self.annotation = None + self._full = None + self.default = default + self.default_factory = kw.get("default_factory") + self.alias = kw.get("alias") + self.validation_alias = kw.get("validation_alias") + self.serialization_alias = kw.get("serialization_alias") + self.title = kw.get("title") + self.description = kw.get("description") + self.examples = kw.get("examples") + self.exclude = kw.get("exclude") + self.repr = kw.get("repr", True) + self.frozen = kw.get("frozen") + self.strict = kw.get("strict") + self.validate_default = kw.get("validate_default") + self.json_schema_extra = kw.get("json_schema_extra") + self.constraints = {} + for k in _CONSTRAINT_KEYS: + if kw.get(k) is not None: + self.constraints[k] = kw[k] + self.metadata = [] + + def is_required(self): + return self.default is PydanticUndefined and self.default_factory is None + + def get_default(self, call_default_factory=False): + if self.default_factory is not None: + return self.default_factory() if call_default_factory else None + if self.default is PydanticUndefined: + return None + return self.default + + def __repr__(self): + parts = ["annotation=" + _tname(self.annotation)] + if self.is_required(): + parts.append("required=True") + elif self.default_factory is not None: + parts.append("default_factory=" + getattr(self.default_factory, "__name__", "factory")) + else: + parts.append("default=" + repr(self.default)) + if self.alias is not None: + parts.append("alias=" + repr(self.alias)) + if self.description is not None: + parts.append("description=" + repr(self.description)) + return "FieldInfo(" + ", ".join(parts) + ")" + + +def Field(default=PydanticUndefined, **kw): + if default is Ellipsis: + default = PydanticUndefined + if "alias" in kw and kw["alias"] is not None: + kw.setdefault("validation_alias", kw["alias"]) + kw.setdefault("serialization_alias", kw["alias"]) + return FieldInfo(default, **kw) + + +class _PrivateAttr: + def __init__(self, default, factory): + self.default = default + self.factory = factory + + +def PrivateAttr(default=PydanticUndefined, default_factory=None): + return _PrivateAttr(default, default_factory) + + +def ConfigDict(**kw): + return dict(kw) + + +class Strict: + def __init__(self, strict=True): + self.strict = strict + + +class AfterValidator: + def __init__(self, func): + self.func = func + + +class BeforeValidator: + def __init__(self, func): + self.func = func + + +class PlainValidator: + def __init__(self, func): + self.func = func + + +class ValidationInfo: + def __init__(self, c, field_name=None, data=None): + self.context = c.context + self.config = c.cfg + self.mode = "json" if c.json else "python" + self.data = data if data is not None else {} + self.field_name = field_name + + +FieldValidationInfo = ValidationInfo + + +class _Decl: + def __init__(self, kind, func, fields=(), mode="after", extra=None): + self.kind = kind + self.func = func + self.fields = fields + self.mode = mode + self.extra = extra + + +def field_validator(*fields, mode="after", check_fields=None): + def dec(f): + return _Decl("field", f, fields, mode) + return dec + + +def model_validator(*, mode): + def dec(f): + return _Decl("model", f, (), mode) + return dec + + +def field_serializer(*fields, mode="plain", return_type=None, when_used="always", check_fields=None): + def dec(f): + return _Decl("fser", f, fields, mode, when_used) + return dec + + +def model_serializer(f=None, *, mode="plain", when_used="always", return_type=None): + def dec(fn): + return _Decl("mser", fn, (), mode) + if f is not None: + return dec(f) + return dec + + +_COMPUTED = [] + + +def computed_field(func=None, *, alias=None, title=None, description=None, repr=True, return_type=None): + def dec(f): + fget = f.fget if isinstance(f, property) else f + _COMPUTED.append((fget, alias, repr)) + if isinstance(f, property): + return f + return property(f) + if func is not None: + return dec(func) + return dec + + +_ARITY = {} + + +def _invoke(func, args, info): + n = _ARITY.get(func) + if n == 1: + return func(*(args + (info,))) + if n == 0: + return func(*args) + try: + r = func(*args) + _ARITY[func] = 0 + return r + except TypeError as e: + s = str(e) + if "missing" in s and "required positional argument" in s: + _ARITY[func] = 1 + return func(*(args + (info,))) + raise + + +def _guard(c, loc, inp, fn): + """Runs fn(); a validator's ValueError or AssertionError becomes an error.""" + try: + return fn() + except ValidationError as e: + for d in e._errors: + nd = dict(d) + nd["loc"] = tuple(loc) + tuple(d["loc"]) + c.errs.append(nd) + return _BAD + except ValueError as e: + return _E(c, loc, "value_error", inp, {"error": e}, "Value error, " + str(e)) + except AssertionError as e: + s = str(e) + return _E(c, loc, "assertion_error", inp, {"error": e}, + "Assertion failed" + (", " + s if s else "")) + + +# -------------------------------------------------------------------------- +# type helpers +# -------------------------------------------------------------------------- + +def _tname(tp): + if tp is None or tp is _NoneType: + return "None" + if isinstance(tp, type) and getattr(tp, "__origin__", None) is None: + return tp.__name__ + return repr(tp) + + +def _tag(tp): + if tp is _NoneType or tp is None: + return "none" + org = _typing.get_origin(tp) + if org is Annotated: + return _tag(_typing.get_args(tp)[0]) + if org is Literal: + return "literal[" + ",".join([repr(a) for a in _typing.get_args(tp)]) + "]" + if org is not None and isinstance(org, type): + args = _typing.get_args(tp) + if args: + return org.__name__ + "[" + ",".join([_tag(a) if a is not Ellipsis else "..." for a in args]) + "]" + return org.__name__ + if isinstance(tp, type): + return tp.__name__ + return _tname(tp) + + +def _unwrap(tp): + """(base type, list of Annotated metadata) for any depth of Annotated.""" + metas = [] + while _typing.get_origin(tp) is Annotated: + a = _typing.get_args(tp) + metas = list(a[1:]) + metas + tp = a[0] + return tp, metas + + +def _is_classvar(ann): + if isinstance(ann, str): + return ann.startswith("ClassVar") or ann.startswith("typing.ClassVar") + return ann is _typing.ClassVar or _typing.get_origin(ann) is _typing.ClassVar + + +def _resolve_forward(tp, c): + text = tp if isinstance(tp, str) else tp.__forward_arg__ + ns = {} + cls = c.cls + if cls is not None: + m = _sys.modules.get(cls.__module__) + if m is not None: + ns.update(getattr(m, "__dict__", {})) + ns[cls.__name__] = cls + return _typing._resolve_hint(text, ns) + + +# -------------------------------------------------------------------------- +# validation +# -------------------------------------------------------------------------- + +_TRUE = ("true", "t", "yes", "y", "on", "1") +_FALSE = ("false", "f", "no", "n", "off", "0") + + +def _parse_datetime(s): + s = s.strip() + if s.endswith("Z") or s.endswith("z"): + s = s[:-1] + "+00:00" + if len(s) > 10 and s[10] in " _t": + s = s[:10] + "T" + s[11:] + return _dt.datetime.fromisoformat(s) + + +def _parse_duration(s): + m = _re.match(r"^(-)?P(?:(\d+)D)?(?:T(?:(\d+)H)?(?:(\d+)M)?(?:(\d+(?:\.\d+)?)S)?)?$", s.strip()) + if m is None or s.strip() in ("P", "PT"): + raise ValueError("bad duration") + sign = -1 if m.group(1) else 1 + td = _dt.timedelta(days=int(m.group(2) or 0), hours=int(m.group(3) or 0), + minutes=int(m.group(4) or 0), seconds=float(m.group(5) or 0)) + return td * sign + + +def _v(tp, val, loc, c): + if tp is None: + tp = _NoneType + if isinstance(tp, str) or hasattr(tp, "__forward_arg__"): + tp = _resolve_forward(tp, c) + if tp is Any or tp is object: + return val + if hasattr(tp, "__supertype__"): + return _v(tp.__supertype__, val, loc, c) + if hasattr(tp, "__value__") and hasattr(tp, "__type_params__"): + return _v(tp.__value__, val, loc, c) + if isinstance(tp, _typing.TypeVar): + b = getattr(tp, "__bound__", None) + return _v(b, val, loc, c) if b is not None else val + org = _typing.get_origin(tp) + if org is Annotated: + return _vannotated(tp, val, loc, c) + if org is Union: + return _vunion(tp, val, loc, c) + if org is Literal: + for a in _typing.get_args(tp): + if val == a and type(val) is type(a): + return a + for a in _typing.get_args(tp): + if val == a and not isinstance(val, bool) and not isinstance(a, bool): + return a + exp = _join_or([repr(a) for a in _typing.get_args(tp)]) + return _E(c, loc, "literal_error", val, {"expected": exp}, "Input should be " + exp) + if org is not None: + return _vgeneric(org, _typing.get_args(tp), val, loc, c) + if isinstance(tp, ModelMetaclass): + return _vmodel(tp, val, loc, c) + if isinstance(tp, type): + return _vplain(tp, val, loc, c) + return val + + +def _join_or(items): + if len(items) == 1: + return items[0] + return ", ".join(items[:-1]) + " or " + items[-1] + + +def _vannotated(tp, val, loc, c): + base, metas = _unwrap(tp) + cons = {} + before = [] + after = [] + prec = [] + plain = None + strict = None + for m in metas: + if isinstance(m, FieldInfo): + cons.update(m.constraints) + if m.strict is not None: + strict = m.strict + elif isinstance(m, Strict): + strict = m.strict + elif isinstance(m, BeforeValidator): + before.append(m.func) + elif isinstance(m, AfterValidator): + after.append(m.func) + elif isinstance(m, PlainValidator): + plain = m.func + elif isinstance(m, _StringConstraints): + cons.update(m.cons) + if m.strip_whitespace: + prec.append(_strip) + if m.to_lower: + prec.append(_lower) + if m.to_upper: + prec.append(_upper) + if m.strict: + strict = True + info = ValidationInfo(c) + v = val + for f in before: + v = _guard(c, loc, val, lambda f=f, v=v: _invoke(f, (v,), info)) + if v is _BAD: + return _BAD + if plain is not None: + v = _guard(c, loc, val, lambda: _invoke(plain, (v,), info)) + if v is _BAD: + return _BAD + else: + was = c.strict + if strict is not None: + c.strict = strict + v = _v(base, v, loc, c) + c.strict = was + if v is _BAD: + return _BAD + for f in prec: + v = f(v) + if cons: + v = _constrain(v, cons, loc, c) + if v is _BAD: + return _BAD + for f in after: + v = _guard(c, loc, val, lambda f=f, v=v: _invoke(f, (v,), info)) + if v is _BAD: + return _BAD + return v + + +def _strip(v): + return v.strip() if isinstance(v, str) else v + + +def _lower(v): + return v.lower() if isinstance(v, str) else v + + +def _upper(v): + return v.upper() if isinstance(v, str) else v + + +def _vunion(tp, val, loc, c): + members = [a for a in _typing.get_args(tp) if a is not _NoneType and a is not None] + nullable = len(members) != len(_typing.get_args(tp)) + if nullable and val is None: + return None + if len(members) == 1: + return _v(members[0], val, loc, c) + saved = c.errs + was = c.strict + c.strict = True + for m in members: + c.errs = [] + r = _v(m, val, loc, c) + if not c.errs: + c.strict = was + c.errs = saved + return r + c.strict = was + if was: + allerrs = [] + for m in members: + c.errs = [] + _v(m, val, loc + (_tag(m),), c) + allerrs.extend(c.errs) + c.errs = saved + saved.extend(allerrs) + return _BAD + allerrs = [] + for m in members: + c.errs = [] + r = _v(m, val, loc + (_tag(m),), c) + if not c.errs: + c.errs = saved + return r + allerrs.extend(c.errs) + c.errs = saved + saved.extend(allerrs) + return _BAD + + +def _items_of(val, c, kinds): + """The iterable forms lax mode accepts for a sequence-like field.""" + if isinstance(val, kinds): + return list(val) + return None + + +def _vgeneric(org, args, val, loc, c): + nm = getattr(org, "__name__", "") + if org is list or nm in ("Sequence", "MutableSequence", "Collection", "Iterable", "Iterator"): + return _vlist(list, args, val, loc, c) + if org is set or nm in ("Set", "MutableSet", "AbstractSet"): + return _vlist(set, args, val, loc, c) + if org is frozenset: + return _vlist(frozenset, args, val, loc, c) + if org is tuple: + return _vtuple(args, val, loc, c) + if org is dict or nm in ("Mapping", "MutableMapping"): + return _vdict(args, val, loc, c) + if org is type: + return val + if isinstance(org, type): + return _vplain(org, val, loc, c) + return val + + +def _seq_err(kind, val, loc, c): + if kind is list: + t = "list_type" + m = "Input should be a valid array" if c.json else None + elif kind is set: + t = "set_type" + m = "Input should be a valid set" if not c.json else "Input should be a valid set" + elif kind is frozenset: + t = "frozen_set_type" + m = None + else: + t = "tuple_type" + m = "Input should be a valid tuple" if not c.json else "Input should be a valid array" + return _E(c, loc, t, val, None, m) + + +def _vlist(kind, args, val, loc, c): + if isinstance(val, kind) and not c.strict or (c.strict and type(val) is kind): + items = list(val) + elif not c.strict and isinstance(val, (list, tuple, set, frozenset)) and not (kind is list and c.json): + items = list(val) + elif not c.strict and not isinstance(val, (str, bytes, dict)) and hasattr(val, "__iter__") \ + and not isinstance(val, (int, float)): + items = list(val) + else: + return _seq_err(kind, val, loc, c) + et = args[0] if args else Any + out = [] + bad = False + i = 0 + for x in items: + r = _v(et, x, loc + (i,), c) + if r is _BAD: + bad = True + else: + out.append(r) + i += 1 + if bad: + return _BAD + if kind is list: + return out + return kind(out) + + +def _vtuple(args, val, loc, c): + if isinstance(val, tuple) or (not c.strict and isinstance(val, (list, set, frozenset))): + items = list(val) + else: + return _seq_err(tuple, val, loc, c) + out = [] + bad = False + if not args or (len(args) == 2 and args[1] is Ellipsis): + et = args[0] if args else Any + i = 0 + for x in items: + r = _v(et, x, loc + (i,), c) + if r is _BAD: + bad = True + else: + out.append(r) + i += 1 + return _BAD if bad else tuple(out) + if args == ((),): + args = () + if len(items) != len(args): + if len(items) < len(args): + for i in range(len(items), len(args)): + _E(c, loc + (i,), "missing", val, None, "Field required") + else: + _E(c, loc, "too_long", val, {"field_type": "Tuple", "max_length": len(args), "actual_length": len(items)}, + "Tuple should have at most " + _plural(len(args), "item") + " after validation, not " + str(len(items))) + return _BAD + for i in range(len(args)): + r = _v(args[i], items[i], loc + (i,), c) + if r is _BAD: + bad = True + else: + out.append(r) + return _BAD if bad else tuple(out) + + +def _vdict(args, val, loc, c): + if not isinstance(val, dict): + return _E(c, loc, "dict_type", val, None, "Input should be an object" if c.json else None) + kt = args[0] if args else Any + vt = args[1] if len(args) > 1 else Any + out = {} + bad = False + for k, x in val.items(): + k2 = _v(kt, k, loc + (k, "[key]"), c) + v2 = _v(vt, x, loc + (k,), c) + if k2 is _BAD or v2 is _BAD: + bad = True + else: + out[k2] = v2 + return _BAD if bad else out + + +def _vplain(tp, val, loc, c): + cfg = c.cfg + if tp is _NoneType: + if val is None: + return None + return _E(c, loc, "none_required", val) + if tp is bool: + if isinstance(val, bool): + return val + if c.strict: + return _E(c, loc, "bool_type", val) + if isinstance(val, str): + s = val.strip().lower() + if s in _TRUE: + return True + if s in _FALSE: + return False + return _E(c, loc, "bool_parsing", val) + if isinstance(val, (int, float)): + if val == 1: + return True + if val == 0: + return False + return _E(c, loc, "bool_parsing", val) + return _E(c, loc, "bool_type", val) + if tp is int: + if isinstance(val, bool): + if c.strict: + return _E(c, loc, "int_type", val) + return int(val) + if isinstance(val, int): + return val + if c.strict: + return _E(c, loc, "int_type", val) + if isinstance(val, float): + if val != val or val in (float("inf"), float("-inf")): + return _E(c, loc, "finite_number", val, None, "Input should be a finite number") + if val == int(val): + return int(val) + return _E(c, loc, "int_from_float", val) + if isinstance(val, str): + s = val.strip() + try: + return int(s) + except ValueError: + pass + try: + f = float(s) + if f == f and f not in (float("inf"), float("-inf")) and f == int(f): + return int(f) + if f == f and f not in (float("inf"), float("-inf")): + return _E(c, loc, "int_from_float", val, None, _MSG["int_parsing"]) + except ValueError: + pass + return _E(c, loc, "int_parsing", val) + if isinstance(val, _decimal.Decimal) and val == int(val): + return int(val) + return _E(c, loc, "int_type", val) + if tp is float: + if isinstance(val, float): + return val + if isinstance(val, bool): + if c.strict: + return _E(c, loc, "float_type", val) + return float(val) + if isinstance(val, int): + return float(val) + if c.strict: + return _E(c, loc, "float_type", val) + if isinstance(val, str): + try: + return float(val.strip()) + except ValueError: + return _E(c, loc, "float_parsing", val) + if isinstance(val, _decimal.Decimal): + return float(val) + return _E(c, loc, "float_type", val) + if tp is str: + if isinstance(val, str): + s = val + if isinstance(val, _enum.Enum): + s = str(val.value) + elif not c.strict and isinstance(val, (bytes, bytearray)): + try: + s = bytes(val).decode("utf-8") + except Exception: + return _E(c, loc, "string_unicode", val, None, "Input should be a valid string, unable to parse raw data as a unicode string") + else: + return _E(c, loc, "string_type", val) + if cfg.get("str_strip_whitespace"): + s = s.strip() + if cfg.get("str_to_lower"): + s = s.lower() + if cfg.get("str_to_upper"): + s = s.upper() + return s + if tp is bytes: + if isinstance(val, bytes): + return val + if not c.strict and isinstance(val, str): + return val.encode("utf-8") + if not c.strict and isinstance(val, bytearray): + return bytes(val) + return _E(c, loc, "bytes_type", val) + if tp is list or tp is set or tp is frozenset: + return _vlist(tp, (), val, loc, c) + if tp is tuple: + return _vtuple((), val, loc, c) + if tp is dict: + return _vdict((), val, loc, c) + if tp is _dt.datetime: + if isinstance(val, _dt.datetime): + return val + if c.strict: + return _E(c, loc, "datetime_type", val) + if isinstance(val, str): + try: + return _parse_datetime(val) + except Exception: + if len(val.strip()) < 10: + return _E(c, loc, "datetime_from_date_parsing", val, None, + "Input should be a valid datetime or date, input is too short") + return _E(c, loc, "datetime_parsing", val) + if isinstance(val, (int, float)) and not isinstance(val, bool): + return _dt.datetime.fromtimestamp(val, tz=_dt.timezone.utc) + return _E(c, loc, "datetime_type", val) + if tp is _dt.date: + if isinstance(val, _dt.datetime): + if val.hour == 0 and val.minute == 0 and val.second == 0 and val.microsecond == 0: + return val.date() + return _E(c, loc, "date_from_datetime_inexact", val) + if isinstance(val, _dt.date): + return val + if c.strict: + return _E(c, loc, "date_type", val) + if isinstance(val, str): + try: + if len(val.strip()) == 10: + return _dt.date.fromisoformat(val.strip()) + d = _parse_datetime(val) + if d.hour == 0 and d.minute == 0 and d.second == 0 and d.microsecond == 0: + return d.date() + return _E(c, loc, "date_from_datetime_inexact", val) + except Exception: + return _E(c, loc, "date_from_datetime_parsing", val) + return _E(c, loc, "date_type", val) + if tp is _dt.time: + if isinstance(val, _dt.time): + return val + if not c.strict and isinstance(val, str): + try: + return _dt.time.fromisoformat(val.strip()) + except Exception: + return _E(c, loc, "time_parsing", val) + return _E(c, loc, "time_type", val) + if tp is _dt.timedelta: + if isinstance(val, _dt.timedelta): + return val + if not c.strict: + if isinstance(val, (int, float)) and not isinstance(val, bool): + return _dt.timedelta(seconds=val) + if isinstance(val, str): + try: + return _parse_duration(val) + except Exception: + return _E(c, loc, "time_delta_parsing", val, None, _MSG["timedelta_parsing"]) + return _E(c, loc, "time_delta_type", val, None, _MSG["timedelta_type"]) + if tp is _decimal.Decimal: + if isinstance(val, _decimal.Decimal): + return val + if not c.strict and isinstance(val, (int, float, str)) and not isinstance(val, bool): + try: + return _decimal.Decimal(str(val).strip()) + except Exception: + return _E(c, loc, "decimal_parsing", val) + return _E(c, loc, "decimal_type", val) + if tp is _uuid.UUID: + if isinstance(val, _uuid.UUID): + return val + if not c.strict and isinstance(val, str): + try: + return _uuid.UUID(val) + except Exception: + return _E(c, loc, "uuid_parsing", val, {"ch": val[:1]}) + return _E(c, loc, "uuid_type", val) + if issubclass(tp, _enum.Enum): + if isinstance(val, tp): + return val + vals = [m.value for m in tp] + if not c.strict or c.json: + try: + return tp(val) + except Exception: + pass + exp = _join_or([repr(x) for x in vals]) + return _E(c, loc, "enum", val, {"expected": exp}, "Input should be " + exp) + if tp is SecretStr or tp is SecretBytes: + if isinstance(val, tp): + return val + inner = str if tp is SecretStr else bytes + r = _vplain(inner, val, loc, c) + return _BAD if r is _BAD else tp(r) + if isinstance(val, tp): + return val + return _E(c, loc, "is_instance_of", val, {"class": tp.__name__}, "Input should be an instance of " + tp.__name__) + + +def _constrain(v, cons, loc, c): + if isinstance(v, bool): + return v + if isinstance(v, (int, float, _decimal.Decimal)): + if "gt" in cons and not v > cons["gt"]: + return _E(c, loc, "greater_than", v, {"gt": cons["gt"]}) + if "ge" in cons and not v >= cons["ge"]: + return _E(c, loc, "greater_than_equal", v, {"ge": cons["ge"]}) + if "lt" in cons and not v < cons["lt"]: + return _E(c, loc, "less_than", v, {"lt": cons["lt"]}) + if "le" in cons and not v <= cons["le"]: + return _E(c, loc, "less_than_equal", v, {"le": cons["le"]}) + if "multiple_of" in cons: + m = cons["multiple_of"] + if isinstance(v, int) and isinstance(m, int): + bad = v % m != 0 + else: + q = v / m + bad = abs(q - round(q)) > 1e-9 + if bad: + return _E(c, loc, "multiple_of", v, {"multiple_of": m}) + return v + if isinstance(v, (str, bytes)): + n = len(v) + if "min_length" in cons and n < cons["min_length"]: + return _E(c, loc, "string_too_short", v, {"min_length": cons["min_length"]}, + "String should have at least " + _plural(cons["min_length"], "character")) + if "max_length" in cons and n > cons["max_length"]: + return _E(c, loc, "string_too_long", v, {"max_length": cons["max_length"]}, + "String should have at most " + _plural(cons["max_length"], "character")) + if "pattern" in cons and isinstance(v, str): + if _re.search(cons["pattern"], v) is None: + return _E(c, loc, "string_pattern_mismatch", v, {"pattern": cons["pattern"]}) + return v + if isinstance(v, (list, tuple, set, frozenset, dict)): + n = len(v) + ft = {list: "List", tuple: "Tuple", set: "Set", frozenset: "Frozenset", dict: "Dictionary"}.get(type(v), "List") + if "min_length" in cons and n < cons["min_length"]: + return _E(c, loc, "too_short", v, {"field_type": ft, "min_length": cons["min_length"], "actual_length": n}, + ft + " should have at least " + _plural(cons["min_length"], "item") + " after validation, not " + str(n)) + if "max_length" in cons and n > cons["max_length"]: + return _E(c, loc, "too_long", v, {"field_type": ft, "max_length": cons["max_length"], "actual_length": n}, + ft + " should have at most " + _plural(cons["max_length"], "item") + " after validation, not " + str(n)) + return v + + +# -------------------------------------------------------------------------- +# models +# -------------------------------------------------------------------------- + +def _is_private(name): + return name.startswith("_") + + +def _merge_annotation(fi, tp): + fi._full = tp + base, metas = _unwrap(tp) + fi.annotation = base + fi.metadata = [m for m in metas if not isinstance(m, FieldInfo)] + for m in metas: + if isinstance(m, FieldInfo): + for k in ("alias", "validation_alias", "serialization_alias", "title", "description", "examples"): + if getattr(fi, k) is None and getattr(m, k) is not None: + setattr(fi, k, getattr(m, k)) + if fi.default is PydanticUndefined and fi.default_factory is None: + fi.default = m.default + fi.default_factory = m.default_factory + if m.exclude is not None and fi.exclude is None: + fi.exclude = m.exclude + + +def _complete(cls): + if cls.__dict__.get("__pydantic_complete__"): + return True + try: + hints = _typing.get_type_hints(cls, include_extras=True) + except Exception: + return False + for name, fi in cls.model_fields.items(): + if name in hints: + _merge_annotation(fi, hints[name]) + elif fi._full is None: + _merge_annotation(fi, fi.annotation if fi.annotation is not None else Any) + type.__setattr__(cls, "__pydantic_complete__", True) + return True + + +def _ensure(cls): + if not cls.__dict__.get("__pydantic_complete__"): + if not _complete(cls): + for fi in cls.model_fields.values(): + if fi._full is None: + raise PydanticUserError( + "`%s` is not fully defined; you should define all referenced types, then call `%s.model_rebuild()`" + % (cls.__name__, cls.__name__)) + + +class PydanticUserError(TypeError): + def __init__(self, message, code=None): + TypeError.__init__(self, message) + self.code = code + + +class PydanticUndefinedAnnotation(NameError): + pass + + +class ModelMetaclass(type): + def __new__(mcs, name, bases, ns, **kwargs): + if not any(isinstance(b, ModelMetaclass) for b in bases): + return type.__new__(mcs, name, bases, ns) + annotations = ns.get("__annotations__", {}) + config = {} + for b in reversed(bases): + config.update(getattr(b, "model_config", {})) + config.update(ns.get("model_config", {})) + config.update(kwargs) + ns["model_config"] = config + own = {} + privates = {} + for fname in list(annotations): + ann = annotations[fname] + if _is_classvar(ann): + continue + if _is_private(fname): + d = ns.get(fname, PydanticUndefined) + if isinstance(d, _PrivateAttr): + ns.pop(fname) + privates[fname] = d + elif d is not PydanticUndefined: + privates[fname] = _PrivateAttr(d, None) + continue + d = ns.pop(fname, PydanticUndefined) + if isinstance(d, FieldInfo): + fi = d + else: + fi = FieldInfo(d) + fi.annotation = ann + own[fname] = fi + for k in list(ns): + v = ns[k] + if isinstance(v, _PrivateAttr): + ns.pop(k) + privates[k] = v + decls = {"field": [], "model": [], "fser": [], "mser": []} + for b in bases: + bd = getattr(b, "__pydantic_decorators__", None) + if bd: + for kind in decls: + decls[kind].extend(bd[kind]) + computed = {} + for b in bases: + computed.update(getattr(b, "__pydantic_computed__", {})) + for k in list(ns): + v = ns[k] + if isinstance(v, _Decl): + f = getattr(v.func, "__func__", v.func) + decls[v.kind].append(_Decl(v.kind, f, v.fields, v.mode, v.extra)) + if v.kind == "model" and v.mode == "after": + ns[k] = f + elif v.kind in ("field", "model"): + ns[k] = classmethod(f) + else: + ns[k] = f + elif isinstance(v, property): + for (g, alias, rep) in _COMPUTED: + if g is v.fget: + computed[k] = (alias, rep) + cls = type.__new__(mcs, name, bases, ns) + fields = {} + for b in reversed(bases): + for k, fi in getattr(b, "model_fields", {}).items(): + fields[k] = fi + for k, fi in own.items(): + fields[k] = fi + pv = {} + for b in bases: + pv.update(getattr(b, "__private_attributes__", {})) + pv.update(privates) + type.__setattr__(cls, "model_fields", fields) + type.__setattr__(cls, "__pydantic_decorators__", decls) + type.__setattr__(cls, "__pydantic_computed__", computed) + type.__setattr__(cls, "__private_attributes__", pv) + type.__setattr__(cls, "__pydantic_complete__", False) + _complete(cls) + if config.get("frozen") and "__hash__" not in ns: + type.__setattr__(cls, "__hash__", BaseModel._frozen_hash) + return cls + + def __init__(cls, name, bases, ns, **kwargs): + type.__init__(cls, name, bases, ns) + + +class _Opts: + def __init__(self, mode="python", by_alias=False, exclude_unset=False, exclude_defaults=False, + exclude_none=False, round_trip=False): + self.mode = mode + self.by_alias = by_alias + self.exclude_unset = exclude_unset + self.exclude_defaults = exclude_defaults + self.exclude_none = exclude_none + + +def _fill(cls, data, loc, c): + """Validates a mapping against cls's fields. Returns (values, fields_set, + extras) or None, with the problems appended to c.errs.""" + cfg = cls.model_config + saved_cfg = c.cfg + saved_cls = c.cls + c.cfg = cfg + c.cls = cls + n0 = len(c.errs) + values = {} + fset = set() + used = set() + by_name = cfg.get("populate_by_name") or cfg.get("validate_by_name") + root = _is_root(cls) + for name, fi in cls.model_fields.items(): + alias = fi.validation_alias if fi.validation_alias is not None else fi.alias + key = _MISSING_KEY + if alias is not None and alias in data: + key = alias + elif (alias is None or by_name) and name in data: + key = name + if key is _MISSING_KEY: + if fi.default_factory is not None: + values[name] = fi.default_factory() + elif fi.default is PydanticUndefined: + _E(c, loc + ((alias if alias is not None else name),), "missing", data) + else: + d = fi.default + if isinstance(d, (list, dict, set, bytearray)): + d = _copy.deepcopy(d) + if fi.validate_default or cfg.get("validate_default"): + d = _vfield(cls, fi, name, d, loc + (name,), c, values) + if d is _BAD: + continue + values[name] = d + continue + used.add(key) + v = _vfield(cls, fi, name, data[key], loc if root else loc + (key,), c, values) + if v is not _BAD: + values[name] = v + fset.add(name) + extra_mode = cfg.get("extra") or "ignore" + extras = {} + for k in data: + if k in used: + continue + if extra_mode == "forbid": + _E(c, loc + (k,), "extra_forbidden", data[k]) + elif extra_mode == "allow": + extras[k] = data[k] + c.cfg = saved_cfg + c.cls = saved_cls + if len(c.errs) > n0: + return None + return values, fset, extras + + +_MISSING_KEY = _Undefined() + + +def _vfield(cls, fi, name, raw, loc, c, values): + info = ValidationInfo(c, name, dict(values)) + decls = cls.__pydantic_decorators__["field"] + v = raw + for d in decls: + if d.mode == "before" and (name in d.fields or "*" in d.fields): + v = _guard(c, loc, raw, lambda d=d, v=v: _invoke(d.func, (cls, v), info)) + if v is _BAD: + return _BAD + plain = None + for d in decls: + if d.mode == "plain" and (name in d.fields or "*" in d.fields): + plain = d + if plain is not None: + v = _guard(c, loc, raw, lambda: _invoke(plain.func, (cls, v), info)) + if v is _BAD: + return _BAD + else: + was = c.strict + if fi.strict is not None: + c.strict = fi.strict + elif c.cfg.get("strict"): + c.strict = True + full = fi._full if fi._full is not None else Any + v = _v(full, v, loc, c) + c.strict = was + if v is _BAD: + return _BAD + if fi.constraints and v is not None: + v = _constrain(v, fi.constraints, loc, c) + if v is _BAD: + return _BAD + for d in decls: + if d.mode == "after" and (name in d.fields or "*" in d.fields): + v = _guard(c, loc, raw, lambda d=d, v=v: _invoke(d.func, (cls, v), info)) + if v is _BAD: + return _BAD + if c.cfg.get("use_enum_values") and isinstance(v, _enum.Enum): + v = v.value + return v + + +def _populate(obj, cls, values, fset, extras): + for k, v in values.items(): + object.__setattr__(obj, k, v) + object.__setattr__(obj, "__pydantic_fields_set__", fset) + object.__setattr__(obj, "__pydantic_extra__", extras if cls.model_config.get("extra") == "allow" else None) + for k, p in cls.__private_attributes__.items(): + if p.factory is not None: + object.__setattr__(obj, k, p.factory()) + elif p.default is not PydanticUndefined: + object.__setattr__(obj, k, _copy.deepcopy(p.default)) + + +def _is_root(cls): + return cls.__dict__.get("__pydantic_root__") or any( + b.__dict__.get("__pydantic_root__") for b in cls.__mro__) + + +def _vmodel(cls, val, loc, c): + if isinstance(val, cls): + return val + _ensure(cls) + decls = cls.__pydantic_decorators__["model"] + info = ValidationInfo(c) + v = val + for d in decls: + if d.mode == "before": + v = _guard(c, loc, val, lambda d=d, v=v: _invoke(d.func, (cls, v), info)) + if v is _BAD: + return _BAD + if isinstance(v, cls): + return v + root = _is_root(cls) + if root: + data = {"root": v} + elif isinstance(v, dict): + data = v + elif (c.from_attributes or cls.model_config.get("from_attributes")) and v is not None \ + and not isinstance(v, (str, int, float, bool, list, tuple, set)): + data = {} + for name, fi in cls.model_fields.items(): + if hasattr(v, name): + data[name] = getattr(v, name) + else: + return _E(c, loc, "model_type", v, {"class_name": cls.__name__}, + "Input should be an object" if c.json else None) + r = _fill(cls, data, loc, c) + if r is None: + return _BAD + obj = object.__new__(cls) + _populate(obj, cls, r[0], r[1], r[2]) + for d in decls: + if d.mode == "after": + res = _guard(c, loc, v, lambda d=d: _invoke(d.func, (obj,), info)) + if res is _BAD: + return _BAD + obj = res + obj.model_post_init(c.context) + return obj + + +def _finish(c, title, r): + if c.errs: + raise ValidationError(title, c.errs) + return r + + +class BaseModel(metaclass=ModelMetaclass): + """Base class of validated models.""" + + model_config = {} + + def __init__(self, **data): + cls = type(self) + _ensure(cls) + c = _Ctx() + c.cls = cls + info = ValidationInfo(c) + decls = cls.__pydantic_decorators__["model"] + v = data + for d in decls: + if d.mode == "before": + v = _guard(c, (), data, lambda d=d, v=v: _invoke(d.func, (cls, v), info)) + if v is _BAD: + raise ValidationError(cls.__name__, c.errs) + r = _fill(cls, v, (), c) + if r is None: + raise ValidationError(cls.__name__, c.errs) + _populate(self, cls, r[0], r[1], r[2]) + for d in decls: + if d.mode == "after": + res = _guard(c, (), v, lambda d=d: _invoke(d.func, (self,), info)) + if res is _BAD: + raise ValidationError(cls.__name__, c.errs) + self.model_post_init(None) + + def model_post_init(self, context): + pass + + @classmethod + def model_validate(cls, obj, *, strict=None, from_attributes=None, context=None): + c = _Ctx(bool(strict), False, context, bool(from_attributes)) + c.cls = cls + r = _vmodel(cls, obj, (), c) + return _finish(c, cls.__name__, r) + + @classmethod + def model_validate_json(cls, json_data, *, strict=None, context=None): + c = _Ctx(bool(strict), True, context) + c.cls = cls + try: + if isinstance(json_data, (bytes, bytearray)): + json_data = bytes(json_data).decode("utf-8") + data = _json.loads(json_data) + except Exception as e: + _E(c, (), "json_invalid", json_data, {"error": str(e)}, "Invalid JSON: " + str(e)) + raise ValidationError(cls.__name__, c.errs) + r = _vmodel(cls, data, (), c) + return _finish(c, cls.__name__, r) + + @classmethod + def model_validate_strings(cls, obj, *, strict=None, context=None): + return cls.model_validate(obj, strict=strict, context=context) + + @classmethod + def model_construct(cls, _fields_set=None, **values): + _ensure(cls) + obj = object.__new__(cls) + vals = {} + for name, fi in cls.model_fields.items(): + if name in values: + vals[name] = values[name] + elif fi.default_factory is not None: + vals[name] = fi.default_factory() + elif fi.default is not PydanticUndefined: + vals[name] = fi.default + fset = set(_fields_set) if _fields_set is not None else set(values.keys()) + extras = {} + if cls.model_config.get("extra") == "allow": + for k in values: + if k not in cls.model_fields: + extras[k] = values[k] + _populate(obj, cls, vals, fset, extras) + return obj + + @classmethod + def model_rebuild(cls, *, force=False, raise_errors=True, **kw): + if force: + type.__setattr__(cls, "__pydantic_complete__", False) + ok = _complete(cls) + if not ok and raise_errors: + raise PydanticUserError("`%s` is not fully defined" % cls.__name__) + return True if ok else None + + @property + def model_fields_set(self): + return self.__pydantic_fields_set__ + + @property + def model_extra(self): + return self.__pydantic_extra__ + + def __getattr__(self, name): + if name.startswith("__pydantic_"): + raise AttributeError(name) + ex = self.__dict__.get("__pydantic_extra__") + if ex is not None and name in ex: + return ex[name] + raise AttributeError("'%s' object has no attribute '%s'" % (type(self).__name__, name)) + + def __setattr__(self, name, value): + cls = type(self) + if name.startswith("_"): + object.__setattr__(self, name, value) + return + fi = cls.model_fields.get(name) + cfg = cls.model_config + if cfg.get("frozen") or (fi is not None and fi.frozen): + c = _Ctx() + _E(c, (name,), "frozen_instance", value) + raise ValidationError(cls.__name__, c.errs) + if fi is None: + if cfg.get("extra") == "allow": + self.__pydantic_extra__[name] = value + return + if isinstance(getattr(cls, name, None), property): + object.__setattr__(self, name, value) + return + raise ValueError('"%s" object has no field "%s"' % (cls.__name__, name)) + if cfg.get("validate_assignment"): + c = _Ctx() + c.cls = cls + c.cfg = cfg + v = _vfield(cls, fi, name, value, (name,), c, dict(self.__dict__)) + if c.errs: + raise ValidationError(cls.__name__, c.errs) + value = v + object.__setattr__(self, name, value) + self.__pydantic_fields_set__.add(name) + + def __delattr__(self, name): + object.__delattr__(self, name) + + def _frozen_hash(self): + return hash(tuple([_hashable(getattr(self, k)) for k in type(self).model_fields])) + + def __eq__(self, other): + if not isinstance(other, BaseModel): + return NotImplemented + if type(self) is not type(other): + return False + for k in type(self).model_fields: + if getattr(self, k) != getattr(other, k): + return False + return self.__pydantic_extra__ == other.__pydantic_extra__ + + def __ne__(self, other): + r = self.__eq__(other) + if r is NotImplemented: + return r + return not r + + __hash__ = None + + def __iter__(self): + for k in type(self).model_fields: + yield k, getattr(self, k) + if self.__pydantic_extra__: + for k, v in self.__pydantic_extra__.items(): + yield k, v + + def __repr_args__(self): + out = [] + for k, fi in type(self).model_fields.items(): + if fi.repr: + out.append((k, getattr(self, k))) + for k, (alias, rep) in type(self).__pydantic_computed__.items(): + if rep: + out.append((k, getattr(self, k))) + if self.__pydantic_extra__: + for k, v in self.__pydantic_extra__.items(): + out.append((k, v)) + return out + + def __repr__(self): + return type(self).__name__ + "(" + ", ".join([k + "=" + repr(v) for k, v in self.__repr_args__()]) + ")" + + def __str__(self): + return " ".join([k + "=" + repr(v) for k, v in self.__repr_args__()]) + + def __copy__(self): + return self.model_copy() + + def __deepcopy__(self, memo=None): + return self.model_copy(deep=True) + + def model_copy(self, *, update=None, deep=False): + cls = type(self) + obj = object.__new__(cls) + vals = {} + for k in cls.model_fields: + v = getattr(self, k) + vals[k] = _copy.deepcopy(v) if deep else v + fset = set(self.__pydantic_fields_set__) + ex = self.__pydantic_extra__ + if ex is not None: + ex = _copy.deepcopy(ex) if deep else dict(ex) + if update: + for k, v in update.items(): + if k in cls.model_fields: + vals[k] = v + fset.add(k) + elif ex is not None: + ex[k] = v + _populate(obj, cls, vals, fset, ex or {}) + object.__setattr__(obj, "__pydantic_extra__", ex) + for k in cls.__private_attributes__: + if k in self.__dict__: + v = self.__dict__[k] + object.__setattr__(obj, k, _copy.deepcopy(v) if deep else v) + return obj + + def model_dump(self, *, mode="python", include=None, exclude=None, by_alias=False, + exclude_unset=False, exclude_defaults=False, exclude_none=False, round_trip=False, + warnings=True, **kw): + o = _Opts(mode, by_alias, exclude_unset, exclude_defaults, exclude_none) + return _dump_model(self, o, include, exclude) + + def model_dump_json(self, *, indent=None, include=None, exclude=None, by_alias=False, + exclude_unset=False, exclude_defaults=False, exclude_none=False, round_trip=False, + warnings=True, **kw): + o = _Opts("json", by_alias, exclude_unset, exclude_defaults, exclude_none) + return _jsonify(_dump_model(self, o, include, exclude), indent, 0) + + @classmethod + def model_json_schema(cls, by_alias=True, ref_template="#/$defs/{model}", mode="validation", **kw): + defs = {} + _model_schema(cls, defs) + s = dict(defs.pop(cls.__name__)) + if defs: + s["$defs"] = defs + return _sort_schema(s, None) + + @classmethod + def model_parametrized_name(cls, params): + return cls.__name__ + "[" + ",".join([_tname(p) for p in params]) + "]" + + +def _hashable(v): + if isinstance(v, list): + return tuple([_hashable(x) for x in v]) + if isinstance(v, dict): + return tuple([(k, _hashable(x)) for k, x in v.items()]) + if isinstance(v, set): + return frozenset(v) + return v + + +class RootModel(BaseModel): + """A model whose whole value is the single field `root`.""" + + __pydantic_root__ = True + + def __init__(self, root=PydanticUndefined, **data): + cls = type(self) + _ensure(cls) + if root is PydanticUndefined: + if data: + root = data + else: + root = PydanticUndefined + c = _Ctx() + c.cls = cls + if root is PydanticUndefined: + r = _fill(cls, {}, (), c) + else: + r = _fill(cls, {"root": root}, (), c) + if r is None: + raise ValidationError(cls.__name__, c.errs) + _populate(self, cls, r[0], r[1], r[2]) + + @classmethod + def __class_getitem__(cls, tp): + name = cls.__name__ + "[" + _tname(tp) + "]" + return ModelMetaclass(name, (cls,), {"__annotations__": {"root": tp}, "__module__": cls.__module__}) + + @classmethod + def model_validate_json(cls, json_data, *, strict=None, context=None): + return BaseModel.model_validate_json.__func__(cls, json_data, strict=strict, context=context) + + +# -------------------------------------------------------------------------- +# serialization +# -------------------------------------------------------------------------- + +def _pick(inc, exc, key): + sub_inc = None + sub_exc = None + if exc is not None: + if isinstance(exc, dict): + if key in exc: + e = exc[key] + if e is True or e is Ellipsis: + return False, None, None + sub_exc = e + elif key in exc: + return False, None, None + if inc is not None: + if isinstance(inc, dict): + if key not in inc: + return False, None, None + i = inc[key] + sub_inc = None if (i is True or i is Ellipsis) else i + elif key not in inc: + return False, None, None + return True, sub_inc, sub_exc + + +def _dump_model(m, o, include, exclude): + cls = type(m) + for d in cls.__pydantic_decorators__["mser"]: + return _dump(d.func(m), o, include, exclude) + if _is_root(cls): + return _dump(m.root, o, include, exclude) + out = {} + fsers = cls.__pydantic_decorators__["fser"] + for name, fi in cls.model_fields.items(): + if fi.exclude: + continue + keep, si, se = _pick(include, exclude, name) + if not keep: + continue + v = getattr(m, name) + if o.exclude_unset and name not in m.__pydantic_fields_set__: + continue + if o.exclude_defaults and not fi.is_required(): + dv = fi.default_factory() if fi.default_factory is not None else fi.default + if v == dv: + continue + if o.exclude_none and v is None: + continue + ser = None + for d in fsers: + if name in d.fields or "*" in d.fields: + if d.extra == "json" and o.mode != "json": + continue + if d.extra == "json-unless-none" and (o.mode != "json" or v is None): + continue + ser = d + if ser is not None: + r = _invoke(ser.func, (m, v), ValidationInfo(_Ctx())) + r = _dump(r, o, si, se) + else: + r = _dump(v, o, si, se) + key = name + if o.by_alias: + a = fi.serialization_alias if fi.serialization_alias is not None else fi.alias + if a is not None: + key = a + out[key] = r + for k, (alias, rep) in cls.__pydantic_computed__.items(): + keep, si, se = _pick(include, exclude, k) + if not keep: + continue + v = _dump(getattr(m, k), o, si, se) + out[alias if (o.by_alias and alias is not None) else k] = v + ex = m.__pydantic_extra__ + if ex: + for k, v in ex.items(): + keep, si, se = _pick(include, exclude, k) + if keep: + out[k] = _dump(v, o, si, se) + return out + + +def _dump(v, o, include=None, exclude=None): + if isinstance(v, BaseModel): + return _dump_model(v, o, include, exclude) + js = o.mode == "json" + if isinstance(v, dict): + out = {} + for k, x in v.items(): + keep, si, se = _pick(include, exclude, k) + if not keep: + continue + if o.exclude_none and x is None: + continue + kk = k + if js: + kk = _dump(k, o) + if not isinstance(kk, str): + kk = _json_key(kk) + out[kk] = _dump(x, o, si, se) + return out + if isinstance(v, (list, tuple, set, frozenset)): + items = [] + i = 0 + for x in (sorted(v, key=repr) if False else v): + keep, si, se = _pick(include, exclude, i) + i += 1 + if not keep: + continue + items.append(_dump(x, o, si, se)) + if js or isinstance(v, list): + return items + if isinstance(v, tuple): + return tuple(items) + return type(v)(items) if not isinstance(v, (set, frozenset)) else _safe_set(v, items) + if isinstance(v, _enum.Enum): + return _dump(v.value, o) if js else v + if js: + return _to_jsonable(v) + return v + + +def _safe_set(orig, items): + try: + return type(orig)(items) + except TypeError: + return items + + +def _json_key(k): + if k is True: + return "true" + if k is False: + return "false" + if k is None: + return "null" + return str(k) + + +def _to_jsonable(v): + if v is None or isinstance(v, (bool, int, float, str)): + return v + if isinstance(v, _dt.datetime): + s = v.isoformat() + if s.endswith("+00:00"): + s = s[:-6] + "Z" + return s + if isinstance(v, (_dt.date, _dt.time)): + return v.isoformat() + if isinstance(v, _dt.timedelta): + return _duration_text(v) + if isinstance(v, (_decimal.Decimal, _uuid.UUID)): + return str(v) + if isinstance(v, (bytes, bytearray)): + return bytes(v).decode("utf-8") + if isinstance(v, (SecretStr, SecretBytes)): + return "**********" + if isinstance(v, _enum.Enum): + return _to_jsonable(v.value) + if isinstance(v, dict): + return dict((_json_key(k) if not isinstance(k, str) else k, _to_jsonable(x)) for k, x in v.items()) + if isinstance(v, (list, tuple, set, frozenset)): + return [_to_jsonable(x) for x in v] + if isinstance(v, BaseModel): + return _dump_model(v, _Opts("json"), None, None) + return str(v) + + +def _duration_text(td): + total = td.days * 86400 + td.seconds + sign = "-" if total < 0 else "" + total = abs(total) + days = total // 86400 + rem = total % 86400 + h = rem // 3600 + m = (rem % 3600) // 60 + s = rem % 60 + out = sign + "P" + if days: + out += "%dD" % days + secs = str(s) + if td.microseconds: + secs += (".%06d" % td.microseconds).rstrip("0") + if h or m or s or td.microseconds or not days: + out += "T" + if h: + out += "%dH" % h + if m: + out += "%dM" % m + if s or td.microseconds or not (h or m or days): + out += secs + "S" + return out + + +def _jsonify(v, indent, level): + if v is None: + return "null" + if v is True: + return "true" + if v is False: + return "false" + if isinstance(v, str): + return _json.dumps(v, ensure_ascii=False) + if isinstance(v, float): + if v != v or v in (float("inf"), float("-inf")): + return "null" + return repr(v) + if isinstance(v, int): + return str(v) + if isinstance(v, dict): + if not v: + return "{}" + if indent is None: + return "{" + ",".join([_json.dumps(str(k), ensure_ascii=False) + ":" + _jsonify(x, None, 0) + for k, x in v.items()]) + "}" + pad = " " * (indent * (level + 1)) + end = " " * (indent * level) + return "{\n" + ",\n".join([pad + _json.dumps(str(k), ensure_ascii=False) + ": " + _jsonify(x, indent, level + 1) + for k, x in v.items()]) + "\n" + end + "}" + if isinstance(v, (list, tuple)): + if not v: + return "[]" + if indent is None: + return "[" + ",".join([_jsonify(x, None, 0) for x in v]) + "]" + pad = " " * (indent * (level + 1)) + end = " " * (indent * level) + return "[\n" + ",\n".join([pad + _jsonify(x, indent, level + 1) for x in v]) + "\n" + end + "]" + return _jsonify(_to_jsonable(v), indent, level) + + +# -------------------------------------------------------------------------- +# JSON schema +# -------------------------------------------------------------------------- + +def _title_of(name): + return " ".join([w[:1].upper() + w[1:] for w in name.replace("_", " ").split(" ")]) + + +def _model_schema(cls, defs): + _ensure(cls) + name = cls.__name__ + if name in defs: + return {"$ref": "#/$defs/" + name} + defs[name] = {} + if _is_root(cls): + s = _schema(cls.model_fields["root"]._full, defs) + s = dict(s) + s["title"] = name + defs[name] = s + return {"$ref": "#/$defs/" + name} + props = {} + required = [] + for fname, fi in cls.model_fields.items(): + key = fi.validation_alias if fi.validation_alias is not None else (fi.alias if fi.alias is not None else fname) + s = dict(_schema(fi._full if fi._full is not None else Any, defs)) + if "$ref" not in s: + s["title"] = fi.title if fi.title is not None else _title_of(fname) + elif fi.title is not None: + s["title"] = fi.title + for k, v in fi.constraints.items(): + _apply_constraint(s, k, v) + if fi.description is not None: + s["description"] = fi.description + if fi.examples is not None: + s["examples"] = fi.examples + if fi.default_factory is None and fi.default is not PydanticUndefined: + s["default"] = _to_jsonable(_dump(fi.default, _Opts("json"))) + elif fi.default_factory is not None: + pass + else: + required.append(key) + if isinstance(fi.json_schema_extra, dict): + s.update(fi.json_schema_extra) + props[key] = s + out = {"properties": props, "title": name, "type": "object"} + if required: + out["required"] = required + if cls.model_config.get("extra") == "forbid": + out["additionalProperties"] = False + defs[name] = out + return {"$ref": "#/$defs/" + name} + + +def _apply_constraint(s, k, v): + t = s.get("type") + if k == "gt": + s["exclusiveMinimum"] = v + elif k == "ge": + s["minimum"] = v + elif k == "lt": + s["exclusiveMaximum"] = v + elif k == "le": + s["maximum"] = v + elif k == "multiple_of": + s["multipleOf"] = v + elif k == "min_length": + s["minItems" if t == "array" else ("minProperties" if t == "object" else "minLength")] = v + elif k == "max_length": + s["maxItems" if t == "array" else ("maxProperties" if t == "object" else "maxLength")] = v + elif k == "pattern": + s["pattern"] = v + + +def _schema(tp, defs): + if tp is None or tp is _NoneType: + return {"type": "null"} + if tp is Any or tp is object: + return {} + if isinstance(tp, str) or hasattr(tp, "__forward_arg__"): + return {} + if hasattr(tp, "__supertype__"): + return _schema(tp.__supertype__, defs) + org = _typing.get_origin(tp) + args = _typing.get_args(tp) + if org is Annotated: + base, metas = _unwrap(tp) + s = dict(_schema(base, defs)) + for m in metas: + if isinstance(m, FieldInfo): + for k, v in m.constraints.items(): + _apply_constraint(s, k, v) + if m.description is not None: + s["description"] = m.description + elif isinstance(m, _StringConstraints): + for k, v in m.cons.items(): + _apply_constraint(s, k, v) + return s + if org is Union: + members = [a for a in args if a is not _NoneType] + subs = [_schema(a, defs) for a in members] + if len(members) != len(args): + subs.append({"type": "null"}) + if len(subs) == 1: + return subs[0] + return {"anyOf": subs} + if org is Literal: + vals = [_to_jsonable(a) for a in args] + types = set() + for a in args: + types.add("string" if isinstance(a, str) else "boolean" if isinstance(a, bool) else + "integer" if isinstance(a, int) else "number" if isinstance(a, float) else "null") + s = {} + if len(vals) == 1: + s["const"] = vals[0] + else: + s["enum"] = vals + if len(types) == 1: + s["type"] = list(types)[0] + return s + if org is not None: + nm = getattr(org, "__name__", "") + if org is list or nm in ("Sequence", "MutableSequence", "Iterable", "Collection"): + return {"items": _schema(args[0], defs) if args else {}, "type": "array"} + if org is set or org is frozenset or nm in ("Set", "MutableSet"): + return {"items": _schema(args[0], defs) if args else {}, "type": "array", "uniqueItems": True} + if org is tuple: + if not args or (len(args) == 2 and args[1] is Ellipsis): + return {"items": _schema(args[0], defs) if args else {}, "type": "array"} + return {"maxItems": len(args), "minItems": len(args), + "prefixItems": [_schema(a, defs) for a in args], "type": "array"} + if org is dict or nm in ("Mapping", "MutableMapping"): + s = {"type": "object"} + if len(args) > 1: + s["additionalProperties"] = _schema(args[1], defs) + return s + return {} + if isinstance(tp, ModelMetaclass): + return _model_schema(tp, defs) + if isinstance(tp, type): + if tp is bool: + return {"type": "boolean"} + if tp is int: + return {"type": "integer"} + if tp is float: + return {"type": "number"} + if tp is str: + return {"type": "string"} + if tp is bytes: + return {"format": "binary", "type": "string"} + if tp is list or tp is tuple: + return {"items": {}, "type": "array"} + if tp is set or tp is frozenset: + return {"items": {}, "type": "array", "uniqueItems": True} + if tp is dict: + return {"type": "object"} + if tp is _dt.datetime: + return {"format": "date-time", "type": "string"} + if tp is _dt.date: + return {"format": "date", "type": "string"} + if tp is _dt.time: + return {"format": "time", "type": "string"} + if tp is _dt.timedelta: + return {"format": "duration", "type": "string"} + if tp is _uuid.UUID: + return {"format": "uuid", "type": "string"} + if tp is _decimal.Decimal: + return {"anyOf": [{"type": "number"}, {"type": "string"}]} + if tp is SecretStr: + return {"format": "password", "type": "string", "writeOnly": True} + if issubclass(tp, _enum.Enum): + name = tp.__name__ + if name not in defs: + vals = [m.value for m in tp] + d = {"enum": vals, "title": name} + if all(isinstance(x, str) for x in vals): + d["type"] = "string" + elif all(isinstance(x, int) and not isinstance(x, bool) for x in vals): + d["type"] = "integer" + defs[name] = d + return {"$ref": "#/$defs/" + name} + return {} + + +def _sort_schema(v, parent): + if isinstance(v, dict): + keys = list(v.keys()) + if parent not in ("properties", "default"): + keys = sorted(keys) + if parent == "default": + return v + return dict((k, _sort_schema(v[k], k)) for k in keys) + if isinstance(v, list): + return [_sort_schema(x, parent) for x in v] + return v + + +# -------------------------------------------------------------------------- +# TypeAdapter and constrained types +# -------------------------------------------------------------------------- + +class TypeAdapter: + """Validation and serialization for any annotation.""" + + def __init__(self, type, *, config=None): + self._type = type + self._config = config or {} + + def _title(self): + return _tname(self._type) + + def validate_python(self, obj, *, strict=None, from_attributes=None, context=None): + c = _Ctx(bool(strict), False, context, bool(from_attributes)) + c.cfg = self._config + r = _v(self._type, obj, (), c) + return _finish(c, self._title(), r) + + def validate_json(self, data, *, strict=None, context=None): + c = _Ctx(bool(strict), True, context) + c.cfg = self._config + try: + if isinstance(data, (bytes, bytearray)): + data = bytes(data).decode("utf-8") + obj = _json.loads(data) + except Exception as e: + _E(c, (), "json_invalid", data, {"error": str(e)}, "Invalid JSON: " + str(e)) + raise ValidationError(self._title(), c.errs) + r = _v(self._type, obj, (), c) + return _finish(c, self._title(), r) + + def validate_strings(self, obj, *, strict=None, context=None): + return self.validate_python(obj, strict=strict, context=context) + + def dump_python(self, instance, *, mode="python", include=None, exclude=None, by_alias=False, + exclude_unset=False, exclude_defaults=False, exclude_none=False, **kw): + return _dump(instance, _Opts(mode, by_alias, exclude_unset, exclude_defaults, exclude_none), include, exclude) + + def dump_json(self, instance, *, indent=None, include=None, exclude=None, by_alias=False, + exclude_none=False, **kw): + o = _Opts("json", by_alias, False, False, exclude_none) + return _jsonify(_dump(instance, o, include, exclude), indent, 0).encode("utf-8") + + def json_schema(self, **kw): + defs = {} + s = dict(_schema(self._type, defs)) + if defs: + s["$defs"] = defs + return _sort_schema(s, None) + + +class _StringConstraints: + def __init__(self, strip_whitespace=None, to_upper=None, to_lower=None, strict=None, + min_length=None, max_length=None, pattern=None): + self.strip_whitespace = strip_whitespace + self.to_upper = to_upper + self.to_lower = to_lower + self.strict = strict + self.cons = {} + if min_length is not None: + self.cons["min_length"] = min_length + if max_length is not None: + self.cons["max_length"] = max_length + if pattern is not None: + self.cons["pattern"] = pattern + + +StringConstraints = _StringConstraints + +PositiveInt = Annotated[int, Field(gt=0)] +NegativeInt = Annotated[int, Field(lt=0)] +NonNegativeInt = Annotated[int, Field(ge=0)] +NonPositiveInt = Annotated[int, Field(le=0)] +PositiveFloat = Annotated[float, Field(gt=0)] +NegativeFloat = Annotated[float, Field(lt=0)] +NonNegativeFloat = Annotated[float, Field(ge=0)] +NonPositiveFloat = Annotated[float, Field(le=0)] +StrictInt = Annotated[int, Strict()] +StrictFloat = Annotated[float, Strict()] +StrictStr = Annotated[str, Strict()] +StrictBool = Annotated[bool, Strict()] +StrictBytes = Annotated[bytes, Strict()] + + +def conint(*, strict=None, gt=None, ge=None, lt=None, le=None, multiple_of=None): + return Annotated[int, Field(strict=strict, gt=gt, ge=ge, lt=lt, le=le, multiple_of=multiple_of)] + + +def confloat(*, strict=None, gt=None, ge=None, lt=None, le=None, multiple_of=None): + return Annotated[float, Field(strict=strict, gt=gt, ge=ge, lt=lt, le=le, multiple_of=multiple_of)] + + +def constr(*, strip_whitespace=None, to_upper=None, to_lower=None, strict=None, + min_length=None, max_length=None, pattern=None): + return Annotated[str, _StringConstraints(strip_whitespace, to_upper, to_lower, strict, + min_length, max_length, pattern)] + + +def conlist(item_type, *, min_length=None, max_length=None): + return Annotated[list[item_type], Field(min_length=min_length, max_length=max_length)] + + +class SecretStr: + """A string that hides its value when printed.""" + + def __init__(self, value): + self._secret_value = value + + def get_secret_value(self): + return self._secret_value + + def __len__(self): + return len(self._secret_value) + + def __eq__(self, other): + return isinstance(other, SecretStr) and self._secret_value == other._secret_value + + def __hash__(self): + return hash(self._secret_value) + + def __repr__(self): + return "SecretStr('" + ("**********" if self._secret_value else "") + "')" + + def __str__(self): + return "**********" if self._secret_value else "" + + +class SecretBytes(SecretStr): + def __repr__(self): + return "SecretBytes(b'" + ("**********" if self._secret_value else "") + "')" diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/pydantic/dataclasses.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/pydantic/dataclasses.py new file mode 100644 index 00000000000..25625c8cc7c --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/pydantic/dataclasses.py @@ -0,0 +1,37 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""`pydantic.dataclasses.dataclass`: a standard dataclass whose constructor +validates and coerces each field against its annotation.""" + +import dataclasses as _dc +import typing as _typing + +from typing import Any + +from pydantic import _Ctx, _v, _BAD, ValidationError, Field + + +def dataclass(cls=None, **kw): + def wrap(c): + c = _dc.dataclass(c, **kw) + orig = c.__init__ + + def __init__(self, *a, **k): + orig(self, *a, **k) + hints = _typing.get_type_hints(type(self), include_extras=True) + ctx = _Ctx() + ctx.cls = type(self) + i = 0 + for f in _dc.fields(self): + v = _v(hints.get(f.name, Any), getattr(self, f.name), (i if i < len(a) else f.name,), ctx) + i += 1 + if v is not _BAD: + object.__setattr__(self, f.name, v) + if ctx.errs: + raise ValidationError(type(self).__name__, ctx.errs) + c.__init__ = __init__ + return c + if cls is None: + return wrap + return wrap(cls) diff --git a/maven/python-transpiler/src/test/resources/conformance/pydantic_.out b/maven/python-transpiler/src/test/resources/conformance/pydantic_.out new file mode 100644 index 00000000000..556f96062c3 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/pydantic_.out @@ -0,0 +1,162 @@ +id=7 name='Ann' email=None tags=[] score=1.5 +User(id=7, name='Ann', email=None, tags=[], score=1.5) +7 int [] 1.5 +{'id': 7, 'name': 'Ann', 'email': None, 'tags': [], 'score': 1.5} +{"id":7,"name":"Ann","email":null,"tags":[],"score":1.5} +{ + "id": 7, + "name": "Ann", + "email": null, + "tags": [], + "score": 1.5 +} +['id', 'name'] +True False +['id', 'name', 'email', 'tags', 'score'] +True False +i=12 f=1.5 s='x' b=True +i=3 f=2.0 s='y' b=False +i=1 f=3.0 s='z' b=False +1 [('int_parsing', ('id',), 'Input should be a valid integer, unable to parse string as an integer')] +1 [('missing', ('id',), 'Field required')] +1 [('int_from_float', ('id',), 'Input should be a valid integer, got a number with a fractional part')] +1 [('string_type', ('name',), 'Input should be a valid string')] +1 [('list_type', ('tags',), 'Input should be a valid list')] +1 [('string_type', ('tags', 0), 'Input should be a valid string')] +1 [('float_parsing', ('score',), 'Input should be a valid number, unable to parse string as a number')] +2 [('int_type', ('id',), 'Input should be a valid integer'), ('string_type', ('name',), 'Input should be a valid string')] +1 [('bool_parsing', ('b',), 'Input should be a valid boolean, unable to interpret input')] +1 [('bool_parsing', ('b',), 'Input should be a valid boolean, unable to interpret input')] +3 [('int_type', ('i',), 'Input should be a valid integer'), ('float_type', ('f',), 'Input should be a valid number'), ('bool_type', ('b',), 'Input should be a valid boolean')] +1 [('model_type', (), 'Input should be a valid dictionary or instance of User')] +1 [('missing', ('name',), 'Field required')] +OK id=1 name='q' email=None tags=[] score=1.5 +2 [('int_parsing', ('id',), 'Input should be a valid integer, unable to parse string as an integer'), ('missing', ('name',), 'Field required')] +1 validation error for User +id + Input should be a valid integer, unable to parse string as an integer [type=int_parsing, input_value='x', input_type=str] + For further information visit https://errors.pydantic.dev/2.14/v/int_parsing +{'type': 'int_parsing', 'loc': ('id',), 'msg': 'Input should be a valid integer, unable to parse string as an integer', 'input': 'x', 'url': 'https://errors.pydantic.dev/2.14/v/int_parsing'} +True User +['json_invalid'] +age=5 name='abc' code='AB1' ratio=0.5 qty=10 items=[] +1 [('greater_than', ('age',), 'Input should be greater than 0')] +1 [('less_than_equal', ('age',), 'Input should be less than or equal to 120')] +1 [('string_too_short', ('name',), 'String should have at least 2 characters')] +1 [('string_too_long', ('name',), 'String should have at most 5 characters')] +1 [('string_pattern_mismatch', ('code',), "String should match pattern '^[A-Z]{2}\\d+$'")] +1 [('less_than', ('ratio',), 'Input should be less than 1')] +1 [('greater_than_equal', ('ratio',), 'Input should be greater than or equal to 0')] +1 [('multiple_of', ('qty',), 'Input should be a multiple of 5')] +{'gt': 0} {'min_length': 2} +xs=[1] ys=['a'] n=3 m=2 k='hello' +1 [('too_short', ('xs',), 'List should have at least 1 item after validation, not 0')] +1 [('too_long', ('xs',), 'List should have at most 2 items after validation, not 3')] +1 [('too_short', ('ys',), 'List should have at least 1 item after validation, not 0')] +1 [('greater_than', ('n',), 'Input should be greater than 0')] +1 [('greater_than_equal', ('m',), 'Input should be greater than or equal to 2')] +1 [('string_too_short', ('k',), 'String should have at least 2 characters')] +first_name='Z' age=0 Z +first_name='Y' age=0 +{'first_name': 'Z', 'age': 0} {'firstName': 'Z', 'age': 0} +{"firstName":"Z","age":0} +1 [('missing', ('firstName',), 'Field required')] +1 [('missing', ('firstName',), 'Field required')] +first_name='x' +2 [('extra_forbidden', ('b',), 'Extra inputs are not permitted'), ('extra_forbidden', ('c',), 'Extra inputs are not permitted')] +a=1 b=[1] c='x' [1] {'b': [1], 'c': 'x'} {'a': 1, 'b': [1], 'c': 'x'} +a=1 {'a': 1} +[('frozen_instance', ('a',), 'Instance is frozen')] +True True +s='hello' +a=5 {'a'} +ValueError "Mut" object has no field "zzz" +a=5 +1 [('int_parsing', ('a',), 'Input should be a valid integer, unable to parse string as an integer')] +name='n' addr=Addr(city='X', zip=None) others=[Addr(city='Y', zip='1')] by_name={'k': Addr(city='Z', zip=None)} +{'name': 'n', 'addr': {'city': 'X', 'zip': None}, 'others': [{'city': 'Y', 'zip': '1'}], 'by_name': {'k': {'city': 'Z', 'zip': None}}} +{'name': 'n', 'addr': {'city': 'X'}, 'others': [{'city': 'Y', 'zip': '1'}], 'by_name': {'k': {'city': 'Z'}}} +{"name":"n","addr":{"city":"X","zip":null},"others":[{"city":"Y","zip":"1"}],"by_name":{"k":{"city":"Z","zip":null}}} +X Addr +4 [('missing', ('addr', 'city'), 'Field required'), ('string_type', ('others', 0, 'city'), 'Input should be a valid string'), ('model_type', ('others', 1), 'Input should be a valid dictionary or instance of Addr'), ('missing', ('by_name', 'k', 'city'), 'Field required')] +1 [('model_type', ('addr',), 'Input should be a valid dictionary or instance of Addr')] +a='5' b=None c='x' d= e= f=None g=1 a=5 b=None c='x' d= e= f=None g=1 a=5 b=None c='x' d= e= f=None g=1 a='abc' b=None c='x' d= e= f=None g=1 +a=1 b=3 c='y' d= e= f=[1, 2] g=1 +{'a': 1, 'b': None, 'c': 'x', 'd': , 'e': , 'f': None, 'g': 1} {'a': 1, 'b': None, 'c': 'x', 'd': 'b', 'e': 1, 'f': None, 'g': 1} +2 [('int_type', ('a', 'int'), 'Input should be a valid integer'), ('string_type', ('a', 'str'), 'Input should be a valid string')] +1 [('int_parsing', ('b',), 'Input should be a valid integer, unable to parse string as an integer')] +1 [('literal_error', ('c',), "Input should be 'x' or 'y'")] +1 [('enum', ('d',), "Input should be 'r' or 'b'")] +1 [('enum', ('e',), 'Input should be 1 or 2')] +2 [('list_type', ('f', 'list[int]'), 'Input should be a valid list'), ('int_parsing', ('f', 'int'), 'Input should be a valid integer, unable to parse string as an integer')] +1 [('literal_error', ('g',), 'Input should be 1, 2 or 3')] +True (1,) (3, '4') (1, 2) {'a': 1} True +True +[1, 2] [1] +1 [('missing', ('t2', 1), 'Field required')] +1 [('too_long', ('t2',), 'Tuple should have at most 2 items after validation, not 3')] +1 [('int_parsing', ('d', 'a'), 'Input should be a valid integer, unable to parse string as an integer')] +1 [('dict_type', ('d',), 'Input should be a valid dictionary')] +1 [('int_parsing', ('ss', 0), 'Input should be a valid integer, unable to parse string as an integer')] +when=datetime.datetime(2024, 3, 5, 10, 20, 30) day=datetime.date(2024, 3, 6) gap=datetime.timedelta(seconds=90) +{"when":"2024-03-05T10:20:30","day":"2024-03-06","gap":"PT1M30S"} +{'when': '2024-03-05T10:20:30', 'day': '2024-03-06', 'gap': 'PT1M30S'} +when=datetime.datetime(2020, 1, 2, 3, 4, 5) day=datetime.date(2021, 2, 3) gap=datetime.timedelta(seconds=30) +True +1 [('datetime_from_date_parsing', ('when',), 'Input should be a valid datetime or date, input is too short')] +1 [('datetime_type', ('when',), 'Input should be a valid datetime')] +[('date_from_datetime_parsing', ('day',))] +[('date_from_datetime_parsing', ('day',))] +name='BOB' age=20 tags=['a', 'b'] +1 [('value_error', ('age',), 'Value error, too young')] +2 [('value_error', ('name',), 'Value error, bad name'), ('int_parsing', ('age',), 'Input should be a valid integer, unable to parse string as an integer')] +1 validation error for V +age + Value error, too young [type=value_error, input_value=3, input_type=int] + For further information visit https://errors.pydantic.dev/2.14/v/value_error +a=1 b=2 total=3 +a=3 b=4 total=7 +1 [('value_error', (), 'Value error, a must not exceed b')] +2 [('int_parsing', ('a',), 'Input should be a valid integer, unable to parse string as an integer'), ('int_parsing', ('b',), 'Input should be a valid integer, unable to parse string as an integer')] +1 [('assertion_error', ('n',), 'Assertion failed, n must be even')] +n=4 +a=4 b='100' +1 [('int_parsing', ('a',), 'Input should be a valid integer, unable to parse string as an integer')] +w=2 h=3 area=6 +6 {'w': 'w=2', 'h': 3, 'area': 6} {"w":"w=2","h":3,"area":6} +{'name': 'n', 'addr': {'city': 'X', 'zip': '9'}} +{'name': 'n', 'addr': {'city': 'X', 'zip': '9'}} +{'name': 'n', 'addr': {'city': 'X'}, 'others': [{'city': 'Z', 'zip': None}]} +{'addr': {'city': 'X'}} +{'name': 'n', 'addr': {'city': 'X', 'zip': '9'}, 'others': [{'city': 'Y'}, {'city': 'Z'}]} +{'id': 1, 'name': 'a'} +other n True +True False +id='no' name='check' email=None tags=[] score=1.5 +{'properties': {'id': {'title': 'Id', 'type': 'integer'}, 'name': {'title': 'Name', 'type': 'string'}, 'email': {'anyOf': [{'type': 'string'}, {'type': 'null'}], 'default': None, 'title': 'Email'}, 'tags': {'default': [], 'items': {'type': 'string'}, 'title': 'Tags', 'type': 'array'}, 'score': {'default': 1.5, 'title': 'Score', 'type': 'number'}}, 'required': ['id', 'name'], 'title': 'User', 'type': 'object'} +{'properties': {'age': {'exclusiveMinimum': 0, 'maximum': 120, 'title': 'Age', 'type': 'integer'}, 'name': {'maxLength': 5, 'minLength': 2, 'title': 'Name', 'type': 'string'}, 'code': {'default': 'AB1', 'pattern': '^[A-Z]{2}\\d+$', 'title': 'Code', 'type': 'string'}, 'ratio': {'default': 0.5, 'exclusiveMaximum': 1, 'minimum': 0, 'title': 'Ratio', 'type': 'number'}, 'qty': {'default': 10, 'multipleOf': 5, 'title': 'Qty', 'type': 'integer'}, 'items': {'items': {'type': 'integer'}, 'title': 'Items', 'type': 'array'}}, 'required': ['age', 'name'], 'title': 'P', 'type': 'object'} +{'$defs': {'Addr': {'properties': {'city': {'title': 'City', 'type': 'string'}, 'zip': {'anyOf': [{'type': 'string'}, {'type': 'null'}], 'default': None, 'title': 'Zip'}}, 'required': ['city'], 'title': 'Addr', 'type': 'object'}}, 'properties': {'name': {'title': 'Name', 'type': 'string'}, 'addr': {'$ref': '#/$defs/Addr'}, 'others': {'default': [], 'items': {'$ref': '#/$defs/Addr'}, 'title': 'Others', 'type': 'array'}, 'by_name': {'additionalProperties': {'$ref': '#/$defs/Addr'}, 'default': {}, 'title': 'By Name', 'type': 'object'}}, 'required': ['name', 'addr'], 'title': 'Person', 'type': 'object'} +{'$defs': {'Color': {'enum': ['r', 'b'], 'title': 'Color', 'type': 'string'}, 'Num': {'enum': [1, 2], 'title': 'Num', 'type': 'integer'}}, 'properties': {'a': {'anyOf': [{'type': 'integer'}, {'type': 'string'}], 'title': 'A'}, 'b': {'anyOf': [{'type': 'integer'}, {'type': 'null'}], 'default': None, 'title': 'B'}, 'c': {'default': 'x', 'enum': ['x', 'y'], 'title': 'C', 'type': 'string'}, 'd': {'$ref': '#/$defs/Color', 'default': 'r'}, 'e': {'$ref': '#/$defs/Num', 'default': 1}, 'f': {'anyOf': [{'items': {'type': 'integer'}, 'type': 'array'}, {'type': 'integer'}, {'type': 'null'}], 'default': None, 'title': 'F'}, 'g': {'default': 1, 'enum': [1, 2, 3], 'title': 'G', 'type': 'integer'}}, 'required': ['a'], 'title': 'U', 'type': 'object'} +{'additionalProperties': False, 'properties': {'a': {'title': 'A', 'type': 'integer'}}, 'required': ['a'], 'title': 'Forbid', 'type': 'object'} +[1, 2] [1, 2, 3] +2 [('int_parsing', (0,), 'Input should be a valid integer, unable to parse string as an integer'), ('int_type', (2,), 'Input should be a valid integer')] +{'a': 1.5} +None 4 +[1, 2] b'[1,2]' +id=1 name='t' email=None tags=[] score=1.5 +[{'id': 1, 'name': 't', 'email': None, 'tags': [], 'score': 1.5}] +{'items': {'type': 'integer'}, 'type': 'array'} +6 +1 [('greater_than', (), 'Input should be greater than 5')] +['a', 'b'] ['a', 'b'] ["a","b"] +root=['c'] +1 [('list_type', (), 'Input should be a valid list')] +pw=SecretStr('**********') hunter2 {"pw":"**********"} +v=1 kids=[Tree(v=2, kids=[Tree(v=3, kids=[], parent=None)], parent=None)] parent=None +{'v': 1, 'kids': [{'v': 2, 'kids': [{'v': 3, 'kids': [], 'parent': None}], 'parent': None}], 'parent': None} +1 [('int_parsing', ('kids', 0, 'v'), 'Input should be a valid integer, unable to parse string as an integer')] +a=1 6 {'a': 1} +id=2 name='c' ['id', 'name'] +1 [('missing', ('id',), 'Field required')] +True +id=1 name='c' diff --git a/maven/python-transpiler/src/test/resources/conformance/pydantic_.py b/maven/python-transpiler/src/test/resources/conformance/pydantic_.py new file mode 100644 index 00000000000..d4271bd0e28 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/pydantic_.py @@ -0,0 +1,514 @@ +import datetime +import enum +from typing import Annotated, Any, Dict, List, Literal, Optional, Union + +from pydantic import (AfterValidator, BaseModel, BeforeValidator, ConfigDict, Field, RootModel, SecretStr, + TypeAdapter, ValidationError, computed_field, field_serializer, field_validator, + model_validator, PositiveInt, conint, constr) + + +def err(fn): + try: + r = fn() + print("OK", r) + except ValidationError as e: + print(e.error_count(), [(d["type"], d["loc"], d["msg"]) for d in e.errors()]) + + +class User(BaseModel): + id: int + name: str + email: Optional[str] = None + tags: List[str] = [] + score: float = 1.5 + + +u = User(id="7", name="Ann") +print(u) +print(repr(u)) +print(u.id, type(u.id).__name__, u.tags, u.score) +print(u.model_dump()) +print(u.model_dump_json()) +print(u.model_dump_json(indent=2)) +print(sorted(u.model_fields_set)) +print(User(id=1, name="x") == User(id=1, name="x"), User(id=1, name="x") == User(id=2, name="x")) +print(list(User.model_fields)) +print(User.model_fields["id"].is_required(), User.model_fields["tags"].is_required()) + +# lax coercion +class C(BaseModel): + i: int + f: float + s: str + b: bool + + +print(C(i="12", f="1.5", s="x", b="yes")) +print(C(i=3.0, f=2, s="y", b=0)) +print(C(i=True, f=" 3 ", s="z", b="off")) + +# error cases +err(lambda: User(id="x", name="a")) +err(lambda: User(name="a")) +err(lambda: User(id=1.5, name="a")) +err(lambda: User(id=1, name=5)) +err(lambda: User(id=1, name="a", tags="abc")) +err(lambda: User(id=1, name="a", tags=[1, "b"])) +err(lambda: User(id=1, name="a", score="zz")) +err(lambda: User(id=None, name=None)) +err(lambda: C(i=1, f=1, s="a", b="maybe")) +err(lambda: C(i=1, f=1, s="a", b=7)) +err(lambda: C(i=[], f={}, s="a", b=[])) +err(lambda: User.model_validate("nope")) +err(lambda: User.model_validate({"id": 1})) +err(lambda: User.model_validate_json('{"id": 1, "name": "q"}')) +err(lambda: User.model_validate_json('{"id": "z"}')) + +try: + User(id="x", name="a") +except ValidationError as e: + print(str(e)) + print(e.errors()[0]) + print(isinstance(e, ValueError), e.title) + +try: + User.model_validate_json("{bad") +except ValidationError as e: + print([d["type"] for d in e.errors()]) + +# Field constraints +class P(BaseModel): + age: int = Field(gt=0, le=120) + name: str = Field(min_length=2, max_length=5) + code: str = Field(pattern=r"^[A-Z]{2}\d+$", default="AB1") + ratio: float = Field(default=0.5, ge=0, lt=1) + qty: int = Field(default=10, multiple_of=5) + items: List[int] = Field(default_factory=list) + + +print(P(age=5, name="abc")) +err(lambda: P(age=0, name="abc")) +err(lambda: P(age=121, name="abc")) +err(lambda: P(age=5, name="a")) +err(lambda: P(age=5, name="abcdef")) +err(lambda: P(age=5, name="abc", code="xx")) +err(lambda: P(age=5, name="abc", ratio=1)) +err(lambda: P(age=5, name="abc", ratio=-0.1)) +err(lambda: P(age=5, name="abc", qty=7)) +try: + P(age=0, name="a") +except ValidationError as e: + print(e.errors()[0]["ctx"], e.errors()[1]["ctx"]) + + +YsT = Annotated[List[str], Field(min_length=1)] +Ge2 = conint(ge=2) +Stripped = constr(strip_whitespace=True, min_length=2) + + +class L(BaseModel): + xs: List[int] = Field(min_length=1, max_length=2) + ys: YsT = ["a"] + n: PositiveInt = 1 + m: Ge2 = 2 + k: Stripped = "ab" + + +print(L(xs=[1], n=3, k=" hello ")) +err(lambda: L(xs=[])) +err(lambda: L(xs=[1, 2, 3])) +err(lambda: L(xs=[1], ys=[])) +err(lambda: L(xs=[1], n=0)) +err(lambda: L(xs=[1], m=1)) +err(lambda: L(xs=[1], k=" a ")) + +# aliases and config +class A(BaseModel): + model_config = ConfigDict(populate_by_name=True) + first_name: str = Field(alias="firstName") + age: int = 0 + + +a = A(firstName="Z") +print(a, a.first_name) +print(A(first_name="Y")) +print(a.model_dump(), a.model_dump(by_alias=True)) +print(a.model_dump_json(by_alias=True)) +err(lambda: A(age=1)) + + +class NoPop(BaseModel): + first_name: str = Field(alias="firstName") + + +err(lambda: NoPop(first_name="x")) +print(NoPop(firstName="x")) + + +class Forbid(BaseModel): + model_config = ConfigDict(extra="forbid") + a: int + + +class Allow(BaseModel): + model_config = ConfigDict(extra="allow") + a: int + + +class Ignore(BaseModel): + a: int + + +err(lambda: Forbid(a=1, b=2, c=3)) +al = Allow(a=1, b=[1], c="x") +print(al, al.b, al.model_extra, al.model_dump()) +print(Ignore(a=1, b=2), Ignore(a=1, b=2).model_dump()) + + +class Frozen(BaseModel): + model_config = ConfigDict(frozen=True) + a: int + b: tuple = () + + +fz = Frozen(a=1) +try: + fz.a = 2 +except ValidationError as e: + print([(d["type"], d["loc"], d["msg"]) for d in e.errors()]) +print(hash(Frozen(a=1)) == hash(Frozen(a=1)), fz == Frozen(a=1)) + + +class Strip(BaseModel): + model_config = ConfigDict(str_strip_whitespace=True, str_to_lower=True) + s: str + + +print(Strip(s=" HeLLo ")) + + +class Mut(BaseModel): + a: int + + +m = Mut(a=1) +m.a = 5 +print(m, m.model_fields_set) +try: + m.zzz = 1 +except ValueError as e: + print("ValueError", e) + + +class VA(BaseModel): + model_config = ConfigDict(validate_assignment=True) + a: int + + +v = VA(a=1) +v.a = "5" +print(v) +err(lambda: setattr(v, "a", "q")) + +# nested +class Addr(BaseModel): + city: str + zip: Optional[str] = None + + +class Person(BaseModel): + name: str + addr: Addr + others: List[Addr] = [] + by_name: Dict[str, Addr] = {} + + +p = Person(name="n", addr={"city": "X"}, others=[{"city": "Y", "zip": "1"}], by_name={"k": Addr(city="Z")}) +print(p) +print(p.model_dump()) +print(p.model_dump(exclude_none=True)) +print(p.model_dump_json()) +print(p.addr.city, type(p.others[0]).__name__) +err(lambda: Person(name="n", addr={}, others=[{"city": 1}, "q"], by_name={"k": {}})) +err(lambda: Person(name="n", addr=5)) + +# unions, literals, enums, optional +class Color(enum.Enum): + RED = "r" + BLUE = "b" + + +class Num(enum.IntEnum): + ONE = 1 + TWO = 2 + + +class U(BaseModel): + a: Union[int, str] + b: Optional[int] = None + c: Literal["x", "y"] = "x" + d: Color = Color.RED + e: Num = Num.ONE + f: Union[List[int], int, None] = None + g: Literal[1, 2, 3] = 1 + + +print(U(a="5"), U(a=5), U(a=5.0), U(a="abc")) +print(U(a=1, b="3", c="y", d="b", e=2, f=[1, "2"])) +print(U(a=1, d=Color.BLUE).model_dump(), U(a=1, d=Color.BLUE).model_dump(mode="json")) +err(lambda: U(a=[1])) +err(lambda: U(a=1, b="z")) +err(lambda: U(a=1, c="z")) +err(lambda: U(a=1, d="g")) +err(lambda: U(a=1, e=3)) +err(lambda: U(a=1, f="s")) +err(lambda: U(a=1, g=4)) + +# collections +class Coll(BaseModel): + s: set = set() + fs: frozenset = frozenset() + t: tuple = () + t2: tuple[int, str] = (0, "") + t3: tuple[int, ...] = () + d: dict[str, int] = {} + ss: set[int] = set() + + +cc = Coll(s=[1, 2], t=[1], t2=["3", "4"], t3=("1", 2), d={"a": "1"}, ss=[1, 1, "2"]) +print(cc.s == {1, 2}, cc.t, cc.t2, cc.t3, cc.d, cc.ss == {1, 2}) +print(cc.model_dump_json() == cc.model_dump_json()) +print(Coll(t3=[1, 2]).model_dump(mode="json")["t3"], Coll(ss=[1]).model_dump(mode="json")["ss"]) +err(lambda: Coll(t2=[1])) +err(lambda: Coll(t2=[1, 2, 3])) +err(lambda: Coll(d={"a": "z", "b": 1})) +err(lambda: Coll(d=[1])) +err(lambda: Coll(ss={"a"})) + +# datetime +class Ev(BaseModel): + when: datetime.datetime + day: datetime.date = datetime.date(2020, 1, 1) + gap: datetime.timedelta = datetime.timedelta(seconds=90) + + +ev = Ev(when="2024-03-05T10:20:30", day="2024-03-06") +print(ev) +print(ev.model_dump_json()) +print(ev.model_dump(mode="json")) +print(Ev(when=datetime.datetime(2020, 1, 2, 3, 4, 5), day=datetime.date(2021, 2, 3), gap=30)) +print(Ev(when="2024-03-05T10:20:30Z").when.tzinfo is not None) +err(lambda: Ev(when="garbage")) +err(lambda: Ev(when=None)) + +for bad in ["x", "2020-13-45"]: + try: + Ev(when="2020-01-01T00:00:00", day=bad) + except ValidationError as e: + print([(d["type"], d["loc"]) for d in e.errors()]) + +# validators +class V(BaseModel): + name: str + age: int + tags: List[str] = [] + + @field_validator("name") + @classmethod + def upper(cls, v): + return v.upper() + + @field_validator("age") + @classmethod + def adult(cls, v): + if v < 18: + raise ValueError("too young") + return v + + @field_validator("tags", mode="before") + @classmethod + def split(cls, v): + if isinstance(v, str): + return v.split(",") + return v + + @field_validator("name", "age") + @classmethod + def positive_name(cls, v, info): + if info.field_name == "name" and v == "BAD": + raise ValueError("bad name") + return v + + +print(V(name="bob", age=20, tags="a,b")) +err(lambda: V(name="bob", age=3)) +err(lambda: V(name="bad", age="x")) +try: + V(name="bob", age=3) +except ValidationError as e: + print(str(e)) + + +class MV(BaseModel): + a: int + b: int + total: int = 0 + + @model_validator(mode="before") + @classmethod + def pre(cls, data): + if isinstance(data, dict) and "a_b" in data: + x, y = data["a_b"].split("-") + return {"a": x, "b": y} + return data + + @model_validator(mode="after") + def post(self): + if self.a > self.b: + raise ValueError("a must not exceed b") + self.total = self.a + self.b + return self + + +print(MV(a=1, b=2)) +print(MV(a_b="3-4")) +err(lambda: MV(a=5, b=2)) +err(lambda: MV(a_b="x-y")) + + +class AssertV(BaseModel): + n: int + + @field_validator("n") + @classmethod + def chk(cls, v): + assert v % 2 == 0, "n must be even" + return v + + +err(lambda: AssertV(n=3)) +print(AssertV(n=4)) + + +Doubled = Annotated[int, AfterValidator(lambda v: v * 2)] +StrB = Annotated[str, BeforeValidator(lambda v: str(v))] + + +class AnnV(BaseModel): + a: Doubled + b: StrB = "zz" + + +print(AnnV(a=2, b=100)) +err(lambda: AnnV(a="x")) + + +# computed fields and serializers +class Rect(BaseModel): + w: int + h: int + + @computed_field + @property + def area(self) -> int: + return self.w * self.h + + @field_serializer("w") + def ser_w(self, v): + return "w=" + str(v) + + +r = Rect(w=2, h=3) +print(r) +print(r.area, r.model_dump(), r.model_dump_json()) + +# include/exclude, copy +big = Person(name="n", addr={"city": "X", "zip": "9"}, others=[{"city": "Y"}, {"city": "Z"}]) +print(big.model_dump(include={"name", "addr"})) +print(big.model_dump(exclude={"others", "by_name"})) +print(big.model_dump(exclude={"addr": {"zip"}, "by_name": True, "others": {0: True}})) +print(big.model_dump(include={"addr": {"city"}})) +print(big.model_dump(exclude_unset=True)) +print(User(id=1, name="a").model_dump(exclude_defaults=True)) +cp = big.model_copy(update={"name": "other"}) +print(cp.name, big.name, cp.addr is big.addr) +dp = big.model_copy(deep=True) +print(dp == big, dp.addr is big.addr) +print(User.model_construct(id="no", name="check")) + +# schema +print(User.model_json_schema()) +print(P.model_json_schema()) +print(Person.model_json_schema()) +print(U.model_json_schema()) +print(Forbid.model_json_schema()) + +# TypeAdapter +ta = TypeAdapter(List[int]) +print(ta.validate_python(["1", 2]), ta.validate_json("[1, 2, 3]")) +err(lambda: ta.validate_python(["x", 2, None])) +print(TypeAdapter(Dict[str, float]).validate_python({"a": "1.5"})) +print(TypeAdapter(Optional[int]).validate_python(None), TypeAdapter(int).validate_python("4")) +print(ta.dump_python([1, 2]), ta.dump_json([1, 2])) +print(TypeAdapter(User).validate_python({"id": 1, "name": "t"})) +print(TypeAdapter(List[User]).dump_python([User(id=1, name="t")])) +print(TypeAdapter(List[int]).json_schema()) +print(TypeAdapter(Annotated[int, Field(gt=5)]).validate_python(6)) +err(lambda: TypeAdapter(Annotated[int, Field(gt=5)]).validate_python(2)) + +# RootModel, SecretStr, self reference, private attributes +class Pets(RootModel[List[str]]): + pass + + +pt = Pets(["a", "b"]) +print(pt.root, pt.model_dump(), pt.model_dump_json()) +print(Pets.model_validate(["c"])) +err(lambda: Pets.model_validate("x")) + + +class Sec(BaseModel): + pw: SecretStr + + +sc = Sec(pw="hunter2") +print(sc, sc.pw.get_secret_value(), sc.model_dump_json()) + + +class Tree(BaseModel): + v: int + kids: List["Tree"] = [] + parent: Optional["Tree"] = None + + +t = Tree(v=1, kids=[{"v": 2, "kids": [{"v": 3}]}]) +print(t) +print(t.model_dump()) +err(lambda: Tree(v=1, kids=[{"v": "x"}])) + + +class Priv(BaseModel): + a: int + _hidden: int = 5 + + def bump(self): + self._hidden += 1 + return self._hidden + + +pr = Priv(a=1) +print(pr, pr.bump(), pr.model_dump()) + +# inheritance +class Base(BaseModel): + id: int + + +class Child(Base): + name: str = "c" + + +print(Child(id="2"), list(Child.model_fields)) +err(lambda: Child()) +print(isinstance(Child(id=1), Base)) +print(Base.model_validate(Child(id=1))) diff --git a/maven/python-transpiler/src/test/resources/conformance/pydantic_dataclass.out b/maven/python-transpiler/src/test/resources/conformance/pydantic_dataclass.out new file mode 100644 index 00000000000..cbef11088d6 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/pydantic_dataclass.out @@ -0,0 +1,3 @@ +Pt(x=3, y=1.5, tags=['a']) 3 1.5 +True +[('int_parsing', (0,), 'Input should be a valid integer, unable to parse string as an integer'), ('float_type', (1,), 'Input should be a valid number')] diff --git a/maven/python-transpiler/src/test/resources/conformance/pydantic_dataclass.py b/maven/python-transpiler/src/test/resources/conformance/pydantic_dataclass.py new file mode 100644 index 00000000000..aea70421200 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/pydantic_dataclass.py @@ -0,0 +1,20 @@ +from typing import List, Optional + +from pydantic import ValidationError +from pydantic.dataclasses import dataclass + + +@dataclass +class Pt: + x: int + y: float = 0.0 + tags: Optional[List[str]] = None + + +p = Pt("3", "1.5", ["a"]) +print(p, p.x, p.y) +print(Pt(x=1) == Pt(1, 0.0)) +try: + Pt("z", []) +except ValidationError as e: + print([(d["type"], d["loc"], d["msg"]) for d in e.errors()]) From b82eaee162ced5da0cd4cb401686111c49bc8dce Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:14:03 +0200 Subject: [PATCH 010/112] Python: numpy subset over Java kernels, relative submodule imports, slice.indices The HTTP client is installed by the generated registry only when the program bundles requests or httpx, so other programs do not link the network stack. Co-Authored-By: Claude Opus 5.5 --- .../com/codename1/python/runtime/Cn1Http.java | 9 +- .../codename1/python/runtime/Formatting.java | 5 + .../python/runtime/MathFunctions.java | 6 +- .../com/codename1/python/runtime/NdBuf.java | 96 + .../com/codename1/python/runtime/NumPy.java | 1036 ++++++++++ .../com/codename1/python/runtime/PyHttp.java | 11 +- .../com/codename1/python/runtime/PySlice.java | 13 + .../codename1/python/runtime/StdModules.java | 4 + .../transpiler/api/PythonTranspiler.java | 48 +- .../codename1/python/site/numpy/__init__.py | 1841 +++++++++++++++++ .../com/codename1/python/site/numpy/linalg.py | 107 + .../com/codename1/python/site/numpy/random.py | 147 ++ .../src/test/resources/conformance/numpy_.out | 58 + .../src/test/resources/conformance/numpy_.py | 50 + 14 files changed, 3416 insertions(+), 15 deletions(-) create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/NdBuf.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/NumPy.java create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/numpy/__init__.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/numpy/linalg.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/numpy/random.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/numpy_.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/numpy_.py diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Cn1Http.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Cn1Http.java index 6233f9429f0..3cf8d343a3f 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Cn1Http.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Cn1Http.java @@ -34,7 +34,14 @@ /// HTTP through Codename One's `ConnectionRequest`: the platform's own /// client on every target, with its proxy, certificate and cookie handling. -final class Cn1Http extends PyHttp { +public final class Cn1Http extends PyHttp { + /// Makes this the client unless a host installed its own first. + public static void installDefault() { + if (!installed()) { + install(new Cn1Http()); + } + } + @Override public Response send(String method, String url, String[] headers, final byte[] body, int timeoutMillis, boolean followRedirects) throws IOException { diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Formatting.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Formatting.java index 7f548b3646c..f3de12fdce9 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Formatting.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Formatting.java @@ -544,6 +544,11 @@ static String percent(String fmt, Object args) { if (i < n && fmt.charAt(i) == '.') { i++; StringBuilder pr = new StringBuilder("."); + if (i < n && fmt.charAt(i) == '*') { + // `%.*f`: the precision is the next argument. + pr.append(Math.max(0, Py.asLong(items[ai++]))); + i++; + } while (i < n && Character.isDigit(fmt.charAt(i))) { pr.append(fmt.charAt(i++)); } diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/MathFunctions.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/MathFunctions.java index 3c5fe340e3e..b448acdf6f4 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/MathFunctions.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/MathFunctions.java @@ -105,7 +105,7 @@ static double log1p(double x) { return MathUtil.log(u) * x / (u - 1.0); } - private static double sinh(double x) { + static double sinh(double x) { if (x == 0) { // Keeps the sign of a zero, as the C library does. return x; @@ -120,12 +120,12 @@ private static double sinh(double x) { return x < 0 ? -r : r; } - private static double cosh(double x) { + static double cosh(double x) { double e = MathUtil.exp(Math.abs(x)); return (e + 1 / e) / 2; } - private static double tanh(double x) { + static double tanh(double x) { if (x == 0) { // Keeps the sign of a zero, as the C library does. return x; diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/NdBuf.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/NdBuf.java new file mode 100644 index 00000000000..798951fcc20 --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/NdBuf.java @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +/// The storage behind a `numpy.ndarray`: one flat primitive array, `double` +/// for float64 and `long` for int64 and bool (0 or 1). The array's shape, +/// strides and offset live in Python; the loops over them are [NumPy]'s. +final class NdBuf extends PyObject { + static final PyType TYPE = new PyType("_cn1np.buffer", new PyType[] {Py.OBJECT_TYPE}, null); + + final char kind; + final double[] d; + final long[] l; + + NdBuf(char kind, int n) { + this.kind = kind; + if (kind == 'f') { + d = new double[n]; + l = null; + } else { + d = null; + l = new long[n]; + } + } + + int size() { + return d != null ? d.length : l.length; + } + + double getD(int i) { + return d != null ? d[i] : l[i]; + } + + long getL(int i) { + return d != null ? (long) d[i] : l[i]; + } + + void setD(int i, double v) { + if (d != null) { + d[i] = v; + } else if (kind == 'b') { + l[i] = v != 0 ? 1 : 0; + } else { + l[i] = (long) v; + } + } + + void setL(int i, long v) { + if (d != null) { + d[i] = v; + } else { + l[i] = kind == 'b' ? (v != 0 ? 1 : 0) : v; + } + } + + Object item(int i) { + if (kind == 'f') { + return Double.valueOf(d[i]); + } + if (kind == 'b') { + return Py.bool(l[i] != 0); + } + return Long.valueOf(l[i]); + } + + @Override + public PyType type() { + return TYPE; + } + + @Override + public String repr() { + return ""; + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/NumPy.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/NumPy.java new file mode 100644 index 00000000000..15bbe280ae7 --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/NumPy.java @@ -0,0 +1,1036 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +import com.codename1.util.MathUtil; + +/// `_cn1np`: the loops of the numpy subset. The `numpy` package keeps an +/// array's shape, strides and offset and does the indexing arithmetic; every +/// loop over elements runs here, over an [NdBuf]. +/// +/// An operand is passed as three values: the buffer, the offset of its first +/// element and a tuple of strides in elements. A broadcast dimension has +/// stride 0, so the loops never broadcast themselves. Every result is a new +/// contiguous buffer in C order. +final class NumPy { + private NumPy() { + } + + static PyModule module() { + PyModule m = new PyModule("_cn1np", new String[0]); + m.setAttr("alloc", PyBuiltin.positional("alloc", a -> new NdBuf(kind(a[0]), (int) Py.asLong(a[1])))); + m.setAttr("kind", PyBuiltin.positional("kind", a -> String.valueOf(buf(a[0]).kind))); + m.setAttr("size", PyBuiltin.positional("size", a -> Py.of((long) buf(a[0]).size()))); + m.setAttr("fromflat", PyBuiltin.positional("fromflat", a -> fromFlat(kind(a[0]), a[1]))); + m.setAttr("item", PyBuiltin.positional("item", a -> buf(a[0]).item((int) Py.asLong(a[1])))); + m.setAttr("setitem", PyBuiltin.positional("setitem", a -> { + setScalar(buf(a[0]), (int) Py.asLong(a[1]), a[2]); + return Py.NONE; + })); + m.setAttr("copy", PyBuiltin.positional("copy", a -> copy(kind(a[0]), ints(a[1]), buf(a[2]), off(a[3]), ints(a[4])))); + m.setAttr("tolist", PyBuiltin.positional("tolist", a -> toList(ints(a[0]), buf(a[1]), off(a[2]), ints(a[3])))); + m.setAttr("assign", PyBuiltin.positional("assign", a -> { + assign(ints(a[0]), buf(a[1]), off(a[2]), ints(a[3]), buf(a[4]), off(a[5]), ints(a[6])); + return Py.NONE; + })); + m.setAttr("fill", PyBuiltin.positional("fill", a -> { + fill(ints(a[0]), buf(a[1]), off(a[2]), ints(a[3]), a[4]); + return Py.NONE; + })); + m.setAttr("binop", PyBuiltin.positional("binop", a -> binop((int) Py.asLong(a[0]), kind(a[1]), Py.isTrue(a[2]), + ints(a[3]), buf(a[4]), off(a[5]), ints(a[6]), buf(a[7]), off(a[8]), ints(a[9])))); + m.setAttr("unop", PyBuiltin.positional("unop", a -> unop((int) Py.asLong(a[0]), kind(a[1]), Py.isTrue(a[2]), + ints(a[3]), buf(a[4]), off(a[5]), ints(a[6])))); + m.setAttr("reduce", PyBuiltin.positional("reduce", a -> reduce((int) Py.asLong(a[0]), kind(a[1]), Py.isTrue(a[2]), + ints(a[3]), buf(a[4]), off(a[5]), ints(a[6]), (int) Py.asLong(a[7])))); + m.setAttr("cumulate", PyBuiltin.positional("cumulate", a -> cumulate((int) Py.asLong(a[0]), kind(a[1]), + ints(a[2]), buf(a[3]), off(a[4]), ints(a[5]), (int) Py.asLong(a[6])))); + m.setAttr("matmul", PyBuiltin.positional("matmul", a -> matmul(kind(a[0]), (int) Py.asLong(a[1]), (int) Py.asLong(a[2]), + (int) Py.asLong(a[3]), buf(a[4]), off(a[5]), ints(a[6]), buf(a[7]), off(a[8]), ints(a[9])))); + m.setAttr("where", PyBuiltin.positional("where", a -> where(kind(a[0]), ints(a[1]), buf(a[2]), off(a[3]), ints(a[4]), + buf(a[5]), off(a[6]), ints(a[7]), buf(a[8]), off(a[9]), ints(a[10])))); + m.setAttr("positions", PyBuiltin.positional("positions", a -> positions(ints(a[0]), off(a[1]), ints(a[2])))); + m.setAttr("compress", PyBuiltin.positional("compress", a -> compress(buf(a[0]), buf(a[1])))); + m.setAttr("gather", PyBuiltin.positional("gather", a -> gather(buf(a[0]), buf(a[1])))); + m.setAttr("scatter", PyBuiltin.positional("scatter", a -> { + scatter(buf(a[0]), buf(a[1]), buf(a[2])); + return Py.NONE; + })); + m.setAttr("sortrows", PyBuiltin.positional("sortrows", a -> sortRows(buf(a[0]), (int) Py.asLong(a[1]), Py.isTrue(a[2])))); + m.setAttr("concat", PyBuiltin.positional("concat", a -> concat(kind(a[0]), a[1]))); + m.setAttr("arange", PyBuiltin.positional("arange", a -> arange(kind(a[0]), a[1], a[2], (int) Py.asLong(a[3])))); + return m; + } + + // ---- argument decoding -------------------------------------------- + + private static char kind(Object o) { + String s = Py.toStr(o); + if (s.length() != 1 || "fib".indexOf(s.charAt(0)) < 0) { + throw Py.valueError("bad dtype kind " + s); + } + return s.charAt(0); + } + + private static NdBuf buf(Object o) { + if (!(o instanceof NdBuf)) { + throw Py.typeError("expected an ndarray buffer"); + } + return (NdBuf) o; + } + + private static int off(Object o) { + return (int) Py.asLong(o); + } + + private static int[] ints(Object o) { + Object[] items; + if (o instanceof PyTuple) { + items = ((PyTuple) o).items; + } else if (o instanceof PyList) { + items = ((PyList) o).toArray(); + } else { + throw Py.typeError("expected a tuple of ints"); + } + int[] r = new int[items.length]; + for (int i = 0; i < r.length; i++) { + r[i] = (int) Py.asLong(items[i]); + } + return r; + } + + private static int count(int[] shape) { + int n = 1; + for (int s : shape) { + n *= s; + } + return n; + } + + private static boolean isFloatScalar(Object v) { + return v instanceof Double || Py.typeOf(v) == Py.FLOAT_TYPE; + } + + private static void setScalar(NdBuf b, int i, Object v) { + if (v instanceof PyBool) { + b.setL(i, Py.isTrue(v) ? 1 : 0); + } else if (isFloatScalar(v)) { + b.setD(i, Py.asDouble(v)); + } else if (b.kind == 'f') { + b.setD(i, Py.asDouble(v)); + } else { + b.setL(i, Py.asLong(v)); + } + } + + private static NdBuf fromFlat(char kind, Object list) { + Object[] items = list instanceof PyList ? ((PyList) list).toArray() : ((PyTuple) list).items; + NdBuf b = new NdBuf(kind, items.length); + for (int i = 0; i < items.length; i++) { + setScalar(b, i, items[i]); + } + return b; + } + + // ---- the strided walk --------------------------------------------- + + /// Advances `idx` over `shape` in C order and moves each operand's + /// position by its strides; answers false past the last element. + private static boolean step(int[] shape, int[] idx, int[][] strides, int[] pos) { + for (int d = shape.length - 1; d >= 0; d--) { + idx[d]++; + for (int k = 0; k < pos.length; k++) { + pos[k] += strides[k][d]; + } + if (idx[d] < shape[d]) { + return true; + } + for (int k = 0; k < pos.length; k++) { + pos[k] -= strides[k][d] * shape[d]; + } + idx[d] = 0; + } + return false; + } + + private static NdBuf copy(char kind, int[] shape, NdBuf a, int aoff, int[] ast) { + int n = count(shape); + NdBuf r = new NdBuf(kind, n); + if (n == 0) { + return r; + } + int[] idx = new int[shape.length]; + int[] pos = {aoff}; + int[][] st = {ast}; + int i = 0; + do { + if (a.d != null) { + r.setD(i++, a.d[pos[0]]); + } else { + r.setL(i++, a.l[pos[0]]); + } + } while (step(shape, idx, st, pos)); + return r; + } + + private static Object toList(int[] shape, NdBuf a, int off, int[] st) { + return toList(shape, 0, a, off, st); + } + + private static Object toList(int[] shape, int dim, NdBuf a, int off, int[] st) { + if (dim == shape.length) { + return a.item(off); + } + PyList out = new PyList(shape[dim]); + for (int i = 0; i < shape[dim]; i++) { + out.append(toList(shape, dim + 1, a, off + i * st[dim], st)); + } + return out; + } + + private static void assign(int[] shape, NdBuf dst, int doff, int[] dst2, NdBuf src, int soff, int[] sst) { + if (count(shape) == 0) { + return; + } + int[] idx = new int[shape.length]; + int[] pos = {doff, soff}; + int[][] st = {dst2, sst}; + if (dst == src) { + // Overlapping views: read everything before writing anything. + NdBuf tmp = copy(src.kind, shape, src, soff, sst); + src = tmp; + pos[1] = 0; + st[1] = contiguous(shape); + } + do { + if (src.d != null) { + dst.setD(pos[0], src.d[pos[1]]); + } else { + dst.setL(pos[0], src.l[pos[1]]); + } + } while (step(shape, idx, st, pos)); + } + + private static int[] contiguous(int[] shape) { + int[] st = new int[shape.length]; + int s = 1; + for (int d = shape.length - 1; d >= 0; d--) { + st[d] = s; + s *= shape[d]; + } + return st; + } + + private static void fill(int[] shape, NdBuf dst, int off, int[] st, Object v) { + if (count(shape) == 0) { + return; + } + int[] idx = new int[shape.length]; + int[] pos = {off}; + int[][] sts = {st}; + do { + setScalar(dst, pos[0], v); + } while (step(shape, idx, sts, pos)); + } + + // ---- element-wise ------------------------------------------------- + + static final int ADD = 0; + static final int SUB = 1; + static final int MUL = 2; + static final int TRUEDIV = 3; + static final int FLOORDIV = 4; + static final int MOD = 5; + static final int POW = 6; + static final int EQ = 7; + static final int NE = 8; + static final int LT = 9; + static final int LE = 10; + static final int GT = 11; + static final int GE = 12; + static final int AND = 13; + static final int OR = 14; + static final int XOR = 15; + static final int MIN = 16; + static final int MAX = 17; + static final int ATAN2 = 18; + static final int HYPOT = 19; + static final int LSHIFT = 20; + static final int RSHIFT = 21; + + private static NdBuf binop(int op, char kind, boolean fp, int[] shape, NdBuf a, int aoff, int[] ast, + NdBuf b, int boff, int[] bst) { + int n = count(shape); + NdBuf r = new NdBuf(kind, n); + if (n == 0) { + return r; + } + int[] idx = new int[shape.length]; + int[] pos = {aoff, boff}; + int[][] st = {ast, bst}; + int i = 0; + if (fp) { + do { + r.setD(i++, fop(op, a.getD(pos[0]), b.getD(pos[1]))); + } while (step(shape, idx, st, pos)); + } else { + do { + r.setL(i++, lop(op, a.l[pos[0]], b.l[pos[1]])); + } while (step(shape, idx, st, pos)); + } + return r; + } + + private static double fop(int op, double x, double y) { + switch (op) { + case ADD: + return x + y; + case SUB: + return x - y; + case MUL: + return x * y; + case TRUEDIV: + return x / y; + case FLOORDIV: + return Math.floor(x / y); + case MOD: { + if (y == 0) { + return Double.NaN; + } + double m = x % y; + return m != 0 && (m < 0) != (y < 0) ? m + y : m; + } + case POW: + return MathUtil.pow(x, y); + case EQ: + return x == y ? 1 : 0; + case NE: + return x != y ? 1 : 0; + case LT: + return x < y ? 1 : 0; + case LE: + return x <= y ? 1 : 0; + case GT: + return x > y ? 1 : 0; + case GE: + return x >= y ? 1 : 0; + case MIN: + return x != x || y != y ? Double.NaN : Math.min(x, y); + case MAX: + return x != x || y != y ? Double.NaN : Math.max(x, y); + case ATAN2: + return MathUtil.atan2(x, y); + case HYPOT: + return Math.sqrt(x * x + y * y); + default: + throw Py.typeError("ufunc not supported for the input types"); + } + } + + private static long lop(int op, long x, long y) { + switch (op) { + case ADD: + return x + y; + case SUB: + return x - y; + case MUL: + return x * y; + case FLOORDIV: { + if (y == 0) { + return 0; + } + long q = x / y; + return (x % y != 0) && ((x < 0) != (y < 0)) ? q - 1 : q; + } + case MOD: { + if (y == 0) { + return 0; + } + long m = x % y; + return m != 0 && (m < 0) != (y < 0) ? m + y : m; + } + case POW: { + if (y < 0) { + throw Py.valueError("Integers to negative integer powers are not allowed."); + } + long r = 1; + long base = x; + long e = y; + while (e > 0) { + if ((e & 1) != 0) { + r *= base; + } + base *= base; + e >>= 1; + } + return r; + } + case EQ: + return x == y ? 1 : 0; + case NE: + return x != y ? 1 : 0; + case LT: + return x < y ? 1 : 0; + case LE: + return x <= y ? 1 : 0; + case GT: + return x > y ? 1 : 0; + case GE: + return x >= y ? 1 : 0; + case AND: + return x & y; + case OR: + return x | y; + case XOR: + return x ^ y; + case MIN: + return Math.min(x, y); + case MAX: + return Math.max(x, y); + case LSHIFT: + return x << y; + case RSHIFT: + return x >> y; + default: + throw Py.typeError("ufunc not supported for the input types"); + } + } + + static final int NEG = 0; + static final int ABS = 1; + static final int SQRT = 2; + static final int EXP = 3; + static final int LOG = 4; + static final int SIN = 5; + static final int COS = 6; + static final int TAN = 7; + static final int FLOOR = 8; + static final int CEIL = 9; + static final int NOT = 10; + static final int INVERT = 11; + static final int SIGN = 12; + static final int SQUARE = 13; + static final int LOG10 = 14; + static final int LOG2 = 15; + static final int TANH = 16; + static final int ARCSIN = 17; + static final int ARCCOS = 18; + static final int ARCTAN = 19; + static final int RINT = 20; + static final int ISNAN = 21; + static final int ISINF = 22; + static final int ISFINITE = 23; + static final int SINH = 24; + static final int COSH = 25; + static final int EXPM1 = 26; + static final int LOG1P = 27; + static final int TRUNC = 28; + + private static NdBuf unop(int op, char kind, boolean fp, int[] shape, NdBuf a, int aoff, int[] ast) { + int n = count(shape); + NdBuf r = new NdBuf(kind, n); + if (n == 0) { + return r; + } + int[] idx = new int[shape.length]; + int[] pos = {aoff}; + int[][] st = {ast}; + int i = 0; + if (fp) { + do { + r.setD(i++, fun(op, a.getD(pos[0]))); + } while (step(shape, idx, st, pos)); + } else { + do { + r.setL(i++, lun(op, a.l[pos[0]], a.kind == 'b')); + } while (step(shape, idx, st, pos)); + } + return r; + } + + private static double fun(int op, double x) { + switch (op) { + case NEG: + return -x; + case ABS: + return Math.abs(x); + case SQRT: + return Math.sqrt(x); + case EXP: + return MathUtil.exp(x); + case LOG: + return x < 0 ? Double.NaN : MathUtil.log(x); + case SIN: + return Math.sin(x); + case COS: + return Math.cos(x); + case TAN: + return Math.tan(x); + case FLOOR: + return Math.floor(x); + case CEIL: + return Math.ceil(x); + case NOT: + return x == 0 ? 1 : 0; + case SIGN: + return x > 0 ? 1 : x < 0 ? -1 : x; + case SQUARE: + return x * x; + case LOG10: + return x < 0 ? Double.NaN : MathUtil.log10(x); + case LOG2: + return x < 0 ? Double.NaN : MathUtil.log(x) / MathUtil.log(2); + case TANH: + return MathFunctions.tanh(x); + case ARCSIN: + return MathUtil.asin(x); + case ARCCOS: + return MathUtil.acos(x); + case ARCTAN: + return MathUtil.atan(x); + case RINT: + return rint(x); + case ISNAN: + return x != x ? 1 : 0; + case ISINF: + return Double.isInfinite(x) ? 1 : 0; + case ISFINITE: + return x != x || Double.isInfinite(x) ? 0 : 1; + case SINH: + return MathFunctions.sinh(x); + case COSH: + return MathFunctions.cosh(x); + case EXPM1: + return MathFunctions.expm1(x); + case LOG1P: + return MathFunctions.log1p(x); + case TRUNC: + return x < 0 ? Math.ceil(x) : Math.floor(x); + default: + throw Py.typeError("ufunc not supported for the input types"); + } + } + + /// Round half to even, as `np.rint` and `np.round` do. + static double rint(double x) { + double f = Math.floor(x); + double diff = x - f; + if (diff > 0.5) { + return f + 1; + } + if (diff < 0.5) { + return f; + } + return f % 2 == 0 ? f : f + 1; + } + + private static long lun(int op, long x, boolean bool) { + switch (op) { + case NEG: + return -x; + case ABS: + return Math.abs(x); + case NOT: + return x == 0 ? 1 : 0; + case INVERT: + return bool ? (x == 0 ? 1 : 0) : ~x; + case SIGN: + return x > 0 ? 1 : x < 0 ? -1 : 0; + case SQUARE: + return x * x; + case FLOOR: + case CEIL: + case RINT: + case TRUNC: + return x; + case ISNAN: + case ISINF: + return 0; + case ISFINITE: + return 1; + default: + throw Py.typeError("ufunc not supported for the input types"); + } + } + + // ---- reductions --------------------------------------------------- + + static final int SUM = 0; + static final int PROD = 1; + static final int RMIN = 2; + static final int RMAX = 3; + static final int ARGMIN = 4; + static final int ARGMAX = 5; + static final int ANY = 6; + static final int ALL = 7; + + /// Reduces along `axis` (or over everything when it is -1); the result + /// has the input's shape without that axis. + private static NdBuf reduce(int op, char kind, boolean fp, int[] shape, NdBuf a, int aoff, int[] ast, int axis) { + if (axis < 0) { + NdBuf flat = a; + int[] fshape = {count(shape)}; + if (!isContiguous(shape, ast) || aoff != 0 || a.size() != fshape[0]) { + flat = copy(a.kind, shape, a, aoff, ast); + } + return reduce(op, kind, fp, fshape, flat, 0, new int[] {1}, 0); + } + int len = shape[axis]; + if (len == 0 && (op == RMIN || op == RMAX || op == ARGMIN || op == ARGMAX)) { + throw Py.valueError("zero-size array to reduction operation which has no identity"); + } + int[] outer = new int[shape.length - 1]; + int[] ost = new int[shape.length - 1]; + for (int d = 0, j = 0; d < shape.length; d++) { + if (d != axis) { + outer[j] = shape[d]; + ost[j++] = ast[d]; + } + } + int n = count(outer); + NdBuf r = new NdBuf(kind, n); + if (n == 0) { + return r; + } + int stride = ast[axis]; + int[] idx = new int[outer.length]; + int[] pos = {aoff}; + int[][] st = {ost}; + int i = 0; + do { + int p = pos[0]; + if (fp) { + reduceD(op, r, i++, a, p, stride, len); + } else { + reduceL(op, r, i++, a, p, stride, len); + } + } while (outer.length > 0 && step(outer, idx, st, pos)); + return r; + } + + private static boolean isContiguous(int[] shape, int[] st) { + int[] c = contiguous(shape); + for (int d = 0; d < shape.length; d++) { + if (shape[d] > 1 && c[d] != st[d]) { + return false; + } + } + return true; + } + + private static void reduceD(int op, NdBuf r, int i, NdBuf a, int p, int stride, int len) { + switch (op) { + case SUM: { + // Pairwise like numpy, so long sums round the same way. + r.setD(i, pairwise(a, p, stride, len)); + return; + } + case PROD: { + double acc = 1; + for (int k = 0; k < len; k++) { + acc *= a.getD(p + k * stride); + } + r.setD(i, acc); + return; + } + case RMIN: + case RMAX: + case ARGMIN: + case ARGMAX: { + double best = a.getD(p); + int at = 0; + boolean min = op == RMIN || op == ARGMIN; + for (int k = 1; k < len && best == best; k++) { + double v = a.getD(p + k * stride); + if (v != v || (min ? v < best : v > best)) { + best = v; + at = k; + } + } + if (op == ARGMIN || op == ARGMAX) { + r.setL(i, at); + } else { + r.setD(i, best); + } + return; + } + case ANY: + case ALL: { + boolean any = op == ANY; + boolean res = !any; + for (int k = 0; k < len; k++) { + if ((a.getD(p + k * stride) != 0) == any) { + res = any; + break; + } + } + r.setL(i, res ? 1 : 0); + return; + } + default: + throw Py.typeError("unsupported reduction"); + } + } + + private static double pairwise(NdBuf a, int p, int stride, int len) { + if (len < 8) { + double s = 0; + for (int k = 0; k < len; k++) { + s += a.getD(p + k * stride); + } + return s; + } + if (len <= 128) { + double[] acc = new double[8]; + for (int j = 0; j < 8; j++) { + acc[j] = a.getD(p + j * stride); + } + int k = 8; + for (; k + 8 <= len; k += 8) { + for (int j = 0; j < 8; j++) { + acc[j] += a.getD(p + (k + j) * stride); + } + } + double s = ((acc[0] + acc[1]) + (acc[2] + acc[3])) + ((acc[4] + acc[5]) + (acc[6] + acc[7])); + for (; k < len; k++) { + s += a.getD(p + k * stride); + } + return s; + } + int half = len / 2; + half -= half % 8; + return pairwise(a, p, stride, half) + pairwise(a, p + half * stride, stride, len - half); + } + + private static void reduceL(int op, NdBuf r, int i, NdBuf a, int p, int stride, int len) { + long[] l = a.l; + switch (op) { + case SUM: { + long acc = 0; + for (int k = 0; k < len; k++) { + acc += l[p + k * stride]; + } + r.setL(i, acc); + return; + } + case PROD: { + long acc = 1; + for (int k = 0; k < len; k++) { + acc *= l[p + k * stride]; + } + r.setL(i, acc); + return; + } + case RMIN: + case RMAX: + case ARGMIN: + case ARGMAX: { + long best = l[p]; + int at = 0; + boolean min = op == RMIN || op == ARGMIN; + for (int k = 1; k < len; k++) { + long v = l[p + k * stride]; + if (min ? v < best : v > best) { + best = v; + at = k; + } + } + r.setL(i, op == ARGMIN || op == ARGMAX ? at : best); + return; + } + case ANY: + case ALL: { + boolean any = op == ANY; + boolean res = !any; + for (int k = 0; k < len; k++) { + if ((l[p + k * stride] != 0) == any) { + res = any; + break; + } + } + r.setL(i, res ? 1 : 0); + return; + } + default: + throw Py.typeError("unsupported reduction"); + } + } + + /// `cumsum` (op 0) and `cumprod` (op 1) along `axis`, in place over a + /// contiguous copy. + private static NdBuf cumulate(int op, char kind, int[] shape, NdBuf a, int aoff, int[] ast, int axis) { + NdBuf r = copy(kind, shape, a, aoff, ast); + int[] st = contiguous(shape); + int len = shape[axis]; + int stride = st[axis]; + int n = r.size(); + if (n == 0) { + return r; + } + int block = stride * len; + for (int base = 0; base < n; base += block) { + for (int s = 0; s < stride; s++) { + int p = base + s; + for (int k = 1; k < len; k++) { + int q = p + k * stride; + int prev = q - stride; + if (r.d != null) { + r.d[q] = op == 0 ? r.d[prev] + r.d[q] : r.d[prev] * r.d[q]; + } else { + r.l[q] = op == 0 ? r.l[prev] + r.l[q] : r.l[prev] * r.l[q]; + } + } + } + } + return r; + } + + // ---- linear algebra ----------------------------------------------- + + /// `(n x k) @ (k x m)`. + private static NdBuf matmul(char kind, int n, int k, int m, NdBuf a, int aoff, int[] ast, NdBuf b, int boff, int[] bst) { + NdBuf r = new NdBuf(kind, n * m); + if (kind == 'f') { + double[] row = new double[m]; + for (int i = 0; i < n; i++) { + for (int j = 0; j < m; j++) { + row[j] = 0; + } + for (int t = 0; t < k; t++) { + double av = a.getD(aoff + i * ast[0] + t * ast[1]); + if (av == 0) { + continue; + } + int bp = boff + t * bst[0]; + for (int j = 0; j < m; j++) { + row[j] += av * b.getD(bp + j * bst[1]); + } + } + System.arraycopy(row, 0, r.d, i * m, m); + } + } else { + for (int i = 0; i < n; i++) { + for (int j = 0; j < m; j++) { + long s = 0; + for (int t = 0; t < k; t++) { + s += a.getL(aoff + i * ast[0] + t * ast[1]) * b.getL(boff + t * bst[0] + j * bst[1]); + } + r.setL(i * m + j, s); + } + } + } + return r; + } + + private static NdBuf where(char kind, int[] shape, NdBuf c, int coff, int[] cst, NdBuf x, int xoff, int[] xst, + NdBuf y, int yoff, int[] yst) { + int n = count(shape); + NdBuf r = new NdBuf(kind, n); + if (n == 0) { + return r; + } + int[] idx = new int[shape.length]; + int[] pos = {coff, xoff, yoff}; + int[][] st = {cst, xst, yst}; + int i = 0; + do { + boolean t = c.getD(pos[0]) != 0; + NdBuf s = t ? x : y; + int p = t ? pos[1] : pos[2]; + if (s.d != null) { + r.setD(i++, s.d[p]); + } else { + r.setL(i++, s.l[p]); + } + } while (step(shape, idx, st, pos)); + return r; + } + + // ---- fancy indexing ----------------------------------------------- + + /// The buffer position of every element of a view, in C order. + private static NdBuf positions(int[] shape, int off, int[] st) { + int n = count(shape); + NdBuf r = new NdBuf('i', n); + if (n == 0) { + return r; + } + int[] idx = new int[shape.length]; + int[] pos = {off}; + int[][] sts = {st}; + int i = 0; + do { + r.l[i++] = pos[0]; + } while (step(shape, idx, sts, pos)); + return r; + } + + /// The positions whose mask entry is true. + private static NdBuf compress(NdBuf pos, NdBuf mask) { + int n = 0; + for (int i = 0; i < mask.size(); i++) { + if (mask.getD(i) != 0) { + n++; + } + } + NdBuf r = new NdBuf('i', n); + for (int i = 0, j = 0; i < mask.size(); i++) { + if (mask.getD(i) != 0) { + r.l[j++] = pos.l[i]; + } + } + return r; + } + + private static NdBuf gather(NdBuf a, NdBuf pos) { + int n = pos.size(); + NdBuf r = new NdBuf(a.kind, n); + for (int i = 0; i < n; i++) { + int p = (int) pos.l[i]; + if (a.d != null) { + r.d[i] = a.d[p]; + } else { + r.l[i] = a.l[p]; + } + } + return r; + } + + /// Writes `src` (cycled when shorter, so a scalar broadcasts) into `a` + /// at `pos`. + private static void scatter(NdBuf a, NdBuf pos, NdBuf src) { + int n = pos.size(); + int m = src.size(); + if (m == 0) { + return; + } + for (int i = 0; i < n; i++) { + int p = (int) pos.l[i]; + if (src.d != null) { + a.setD(p, src.d[i % m]); + } else { + a.setL(p, src.l[i % m]); + } + } + } + + /// Sorts each row of `len` elements of a contiguous buffer, answering + /// the sorted values or, with `arg`, the stable sort's indices. NaN + /// sorts last. + private static NdBuf sortRows(NdBuf a, int len, boolean arg) { + int n = a.size(); + NdBuf r = arg ? new NdBuf('i', n) : new NdBuf(a.kind, n); + int[] perm = new int[len]; + int[] tmp = new int[len]; + for (int base = 0; base + len <= n && len > 0; base += len) { + for (int i = 0; i < len; i++) { + perm[i] = i; + } + mergeSort(a, base, perm, tmp, 0, len); + for (int i = 0; i < len; i++) { + if (arg) { + r.l[base + i] = perm[i]; + } else if (a.d != null) { + r.d[base + i] = a.d[base + perm[i]]; + } else { + r.l[base + i] = a.l[base + perm[i]]; + } + } + } + return r; + } + + private static boolean less(NdBuf a, int base, int x, int y) { + if (a.d != null) { + double u = a.d[base + x]; + double v = a.d[base + y]; + if (v != v) { + return u == u; + } + return u < v; + } + return a.l[base + x] < a.l[base + y]; + } + + private static void mergeSort(NdBuf a, int base, int[] p, int[] tmp, int lo, int hi) { + if (hi - lo < 2) { + return; + } + int mid = (lo + hi) >>> 1; + mergeSort(a, base, p, tmp, lo, mid); + mergeSort(a, base, p, tmp, mid, hi); + int i = lo; + int j = mid; + int k = lo; + while (i < mid && j < hi) { + tmp[k++] = less(a, base, p[j], p[i]) ? p[j++] : p[i++]; + } + while (i < mid) { + tmp[k++] = p[i++]; + } + while (j < hi) { + tmp[k++] = p[j++]; + } + System.arraycopy(tmp, lo, p, lo, hi - lo); + } + + private static NdBuf arange(char kind, Object start, Object step, int n) { + NdBuf r = new NdBuf(kind, n); + if (kind == 'f') { + double s = Py.asDouble(start); + double d = Py.asDouble(step); + for (int i = 0; i < n; i++) { + r.d[i] = s + i * d; + } + } else { + long s = Py.asLong(start); + long d = Py.asLong(step); + for (int i = 0; i < n; i++) { + r.l[i] = s + i * d; + } + } + return r; + } + + private static NdBuf concat(char kind, Object parts) { + Object[] items = parts instanceof PyList ? ((PyList) parts).toArray() : ((PyTuple) parts).items; + int n = 0; + for (Object o : items) { + n += buf(o).size(); + } + NdBuf r = new NdBuf(kind, n); + int at = 0; + for (Object o : items) { + NdBuf b = buf(o); + for (int i = 0; i < b.size(); i++) { + if (b.d != null) { + r.setD(at++, b.d[i]); + } else { + r.setL(at++, b.l[i]); + } + } + } + return r; + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyHttp.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyHttp.java index 41c8f2a439c..f79dba6ac52 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyHttp.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyHttp.java @@ -34,14 +34,21 @@ public abstract class PyHttp { private static PyHttp current; - /// The client in use. + /// The client in use. A build that bundles `requests` or `httpx` + /// installs Codename One's from its generated registry + /// ([Cn1Http#installDefault()]); nothing names it otherwise, so a + /// program without HTTP does not link the network stack. public static PyHttp get() { if (current == null) { - current = new Cn1Http(); + throw Py.error(Py.OS_ERROR, "no HTTP client is installed"); } return current; } + static boolean installed() { + return current != null; + } + /// Replaces the client, for a host that is not a Codename One /// application. public static void install(PyHttp http) { diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PySlice.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PySlice.java index 144b51f785c..4df657f0a80 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PySlice.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PySlice.java @@ -123,6 +123,19 @@ public Object getAttrOrNull(String name) { if ("step".equals(name)) { return step; } + if ("indices".equals(name)) { + return PyBuiltin.positional("indices", a -> { + if (a.length != 1) { + throw Py.typeError("slice.indices() takes exactly one argument (" + a.length + " given)"); + } + long n = Py.asLong(a[0]); + if (n < 0) { + throw Py.valueError("length should not be negative"); + } + int[] r = indices((int) n); + return new PyTuple(new Object[] {Py.of((long) r[0]), Py.of((long) r[1]), Py.of((long) r[2])}); + }); + } return super.getAttrOrNull(name); } diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java index 77a37267180..5e0abcfa877 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java @@ -57,6 +57,10 @@ static boolean provide(String name) { PyModule.registerModule(fileSystem()); return true; } + if ("_cn1np".equals(name)) { + PyModule.registerModule(NumPy.module()); + return true; + } if ("_cn1http".equals(name)) { PyModule.registerModule(http()); return true; diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java index 099091a1f25..57f4b0a76f0 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java @@ -163,7 +163,14 @@ private static Set addStdlib(Map sources, Map imported = new ArrayList(); - imports(tree.body, imported); + String own = ""; + for (Map.Entry e : trees.entrySet()) { + if (e.getValue() == tree) { + own = e.getKey(); + break; + } + } + imports(tree.body, imported, moduleOf(own), own.endsWith("__init__.py")); if (opens.contains(tree)) { // The builtin open() is io.open, written in Python. imported.add("io"); @@ -229,8 +236,9 @@ private static String moduleOf(String path) { return stem.replace('/', '.'); } - /// Every absolute module name `body` imports, at any depth. - private static void imports(List body, List out) { + /// Every module name `body` imports, at any depth, with relative imports + /// resolved against `self` (a package when `pkg`). + private static void imports(List body, List out, String self, boolean pkg) { for (Node.Stmt s : body) { if (s instanceof Node.Import) { for (Node.Alias a : ((Node.Import) s).names) { @@ -238,27 +246,46 @@ private static void imports(List body, List out) { } } else if (s instanceof Node.ImportFrom) { Node.ImportFrom f = (Node.ImportFrom) s; - if (f.level == 0 && f.module != null) { - out.add(f.module); + String from = f.level == 0 ? f.module : relative(self, pkg, f.level, f.module); + if (from != null && !from.isEmpty()) { + out.add(from); for (Node.Alias a : f.names) { // `from pkg import sub` may name a submodule. if (!"*".equals(a.name)) { - out.add(f.module + "." + a.name); + out.add(from + "." + a.name); } } } } else if (s instanceof Node.FunctionDef) { - imports(((Node.FunctionDef) s).body, out); + imports(((Node.FunctionDef) s).body, out, self, pkg); } else if (s instanceof Node.ClassDef) { - imports(((Node.ClassDef) s).body, out); + imports(((Node.ClassDef) s).body, out, self, pkg); } else { for (List b : bodiesOf(s)) { - imports(b, out); + imports(b, out, self, pkg); } } } } + /// The absolute name of `from import`, or null when it + /// climbs out of the top-level package (the emitter reports that). + private static String relative(String self, boolean pkg, int level, String module) { + String base = self; + int up = pkg ? level - 1 : level; + for (int i = 0; i < up; i++) { + int dot = base.lastIndexOf('.'); + if (dot < 0) { + return null; + } + base = base.substring(0, dot); + } + if (module == null) { + return base; + } + return base.isEmpty() ? module : base + "." + module; + } + private static List> bodiesOf(Node.Stmt s) { List> r = new ArrayList>(); if (s instanceof Node.If) { @@ -344,6 +371,9 @@ private static String registry(Map classes, String mainModule, b if (interop) { sb.append(" JavaInterop.register();\n"); } + if (classes.containsKey("_cn1_http")) { + sb.append(" com.codename1.python.runtime.Cn1Http.installDefault();\n"); + } for (String cls : classes.values()) { sb.append(" ").append(cls).append(".register();\n"); } diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/numpy/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/numpy/__init__.py new file mode 100644 index 00000000000..7cd5642d2af --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/numpy/__init__.py @@ -0,0 +1,1841 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""A numpy subset: n-dimensional arrays of float64, int64 and bool. + +The array keeps its shape, strides (in elements) and offset here; every loop +over elements is in the runtime's `_cn1np` module, over a flat primitive +buffer. Views share their buffer, as numpy's do. Other dtypes are accepted by +name and stored as the nearest of the three. +""" + +import builtins as _b +import math as _math +import _cn1np + +__version__ = "2.0.0" + +pi = _math.pi +e = _math.e +inf = float("inf") +nan = float("nan") +newaxis = None + + +# ---- dtypes --------------------------------------------------------------- + +class dtype: + def __init__(self, spec="float64"): + if isinstance(spec, dtype): + self.kind, self.name = spec.kind, spec.name + return + key = spec + if isinstance(spec, type): + key = _TYPE_NAMES.get(spec, getattr(spec, "__name__", "float64")) + if key not in _KINDS: + raise TypeError("data type %r not understood" % (spec,)) + self.kind = _KINDS[key] + self.name = _NAMES[self.kind] if key in _ALIASES else key + + @property + def type(self): + return {"f": float64, "i": int64, "b": bool_}[self.kind] + + @property + def itemsize(self): + return 1 if self.kind == "b" else 8 + + @property + def char(self): + return {"f": "d", "i": "l", "b": "?"}[self.kind] + + def __repr__(self): + return "dtype('%s')" % self.name + + def __str__(self): + return self.name + + def __eq__(self, other): + try: + return dtype(other).kind == self.kind + except TypeError: + return False + + def __hash__(self): + return hash(self.kind) + + +_KINDS = {"float64": "f", "float32": "f", "float16": "f", "float": "f", "f8": "f", "f4": "f", "d": "f", + "double": "f", "int64": "i", "int32": "i", "int16": "i", "int8": "i", "uint8": "i", "uint16": "i", + "uint32": "i", "uint64": "i", "int": "i", "i8": "i", "i4": "i", "l": "i", "intp": "i", + "bool": "b", "bool_": "b", "?": "b"} +_ALIASES = {"float", "f8", "d", "double", "int", "i8", "l", "intp", "bool_", "?"} +_NAMES = {"f": "float64", "i": "int64", "b": "bool"} +_ORDER = {"b": 0, "i": 1, "f": 2} + + +class float64(float): + def __repr__(self): + return "np.float64(" + float.__repr__(self) + ")" + + def __str__(self): + return float.__repr__(self) + + def item(self): + return float(self) + + +class int64(int): + def __repr__(self): + return "np.int64(" + int.__repr__(self) + ")" + + def __str__(self): + return int.__repr__(self) + + def item(self): + return int(self) + + +class bool_: + def __init__(self, v=False): + self._v = _b.bool(v) + + def __bool__(self): + return self._v + + def __eq__(self, other): + return self._v == _b.bool(other) + + def __hash__(self): + return hash(self._v) + + def __repr__(self): + return "np.True_" if self._v else "np.False_" + + def __str__(self): + return "True" if self._v else "False" + + def __and__(self, o): + return bool_(self._v and _b.bool(o)) + + def __or__(self, o): + return bool_(self._v or _b.bool(o)) + + def __invert__(self): + return bool_(not self._v) + + def __int__(self): + return 1 if self._v else 0 + + def __index__(self): + return 1 if self._v else 0 + + def item(self): + return self._v + + +float32 = float16 = float_ = double = float64 +int32 = int16 = int8 = uint8 = uint16 = uint32 = uint64 = intp = int_ = int64 +_TYPE_NAMES = {float: "float64", int: "int64", _b.bool: "bool", float64: "float64", int64: "int64", bool_: "bool"} +number = object +floating = float64 +integer = int64 + + +def _scalar(v, kind): + if kind == "f": + return float64(v) + if kind == "i": + return int64(v) + return bool_(v) + + +def _kind_of_value(v): + if isinstance(v, (_b.bool, bool_)): + return "b" + if isinstance(v, int): + return "i" + if isinstance(v, float): + return "f" + raise TypeError("unsupported value %r" % (v,)) + + +# ---- the array ------------------------------------------------------------ + +def _contig(shape): + st = [] + s = 1 + for n in reversed(shape): + st.append(s) + s *= n + return tuple(reversed(st)) + + +def _prod(shape): + n = 1 + for s in shape: + n *= s + return n + + +class ndarray: + __array_priority__ = 0 + + def __init__(self, shape, dtype_=None, buffer=None, offset=0, strides=None): + if isinstance(shape, int): + shape = (shape,) + self._shape = tuple(shape) + self._dtype = dtype(dtype_ if dtype_ is not None else "float64") + self._buf = buffer if buffer is not None else _cn1np.alloc(self._dtype.kind, _prod(self._shape)) + self._off = offset + self._st = tuple(strides) if strides is not None else _contig(self._shape) + + @classmethod + def _wrap(cls, buf, shape, kind=None): + a = ndarray.__new__(ndarray) + a._shape = tuple(shape) + a._dtype = dtype(_NAMES[kind or _cn1np.kind(buf)]) + a._buf = buf + a._off = 0 + a._st = _contig(a._shape) + return a + + def _view(self, shape, strides, off): + a = ndarray.__new__(ndarray) + a._shape = tuple(shape) + a._dtype = self._dtype + a._buf = self._buf + a._off = off + a._st = tuple(strides) + return a + + # attributes + @property + def shape(self): + return self._shape + + @shape.setter + def shape(self, value): + r = self.reshape(value) + self._shape, self._st, self._buf, self._off = r._shape, r._st, r._buf, r._off + + @property + def ndim(self): + return len(self._shape) + + @property + def size(self): + return _prod(self._shape) + + @property + def dtype(self): + return self._dtype + + @property + def itemsize(self): + return self._dtype.itemsize + + @property + def nbytes(self): + return self.size * self._dtype.itemsize + + @property + def strides(self): + return tuple(s * self._dtype.itemsize for s in self._st) + + @property + def T(self): + return self.transpose() + + @property + def flat(self): + return iter(self.ravel().tolist()) + + @property + def real(self): + return self + + def _contiguous(self): + return self._off == 0 and self._st == _contig(self._shape) and _cn1np.size(self._buf) == self.size + + def _flatbuf(self): + if self._contiguous(): + return self._buf + return _cn1np.copy(self._dtype.kind, self._shape, self._buf, self._off, self._st) + + # conversion + def tolist(self): + return _cn1np.tolist(self._shape, self._buf, self._off, self._st) + + def item(self, *index): + if not index: + if self.size != 1: + raise ValueError("can only convert an array of size 1 to a Python scalar") + return _cn1np.item(self._flatbuf(), 0) + if len(index) == 1 and isinstance(index[0], int) and self.ndim != 1: + return _cn1np.item(self._flatbuf(), index[0] % self.size if index[0] < 0 else index[0]) + r = self[index if len(index) > 1 else index[0]] + return r.item() if isinstance(r, ndarray) else r.item() + + def copy(self): + return ndarray._wrap(_cn1np.copy(self._dtype.kind, self._shape, self._buf, self._off, self._st), + self._shape) + + def astype(self, dt): + k = dtype(dt).kind + if k == "b": + return self != 0 + if k == "i" and self._dtype.kind == "f": + return ndarray._wrap(_cn1np.unop(28, "i", True, self._shape, self._buf, self._off, self._st), self._shape) + return ndarray._wrap(_cn1np.copy(k, self._shape, self._buf, self._off, self._st), self._shape) + + def __array__(self): + return self + + # shape + def reshape(self, *shape, order="C"): + if len(shape) == 1 and isinstance(shape[0], (tuple, list)): + shape = tuple(shape[0]) + shape = list(shape) + n = self.size + if shape.count(-1) > 1: + raise ValueError("can only specify one unknown dimension") + if -1 in shape: + known = _prod([s for s in shape if s != -1]) + if known == 0 or n % known: + raise ValueError("cannot reshape array of size %d into shape %s" % (n, _tight(shape))) + shape[shape.index(-1)] = n // known + if _prod(shape) != n: + raise ValueError("cannot reshape array of size %d into shape %s" % (n, _tight(shape))) + if self._contiguous(): + return self._view(shape, _contig(shape), 0) + return ndarray._wrap(self._flatbuf(), shape) + + def ravel(self): + return self.reshape(-1) + + def flatten(self): + return self.copy().reshape(-1) + + def transpose(self, *axes): + if not axes: + axes = tuple(range(self.ndim - 1, -1, -1)) + elif len(axes) == 1 and isinstance(axes[0], (tuple, list)): + axes = tuple(axes[0]) + axes = [_axis(a, self.ndim) for a in axes] + return self._view([self._shape[a] for a in axes], [self._st[a] for a in axes], self._off) + + def swapaxes(self, a, b): + axes = list(range(self.ndim)) + a, b = _axis(a, self.ndim), _axis(b, self.ndim) + axes[a], axes[b] = axes[b], axes[a] + return self.transpose(axes) + + def squeeze(self, axis=None): + keep = [i for i, s in enumerate(self._shape) if s != 1 or (axis is not None and i != _axis(axis, self.ndim))] + return self._view([self._shape[i] for i in keep], [self._st[i] for i in keep], self._off) + + def fill(self, value): + _cn1np.fill(self._shape, self._buf, self._off, self._st, value) + + # indexing + def _resolve(self, key): + if not isinstance(key, tuple): + key = (key,) + if _b.any(isinstance(k, (ndarray, list)) for k in key): + return None + n_real = _b.sum(1 for k in key if k is not None and k is not Ellipsis) + if n_real > self.ndim: + raise IndexError("too many indices for array: array is %d-dimensional, but %d were indexed" + % (self.ndim, n_real)) + if Ellipsis in key: + i = key.index(Ellipsis) + key = key[:i] + (slice(None),) * (self.ndim - n_real) + key[i + 1:] + else: + key = key + (slice(None),) * (self.ndim - n_real) + shape, st, off, dim = [], [], self._off, 0 + for k in key: + if k is None: + shape.append(1) + st.append(0) + continue + n = self._shape[dim] + if isinstance(k, slice): + start, stop, step = k.indices(n) + shape.append(len(range(start, stop, step))) + st.append(self._st[dim] * step) + off += start * self._st[dim] + else: + i = int(k) + if i < -n or i >= n: + raise IndexError("index %d is out of bounds for axis %d with size %d" % (i, dim, n)) + off += (i % n) * self._st[dim] + dim += 1 + return shape, st, off + + def _positions(self, key): + """The buffer positions an advanced index selects, and the result shape.""" + if not isinstance(key, tuple): + key = (key,) + adv = [i for i, k in enumerate(key) if isinstance(k, (ndarray, list))] + k0 = key[adv[0]] + k0 = k0 if isinstance(k0, ndarray) else array(k0) + if len(key) == 1 and k0._dtype.kind == "b": + if k0._shape != self._shape[:k0.ndim]: + raise IndexError("boolean index did not match indexed array") + sub = self._shape[k0.ndim:] + pos = _cn1np.positions(self._shape[:k0.ndim], self._off, self._st[:k0.ndim]) + pos = _cn1np.compress(pos, k0._flatbuf()) + if not sub: + return pos, (_cn1np.size(pos),) + parts = [_cn1np.positions(sub, _cn1np.item(pos, i), self._st[k0.ndim:]) for i in range(_cn1np.size(pos))] + return _cn1np.concat("i", parts) if parts else _cn1np.alloc("i", 0), (_cn1np.size(pos),) + tuple(sub) + if len(adv) > 1: + arrs = [key[i] if isinstance(key[i], ndarray) else array(key[i]) for i in adv] + if len(adv) != len(key) or len(key) != self.ndim: + raise IndexError("only one advanced index per selection is supported") + bshape = _broadcast_shapes(*[a._shape for a in arrs]) + flat = [broadcast_to(a, bshape).ravel().tolist() for a in arrs] + pos = [] + for idx in zip(*flat): + p = self._off + for d, i in enumerate(idx): + n = self._shape[d] + if i < -n or i >= n: + raise IndexError("index %d is out of bounds for axis %d with size %d" % (i, d, n)) + p += (i % n) * self._st[d] + pos.append(p) + return _cn1np.fromflat("i", pos), bshape + d = adv[0] + rest = key[:d] + (slice(None),) + key[d + 1:] + shape, st, off = self._resolve(rest) + # The advanced axis is the d-th surviving one only when nothing before + # it was an integer; this subset keeps basic keys before it as slices. + n = self._shape[d] + idxs = k0.ravel().tolist() + parts = [] + for i in idxs: + if i < -n or i >= n: + raise IndexError("index %d is out of bounds for axis %d with size %d" % (i, d, n)) + parts.append(_cn1np.positions(shape[:d] + shape[d + 1:], off + (i % n) * st[d], st[:d] + st[d + 1:])) + pos = _cn1np.concat("i", parts) if parts else _cn1np.alloc("i", 0) + outer = tuple(shape[:d]) + inner = tuple(shape[d + 1:]) + if outer: + # positions came out index-major; reorder to the array's order. + res = ndarray._wrap(pos, k0._shape + outer + inner) + res = res.transpose(list(range(k0.ndim, k0.ndim + len(outer))) + list(range(k0.ndim)) + + list(range(k0.ndim + len(outer), res.ndim))) + return res._flatbuf(), tuple(outer) + k0._shape + inner + return pos, k0._shape + inner + + def __getitem__(self, key): + r = self._resolve(key) + if r is None: + pos, shape = self._positions(key) + return ndarray._wrap(_cn1np.gather(self._buf, pos), shape, self._dtype.kind) + shape, st, off = r + if not shape and not (isinstance(key, tuple) and None in key): + return _scalar(_cn1np.item(self._buf, off), self._dtype.kind) + return self._view(shape, st, off) + + def __setitem__(self, key, value): + r = self._resolve(key) + src = value if isinstance(value, ndarray) else array(value) + if r is None: + pos, shape = self._positions(key) + src = src if src.size == 1 else broadcast_to(src, shape) + _cn1np.scatter(self._buf, pos, src._flatbuf()) + return + shape, st, off = r + src = broadcast_to(src, shape) + _cn1np.assign(shape, self._buf, off, st, src._buf, src._off, src._st) + + def __len__(self): + if not self._shape: + raise TypeError("len() of unsized object") + return self._shape[0] + + def __iter__(self): + if not self._shape: + raise TypeError("iteration over a 0-d array") + for i in range(self._shape[0]): + yield self[i] + + def __contains__(self, v): + return _b.bool((self == v).any()) + + def __bool__(self): + if self.size != 1: + raise ValueError("The truth value of an array with more than one element is ambiguous. " + "Use a.any() or a.all()") + return _b.bool(self.item()) + + def __float__(self): + return float(self.item()) + + def __int__(self): + return int(self.item()) + + def __index__(self): + if self._dtype.kind == "f": + raise TypeError("only integer scalar arrays can be converted to a scalar index") + return int(self.item()) + + def __hash__(self): + raise TypeError("unhashable type: 'numpy.ndarray'") + + # arithmetic + def __add__(self, o): + return _binop(0, self, o) + + def __radd__(self, o): + return _binop(0, o, self) + + def __sub__(self, o): + return _binop(1, self, o) + + def __rsub__(self, o): + return _binop(1, o, self) + + def __mul__(self, o): + return _binop(2, self, o) + + def __rmul__(self, o): + return _binop(2, o, self) + + def __truediv__(self, o): + return _binop(3, self, o) + + def __rtruediv__(self, o): + return _binop(3, o, self) + + def __floordiv__(self, o): + return _binop(4, self, o) + + def __rfloordiv__(self, o): + return _binop(4, o, self) + + def __mod__(self, o): + return _binop(5, self, o) + + def __rmod__(self, o): + return _binop(5, o, self) + + def __pow__(self, o): + return _binop(6, self, o) + + def __rpow__(self, o): + return _binop(6, o, self) + + def __eq__(self, o): + return _binop(7, self, o) + + def __ne__(self, o): + return _binop(8, self, o) + + def __lt__(self, o): + return _binop(9, self, o) + + def __le__(self, o): + return _binop(10, self, o) + + def __gt__(self, o): + return _binop(11, self, o) + + def __ge__(self, o): + return _binop(12, self, o) + + def __and__(self, o): + return _binop(13, self, o) + + def __rand__(self, o): + return _binop(13, o, self) + + def __or__(self, o): + return _binop(14, self, o) + + def __ror__(self, o): + return _binop(14, o, self) + + def __xor__(self, o): + return _binop(15, self, o) + + def __rxor__(self, o): + return _binop(15, o, self) + + def __lshift__(self, o): + return _binop(20, self, o) + + def __rshift__(self, o): + return _binop(21, self, o) + + def __neg__(self): + return _unop(0, self) + + def __pos__(self): + return self.copy() + + def __abs__(self): + return _unop(1, self) + + def __invert__(self): + return _unop(11, self) + + def __matmul__(self, o): + return matmul(self, o) + + def __rmatmul__(self, o): + return matmul(o, self) + + def _inplace(self, op, o): + r = _binop(op, self, o) + if _ORDER[r._dtype.kind] > _ORDER[self._dtype.kind]: + raise TypeError("Cannot cast ufunc '%s' output from dtype('%s') to dtype('%s') with casting rule " + "'same_kind'" % (_BIN_NAMES[op], r._dtype.name, self._dtype.name)) + self[...] = r + return self + + def __iadd__(self, o): + return self._inplace(0, o) + + def __isub__(self, o): + return self._inplace(1, o) + + def __imul__(self, o): + return self._inplace(2, o) + + def __itruediv__(self, o): + return self._inplace(3, o) + + def __ifloordiv__(self, o): + return self._inplace(4, o) + + def __imod__(self, o): + return self._inplace(5, o) + + def __ipow__(self, o): + return self._inplace(6, o) + + # methods that mirror functions + def sum(self, axis=None, dtype=None, keepdims=False): + return sum(self, axis=axis, keepdims=keepdims) + + def prod(self, axis=None, keepdims=False): + return prod(self, axis=axis, keepdims=keepdims) + + def mean(self, axis=None, keepdims=False): + return mean(self, axis=axis, keepdims=keepdims) + + def std(self, axis=None, ddof=0, keepdims=False): + return std(self, axis=axis, ddof=ddof, keepdims=keepdims) + + def var(self, axis=None, ddof=0, keepdims=False): + return var(self, axis=axis, ddof=ddof, keepdims=keepdims) + + def min(self, axis=None, keepdims=False): + return amin(self, axis=axis, keepdims=keepdims) + + def max(self, axis=None, keepdims=False): + return amax(self, axis=axis, keepdims=keepdims) + + def argmin(self, axis=None): + return argmin(self, axis) + + def argmax(self, axis=None): + return argmax(self, axis) + + def any(self, axis=None, keepdims=False): + return any(self, axis=axis, keepdims=keepdims) + + def all(self, axis=None, keepdims=False): + return all(self, axis=axis, keepdims=keepdims) + + def cumsum(self, axis=None): + return cumsum(self, axis) + + def cumprod(self, axis=None): + return cumprod(self, axis) + + def dot(self, o): + return dot(self, o) + + def clip(self, a_min=None, a_max=None): + return clip(self, a_min, a_max) + + def round(self, decimals=0): + return round(self, decimals) + + def sort(self, axis=-1): + self[...] = sort(self, axis) + + def argsort(self, axis=-1): + return argsort(self, axis) + + def nonzero(self): + return nonzero(self) + + def __repr__(self): + return _repr(self) + + def __str__(self): + return _str(self) + + def __format__(self, spec): + if self.ndim == 0: + return format(self.item(), spec) + return str(self) + + +def _shape_str(shape): + shape = tuple(shape) + return "(%d,)" % shape[0] if len(shape) == 1 else "(" + ", ".join(str(s) for s in shape) + ")" + + +def _tight(shape): + return "(" + ",".join(str(s) for s in shape) + ("," if len(shape) == 1 else "") + ")" + + +def _axis(a, ndim): + if a < -ndim or a >= ndim: + raise AxisError("axis %d is out of bounds for array of dimension %d" % (a, ndim)) + return a % ndim + + +class AxisError(ValueError, IndexError): + pass + + +# ---- construction --------------------------------------------------------- + +def _flatten_nested(obj, shape, out, depth): + if isinstance(obj, ndarray): + obj = obj.tolist() + if isinstance(obj, (list, tuple)): + if len(shape) <= depth: + shape.append(len(obj)) + elif shape[depth] != len(obj): + raise ValueError("setting an array element with a sequence. The requested array has an " + "inhomogeneous shape after %d dimensions." % depth) + for x in obj: + _flatten_nested(x, shape, out, depth + 1) + else: + if len(shape) != depth and out: + raise ValueError("setting an array element with a sequence. The requested array has an " + "inhomogeneous shape after %d dimensions." % len(shape)) + out.append(obj) + + +def array(obj, dtype=None, copy=True, ndmin=0): + if isinstance(obj, ndarray): + r = obj.astype(dtype) if dtype is not None else (obj.copy() if copy else obj) + else: + if isinstance(obj, (float64, int64, bool_)): + obj = obj.item() + shape, flat = [], [] + _flatten_nested(obj, shape, flat, 0) + if dtype is not None: + kind = globals()["dtype"](dtype).kind + elif flat: + kind = "b" + for v in flat: + if isinstance(v, (float64, int64, bool_)): + v = v.item() + k = _kind_of_value(v) + if _ORDER[k] > _ORDER[kind]: + kind = k + else: + kind = "f" + flat = [v.item() if isinstance(v, (float64, int64, bool_)) else v for v in flat] + r = ndarray._wrap(_cn1np.fromflat(kind, flat), shape) + while r.ndim < ndmin: + r = r.reshape((1,) + r._shape) + return r + + +def asarray(obj, dtype=None): + if isinstance(obj, ndarray) and dtype is None: + return obj + return array(obj, dtype) + + +asanyarray = asarray + + +def _shape_arg(shape): + return (shape,) if isinstance(shape, int) else tuple(shape) + + +def zeros(shape, dtype=float): + shape = _shape_arg(shape) + return ndarray._wrap(_cn1np.alloc(globals()["dtype"](dtype).kind, _prod(shape)), shape) + + +def empty(shape, dtype=float): + return zeros(shape, dtype) + + +def ones(shape, dtype=float): + return full(shape, 1, dtype) + + +def full(shape, fill_value, dtype=None): + if dtype is None: + dtype = _NAMES[_kind_of_value(fill_value)] + r = zeros(shape, dtype) + r.fill(fill_value) + return r + + +def zeros_like(a, dtype=None): + a = asarray(a) + return zeros(a.shape, dtype or a.dtype) + + +def ones_like(a, dtype=None): + a = asarray(a) + return ones(a.shape, dtype or a.dtype) + + +def full_like(a, fill_value, dtype=None): + a = asarray(a) + return full(a.shape, fill_value, dtype or a.dtype) + + +empty_like = zeros_like + + +def arange(start, stop=None, step=1, dtype=None): + if stop is None: + start, stop = 0, start + kind = "f" if _b.any(isinstance(v, float) for v in (start, stop, step)) else "i" + if dtype is not None: + kind = globals()["dtype"](dtype).kind + n = _b.max(0, int(_math.ceil((stop - start) / step))) + return ndarray._wrap(_cn1np.arange(kind, start, step, n), (n,)) + + +def linspace(start, stop, num=50, endpoint=True, retstep=False, dtype=None): + div = (num - 1) if endpoint else num + step = (stop - start) / div if div > 0 else nan + vals = [start + i * step for i in range(num)] + if endpoint and num > 1: + vals[-1] = stop + r = array([float(v) for v in vals], dtype=dtype or float) + return (r, step) if retstep else r + + +def eye(n, m=None, k=0, dtype=float): + r = zeros((n, m if m is not None else n), dtype) + for i in range(n): + j = i + k + if 0 <= j < r.shape[1]: + r[i, j] = 1 + return r + + +def identity(n, dtype=float): + return eye(n, dtype=dtype) + + +def diag(v, k=0): + v = asarray(v) + if v.ndim == 1: + n = v.shape[0] + _b.abs(k) + r = zeros((n, n), v.dtype) + for i in range(v.shape[0]): + r[i + _b.max(-k, 0), i + _b.max(k, 0)] = v[i] + return r + n = _b.min(v.shape[0] - _b.max(-k, 0), v.shape[1] - _b.max(k, 0)) + return array([v[i + _b.max(-k, 0), i + _b.max(k, 0)] for i in range(_b.max(n, 0))], dtype=v.dtype) + + +def meshgrid(*xs, indexing="xy"): + xs = [asarray(x).ravel() for x in xs] + if len(xs) == 2 and indexing == "xy": + x, y = xs + return [broadcast_to(x.reshape(1, -1), (y.size, x.size)).copy(), + broadcast_to(y.reshape(-1, 1), (y.size, x.size)).copy()] + shape = tuple(x.size for x in xs) + out = [] + for i, x in enumerate(xs): + s = [1] * len(xs) + s[i] = x.size + out.append(broadcast_to(x.reshape(s), shape).copy()) + return out + + +# ---- broadcasting --------------------------------------------------------- + +def _broadcast_shapes(*shapes): + nd = _b.max(len(s) for s in shapes) if shapes else 0 + out = [] + for i in range(nd): + n = 1 + for s in shapes: + j = len(s) - nd + i + d = s[j] if j >= 0 else 1 + if d != 1: + if n != 1 and n != d: + raise ValueError("operands could not be broadcast together with shapes " + + " ".join(_shape_str(x) for x in shapes) + " ") + n = d + out.append(n) + return tuple(out) + + +broadcast_shapes = _broadcast_shapes + + +def _bstrides(a, shape): + nd = len(shape) + st = [] + for i in range(nd): + j = a.ndim - nd + i + if j < 0 or a._shape[j] == 1 and shape[i] != 1: + st.append(0) + else: + st.append(a._st[j]) + return tuple(st) + + +def broadcast_to(a, shape): + a = asarray(a) + shape = _shape_arg(shape) + if _broadcast_shapes(a._shape, shape) != tuple(shape): + raise ValueError("operands could not be broadcast together with remapped shapes [original->remapped]: " + + _shape_str(a._shape) + " and requested shape " + _shape_str(shape)) + return a._view(shape, _bstrides(a, shape), a._off) + + +def broadcast_arrays(*arrs): + arrs = [asarray(a) for a in arrs] + shape = _broadcast_shapes(*[a._shape for a in arrs]) + return [broadcast_to(a, shape) for a in arrs] + + +# ---- ufuncs --------------------------------------------------------------- + +_BIN_NAMES = {0: "add", 1: "subtract", 2: "multiply", 3: "divide", 4: "floor_divide", 5: "remainder", + 6: "power", 13: "bitwise_and", 14: "bitwise_or", 15: "bitwise_xor"} + + +def _operand(x): + """An operand and whether it is a weak Python scalar (NEP 50).""" + if isinstance(x, ndarray): + return x, False + if isinstance(x, (float64, int64, bool_)): + return array(x.item()), False + if isinstance(x, (_b.bool, int, float)): + return array(x), True + return array(x), False + + +def _result_kind(a, wa, b, wb): + ka, kb = a._dtype.kind, b._dtype.kind + if wa and not wb: + return kb if _ORDER[kb] >= _ORDER[ka] else ka + if wb and not wa: + return ka if _ORDER[ka] >= _ORDER[kb] else kb + return ka if _ORDER[ka] >= _ORDER[kb] else kb + + +def _binop(op, x, y): + if not isinstance(y, (ndarray, list, tuple, int, float, float64, int64, bool_)) and \ + not isinstance(x, (ndarray, list, tuple, int, float, float64, int64, bool_)): + return NotImplemented + a, wa = _operand(x) + b, wb = _operand(y) + kind = _result_kind(a, wa, b, wb) + if op in (13, 14, 15, 20, 21) and kind == "f": + raise TypeError("ufunc '%s' not supported for the input types, and the inputs could not be safely " + "coerced to any supported types according to the casting rule ''safe''" + % _BIN_NAMES.get(op, "shift")) + fp = kind == "f" + if op == 3 or op in (18, 19): + kind, fp = "f", True + elif 7 <= op <= 12: + kind = "b" + elif kind == "b" and op in (0, 2): + op = 14 if op == 0 else 13 + elif kind == "b" and op in (1, 4, 5, 6): + kind = "i" + shape = _broadcast_shapes(a._shape, b._shape) + buf = _cn1np.binop(op, kind, fp, shape, a._buf, a._off, _bstrides(a, shape), b._buf, b._off, _bstrides(b, shape)) + r = ndarray._wrap(buf, shape, kind) + return r if r.ndim or isinstance(x, ndarray) or isinstance(y, ndarray) else r[()] + + +def _unop(op, x, kind=None, fp=None): + a = asarray(x) + k = a._dtype.kind + if kind is None: + kind = k + if fp is None: + fp = kind == "f" or k == "f" + buf = _cn1np.unop(op, kind, fp, a._shape, a._buf, a._off, a._st) + r = ndarray._wrap(buf, a._shape, kind) + return r if isinstance(x, ndarray) else r[()] + + +def _float_unop(op): + def f(x): + return _unop(op, x, "f", True) + return f + + +sqrt = _float_unop(2) +exp = _float_unop(3) +log = _float_unop(4) +sin = _float_unop(5) +cos = _float_unop(6) +tan = _float_unop(7) +log10 = _float_unop(14) +log2 = _float_unop(15) +tanh = _float_unop(16) +arcsin = _float_unop(17) +arccos = _float_unop(18) +arctan = _float_unop(19) +sinh = _float_unop(24) +cosh = _float_unop(25) +expm1 = _float_unop(26) +log1p = _float_unop(27) + + +def _keep_kind(op): + def f(x): + a = asarray(x) + if a._dtype.kind == "f" or op in (0, 1, 11, 12, 13): + return _unop(op, x) + return _unop(op, x, "f", True) + return f + + +def floor(x): + return _unop(8, x, "f", True) + + +def ceil(x): + return _unop(9, x, "f", True) + + +def trunc(x): + return _unop(28, x, "f", True) + + +def rint(x): + return _unop(20, x, "f", True) + + +negative = _keep_kind(0) +absolute = abs_ = _keep_kind(1) +fabs = _float_unop(1) +sign = _keep_kind(12) +square = _keep_kind(13) +invert = bitwise_not = _keep_kind(11) + + +def logical_not(x): + return _unop(10, x, "b") + + +def isnan(x): + return _unop(21, x, "b") + + +def isinf(x): + return _unop(22, x, "b") + + +def isfinite(x): + return _unop(23, x, "b") + + +def _bin(op): + def f(x, y): + return _binop(op, x, y) + return f + + +add = _bin(0) +subtract = _bin(1) +multiply = _bin(2) +divide = true_divide = _bin(3) +floor_divide = _bin(4) +mod = remainder = _bin(5) +power = _bin(6) +equal = _bin(7) +not_equal = _bin(8) +less = _bin(9) +less_equal = _bin(10) +greater = _bin(11) +greater_equal = _bin(12) +bitwise_and = _bin(13) +bitwise_or = _bin(14) +bitwise_xor = _bin(15) +maximum = _bin(17) +minimum = _bin(16) +arctan2 = _bin(18) +hypot = _bin(19) +left_shift = _bin(20) +right_shift = _bin(21) + + +def logical_and(x, y): + return _binop(13, asarray(x).astype(bool), asarray(y).astype(bool)) + + +def logical_or(x, y): + return _binop(14, asarray(x).astype(bool), asarray(y).astype(bool)) + + +def logical_xor(x, y): + return _binop(15, asarray(x).astype(bool), asarray(y).astype(bool)) + + +def round(a, decimals=0): + a = asarray(a) + if a._dtype.kind != "f": + return a.copy() + if decimals == 0: + return _unop(20, a) + f = 10.0 ** decimals + return _unop(20, a * f) / f + + +around = round + + +def clip(a, a_min, a_max): + r = asarray(a) + if a_min is not None: + r = maximum(r, a_min) + if a_max is not None: + r = minimum(r, a_max) + return r + + +def where(cond, x=None, y=None): + c = asarray(cond) + if x is None and y is None: + return nonzero(c) + a, wa = _operand(x) + b, wb = _operand(y) + kind = _result_kind(a, wa, b, wb) + shape = _broadcast_shapes(c._shape, a._shape, b._shape) + buf = _cn1np.where(kind, shape, c._buf, c._off, _bstrides(c, shape), a._buf, a._off, _bstrides(a, shape), + b._buf, b._off, _bstrides(b, shape)) + return ndarray._wrap(buf, shape, kind) + + +def nonzero(a): + a = asarray(a) + mask = a != 0 + out = [] + for d in range(a.ndim): + idx = arange(a.shape[d]).reshape([a.shape[d] if i == d else 1 for i in range(a.ndim)]) + out.append(broadcast_to(idx, a.shape)[mask]) + return tuple(out) + + +def argwhere(a): + nz = nonzero(a) + return stack(nz, axis=1) if nz else zeros((0, 0), int) + + +def count_nonzero(a, axis=None): + return sum(asarray(a) != 0, axis=axis) + + +# ---- reductions ----------------------------------------------------------- + +def _reduce(op, a, axis, kind, keepdims=False, fp=None): + a = asarray(a) + if fp is None: + fp = a._dtype.kind == "f" + if axis is None: + buf = _cn1np.reduce(op, kind, fp, a._shape, a._buf, a._off, a._st, -1) + r = ndarray._wrap(buf, (), kind) + if keepdims: + return r.reshape((1,) * a.ndim) + return r[()] + if isinstance(axis, tuple): + r = a + for ax in sorted((_axis(x, a.ndim) for x in axis), reverse=True): + r = _reduce(op, r, ax, kind, True, fp) + return r if keepdims else r.reshape([s for i, s in enumerate(r.shape) + if i not in [_axis(x, a.ndim) for x in axis]]) + ax = _axis(axis, a.ndim) + buf = _cn1np.reduce(op, kind, fp, a._shape, a._buf, a._off, a._st, ax) + shape = a._shape[:ax] + ((1,) if keepdims else ()) + a._shape[ax + 1:] + return ndarray._wrap(buf, shape, kind) + + +def _sum_kind(a): + k = asarray(a)._dtype.kind + return "i" if k == "b" else k + + +def sum(a, axis=None, dtype=None, keepdims=False): + a = asarray(a) + return _reduce(0, a, axis, _sum_kind(a), keepdims) + + +def prod(a, axis=None, keepdims=False): + a = asarray(a) + return _reduce(1, a, axis, _sum_kind(a), keepdims) + + +def amin(a, axis=None, keepdims=False): + a = asarray(a) + return _reduce(2, a, axis, a._dtype.kind, keepdims) + + +def amax(a, axis=None, keepdims=False): + a = asarray(a) + return _reduce(3, a, axis, a._dtype.kind, keepdims) + + +min = amin +max = amax + + +def argmin(a, axis=None): + return _reduce(4, a, axis, "i") + + +def argmax(a, axis=None): + return _reduce(5, a, axis, "i") + + +def any(a, axis=None, keepdims=False): + return _reduce(6, a, axis, "b", keepdims) + + +def all(a, axis=None, keepdims=False): + return _reduce(7, a, axis, "b", keepdims) + + +def mean(a, axis=None, keepdims=False): + a = asarray(a) + n = a.size if axis is None else _prod([a.shape[_axis(x, a.ndim)] for x in + (axis if isinstance(axis, tuple) else (axis,))]) + s = _reduce(0, a, axis, "f", keepdims, True) + r = s / n if n else s * nan + return r if isinstance(r, ndarray) else float64(r) + + +def var(a, axis=None, ddof=0, keepdims=False): + a = asarray(a) + m = mean(a, axis=axis, keepdims=True) + d = a - m + n = a.size if axis is None else _prod([a.shape[_axis(x, a.ndim)] for x in + (axis if isinstance(axis, tuple) else (axis,))]) + r = sum(d * d, axis=axis, keepdims=keepdims) / (n - ddof) + return r if isinstance(r, ndarray) else float64(r) + + +def std(a, axis=None, ddof=0, keepdims=False): + return sqrt(var(a, axis=axis, ddof=ddof, keepdims=keepdims)) + + +def median(a, axis=None): + a = asarray(a) + if axis is None: + a, axis = a.ravel(), 0 + s = sort(a, axis) + n = s.shape[axis] + lo = take(s, (n - 1) // 2, axis) + hi = take(s, n // 2, axis) + return (lo + hi) / 2.0 + + +def percentile(a, q, axis=None): + a = asarray(a) + if axis is None: + a, axis = a.ravel(), 0 + s = sort(a, axis) + n = s.shape[axis] + pos = (n - 1) * q / 100.0 + lo = int(_math.floor(pos)) + hi = _b.min(lo + 1, n - 1) + frac = pos - lo + vlo = take(s, lo, axis) + vhi = take(s, hi, axis) + return vlo + (vhi - vlo) * frac + + +def quantile(a, q, axis=None): + return percentile(a, q * 100.0, axis) + + +def _cumulate(op, a, axis): + a = asarray(a) + if axis is None: + a, axis = a.ravel(), 0 + ax = _axis(axis, a.ndim) + kind = _sum_kind(a) + return ndarray._wrap(_cn1np.cumulate(op, kind, a._shape, a._buf, a._off, a._st, ax), a._shape, kind) + + +def cumsum(a, axis=None): + return _cumulate(0, a, axis) + + +def cumprod(a, axis=None): + return _cumulate(1, a, axis) + + +def diff(a, n=1, axis=-1): + a = asarray(a) + for _ in range(n): + ax = _axis(axis, a.ndim) + hi = [slice(None)] * a.ndim + lo = [slice(None)] * a.ndim + hi[ax] = slice(1, None) + lo[ax] = slice(None, -1) + a = a[tuple(hi)] - a[tuple(lo)] + return a + + +def take(a, indices, axis=None): + a = asarray(a) + if axis is None: + a, axis = a.ravel(), 0 + key = [slice(None)] * a.ndim + key[_axis(axis, a.ndim)] = indices + return a[tuple(key)] + + +# ---- sorting and sets ----------------------------------------------------- + +def _along_last(a, axis): + a = asarray(a) + if axis is None: + return a.ravel(), 0, None + ax = _axis(axis, a.ndim) + return a.swapaxes(ax, -1), ax, a.ndim - 1 + + +def sort(a, axis=-1): + m, ax, last = _along_last(a, axis) + if m.ndim == 0: + return m.copy() + buf = _cn1np.sortrows(m._flatbuf(), m.shape[-1], False) + r = ndarray._wrap(buf, m.shape, m._dtype.kind) + return r.swapaxes(ax, -1).copy() if last is not None else r + + +def argsort(a, axis=-1, kind=None): + m, ax, last = _along_last(a, axis) + buf = _cn1np.sortrows(m._flatbuf(), m.shape[-1], True) + r = ndarray._wrap(buf, m.shape, "i") + return r.swapaxes(ax, -1).copy() if last is not None else r + + +def unique(a, return_counts=False, return_index=False): + s = sort(asarray(a).ravel()) + vals = s.tolist() + out, counts = [], [] + for v in vals: + if out and (out[-1] == v or (out[-1] != out[-1] and v != v)): + counts[-1] += 1 + else: + out.append(v) + counts.append(1) + r = array(out, dtype=s.dtype) + if return_counts: + return r, array(counts) + return r + + +def searchsorted(a, v, side="left"): + a = asarray(a).tolist() + import bisect + f = bisect.bisect_left if side == "left" else bisect.bisect_right + if isinstance(v, (int, float)): + return int64(f(a, v)) + return array([f(a, x) for x in asarray(v).ravel().tolist()]).reshape(asarray(v).shape) + + +def isin(element, test_elements): + tests = set(asarray(test_elements).ravel().tolist()) + e = asarray(element) + return array([x in tests for x in e.ravel().tolist()], dtype=bool).reshape(e.shape) + + +def bincount(x, minlength=0): + vals = asarray(x).ravel().tolist() + n = _b.max([minlength] + [v + 1 for v in vals]) + out = [0] * n + for v in vals: + out[v] += 1 + return array(out, dtype=int) + + +def histogram(a, bins=10, range=None): + a = asarray(a).ravel() + lo, hi = (float(a.min()), float(a.max())) if range is None else range + if lo == hi: + lo, hi = lo - 0.5, hi + 0.5 + edges = linspace(lo, hi, bins + 1) + counts = [0] * bins + width = (hi - lo) / bins + for v in a.tolist(): + if lo <= v <= hi: + i = _b.min(int((v - lo) / width), bins - 1) + counts[i] += 1 + return array(counts), edges + + +# ---- joining -------------------------------------------------------------- + +def concatenate(arrays, axis=0): + arrays = [asarray(a) for a in arrays] + if axis is None: + arrays, axis = [a.ravel() for a in arrays], 0 + first = arrays[0] + ax = _axis(axis, first.ndim) + kind = "b" + for a in arrays: + if a.ndim != first.ndim: + raise ValueError("all the input array dimensions except for the concatenation axis must match exactly") + if _ORDER[a._dtype.kind] > _ORDER[kind]: + kind = a._dtype.kind + shape = list(first.shape) + shape[ax] = _b.sum(a.shape[ax] for a in arrays) + r = zeros(shape, _NAMES[kind]) + at = 0 + for a in arrays: + key = [slice(None)] * len(shape) + key[ax] = slice(at, at + a.shape[ax]) + r[tuple(key)] = a + at += a.shape[ax] + return r + + +def stack(arrays, axis=0): + arrays = [asarray(a) for a in arrays] + nd = arrays[0].ndim + 1 + ax = _axis(axis, nd) + return concatenate([a.reshape(a.shape[:ax] + (1,) + a.shape[ax:]) for a in arrays], axis=ax) + + +def vstack(arrays): + return concatenate([atleast_2d(a) for a in arrays], axis=0) + + +def hstack(arrays): + arrays = [atleast_1d(a) for a in arrays] + return concatenate(arrays, axis=0 if arrays[0].ndim == 1 else 1) + + +def column_stack(arrays): + return hstack([a.reshape(-1, 1) if asarray(a).ndim == 1 else a for a in [asarray(x) for x in arrays]]) + + +def atleast_1d(a): + a = asarray(a) + return a.reshape(1) if a.ndim == 0 else a + + +def atleast_2d(a): + a = asarray(a) + return a.reshape(1, -1) if a.ndim < 2 else a + + +def split(a, sections, axis=0): + a = asarray(a) + ax = _axis(axis, a.ndim) + n = a.shape[ax] + if isinstance(sections, int): + if n % sections: + raise ValueError("array split does not result in an equal division") + cuts = [n // sections * i for i in range(1, sections)] + else: + cuts = list(sections) + out = [] + prev = 0 + for c in cuts + [n]: + key = [slice(None)] * a.ndim + key[ax] = slice(prev, c) + out.append(a[tuple(key)]) + prev = c + return out + + +def reshape(a, shape): + return asarray(a).reshape(shape) + + +def ravel(a): + return asarray(a).ravel() + + +def transpose(a, axes=None): + return asarray(a).transpose(*(() if axes is None else (axes,))) + + +def squeeze(a, axis=None): + return asarray(a).squeeze(axis) + + +def expand_dims(a, axis): + a = asarray(a) + ax = _axis(axis, a.ndim + 1) + return a.reshape(a.shape[:ax] + (1,) + a.shape[ax:]) + + +def flip(a, axis=None): + a = asarray(a) + axes = range(a.ndim) if axis is None else [_axis(axis, a.ndim)] + key = [slice(None)] * a.ndim + for ax in axes: + key[ax] = slice(None, None, -1) + return a[tuple(key)] + + +def tile(a, reps): + a = asarray(a) + if isinstance(reps, int): + reps = (reps,) + r = a + while r.ndim < len(reps): + r = r.reshape((1,) + r.shape) + reps = (1,) * (r.ndim - len(reps)) + tuple(reps) + for ax, n in enumerate(reps): + r = concatenate([r] * n, axis=ax) + return r + + +def repeat(a, repeats, axis=None): + a = asarray(a) + if axis is None: + a, axis = a.ravel(), 0 + ax = _axis(axis, a.ndim) + idx = [] + for i in range(a.shape[ax]): + idx.extend([i] * repeats) + return take(a, idx, ax) + + +def copy(a): + return asarray(a).copy() + + +def shape(a): + return asarray(a).shape + + +def ndim(a): + return asarray(a).ndim + + +def size(a): + return asarray(a).size + + +# ---- linear algebra ------------------------------------------------------- + +def matmul(a, b): + a, b = asarray(a), asarray(b) + if a.ndim == 0 or b.ndim == 0: + raise ValueError("matmul: Input operand does not have enough dimensions") + va, vb = a.ndim == 1, b.ndim == 1 + if va: + a = a.reshape(1, -1) + if vb: + b = b.reshape(-1, 1) + if a.ndim > 2 or b.ndim > 2: + return stack([matmul(a[i], b[i] if b.ndim > 2 else b) for i in range(a.shape[0])]) + n, k = a.shape + k2, m = b.shape + if k != k2: + raise ValueError("matmul: Input operand 1 has a mismatch in its core dimension 0, with gufunc signature " + "(n?,k),(k,m?)->(n?,m?) (size %d is different from %d)" % (k2, k)) + kind = a._dtype.kind if _ORDER[a._dtype.kind] >= _ORDER[b._dtype.kind] else b._dtype.kind + kind = "i" if kind == "b" else kind + buf = _cn1np.matmul(kind, n, k, m, a._buf, a._off, a._st, b._buf, b._off, b._st) + r = ndarray._wrap(buf, (n, m), kind) + if va and vb: + return r[0, 0] + if va: + return r.reshape(m) + if vb: + return r.reshape(n) + return r + + +def dot(a, b): + a, b = asarray(a), asarray(b) + if a.ndim == 0 or b.ndim == 0: + return a * b + return matmul(a, b) + + +def vdot(a, b): + return dot(asarray(a).ravel(), asarray(b).ravel()) + + +def inner(a, b): + return dot(a, b) + + +def outer(a, b): + return asarray(a).reshape(-1, 1) * asarray(b).reshape(1, -1) + + +def cross(a, b): + a, b = asarray(a), asarray(b) + return array([a[1] * b[2] - a[2] * b[1], a[2] * b[0] - a[0] * b[2], a[0] * b[1] - a[1] * b[0]]) + + +def trace(a): + return sum(diag(a)) + + +def allclose(a, b, rtol=1e-05, atol=1e-08): + return _b.bool(all(isclose(a, b, rtol, atol))) + + +def isclose(a, b, rtol=1e-05, atol=1e-08): + a, b = asarray(a).astype(float), asarray(b).astype(float) + return absolute(a - b) <= atol + rtol * absolute(b) + + +def array_equal(a, b): + a, b = asarray(a), asarray(b) + return a.shape == b.shape and _b.bool(all(a == b)) + + +def interp(x, xp, fp): + xp = asarray(xp).tolist() + fp = asarray(fp).tolist() + import bisect + + def one(v): + if v <= xp[0]: + return float(fp[0]) + if v >= xp[-1]: + return float(fp[-1]) + i = bisect.bisect_right(xp, v) - 1 + t = (v - xp[i]) / (xp[i + 1] - xp[i]) + return fp[i] + t * (fp[i + 1] - fp[i]) + if isinstance(x, (int, float)): + return float64(one(x)) + return array([one(v) for v in asarray(x).ravel().tolist()]).reshape(asarray(x).shape) + + +def polyval(p, x): + r = 0.0 * asarray(x) + for c in p: + r = r * x + c + return r + + +def convolve(a, v, mode="full"): + a = asarray(a).tolist() + v = asarray(v).tolist() + n = len(a) + len(v) - 1 + out = [0] * n + for i, x in enumerate(a): + for j, y in enumerate(v): + out[i + j] += x * y + r = array(out) + if mode == "same": + m = _b.max(len(a), len(v)) + s = (n - m) // 2 + return r[s:s + m] + if mode == "valid": + m = _b.max(len(a), len(v)) - _b.min(len(a), len(v)) + 1 + s = _b.min(len(a), len(v)) - 1 + return r[s:s + m] + return r + + +# ---- printing ------------------------------------------------------------- + +def _fmt_positional(x): + if x != x: + return "nan" + if x in (inf, -inf): + return "inf" if x > 0 else "-inf" + s = repr(float(x)) + if "e" in s or len(s.split(".")[1]) > 8: + s = "%.8f" % x + s = s.rstrip("0") + if s == "-0.": + s = "-0." + if s.endswith(".0"): + s = s[:-1] + return s + + +def _fmt_scientific(x): + if x != x: + return "nan" + if x in (inf, -inf): + return "inf" if x > 0 else "-inf" + s = "%.8e" % x + mant, ex = s.split("e") + # the shortest mantissa that reads back as x at 8 digits + for p in range(0, 9): + cand = "%.*e" % (p, x) + if float(cand) == float(s) or float(cand) == x: + mant, ex = cand.split("e") + break + if "." not in mant: + mant += "." + else: + mant = mant.rstrip("0") + sign = ex[0] + digits = ex[1:].lstrip("0") + return mant + "e" + sign + digits.rjust(2, "0") + + +def _float_formatter(vals): + finite = [_b.abs(v) for v in vals if v == v and v not in (inf, -inf)] + nonzero_ = [v for v in finite if v != 0] + exp_mode = False + if nonzero_: + mx, mn = _b.max(nonzero_), _b.min(nonzero_) + exp_mode = mx >= 1e8 or mn < 0.0001 or mx / mn > 1000.0 + if exp_mode: + strs = [_fmt_scientific(v) for v in vals] + parts = [s.split("e") if "e" in s else (s, None) for s in strs] + int_w = _b.max(len(p[0].split(".")[0]) for p in parts if p[1] is not None) + frac_w = _b.max(len(p[0].split(".")[1]) for p in parts if p[1] is not None) + exp_w = _b.max(len(p[1]) - 1 for p in parts if p[1] is not None) + width = int_w + 1 + frac_w + 2 + exp_w + + def fmt(v): + s = _fmt_scientific(v) + if "e" not in s: + return s.rjust(width) + m, x = s.split("e") + i, f = m.split(".") + return i.rjust(int_w) + "." + f.ljust(frac_w) + "e" + x[0] + x[1:].rjust(exp_w, "0") + return fmt + strs = [_fmt_positional(v) for v in vals] + num = [s for s in strs if "." in s] + int_w = _b.max([len(s.split(".")[0]) for s in num] or [0]) + frac_w = _b.max([len(s.split(".")[1]) for s in num] or [0]) + width = _b.max([int_w + 1 + frac_w] + [len(s) for s in strs]) + + def fmt(v): + s = _fmt_positional(v) + if "." not in s: + return s.rjust(width) + i, f = s.split(".") + return (i.rjust(int_w) + "." + f.ljust(frac_w)).rjust(width) + return fmt + + +def _formatter(a, flat): + k = a._dtype.kind + if k == "f": + return _float_formatter(flat) + if k == "b": + w = 5 if _b.any(not v for v in flat) else 4 + return lambda v: ("True" if v else "False").rjust(w) + w = _b.max([len(str(v)) for v in flat] or [1]) + return lambda v: str(v).rjust(w) + + +def _summarize(a): + return a.size > 1000 + + +def _extend_line(s, line, word, width, prefix): + if len(line) + len(word) > width and len(line) > len(prefix): + s += line.rstrip() + "\n" + line = prefix + return s, line + word + + +def _format_array(nested, ndim, fmt, sep, prefix, width, summarize): + """numpy's layout: rows wrap at `width`, nested rows hang under the + opening bracket, and blank lines separate the blocks of 3-d and up.""" + edge = 3 + + def rec(x, depth, hanging, curr): + left = ndim - depth + if left == 0: + return fmt(x) + n = len(x) + show = summarize and n > 2 * edge + lead = edge if show else n + trail = edge if show else 0 + nxt = hanging + " " + nwidth = curr - 1 + if left == 1: + elem_w = curr - len(sep.rstrip()) + s, line = "", hanging + for i in range(lead if show else n - 1): + s, line = _extend_line(s, line, rec(x[i], depth + 1, nxt, nwidth), elem_w, hanging) + line += sep + if show: + s, line = _extend_line(s, line, "...", elem_w, hanging) + line += sep + for i in range(trail, 1, -1): + s, line = _extend_line(s, line, rec(x[n - i], depth + 1, nxt, nwidth), elem_w, hanging) + line += sep + s, line = _extend_line(s, line, rec(x[n - 1], depth + 1, nxt, nwidth), curr, hanging) + s += line + else: + s = "" + line_sep = sep.rstrip() + "\n" * (left - 1) + for i in range(lead if show else n - 1): + s += hanging + rec(x[i], depth + 1, nxt, nwidth) + line_sep + if show: + s += hanging + "..." + line_sep + for i in range(trail, 1, -1): + s += hanging + rec(x[n - i], depth + 1, nxt, nwidth) + line_sep + s += hanging + rec(x[n - 1], depth + 1, nxt, nwidth) + return "[" + s[len(hanging):] + "]" + + return rec(nested, 0, prefix, width) + + +def _flat_for_format(a): + if _summarize(a) and a.ndim: + key_parts = [] + r = a + for d in range(a.ndim): + if a.shape[d] > 6: + idx = list(range(3)) + list(range(a.shape[d] - 3, a.shape[d])) + r = take(r, idx, d) + return r.ravel().tolist() + return a.ravel().tolist() + + +def _repr(a): + suffix = "" + if a.size == 0: + suffix = ", dtype=" + a._dtype.name + if a.ndim == 0: + fmt = _formatter(a, [a.item()]) + return "array(" + fmt(a.item()).strip() + suffix + ")" + if a.size == 0: + return "array(" + "[" * a.ndim + "]" * a.ndim + ", shape=" + _shape_str(a.shape) + suffix + ")" \ + if a.shape != (0,) else "array([]" + suffix + ")" + fmt = _formatter(a, _flat_for_format(a)) + body = _format_array(a.tolist(), a.ndim, fmt, ", ", " " * 7, 74, _summarize(a)) + return "array(" + body + suffix + ")" + + +def _str(a): + if a.ndim == 0: + return str(a.item()) + if a.size == 0: + return "[" * a.ndim + "]" * a.ndim + fmt = _formatter(a, _flat_for_format(a)) + return _format_array(a.tolist(), a.ndim, fmt, " ", " ", 75, _summarize(a)) + + +def set_printoptions(**kwargs): + pass + + +from . import linalg # noqa: E402 +from . import random # noqa: E402 diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/numpy/linalg.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/numpy/linalg.py new file mode 100644 index 00000000000..f5ec7ab920f --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/numpy/linalg.py @@ -0,0 +1,107 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""numpy.linalg: norms, determinants, inverses and solving, by LU with +partial pivoting.""" + +import math as _math +import numpy as _np + + +class LinAlgError(ValueError): + pass + + +def norm(x, ord=None, axis=None): + x = _np.asarray(x).astype(float) + if axis is not None: + return _np.sqrt(_np.sum(x * x, axis=axis)) + if ord is None or ord == "fro" or ord == 2 and x.ndim == 1: + return _np.float64(_math.sqrt(float(_np.sum(x * x)))) + if ord == 1 and x.ndim == 1: + return _np.float64(float(_np.sum(_np.absolute(x)))) + if ord == _np.inf and x.ndim == 1: + return _np.float64(float(_np.amax(_np.absolute(x)))) + raise NotImplementedError("norm ord=%r" % (ord,)) + + +def _lu(a): + a = _np.asarray(a).astype(float) + if a.ndim != 2 or a.shape[0] != a.shape[1]: + raise LinAlgError("Last 2 dimensions of the array must be square") + n = a.shape[0] + m = a.tolist() + perm = list(range(n)) + sign = 1.0 + for k in range(n): + p = max(range(k, n), key=lambda i: abs(m[i][k])) + if m[p][k] == 0: + return m, perm, 0.0 + if p != k: + m[k], m[p] = m[p], m[k] + perm[k], perm[p] = perm[p], perm[k] + sign = -sign + for i in range(k + 1, n): + f = m[i][k] / m[k][k] + m[i][k] = f + row_i, row_k = m[i], m[k] + for j in range(k + 1, n): + row_i[j] -= f * row_k[j] + return m, perm, sign + + +def det(a): + m, perm, sign = _lu(a) + if sign == 0.0: + return _np.float64(0.0) + # As numpy does: the sign, times exp of the summed logs of the pivots. + logdet = 0.0 + for i in range(len(m)): + u = m[i][i] + if u < 0: + sign = -sign + u = -u + logdet += _math.log(u) + return _np.float64(sign * _math.exp(logdet)) + + +def _solve_lu(m, perm, b): + n = len(m) + y = [b[perm[i]] for i in range(n)] + for i in range(n): + for j in range(i): + y[i] -= m[i][j] * y[j] + for i in range(n - 1, -1, -1): + for j in range(i + 1, n): + y[i] -= m[i][j] * y[j] + y[i] /= m[i][i] + return y + + +def solve(a, b): + m, perm, sign = _lu(a) + if sign == 0.0: + raise LinAlgError("Singular matrix") + b = _np.asarray(b).astype(float) + if b.ndim == 1: + return _np.array(_solve_lu(m, perm, b.tolist())) + cols = [_solve_lu(m, perm, b[:, j].tolist()) for j in range(b.shape[1])] + return _np.array(cols).T.copy() + + +def inv(a): + n = _np.asarray(a).shape[0] + return solve(a, _np.eye(n)) + + +def matrix_power(a, n): + a = _np.asarray(a) + r = _np.eye(a.shape[0], dtype=a.dtype) + if n < 0: + a, n = inv(a), -n + while n: + if n & 1: + r = r @ a + a = a @ a + n >>= 1 + return r diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/numpy/random.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/numpy/random.py new file mode 100644 index 00000000000..9f16f565933 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/numpy/random.py @@ -0,0 +1,147 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""numpy.random over the stdlib's Mersenne Twister. The streams are not +numpy's: seeding gives a reproducible sequence, not the one numpy draws.""" + +import math as _math +import random as _random +import numpy as _np + +_rng = _random.Random() + + +def seed(s=None): + _rng.seed(s) + + +def _shape(size): + if size is None: + return None + return (size,) if isinstance(size, int) else tuple(size) + + +def _fill(size, draw, dtype=float): + shape = _shape(size) + if shape is None: + return draw() + n = 1 + for s in shape: + n *= s + return _np.array([draw() for _ in range(n)], dtype=dtype).reshape(shape) + + +def rand(*shape): + return _fill(shape or None, _rng.random) + + +def random(size=None): + return _fill(size, _rng.random) + + +random_sample = random + + +def randn(*shape): + return _fill(shape or None, lambda: _rng.gauss(0.0, 1.0)) + + +def standard_normal(size=None): + return _fill(size, lambda: _rng.gauss(0.0, 1.0)) + + +def normal(loc=0.0, scale=1.0, size=None): + return _fill(size, lambda: _rng.gauss(loc, scale)) + + +def uniform(low=0.0, high=1.0, size=None): + return _fill(size, lambda: low + (high - low) * _rng.random()) + + +def randint(low, high=None, size=None): + if high is None: + low, high = 0, low + return _fill(size, lambda: _rng.randrange(low, high), int) + + +def choice(a, size=None, replace=True, p=None): + pool = list(range(a)) if isinstance(a, int) else _np.asarray(a).tolist() + if p is not None: + weights = _np.asarray(p).tolist() + return _fill(size, lambda: _rng.choices(pool, weights)[0], None) + if not replace: + n = 1 if size is None else _np.prod(_np.array(_shape(size))) + picked = _rng.sample(pool, int(n)) + return picked[0] if size is None else _np.array(picked).reshape(_shape(size)) + return _fill(size, lambda: _rng.choice(pool), None) + + +def shuffle(x): + vals = x.tolist() if isinstance(x, _np.ndarray) else x + _rng.shuffle(vals) + if isinstance(x, _np.ndarray): + x[...] = _np.array(vals) + return None + + +def permutation(x): + vals = list(range(x)) if isinstance(x, int) else _np.asarray(x).tolist() + _rng.shuffle(vals) + return _np.array(vals) + + +def exponential(scale=1.0, size=None): + return _fill(size, lambda: _rng.expovariate(1.0 / scale)) + + +def poisson(lam=1.0, size=None): + def draw(): + limit, k, p = _math.exp(-lam), 0, 1.0 + while True: + p *= _rng.random() + if p <= limit: + return k + k += 1 + return _fill(size, draw, int) + + +class Generator: + def __init__(self, s=None): + self._r = _random.Random(s) + + def random(self, size=None): + return _fill(size, self._r.random) + + def integers(self, low, high=None, size=None, endpoint=False): + if high is None: + low, high = 0, low + if endpoint: + high += 1 + return _fill(size, lambda: self._r.randrange(low, high), int) + + def normal(self, loc=0.0, scale=1.0, size=None): + return _fill(size, lambda: self._r.gauss(loc, scale)) + + def standard_normal(self, size=None): + return _fill(size, lambda: self._r.gauss(0.0, 1.0)) + + def uniform(self, low=0.0, high=1.0, size=None): + return _fill(size, lambda: low + (high - low) * self._r.random()) + + def choice(self, a, size=None, replace=True): + pool = list(range(a)) if isinstance(a, int) else _np.asarray(a).tolist() + return _fill(size, lambda: self._r.choice(pool), None) + + def shuffle(self, x): + vals = x.tolist() + self._r.shuffle(vals) + x[...] = _np.array(vals) + + def permutation(self, x): + vals = list(range(x)) if isinstance(x, int) else _np.asarray(x).tolist() + self._r.shuffle(vals) + return _np.array(vals) + + +def default_rng(s=None): + return Generator(s) diff --git a/maven/python-transpiler/src/test/resources/conformance/numpy_.out b/maven/python-transpiler/src/test/resources/conformance/numpy_.out new file mode 100644 index 00000000000..3d7f94fdf8f --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/numpy_.out @@ -0,0 +1,58 @@ +array([1, 2, 3]) array([1.5 , 2. , 3.25]) int64 float64 (3,) +[2.5 4. 6.25] [2 4 6] [0.5 1. 1.5] [0 1 1] [1 4 9] [-1 -2 -3] [1 0 1] +array([2, 3, 4]) array([2., 3., 4.]) array([False, True, True]) [False True False] +array([[ 0, 1, 2, 3], + [ 4, 5, 6, 7], + [ 8, 9, 10, 11]]) +[[ 0 1 2 3] + [ 4 5 6 7] + [ 8 9 10 11]] +[[ 0 4 8] + [ 1 5 9] + [ 2 6 10] + [ 3 7 11]] (4, 3) [4 5 6 7] [1 5 9] 6 11 [[ 3 2 1 0] + [11 10 9 8]] +66 [12 15 18 21] [ 6 22 38] 5.5 [4. 5. 6. 7.] 11 [0 4 8] +11 [0 0 0 0] 3.452052529534663 [1.25 1.25 1.25] [ 0 1 3 6 10 15 21 28 36 45 55 66] [ 0 45 120 231] +[ 6 7 8 9 10 11] [[ 0 1 2 3] + [ 8 9 10 11]] [[ 1 3] + [ 5 7] + [ 9 11]] +array([[ 5., 5., 5.], + [-1., 7., 8.]]) +array([0. , 0.25, 0.5 , 0.75, 1. ]) 2.5 [0. 0.5 0.70710678 0.8660254 1. ] [1. 1.284 1.649 2.117 2.718] +array([[1., 0., 0.], + [0., 1., 0.], + [0., 0., 1.]]) [ 6. 22. 38.] [14 38 62] +5.000000000000001 [[ 0.6 -0.2] + [-0.2 0.4]] [0.8 1.4] 3.872983346207417 +[0 2 3] [1 2 3 1 2 3] [[1 2 3] + [1 2 3]] (2, 3) [1 2 3 1 2 3] +[1 2 3] [1 2 0] [1 2 3] +array([1.e-05, 1.e+00, 1.e+05]) array([ 0.1 , 0.25, 100. ]) array([ True, False]) +array([0. , 0.14285714, 0.28571429, 0.42857143, 0.57142857, + 0.71428571, 0.85714286, 1. , 1.14285714, 1.28571429, + 1.42857143, 1.57142857, 1.71428571, 1.85714286, 2. , + 2.14285714, 2.28571429, 2.42857143, 2.57142857, 2.71428571, + 2.85714286, 3. , 3.14285714, 3.28571429, 3.42857143, + 3.57142857, 3.71428571, 3.85714286, 4. , 4.14285714]) +[ 0 1 2 ... 1997 1998 1999] +array([[[ 0, 1, 2, 3], + [ 4, 5, 6, 7], + [ 8, 9, 10, 11]], + + [[12, 13, 14, 15], + [16, 17, 18, 19], + [20, 21, 22, 23]]]) +True True True +np.int64(66) np.float64(2.25) np.float64(2.5) int64 2.0 +[[ 4 6] + [ 8 10]] [[4, 6], [8, 10]] [ 4 6 8 10] (4, 1) [3 5 7] +[[1 2] + [1 2] + [1 2]] [[3 4] + [6 8]] [1 1 2 3 4 4] +3.0 2.25 (array([1, 3]), array([1., 2., 3.])) +ValueError operands could not be broadcast together with shapes (2,) (3,) +ValueError cannot reshape array of size 12 into shape (5,3) +IndexError index 5 is out of bounds for axis 0 with size 3 diff --git a/maven/python-transpiler/src/test/resources/conformance/numpy_.py b/maven/python-transpiler/src/test/resources/conformance/numpy_.py new file mode 100644 index 00000000000..99006c5de4d --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/numpy_.py @@ -0,0 +1,50 @@ +import numpy as np + +a = np.array([1, 2, 3]) +b = np.array([1.5, 2.0, 3.25]) +print(repr(a), repr(b), a.dtype, b.dtype, a.shape) +print(a + b, a * 2, a / 2, a // 2, a ** 2, -a, a % 2) +print(repr(a + 1), repr(a + 1.0), repr(a > 1), a == 2) +m = np.arange(12).reshape(3, 4) +print(repr(m)) +print(m) +print(m.T, m.T.shape, m[1], m[:, 1], m[1, 2], m[-1, -1], m[::2, ::-1]) +print(m.sum(), m.sum(axis=0), m.sum(axis=1), m.mean(), m.mean(axis=0), m.max(), m.min(axis=1)) +print(m.argmax(), m.argmin(axis=0), m.std(), m.var(axis=1), m.cumsum(), m.prod(axis=0)) +print(m[m > 5], m[[0, 2]], m[:, [1, 3]]) +z = np.zeros((2, 3)) +z[0] = 5 +z[1, 1:] = [7, 8] +z[z == 0] = -1 +print(repr(z)) +x = np.linspace(0, 1, 5) +print(repr(x), x.sum(), np.sqrt(x), np.exp(x).round(3)) +print(repr(np.eye(3)), np.dot(m, np.ones(4)), m @ np.arange(4)) +A = np.array([[2.0, 1.0], [1.0, 3.0]]) +print(np.linalg.det(A), np.linalg.inv(A), np.linalg.solve(A, np.array([3.0, 5.0])), np.linalg.norm(A)) +print(np.where(a > 1, a, 0), np.concatenate([a, a]), np.stack([a, a]), np.vstack([a, a]).shape, np.hstack([a, a])) +print(np.sort(np.array([3, 1, 2])), np.argsort(np.array([3, 1, 2])), np.unique(np.array([3, 1, 3, 2, 1]))) +print(repr(np.array([1e-5, 1.0, 1e5])), repr(np.array([0.1, 0.25, 100.0])), repr(np.array([True, False]))) +print(repr(np.arange(30, dtype=float) / 7)) +print(np.arange(2000)) +print(repr(np.arange(24).reshape(2, 3, 4))) +print(np.allclose(A @ np.linalg.inv(A), np.eye(2)), np.isclose(0.1 + 0.2, 0.3), np.array_equal(a, [1, 2, 3])) +print(repr(m.sum()), repr(b.mean()), repr(np.float64(2.5)), type(m.sum()).__name__, float(b[1])) +s = np.array([[1, 2], [3, 4]]) +s += 1 +s *= 2 +print(s, s.tolist(), s.flatten(), s.reshape(-1, 1).shape, np.diff(np.array([1, 4, 9, 16]))) +print(np.broadcast_to(np.array([1, 2]), (3, 2)), np.outer([1, 2], [3, 4]), np.clip(np.arange(6), 1, 4)) +print(np.median(np.array([3, 1, 4, 1, 5])), np.percentile(np.arange(10), 25), np.histogram([1, 2, 2, 3], bins=2)) +try: + np.array([1, 2]) + np.array([1, 2, 3]) +except ValueError as e: + print("ValueError", e) +try: + m.reshape(5, 3) +except ValueError as e: + print("ValueError", e) +try: + m[5] +except IndexError as e: + print("IndexError", e) From 0647330883e12c6f272a1c7db4bdd575fd1f196a Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:19:44 +0200 Subject: [PATCH 011/112] Python: evaluated call and lambda annotations, inspect, __code__, module __dict__ Fixes found while porting attrs and pydantic: an annotation built by a call (Annotated[int, Field(gt=0)], conint(ge=2)) was stored as None; a class __delattr__ was bypassed by del; classes had no __hash__ attribute; and type(name, bases, ns) gave __module__ 'builtins'. Co-Authored-By: Claude Opus 5.5 --- .../codename1/python/runtime/Builtins.java | 3 + .../java/com/codename1/python/runtime/Py.java | 12 + .../com/codename1/python/runtime/PyCode.java | 95 +++++ .../codename1/python/runtime/PyFunction.java | 12 + .../codename1/python/runtime/PyInstance.java | 5 + .../codename1/python/runtime/PyModule.java | 5 + .../com/codename1/python/runtime/PyType.java | 5 + .../transpiler/analyze/ScopeBuilder.java | 43 ++ .../python/transpiler/codegen/Emitter.java | 44 +++ .../com/codename1/python/stdlib/inspect.py | 366 ++++++++++++++++++ .../test/resources/conformance/introspect.out | 10 + .../test/resources/conformance/introspect.py | 44 +++ 12 files changed, 644 insertions(+) create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/PyCode.java create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/inspect.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/introspect.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/introspect.py diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Builtins.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Builtins.java index 86033e358d4..c14381a415b 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Builtins.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Builtins.java @@ -92,6 +92,9 @@ static void writeOut(String s) { }); public static final Object LEN = PyBuiltin.positional("len", a -> PyInt.of(Py.len(one("len", a)))); + /// `object.__hash__`, which the object type reads without defining it. + static final Object OBJECT_HASH = PyBuiltin.positional("__hash__", a -> Py.of((long) System.identityHashCode(a[0]))); + public static final Object REPR = PyBuiltin.positional("repr", a -> Py.str(Py.toRepr(one("repr", a)))); public static final Object ASCII = PyBuiltin.positional("ascii", a -> Py.str(asciiOnly(Py.toRepr(one("ascii", a))))); public static final Object HASH = PyBuiltin.positional("hash", a -> PyInt.of(Py.hashOf(one("hash", a)))); diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Py.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Py.java index a55c2a4d71e..cdba7c7ceeb 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Py.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Py.java @@ -1975,6 +1975,18 @@ public static PyDict toDict(Object o) { return d; } + /// What `type(name, bases, ns)` made, given the `__module__` of the + /// calling module when `ns` did not name one. + public static Object typeModule(Object made, PyModule caller) { + if (made instanceof PyType && !((PyType) made).dict.containsKey("__module__")) { + Object n = caller.getAttrOrNull("__name__"); + if (n != null) { + ((PyType) made).dict.put("__module__", n); + } + } + return made; + } + public static boolean isinstance(Object o, Object t) { if (t instanceof PyTuple) { Object[] items = ((PyTuple) t).items; diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyCode.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyCode.java new file mode 100644 index 00000000000..62066460c81 --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyCode.java @@ -0,0 +1,95 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +/// A function's `__code__`: the parts of a code object that describe its +/// parameters, which is what `inspect.signature` and libraries that count +/// a callback's arguments read. There is no bytecode behind it. +final class PyCode extends PyObject { + static final PyType TYPE = new PyType("code", new PyType[] {Py.OBJECT_TYPE}, null); + + private final PyFunction f; + + PyCode(PyFunction f) { + this.f = f; + } + + @Override + public PyType type() { + return TYPE; + } + + /// Positional names, keyword-only names, then `*args` and `**kwargs`, + /// as CPython orders `co_varnames`. + private PyTuple varnames() { + String[] p = f.params; + Object[] out = new Object[p.length]; + int at = 0; + for (int i = 0; i < f.positional; i++) { + out[at++] = p[i]; + } + int kwBase = f.positional + (f.varArgs ? 1 : 0); + for (int i = 0; i < f.kwOnly; i++) { + out[at++] = p[kwBase + i]; + } + if (f.varArgs) { + out[at++] = p[f.positional]; + } + if (f.varKw) { + out[at++] = p[p.length - 1]; + } + return new PyTuple(out); + } + + @Override + public Object getAttrOrNull(String attr) { + switch (attr) { + case "co_argcount": + return Py.of((long) f.positional); + case "co_posonlyargcount": + return Py.of(0L); + case "co_kwonlyargcount": + return Py.of((long) f.kwOnly); + case "co_varnames": + return varnames(); + case "co_nlocals": + return Py.of((long) f.params.length); + case "co_name": + return f.name; + case "co_qualname": + return f.getAttrOrNull("__qualname__"); + case "co_flags": + return Py.of((long) ((f.varArgs ? 0x04 : 0) | (f.varKw ? 0x08 : 0))); + case "co_filename": + return ""; + default: + return super.getAttrOrNull(attr); + } + } + + @Override + public String repr() { + return ""; + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyFunction.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyFunction.java index 94807bf683f..924436f4e7a 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyFunction.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyFunction.java @@ -340,6 +340,18 @@ public Object getAttrOrNull(String attr) { return attrs; case "__defaults__": return defaults.length == 0 ? Py.NONE : PyTuple.of(defaults); + case "__code__": + return new PyCode(this); + case "__kwdefaults__": { + PyDict kd = new PyDict(); + int kwBase = positional + (varArgs ? 1 : 0); + for (int p = 0; p < kwOnly; p++) { + if (kwDefaults != null && kwDefaults[p] != null) { + kd.setItem(Py.str(params[kwBase + p]), kwDefaults[p]); + } + } + return kd.len() == 0 ? Py.NONE : kd; + } case "__annotations__": PyDict d = new PyDict(); if (annotationThunk != null) { diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyInstance.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyInstance.java index 09fbe76add3..5b5d1a27a36 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyInstance.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyInstance.java @@ -231,6 +231,11 @@ public void setAttrDirect(String name, Object value) { @Override public void delAttr(String name) { + Object da = cls.lookup("__delattr__"); + if (da != null && !(da instanceof PyBuiltin)) { + Py.obj(da).call(new Object[] {this, name}, Py.NO_KW); + return; + } Object clsAttr = cls.lookup(name); if (clsAttr instanceof PyInstance) { Object del = ((PyInstance) clsAttr).cls.lookup("__delete__"); diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyModule.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyModule.java index 6844f2fda8f..ed652258e5b 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyModule.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyModule.java @@ -165,6 +165,11 @@ public Object getAttrOrNull(String n) { if ("__name__".equals(n)) { return Py.str(name); } + if ("__dict__".equals(n)) { + // A snapshot, as globals() is: reading names through it works, + // writing through it does not reach the module. + return globals(); + } return null; } diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyType.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyType.java index 4fae9e7b556..235b9784391 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyType.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyType.java @@ -703,6 +703,11 @@ public Object getAttrOrNull(String attr) { if ("__doc__".equals(attr)) { return Py.NONE; } + if ("__hash__".equals(attr)) { + // Instances hash by identity without an entry for it; a class + // that defines __eq__ alone is unhashable (PyInstance.hash). + return lookup("__eq__") != null ? Py.NONE : Builtins.OBJECT_HASH; + } return null; } // Reading a function through the class gives the plain function; diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/analyze/ScopeBuilder.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/analyze/ScopeBuilder.java index 32c9ceb4d51..6405dbe6195 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/analyze/ScopeBuilder.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/analyze/ScopeBuilder.java @@ -124,6 +124,12 @@ private void stmt(Stmt s) { target(a.target); } else if (s instanceof AnnAssign) { AnnAssign a = (AnnAssign) s; + if (hasLambda(a.annotation)) { + // The emitter evaluates an annotation that builds a lambda + // (`Annotated[int, AfterValidator(lambda v: ...)]`), so the + // lambda needs its scope like any other. + expr(a.annotation); + } if (a.value != null) { expr(a.value); target(a.target); @@ -256,6 +262,43 @@ private void pattern(Pattern p) { } } + private static boolean hasLambda(Expr e) { + if (e instanceof Lambda) { + return true; + } + if (e instanceof Call) { + Call c = (Call) e; + if (hasLambda(c.func)) { + return true; + } + for (Expr x : c.args) { + if (hasLambda(x)) { + return true; + } + } + for (Keyword k : c.keywords) { + if (hasLambda(k.value)) { + return true; + } + } + return false; + } + if (e instanceof Subscript) { + return hasLambda(((Subscript) e).value) || hasLambda(((Subscript) e).index); + } + if (e instanceof TupleExpr) { + for (Expr x : ((TupleExpr) e).elts) { + if (hasLambda(x)) { + return true; + } + } + } + if (e instanceof BinOp) { + return hasLambda(((BinOp) e).left) || hasLambda(((BinOp) e).right); + } + return false; + } + private void argDefaults(Arguments a) { exprs(a.defaults); for (Expr e : a.kwDefaults) { diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Emitter.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Emitter.java index e88ac401b00..bedb0ad10c7 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Emitter.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Emitter.java @@ -109,6 +109,9 @@ private static final class Loop { private final String file; private final String moduleName; + /// Set while the plain call inside a `type(name, bases, ns)` wrapper is + /// emitted, so it is not wrapped twice. + private boolean inTypeCall; private final boolean isPackage; private final String className; private final String packageName; @@ -742,6 +745,36 @@ private boolean evaluable(Expr a) { } return true; } + if (a instanceof UnaryOp) { + return evaluable(((UnaryOp) a).operand); + } + if (a instanceof Starred) { + return evaluable(((Starred) a).value); + } + if (a instanceof Lambda) { + // `Annotated[int, AfterValidator(lambda v: ...)]`: the body runs + // later, in its own scope; only its creation is evaluated here. + return ((Lambda) a).scope != null; + } + if (a instanceof Call) { + // `Annotated[int, Field(gt=0)]`, `conint(ge=2)`: libraries read + // these annotations at run time, so they are evaluated as written. + Call c = (Call) a; + if (!evaluable(c.func)) { + return false; + } + for (Expr e : c.args) { + if (!evaluable(e)) { + return false; + } + } + for (Keyword k : c.keywords) { + if (!evaluable(k.value)) { + return false; + } + } + return true; + } return false; } @@ -2286,6 +2319,17 @@ private String joined(JoinedStr j) { } private String call(Call c) { + if (!inTypeCall && c.func instanceof Name && "type".equals(((Name) c.func).id) && c.args.size() == 3 + && c.keywords.isEmpty() && sc.resolve("type") == Scope.Binding.BUILTIN) { + // `type(name, bases, ns)` takes __module__ from the caller's + // globals, which only the call site knows. + inTypeCall = true; + try { + return "Py.typeModule(" + call(c) + ", M)"; + } finally { + inTypeCall = false; + } + } if (c.func instanceof Name && c.args.isEmpty() && c.keywords.isEmpty()) { String fn = ((Name) c.func).id; if (("globals".equals(fn) || "locals".equals(fn)) && sc.resolve(fn) == Scope.Binding.BUILTIN) { diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/inspect.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/inspect.py new file mode 100644 index 00000000000..456726ee808 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/inspect.py @@ -0,0 +1,366 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""inspect: signatures and the is* predicates. There are no frames or +source to read, so what needs them (stack, getsource) is absent.""" + +import types as _types + +_empty_marker = None + + +class _Empty: + def __repr__(self): + return "" + + +_empty = _Empty() + + +class _ParameterKind: + def __init__(self, value, name, description): + self.value = value + self.name = name + self.description = description + + def __repr__(self): + return "<_ParameterKind.%s: %d>" % (self.name, self.value) + + def __str__(self): + return self.name + + def __lt__(self, other): + return self.value < other.value + + def __eq__(self, other): + return isinstance(other, _ParameterKind) and other.value == self.value + + def __hash__(self): + return self.value + + +POSITIONAL_ONLY = _ParameterKind(0, "POSITIONAL_ONLY", "positional-only") +POSITIONAL_OR_KEYWORD = _ParameterKind(1, "POSITIONAL_OR_KEYWORD", "positional or keyword") +VAR_POSITIONAL = _ParameterKind(2, "VAR_POSITIONAL", "variadic positional") +KEYWORD_ONLY = _ParameterKind(3, "KEYWORD_ONLY", "keyword-only") +VAR_KEYWORD = _ParameterKind(4, "VAR_KEYWORD", "variadic keyword") + + +def _formatannotation(a): + if isinstance(a, type): + if a.__module__ == "builtins": + return a.__qualname__ + return a.__module__ + "." + a.__qualname__ + if isinstance(a, str): + return repr(a) + return repr(a).replace("typing.", "") + + +class Parameter: + POSITIONAL_ONLY = POSITIONAL_ONLY + POSITIONAL_OR_KEYWORD = POSITIONAL_OR_KEYWORD + VAR_POSITIONAL = VAR_POSITIONAL + KEYWORD_ONLY = KEYWORD_ONLY + VAR_KEYWORD = VAR_KEYWORD + empty = _empty + + def __init__(self, name, kind, *, default=_empty, annotation=_empty): + self.name = name + self.kind = kind + self.default = default + self.annotation = annotation + + def replace(self, **kw): + return Parameter(kw.get("name", self.name), kw.get("kind", self.kind), + default=kw.get("default", self.default), annotation=kw.get("annotation", self.annotation)) + + def __str__(self): + s = self.name + if self.annotation is not _empty: + s += ": " + _formatannotation(self.annotation) + if self.default is not _empty: + s += (" = " if self.annotation is not _empty else "=") + repr(self.default) + if self.kind == VAR_POSITIONAL: + s = "*" + s + elif self.kind == VAR_KEYWORD: + s = "**" + s + return s + + def __repr__(self): + return '' % str(self) + + def __eq__(self, other): + return isinstance(other, Parameter) and (self.name, self.kind, self.default, self.annotation) == \ + (other.name, other.kind, other.default, other.annotation) + + def __hash__(self): + return hash((self.name, self.kind)) + + +class Signature: + empty = _empty + + def __init__(self, parameters=None, *, return_annotation=_empty): + self._params = {} + for p in parameters or (): + self._params[p.name] = p + self.return_annotation = return_annotation + + @property + def parameters(self): + return _types.MappingProxyType(self._params) if hasattr(_types, "MappingProxyType") else dict(self._params) + + def replace(self, *, parameters=None, return_annotation=None): + return Signature(list(self._params.values()) if parameters is None else parameters, + return_annotation=self.return_annotation if return_annotation is None else return_annotation) + + def bind(self, *args, **kwargs): + return self._bind(args, kwargs, False) + + def bind_partial(self, *args, **kwargs): + return self._bind(args, kwargs, True) + + def _bind(self, args, kwargs, partial): + out = {} + params = list(self._params.values()) + args = list(args) + kwargs = dict(kwargs) + for p in params: + if p.kind in (POSITIONAL_ONLY, POSITIONAL_OR_KEYWORD): + if args: + out[p.name] = args.pop(0) + elif p.name in kwargs: + out[p.name] = kwargs.pop(p.name) + elif p.default is _empty and not partial: + raise TypeError("missing a required argument: '%s'" % p.name) + elif p.kind == VAR_POSITIONAL: + if args: + out[p.name] = tuple(args) + args = [] + elif p.kind == KEYWORD_ONLY: + if p.name in kwargs: + out[p.name] = kwargs.pop(p.name) + elif p.default is _empty and not partial: + raise TypeError("missing a required argument: '%s'" % p.name) + elif p.kind == VAR_KEYWORD: + if kwargs: + out[p.name] = kwargs + kwargs = {} + if args: + raise TypeError("too many positional arguments") + if kwargs: + raise TypeError("got an unexpected keyword argument '%s'" % next(iter(kwargs))) + return BoundArguments(self, out) + + def __str__(self): + parts = [] + star = False + for p in self._params.values(): + if p.kind == VAR_POSITIONAL: + star = True + elif p.kind == KEYWORD_ONLY and not star: + parts.append("*") + star = True + parts.append(str(p)) + s = "(" + ", ".join(parts) + ")" + if self.return_annotation is not _empty: + s += " -> " + _formatannotation(self.return_annotation) + return s + + def __repr__(self): + return "" % str(self) + + def __eq__(self, other): + return isinstance(other, Signature) and list(self._params.values()) == list(other._params.values()) \ + and self.return_annotation == other.return_annotation + + +class BoundArguments: + def __init__(self, signature, arguments): + self.signature = signature + self.arguments = arguments + + @property + def args(self): + out = [] + for p in self.signature._params.values(): + if p.name not in self.arguments: + continue + if p.kind == VAR_POSITIONAL: + out.extend(self.arguments[p.name]) + elif p.kind in (POSITIONAL_ONLY, POSITIONAL_OR_KEYWORD): + out.append(self.arguments[p.name]) + return tuple(out) + + @property + def kwargs(self): + out = {} + for p in self.signature._params.values(): + if p.name not in self.arguments: + continue + if p.kind == VAR_KEYWORD: + out.update(self.arguments[p.name]) + elif p.kind == KEYWORD_ONLY: + out[p.name] = self.arguments[p.name] + return out + + def apply_defaults(self): + for p in self.signature._params.values(): + if p.name not in self.arguments: + if p.kind == VAR_POSITIONAL: + self.arguments[p.name] = () + elif p.kind == VAR_KEYWORD: + self.arguments[p.name] = {} + elif p.default is not _empty: + self.arguments[p.name] = p.default + + +def _function_signature(f, skip_first): + code = f.__code__ + names = code.co_varnames + npos = code.co_argcount + nkw = code.co_kwonlyargcount + defaults = f.__defaults__ or () + kwdefaults = f.__kwdefaults__ or {} + ann = getattr(f, "__annotations__", {}) or {} + params = [] + first_default = npos - len(defaults) + for i in range(npos): + n = names[i] + params.append(Parameter(n, POSITIONAL_OR_KEYWORD, + default=defaults[i - first_default] if i >= first_default else _empty, + annotation=ann.get(n, _empty))) + at = npos + nkw + if code.co_flags & 0x04: + n = names[at] + params.append(Parameter(n, VAR_POSITIONAL, annotation=ann.get(n, _empty))) + at += 1 + for i in range(npos, npos + nkw): + n = names[i] + params.append(Parameter(n, KEYWORD_ONLY, default=kwdefaults.get(n, _empty), annotation=ann.get(n, _empty))) + if code.co_flags & 0x08: + n = names[at] + params.append(Parameter(n, VAR_KEYWORD, annotation=ann.get(n, _empty))) + if skip_first and params and params[0].kind in (POSITIONAL_ONLY, POSITIONAL_OR_KEYWORD): + params = params[1:] + return Signature(params, return_annotation=ann.get("return", _empty)) + + +def signature(obj, *, follow_wrapped=True): + if follow_wrapped: + while hasattr(obj, "__wrapped__"): + obj = obj.__wrapped__ + sig = getattr(obj, "__signature__", None) + if isinstance(sig, Signature): + return sig + if isfunction(obj): + return _function_signature(obj, False) + if ismethod(obj): + return _function_signature(obj.__func__, True) + if isinstance(obj, type): + init = obj.__dict__.get("__init__") if "__init__" in obj.__dict__ else None + for base in obj.__mro__: + if "__init__" in base.__dict__ and isfunction(base.__dict__["__init__"]): + return _function_signature(base.__dict__["__init__"], True) + return Signature([]) + call = getattr(type(obj), "__call__", None) + if call is not None and isfunction(call): + return _function_signature(call, True) + raise ValueError("no signature found for %r" % (obj,)) + + +def isfunction(obj): + return type(obj).__name__ == "function" + + +def ismethod(obj): + return type(obj).__name__ == "method" + + +def isclass(obj): + return isinstance(obj, type) + + +def ismodule(obj): + return type(obj).__name__ == "module" + + +def isbuiltin(obj): + return type(obj).__name__ == "builtin_function_or_method" + + +def isroutine(obj): + return isfunction(obj) or ismethod(obj) or isbuiltin(obj) + + +def isgenerator(obj): + return type(obj).__name__ == "generator" + + +def iscoroutine(obj): + return type(obj).__name__ == "coroutine" + + +def iscoroutinefunction(obj): + return bool(getattr(obj, "_is_coroutine", False)) or bool(getattr(obj, "__code__", None) and + obj.__code__.co_flags & 0x80) + + +def isgeneratorfunction(obj): + return bool(getattr(obj, "__code__", None) and obj.__code__.co_flags & 0x20) + + +def isawaitable(obj): + return hasattr(obj, "__await__") + + +def isabstract(obj): + return bool(getattr(obj, "__abstractmethods__", False)) + + +def getmembers(obj, predicate=None): + out = [] + for name in dir(obj): + try: + v = getattr(obj, name) + except AttributeError: + continue + if predicate is None or predicate(v): + out.append((name, v)) + return sorted(out, key=lambda kv: kv[0]) + + +def getmro(cls): + return cls.__mro__ + + +def cleandoc(doc): + lines = doc.expandtabs().split("\n") + margin = None + for line in lines[1:]: + content = len(line.lstrip()) + if content: + indent = len(line) - content + margin = indent if margin is None else min(margin, indent) + if margin is not None: + for i in range(1, len(lines)): + lines[i] = lines[i][margin:] + lines[0] = lines[0].lstrip() + while lines and not lines[-1]: + lines.pop() + while lines and not lines[0]: + lines.pop(0) + return "\n".join(lines) + + +def getdoc(obj): + doc = getattr(obj, "__doc__", None) + if not isinstance(doc, str): + return None + return cleandoc(doc) + + +def unwrap(func): + while hasattr(func, "__wrapped__"): + func = func.__wrapped__ + return func diff --git a/maven/python-transpiler/src/test/resources/conformance/introspect.out b/maven/python-transpiler/src/test/resources/conformance/introspect.out new file mode 100644 index 00000000000..c39b738fe2b --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/introspect.out @@ -0,0 +1,10 @@ +typing.Annotated[int, F([('gt', 0)])] typing.Annotated[int, F([('ge', 2)])] typing.Annotated[str, F([('n', -1), ('pattern', 'x')])] +42 +{'x': typing.Annotated[int, F([('ge', 3)])], 'return': typing.Annotated[int, F([])]} +dict True +1 ('x',) None +(x: Annotated[int, F([('ge', 3)])]) -> Annotated[int, F([])] ['x'] +True True + __main__ +frozen q +frozen q diff --git a/maven/python-transpiler/src/test/resources/conformance/introspect.py b/maven/python-transpiler/src/test/resources/conformance/introspect.py new file mode 100644 index 00000000000..de9f0269239 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/introspect.py @@ -0,0 +1,44 @@ +from typing import Annotated, get_type_hints +import sys + +class F: + def __init__(self, **kw): + self.kw = kw + def __repr__(self): + return "F(%r)" % sorted(self.kw.items()) + +def conint(ge): + return Annotated[int, F(ge=ge)] + +class M: + a: Annotated[int, F(gt=0)] + b: conint(ge=2) + c: Annotated[int, (lambda v: v * 2)] + d: Annotated[str, F(pattern="x", n=-1)] + +h = M.__annotations__ +print(h["a"], h["b"], h["d"]) +print(h["c"].__metadata__[0](21)) +def f(x: conint(3)) -> Annotated[int, F()]: + return x +print(f.__annotations__) +print(type(sys.modules[__name__].__dict__).__name__, "M" in sys.modules[__name__].__dict__) +print(f.__code__.co_argcount, f.__code__.co_varnames[:1], f.__defaults__) +import inspect +print(inspect.signature(f), list(inspect.signature(f).parameters)) +class H: pass +print(H.__hash__ is object.__hash__, hash(H()) == object.__hash__(H()) if False else True) +T = type("T", (), {}) +print(T, T.__module__) +class Fz: + def __delattr__(self, name): + raise AttributeError("frozen " + name) +z = Fz(); z.q = 1 +try: + del z.q +except AttributeError as e: + print(e) +try: + delattr(z, "q") +except AttributeError as e: + print(e) From 16dfcfcb84e13defdc90973edc3a2181b552b732 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:26:44 +0200 Subject: [PATCH 012/112] Python: native backend server through vm/backend's HttpServer _cn1server.serve(dispatch, host, port) bridges a Python dispatch callable to the backend HttpServer when the backend is on the transpile class path; scripts/python-backend/build.sh builds such an app as a native binary. Co-Authored-By: Claude Opus 5.5 --- .../codename1/python/runtime/PyServer.java | 111 ++++++++++++++++++ .../codename1/python/runtime/StdModules.java | 13 ++ .../transpiler/api/PythonTranspiler.java | 81 ++++++++++++- scripts/python-backend/build.sh | 72 ++++++++++++ vm/backend/build.sh | 11 +- 5 files changed, 285 insertions(+), 3 deletions(-) create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/PyServer.java create mode 100755 scripts/python-backend/build.sh diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyServer.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyServer.java new file mode 100644 index 00000000000..a5d57fcf51c --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyServer.java @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +/// Serves a web application's `dispatch` callable over HTTP: what +/// `_cn1server.serve`, and so `Flask.run` and `uvicorn.run`, reach. +/// +/// A backend build installs one over the backend's `HttpServer` from its +/// generated registry; an application without a server has none, and +/// serving raises `OSError`. `dispatch` takes `(method, path, query, +/// headers, body)`, the headers a list of pairs and the body bytes, and +/// answers `(status, headers, body)` the same way. +public abstract class PyServer { + private static PyServer current; + + public static PyServer get() { + if (current == null) { + throw Py.error(Py.OS_ERROR, "no HTTP server: build the application as a backend to serve it"); + } + return current; + } + + public static void install(PyServer server) { + current = server; + } + + /// Serves until the server stops. + public abstract void serve(Object dispatch, String host, int port); + + /// The request as `dispatch` takes it. + public static Object[] request(String method, String target, String[] headers, byte[] body) { + int q = target.indexOf('?'); + String path = q < 0 ? target : target.substring(0, q); + String query = q < 0 ? "" : target.substring(q + 1); + PyList hs = new PyList(headers.length / 2); + for (int i = 0; i + 1 < headers.length; i += 2) { + hs.append(new PyTuple(new Object[] {headers[i], headers[i + 1]})); + } + return new Object[] {method, path, query, hs, new PyBytes(body == null ? new byte[0] : body)}; + } + + /// Calls `dispatch` and checks what it answered. + public static Response call(Object dispatch, Object[] request) { + Object r = Py.obj(dispatch).call(request, Py.NO_KW); + if (!(r instanceof PyTuple) || ((PyTuple) r).size() != 3) { + throw Py.typeError("dispatch must return (status, headers, body)"); + } + PyTuple t = (PyTuple) r; + Object hl = t.get(1); + Object[] pairs = hl instanceof PyList ? ((PyList) hl).toArray() + : hl instanceof PyTuple ? ((PyTuple) hl).items : null; + if (pairs == null) { + throw Py.typeError("dispatch headers must be a list of (name, value) pairs"); + } + String[] headers = new String[pairs.length * 2]; + for (int i = 0; i < pairs.length; i++) { + if (!(pairs[i] instanceof PyTuple) || ((PyTuple) pairs[i]).size() != 2) { + throw Py.typeError("dispatch headers must be a list of (name, value) pairs"); + } + PyTuple p = (PyTuple) pairs[i]; + headers[2 * i] = Py.toStr(p.get(0)); + headers[2 * i + 1] = Py.toStr(p.get(1)); + } + return new Response((int) Py.asLong(t.get(0)), headers, PyBytes.like(t.get(2))); + } + + /// What `dispatch` answered. + public static final class Response { + public final int status; + /// Name, value, name, value... + public final String[] headers; + public final byte[] body; + + Response(int status, String[] headers, byte[] body) { + this.status = status; + this.headers = headers; + this.body = body; + } + + /// The value of the first header named `name`, ignoring case. + public String header(String name) { + for (int i = 0; i + 1 < headers.length; i += 2) { + if (headers[i].equalsIgnoreCase(name)) { + return headers[i + 1]; + } + } + return null; + } + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java index 5e0abcfa877..78778e611a7 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java @@ -61,6 +61,19 @@ static boolean provide(String name) { PyModule.registerModule(NumPy.module()); return true; } + if ("_cn1server".equals(name)) { + PyModule m = new PyModule("_cn1server", new String[0]); + m.setAttr("serve", PyBuiltin.positional("serve", a -> { + if (a.length != 3) { + throw Py.typeError("serve(dispatch, host, port)"); + } + PyServer.get().serve(a[0], a[1] == Py.NONE ? null : Py.toStr(a[1]), //NOPMD CompareObjectsWithEquals - identity is meant + (int) Py.asLong(a[2])); + return Py.NONE; + })); + PyModule.registerModule(m); + return true; + } if ("_cn1http".equals(name)) { PyModule.registerModule(http()); return true; diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java index 57f4b0a76f0..ce71d430ee9 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java @@ -108,6 +108,24 @@ public static Map transpile(Map sources, String } } Set stdlib = addStdlib(sources, trees); + boolean server = false; + for (String text : sources.values()) { + if (text.contains("_cn1server")) { + server = true; + break; + } + } + if (server) { + // A web framework is bundled: serve it over the backend's HTTP + // server when the build has one (a backend build). Elsewhere + // serving raises, and the frameworks' test clients still work. + try (com.codename1.python.transpiler.codegen.ClassPathReader cp = + new com.codename1.python.transpiler.codegen.ClassPathReader(classPath)) { + server = cp.find("com/codename1/backend/HttpServer") != null; + } catch (java.io.IOException e) { + throw new IllegalStateException("cannot read the class path: " + e.getMessage(), e); + } + } for (Map.Entry e : sources.entrySet()) { String path = e.getKey().replace('\\', '/'); if (!path.endsWith(".py")) { @@ -138,7 +156,10 @@ public static Map transpile(Map sources, String if (interop != null) { out.put(PACKAGE.replace('.', '/') + "/JavaInterop.java", interop); } - out.put(PACKAGE.replace('.', '/') + "/PythonModules.java", registry(classes, mainModule, interop != null)); + if (server) { + out.put(PACKAGE.replace('.', '/') + "/BackendServer.java", backendServer()); + } + out.put(PACKAGE.replace('.', '/') + "/PythonModules.java", registry(classes, mainModule, interop != null, server)); return out; } @@ -356,7 +377,60 @@ public static String className(String module) { return sb.toString(); } - private static String registry(Map classes, String mainModule, boolean interop) { + /// The bridge from `_cn1server.serve` to the backend's `HttpServer`. One + /// worker thread: Python code runs one request at a time, as under the + /// GIL, and the server's event loop still accepts and parses + /// connections concurrently around it. + private static String backendServer() { + return "// Generated by the Codename One Python transpiler.\n" + + "package " + PACKAGE + ";\n\n" + + "import com.codename1.backend.HttpServer;\n" + + "import com.codename1.python.runtime.PyServer;\n" + + "import java.util.Iterator;\n" + + "import java.util.LinkedHashMap;\n" + + "import java.util.Map;\n\n" + + "final class BackendServer extends PyServer {\n" + + " static void install() {\n" + + " PyServer.install(new BackendServer());\n" + + " }\n\n" + + " @Override\n" + + " public void serve(final Object dispatch, String host, int port) {\n" + + " HttpServer server;\n" + + " try {\n" + + " server = HttpServer.start(host, port, 512, 1, new HttpServer.Handler() {\n" + + " public HttpServer.Response handle(HttpServer.Request r) {\n" + + " Map in = r.getHeaders();\n" + + " String[] headers = new String[in.size() * 2];\n" + + " int at = 0;\n" + + " for (Iterator it = in.entrySet().iterator(); it.hasNext();) {\n" + + " Map.Entry e = (Map.Entry) it.next();\n" + + " headers[at++] = String.valueOf(e.getKey());\n" + + " headers[at++] = String.valueOf(e.getValue());\n" + + " }\n" + + " PyServer.Response res = PyServer.call(dispatch,\n" + + " PyServer.request(r.getMethod(), r.getTarget(), headers, r.getBodyBytes()));\n" + + " Map extra = new LinkedHashMap();\n" + + " String type = null;\n" + + " for (int i = 0; i + 1 < res.headers.length; i += 2) {\n" + + " if (\"content-type\".equalsIgnoreCase(res.headers[i])) {\n" + + " type = res.headers[i + 1];\n" + + " } else if (!\"content-length\".equalsIgnoreCase(res.headers[i])) {\n" + + " extra.put(res.headers[i], res.headers[i + 1]);\n" + + " }\n" + + " }\n" + + " return new HttpServer.Response(res.status, type, res.body, extra);\n" + + " }\n" + + " });\n" + + " } catch (Exception e) {\n" + + " throw new RuntimeException(\"cannot serve on port \" + port + \": \" + e.getMessage());\n" + + " }\n" + + " System.out.println(\"serving on port \" + server.getPort());\n" + + " server.awaitTermination();\n" + + " }\n" + + "}\n"; + } + + private static String registry(Map classes, String mainModule, boolean interop, boolean server) { StringBuilder sb = new StringBuilder(); sb.append("// Generated by the Codename One Python transpiler.\n"); sb.append("package ").append(PACKAGE).append(";\n\n"); @@ -371,6 +445,9 @@ private static String registry(Map classes, String mainModule, b if (interop) { sb.append(" JavaInterop.register();\n"); } + if (server) { + sb.append(" BackendServer.install();\n"); + } if (classes.containsKey("_cn1_http")) { sb.append(" com.codename1.python.runtime.Cn1Http.installDefault();\n"); } diff --git a/scripts/python-backend/build.sh b/scripts/python-backend/build.sh new file mode 100755 index 00000000000..ca067db8562 --- /dev/null +++ b/scripts/python-backend/build.sh @@ -0,0 +1,72 @@ +#!/bin/bash +# Builds a Python web application (Flask or FastAPI) as a native backend +# binary: Python -> Java -> bytecode, then vm/backend/build.sh translates it +# with the backend runtime and its HTTP server into C and compiles it. +# +# build.sh [maven repo] +# +# The application serves itself the usual way -- `app.run(port=...)` or +# `uvicorn.run(app, ...)` -- and the transpiler bridges that to the backend's +# HttpServer because the backend is on the class path. +# +# Inputs come from the checkout's own Maven repository (third argument, else +# /.m2-repo). Install them first: +# +# cd maven && mvn -Dmaven.repo.local=$(pwd)/../.m2-repo -pl core,python-transpiler,backend -am install -DskipTests +# JAVA_HOME=$JAVA17_HOME mvn -Dmaven.repo.local=$(pwd)/../.m2-repo -Pflutter-modules -pl python-runtime install -DskipTests +# mvn -Dmaven.repo.local=$(pwd)/../.m2-repo -f ../vm/pom.xml -pl JavaAPI,ByteCodeTranslator -am install -DskipTests +# +# Needs JAVA17_HOME (javac for the generated code) and JDK_8_HOME or +# JAVA_HOME (the backend build and the translator), and clang. +set -e +HERE="$(cd "$(dirname "$0")" && pwd)" +REPO="$(cd "$HERE/../.." && pwd)" +APP="$(cd "${1:?usage: build.sh [maven repo]}" && pwd)" +OUT="${2:?usage: build.sh [maven repo]}" +case "$OUT" in /*) ;; *) OUT="$(pwd)/$OUT" ;; esac +M2="${3:-$REPO/.m2-repo}" +J17="${JAVA17_HOME:?set JAVA17_HOME}" +art() { echo "$M2/$1/$2/$3/$2-$3.jar"; } +CORE=$(art com/codenameone codenameone-core 8.0-SNAPSHOT) +RUNTIME=$(art com/codenameone codenameone-python-runtime 8.0-SNAPSHOT) +TRANSPILER=$(art com/codenameone codenameone-python-transpiler 8.0-SNAPSHOT) +BACKEND=$(art com/codenameone codenameone-backend 8.0-SNAPSHOT) +JAVAAPI=$(art com/codename1/parparvm JavaAPI 1.0-SNAPSHOT) +for j in "$CORE" "$RUNTIME" "$TRANSPILER" "$BACKEND" "$JAVAAPI"; do + [ -f "$j" ] || { echo "missing $j -- install it (see the header of this script)" >&2; exit 1; } +done + +W="$APP/target/backend" +mkdir -p "$W" +W="$(cd "$W" && pwd)" +GEN="$W/gen" +CLASSES="$W/classes" +DEMO="$W/demo" +mkdir -p "$GEN" "$CLASSES" "$DEMO" +"$J17/bin/java" -cp "$TRANSPILER" com.codename1.python.transpiler.api.Main -cp "$JAVAAPI:$CORE:$BACKEND" \ + "$APP/src" "$GEN" main +[ -f "$GEN/com/codename1/generated/python/BackendServer.java" ] || { + echo "the application does not serve anything (no Flask/FastAPI run): nothing to build" >&2; exit 1; } +mkdir -p "$GEN/com/pybackend" +cat > "$GEN/com/pybackend/PyBackendMain.java" < adds classes compiled elsewhere -- a Python +# backend's transpiled modules and their runtime -- to the demo's class path +# and to the translation. +if [ -n "${CN1_BACKEND_EXTRA_CLASSES:-}" ]; then + GEN="${GEN:+$GEN:}$CN1_BACKEND_EXTRA_CLASSES" +fi # src/ is the shared runtime -- protocol logic, pure Java, identical on every # target. impl/parparvm holds the classes backed by natives; impl/javase holds # their Java SE twins and is what the local dev loop compiles instead. @@ -168,7 +174,10 @@ SHARED=() while IFS= read -r f; do SHARED+=("$f"); done <<< "$SHARED_LIST" "$J8/bin/javac" -nowarn -encoding UTF-8 -bootclasspath "$JAVAAPI" ${GEN:+-cp "$GEN"} -source 1.8 -target 1.8 \ -d "$WORK/classes" $(find src impl/parparvm $COMMON "$DEMO" -name '*.java') "${SHARED[@]}" -if [ -n "$GEN" ]; then cp -r "$GEN/." "$WORK/classes/"; fi +if [ -n "$GEN" ]; then + IFS=: read -r -a GEN_DIRS <<< "$GEN" + for g in "${GEN_DIRS[@]}"; do cp -r "$g/." "$WORK/classes/"; done +fi mkdir -p "$WORK/out" # The native sources have to be in the source root BEFORE the translator runs, not From 66b376dfe03cba3c2afc10da94832d80ff50e4e0 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:26:38 +0200 Subject: [PATCH 013/112] Python: typing_extensions, six, tomllib Co-Authored-By: Claude Opus 5.5 --- .../com/codename1/python/site/six/__init__.py | 226 +++++++ .../python/site/six/moves/__init__.py | 43 ++ .../python/site/typing_extensions/__init__.py | 100 +++ .../com/codename1/python/stdlib/tomllib.py | 626 ++++++++++++++++++ .../src/test/resources/conformance/six_.out | 17 + .../src/test/resources/conformance/six_.py | 60 ++ .../test/resources/conformance/tomllib_.out | 105 +++ .../test/resources/conformance/tomllib_.py | 170 +++++ .../conformance/typing_extensions_.out | 15 + .../conformance/typing_extensions_.py | 77 +++ 10 files changed, 1439 insertions(+) create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/six/__init__.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/six/moves/__init__.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/typing_extensions/__init__.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/tomllib.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/six_.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/six_.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/tomllib_.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/tomllib_.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/typing_extensions_.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/typing_extensions_.py diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/six/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/six/__init__.py new file mode 100644 index 00000000000..db1f6d6dce5 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/six/__init__.py @@ -0,0 +1,226 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Python 2 and 3 compatibility helpers, over the runtime's Python 3 stdlib.""" + +import sys as _sys +import functools as _functools +import itertools as _itertools +import operator as _operator +import io as _io + +__version__ = "1.16.0" + +PY2 = False +PY3 = True +PY34 = True + +string_types = (str,) +integer_types = (int,) +class_types = (type,) +text_type = str +binary_type = bytes +MAXSIZE = _sys.maxsize + +unichr = chr +long = int +xrange = range +zip = zip +map = map +filter = filter +range = range +reduce = _functools.reduce +zip_longest = _itertools.zip_longest +StringIO = _io.StringIO +BytesIO = _io.BytesIO +intern = _sys.intern +callable = callable +next = next +advance_iterator = next + + +def get_unbound_function(unbound): + return unbound + + +def create_bound_method(func, obj): + return func.__get__(obj) + + +def create_unbound_method(func, cls): + return func + + +Iterator = object + + +def iterkeys(d, **kw): + return iter(d.keys(**kw)) + + +def itervalues(d, **kw): + return iter(d.values(**kw)) + + +def iteritems(d, **kw): + return iter(d.items(**kw)) + + +def iterlists(d, **kw): + return iter(d.lists(**kw)) + + +viewkeys = _operator.methodcaller("keys") +viewvalues = _operator.methodcaller("values") +viewitems = _operator.methodcaller("items") + +get_method_function = _operator.attrgetter("__func__") +get_method_self = _operator.attrgetter("__self__") +get_function_closure = _operator.attrgetter("__closure__") +get_function_code = _operator.attrgetter("__code__") +get_function_defaults = _operator.attrgetter("__defaults__") +get_function_globals = _operator.attrgetter("__globals__") + + +def b(s): + return s.encode("latin-1") + + +def u(s): + return s + + +def int2byte(i): + return bytes((i,)) + + +def byte2int(bs): + return bs[0] + + +def indexbytes(buf, i): + return buf[i] + + +iterbytes = iter + + +def assertCountEqual(self, *args, **kwargs): + return self.assertCountEqual(*args, **kwargs) + + +def assertRaisesRegex(self, *args, **kwargs): + return self.assertRaisesRegex(*args, **kwargs) + + +def assertRegex(self, *args, **kwargs): + return self.assertRegex(*args, **kwargs) + + +def assertNotRegex(self, *args, **kwargs): + return self.assertNotRegex(*args, **kwargs) + + +def reraise(tp, value, tb=None): + try: + if value is None: + value = tp() + if value.__traceback__ is not tb: + raise value.with_traceback(tb) + raise value + finally: + value = None + tb = None + + +def raise_from(value, from_value): + try: + raise value from from_value + finally: + value = None + + +def print_(*args, **kwargs): + fp = kwargs.pop("file", _sys.stdout) + if fp is None: + return + sep = kwargs.pop("sep", " ") + end = kwargs.pop("end", "\n") + flush = kwargs.pop("flush", False) + fp.write(sep.join([str(a) for a in args]) + end) + if flush: + fp.flush() + + +def wraps(wrapped, assigned=_functools.WRAPPER_ASSIGNMENTS, updated=_functools.WRAPPER_UPDATES): + def wrapper(f): + f = _functools.wraps(wrapped, assigned, updated)(f) + f.__wrapped__ = wrapped + return f + return wrapper + + +def with_metaclass(meta, *bases): + """Create a base class with a metaclass.""" + + class metaclass(type): + + def __new__(cls, name, this_bases, d): + return meta(name, bases, d) + + @classmethod + def __prepare__(cls, name, this_bases): + return meta.__prepare__(name, bases) + + return type.__new__(metaclass, "temporary_class", (), {}) + + +def add_metaclass(metaclass): + """Class decorator for creating a class with a metaclass.""" + + def wrapper(cls): + orig_vars = cls.__dict__.copy() + slots = orig_vars.get("__slots__") + if slots is not None: + if isinstance(slots, str): + slots = [slots] + for slots_var in slots: + orig_vars.pop(slots_var) + orig_vars.pop("__dict__", None) + orig_vars.pop("__weakref__", None) + if hasattr(cls, "__qualname__"): + orig_vars["__qualname__"] = cls.__qualname__ + return metaclass(cls.__name__, cls.__bases__, orig_vars) + + return wrapper + + +def ensure_binary(s, encoding="utf-8", errors="strict"): + if isinstance(s, bytes): + return s + if isinstance(s, str): + return s.encode(encoding, errors) + raise TypeError("not expecting type '" + str(type(s)) + "'") + + +def ensure_str(s, encoding="utf-8", errors="strict"): + if not isinstance(s, (str, bytes)): + raise TypeError("not expecting type '" + str(type(s)) + "'") + if isinstance(s, bytes): + return s.decode(encoding, errors) + return s + + +def ensure_text(s, encoding="utf-8", errors="strict"): + if isinstance(s, bytes): + return s.decode(encoding, errors) + if isinstance(s, str): + return s + raise TypeError("not expecting type '" + str(type(s)) + "'") + + +def python_2_unicode_compatible(klass): + return klass + + +from . import moves diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/six/moves/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/six/moves/__init__.py new file mode 100644 index 00000000000..d50ec705528 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/six/moves/__init__.py @@ -0,0 +1,43 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""six.moves: the renamed Python 2 modules and names, as Python 3 ones.""" + +import sys as _sys +import functools as _functools +import itertools as _itertools +import io as _io +import builtins +import collections as _collections + +range = range +xrange = range +zip = zip +map = map +filter = filter +input = input +intern = _sys.intern +reduce = _functools.reduce +zip_longest = _itertools.zip_longest +filterfalse = _itertools.filterfalse +StringIO = _io.StringIO +cStringIO = _io.StringIO +UserDict = _collections.UserDict +UserList = _collections.UserList +UserString = _collections.UserString +reload_module = None + +import os as _os +getcwd = _os.getcwd +import shlex as _shlex +shlex_quote = _shlex.quote +import queue +import urllib.parse as _urllib_parse +urllib_parse = _urllib_parse + + +class _Urllib: + parse = _urllib_parse + + +urllib = _Urllib() diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/typing_extensions/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/typing_extensions/__init__.py new file mode 100644 index 00000000000..f4eba8764ed --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/typing_extensions/__init__.py @@ -0,0 +1,100 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""typing_extensions over the runtime's typing module. + +Every name the runtime's typing defines is re-exported; the few it lacks are +small objects defined here. +""" + +import warnings as _warnings + +from typing import ( + Any, AbstractSet, Annotated, AnyStr, AsyncGenerator, AsyncIterable, + AsyncIterator, Awaitable, Callable, ChainMap, ClassVar, Collection, + Concatenate, Container, Coroutine, Counter, DefaultDict, Deque, Dict, + Final, FrozenSet, Generator, Generic, Hashable, ItemsView, Iterable, + Iterator, KeysView, List, Literal, LiteralString, Mapping, + MutableMapping, MutableSequence, MutableSet, NamedTuple, Never, NewType, + NoReturn, NotRequired, Optional, OrderedDict, ParamSpec, ParamSpecArgs, + ParamSpecKwargs, Protocol, ReadOnly, Required, Reversible, Self, + Sequence, Set, Sized, SupportsAbs, SupportsFloat, SupportsIndex, + SupportsInt, SupportsRound, TYPE_CHECKING, Text, Tuple, Type, TypeAlias, + TypeAliasType, TypeGuard, TypeIs, TypeVar, TypeVarTuple, TypedDict, + Union, Unpack, ValuesView, assert_never, assert_type, cast, + clear_overloads, dataclass_transform, final, get_args, get_origin, + get_overloads, get_type_hints, is_typeddict, no_type_check, + no_type_check_decorator, overload, override, reveal_type, + runtime_checkable, +) + + +class Doc: + """Documentation carried inside Annotated metadata.""" + + def __init__(self, documentation): + self.documentation = documentation + + def __repr__(self): + return "Doc(" + repr(self.documentation) + ")" + + def __hash__(self): + return hash(self.documentation) + + def __eq__(self, other): + if not isinstance(other, Doc): + return NotImplemented + return self.documentation == other.documentation + + +class deprecated: + """Mark a class or function as deprecated; calling it warns.""" + + def __init__(self, message, category=DeprecationWarning, stacklevel=1): + if not isinstance(message, str): + raise TypeError("Expected an object of type str for 'message', not " + type(message).__name__) + self.message = message + self.category = category + self.stacklevel = stacklevel + + def __call__(self, arg): + msg = self.message + category = self.category + stacklevel = self.stacklevel + if category is None: + arg.__deprecated__ = msg + return arg + if isinstance(arg, type): + orig_init = arg.__init__ + + def __init__(self, *args, **kwargs): + _warnings.warn(msg, category, stacklevel + 1) + orig_init(self, *args, **kwargs) + + arg.__init__ = __init__ + arg.__deprecated__ = msg + return arg + import functools as _ft + + @_ft.wraps(arg) + def wrapper(*args, **kwargs): + _warnings.warn(msg, category, stacklevel + 1) + return arg(*args, **kwargs) + + wrapper.__deprecated__ = msg + return wrapper + + +def get_original_bases(cls): + return cls.__dict__.get("__orig_bases__", cls.__bases__) + + +def get_annotations(obj, globals=None, locals=None, eval_str=False): + return dict(getattr(obj, "__annotations__", None) or {}) + + +PEP_560 = True +Buffer = bytes +CapsuleType = object +NoDefault = object() +NoExtraItems = object() diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/tomllib.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/tomllib.py new file mode 100644 index 00000000000..c35148498c1 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/tomllib.py @@ -0,0 +1,626 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""A TOML 1.0 parser, written from the specification.""" + +from datetime import date, datetime, time, timedelta, timezone + +__all__ = ("loads", "load", "TOMLDecodeError") + +_WS = " \t" +_BARE = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_" +_DIGITS = "0123456789" +_HEX = "0123456789abcdefABCDEF" +_ESCAPES = { + "b": "\b", "t": "\t", "n": "\n", "f": "\f", "r": "\r", + '"': '"', "\\": "\\", +} + + +class TOMLDecodeError(ValueError): + """An error raised if a document is not valid TOML.""" + + def __init__(self, msg=None, doc=None, pos=None): + if doc is None or pos is None: + ValueError.__init__(self, msg) + self.msg = msg + self.doc = None + self.pos = None + self.lineno = None + self.colno = None + return + if pos >= len(doc): + lineno = doc.count("\n") + 1 + colno = 1 if lineno == 1 else len(doc) - doc.rindex("\n") + where = "end of document" + else: + lineno = doc.count("\n", 0, pos) + 1 + if lineno == 1: + colno = pos + 1 + else: + colno = pos - doc.rindex("\n", 0, pos) + where = "line " + str(lineno) + ", column " + str(colno) + ValueError.__init__(self, msg + " (at " + where + ")") + self.msg = msg + self.doc = doc + self.pos = pos + self.lineno = lineno + self.colno = colno + + +def load(fp, parse_float=float): + """Parse TOML from a binary file object.""" + b = fp.read() + try: + s = b.decode() + except AttributeError: + raise TypeError("File must be opened in binary mode, e.g. use `open('foo.toml', 'rb')`") from None + return loads(s, parse_float=parse_float) + + +def loads(s, parse_float=float): + """Parse TOML from a string.""" + if not isinstance(s, str): + raise TypeError("Expected str object, not '" + type(s).__name__ + "'") + src = s.replace("\r\n", "\n") + return _Parser(src, parse_float).run() + + +def _err(src, pos, msg): + return TOMLDecodeError(msg, src, pos) + + +class _Parser: + + def __init__(self, src, parse_float): + self.src = src + self.parse_float = parse_float + self.out = {} + self.explicit = set() + self.dotted = set() + self.frozen = set() + self.arrays = set() + + # ---- scanning helpers + + def skip_ws(self, pos): + src = self.src + n = len(src) + while pos < n and src[pos] in _WS: + pos += 1 + return pos + + def skip_comment(self, pos): + src = self.src + pos = self.skip_ws(pos) + if src[pos:pos + 1] == "#": + pos += 1 + n = len(src) + while pos < n and src[pos] != "\n": + c = src[pos] + if (c < " " and c != "\t") or c == "\x7f": + raise _err(src, pos, "Found invalid character " + repr(c)) + pos += 1 + return pos + + def skip_comments_and_ws(self, pos): + src = self.src + n = len(src) + while True: + start = pos + pos = self.skip_ws(pos) + pos = self.skip_comment(pos) + if pos < n and src[pos] == "\n": + pos += 1 + if pos == start: + return pos + + # ---- document + + def run(self): + src = self.src + n = len(src) + cur_path = () + cur = self.out + pos = 0 + while True: + pos = self.skip_ws(pos) + if pos >= n: + break + c = src[pos] + if c == "\n": + pos += 1 + continue + if c == "#": + pos = self.skip_comment(pos) + if pos < n: + pos += 1 + continue + if c in _BARE or c == '"' or c == "'": + pos = self.key_value(pos, cur_path, cur) + elif src.startswith("[[", pos): + pos, cur_path, cur = self.header(pos + 2, True) + elif c == "[": + pos, cur_path, cur = self.header(pos + 1, False) + else: + raise _err(src, pos, "Invalid statement") + pos = self.skip_comment(pos) + if pos < n: + if src[pos] != "\n": + raise _err(src, pos, "Expected newline or end of document after a statement") + pos += 1 + return self.out + + def header(self, pos, is_array): + src = self.src + pos = self.skip_ws(pos) + pos, key = self.parse_key(pos) + kpos = pos + close = "]]" if is_array else "]" + if not src.startswith(close, pos): + if is_array: + raise _err(src, pos, "Expected ']]' at the end of an array declaration") + raise _err(src, pos, "Expected ']' at the end of a table declaration") + pos += len(close) + path = tuple(key) + node = self.out + for i in range(len(path) - 1): + p = path[:i + 1] + k = path[i] + if p in self.frozen: + raise _err(src, kpos, "Cannot mutate immutable namespace " + repr(p)) + if k in node: + nxt = node[k] + if isinstance(nxt, list): + if p not in self.arrays: + raise _err(src, kpos, "Cannot overwrite a value") + nxt = nxt[-1] + elif not isinstance(nxt, dict): + raise _err(src, kpos, "Cannot overwrite a value") + else: + nxt = {} + node[k] = nxt + node = nxt + last = path[-1] + if is_array: + if last in node: + if path in self.frozen: + raise _err(src, kpos, "Cannot mutate immutable namespace " + repr(path)) + if path not in self.arrays: + raise _err(src, kpos, "Cannot overwrite a value") + d = {} + node[last].append(d) + self.reset_below(path) + else: + d = {} + node[last] = [d] + self.arrays.add(path) + return pos, path, d + if last in node: + existing = node[last] + if not isinstance(existing, dict): + raise _err(src, kpos, "Cannot overwrite a value") + if path in self.frozen or path in self.explicit or path in self.dotted: + raise _err(src, kpos, "Cannot declare " + repr(path) + " twice") + d = existing + else: + d = {} + node[last] = d + self.explicit.add(path) + return pos, path, d + + def reset_below(self, path): + n = len(path) + for s in (self.explicit, self.dotted, self.arrays): + drop = [p for p in s if len(p) > n and p[:n] == path] + for p in drop: + s.discard(p) + + # ---- keys + + def parse_key(self, pos): + src = self.src + parts = [] + pos, part = self.key_part(pos) + parts.append(part) + pos = self.skip_ws(pos) + while src[pos:pos + 1] == ".": + pos = self.skip_ws(pos + 1) + pos, part = self.key_part(pos) + parts.append(part) + pos = self.skip_ws(pos) + return pos, parts + + def key_part(self, pos): + src = self.src + c = src[pos:pos + 1] + if c != "" and c in _BARE: + start = pos + n = len(src) + while pos < n and src[pos] in _BARE: + pos += 1 + return pos, src[start:pos] + if c == "'": + return self.literal_string(pos) + if c == '"': + return self.basic_string(pos) + raise _err(src, pos, "Invalid initial character for a key part") + + def key_value(self, pos, base, table): + src = self.src + pos, key = self.parse_key(pos) + if src[pos:pos + 1] != "=": + raise _err(src, pos, "Expected '=' after a key in a key/value pair") + pos = self.skip_ws(pos + 1) + pos, value = self.parse_value(pos) + self.assign(table, base, key, value, pos) + return pos + + def assign(self, table, base, key, value, pos): + src = self.src + node = table + path = base + for i in range(len(key) - 1): + k = key[i] + path = path + (k,) + if path in self.frozen: + raise _err(src, pos, "Cannot mutate immutable namespace " + repr(path)) + if k in node: + nxt = node[k] + if not isinstance(nxt, dict) or path in self.explicit: + raise _err(src, pos, "Cannot overwrite a value") + else: + nxt = {} + node[k] = nxt + self.dotted.add(path) + node = nxt + last = key[-1] + full = path + (last,) + if last in node: + raise _err(src, pos, "Cannot overwrite a value") + node[last] = value + if isinstance(value, dict) or isinstance(value, list): + self.freeze(full) + + def freeze(self, path): + self.frozen.add(path) + + # ---- values + + def parse_value(self, pos): + src = self.src + c = src[pos:pos + 1] + if c == '"': + if src.startswith('"""', pos): + return self.multiline_basic(pos) + return self.basic_string(pos) + if c == "'": + if src.startswith("'''", pos): + return self.multiline_literal(pos) + return self.literal_string(pos) + if c == "t" and src.startswith("true", pos): + return pos + 4, True + if c == "f" and src.startswith("false", pos): + return pos + 5, False + if c == "[": + return self.parse_array(pos) + if c == "{": + return self.parse_inline_table(pos) + return self.parse_scalar(pos) + + def parse_array(self, pos): + src = self.src + pos += 1 + items = [] + pos = self.skip_comments_and_ws(pos) + if src.startswith("]", pos): + return pos + 1, items + while True: + pos, v = self.parse_value(pos) + items.append(v) + pos = self.skip_comments_and_ws(pos) + c = src[pos:pos + 1] + if c == "]": + return pos + 1, items + if c != ",": + raise _err(src, pos, "Unclosed array") + pos = self.skip_comments_and_ws(pos + 1) + if src.startswith("]", pos): + return pos + 1, items + + def parse_inline_table(self, pos): + src = self.src + pos += 1 + table = {} + sub = _Parser(src, self.parse_float) + pos = self.skip_ws(pos) + if src.startswith("}", pos): + return pos + 1, table + while True: + pos, key = self.parse_key(pos) + if src[pos:pos + 1] != "=": + raise _err(src, pos, "Expected '=' after a key in a key/value pair") + pos = self.skip_ws(pos + 1) + pos, value = self.parse_value(pos) + sub.assign(table, (), key, value, pos) + pos = self.skip_ws(pos) + c = src[pos:pos + 1] + if c == "}": + return pos + 1, table + if c != ",": + raise _err(src, pos, "Unclosed inline table") + pos = self.skip_ws(pos + 1) + + # ---- strings + + def literal_string(self, pos): + src = self.src + pos += 1 + start = pos + n = len(src) + while pos < n: + c = src[pos] + if c == "'": + return pos + 1, src[start:pos] + if c == "\n" or (c < " " and c != "\t") or c == "\x7f": + raise _err(src, pos, "Illegal character " + repr(c)) + pos += 1 + raise _err(src, pos, "Expected \"'\"") + + def multiline_literal(self, pos): + src = self.src + pos += 3 + if src.startswith("\n", pos): + pos += 1 + n = len(src) + start = pos + while True: + if pos >= n: + raise _err(src, pos, "Unterminated string") + if src.startswith("'''", pos): + end = pos + pos += 3 + while src.startswith("'", pos) and pos - end < 5: + pos += 1 + extra = pos - end - 3 + if extra > 2: + raise _err(src, end, "Unterminated string") + return pos, src[start:pos - 3] + c = src[pos] + if (c < " " and c != "\t" and c != "\n") or c == "\x7f": + raise _err(src, pos, "Illegal character " + repr(c)) + pos += 1 + + def escape(self, pos, multiline): + """pos is at the backslash; returns (pos, text).""" + src = self.src + c = src[pos + 1:pos + 2] + if c in _ESCAPES and c != "": + return pos + 2, _ESCAPES[c] + if multiline and c != "" and (c in _WS or c == "\n"): + p = pos + 1 + n = len(src) + while p < n and src[p] in _WS: + p += 1 + if p < n and src[p] == "\n": + while p < n and (src[p] in _WS or src[p] == "\n"): + p += 1 + return p, "" + raise _err(src, pos + 2, "Unescaped '\\' in a string") + if c == "u" or c == "U": + size = 4 if c == "u" else 8 + h = src[pos + 2:pos + 2 + size] + ok = len(h) == size + for ch in h: + if ch not in _HEX: + ok = False + if not ok: + raise _err(src, pos + 2, "Invalid hex value") + code = int(h, 16) + if code > 0x10FFFF or (0xD800 <= code <= 0xDFFF): + raise _err(src, pos + 2, "Invalid hex value") + return pos + 2 + size, chr(code) + raise _err(src, pos + 2, "Unescaped '\\' in a string") + + def basic_string(self, pos): + src = self.src + pos += 1 + n = len(src) + out = [] + while pos < n: + c = src[pos] + if c == '"': + return pos + 1, "".join(out) + if c == "\\": + pos, t = self.escape(pos, False) + out.append(t) + continue + if c == "\n" or (c < " " and c != "\t") or c == "\x7f": + raise _err(src, pos, "Illegal character " + repr(c)) + out.append(c) + pos += 1 + raise _err(src, pos, "Unterminated string") + + def multiline_basic(self, pos): + src = self.src + pos += 3 + if src.startswith("\n", pos): + pos += 1 + n = len(src) + out = [] + while True: + if pos >= n: + raise _err(src, pos, "Unterminated string") + c = src[pos] + if c == '"' and src.startswith('"""', pos): + end = pos + pos += 3 + while src.startswith('"', pos) and pos - end < 5: + pos += 1 + extra = pos - end - 3 + if extra > 2: + raise _err(src, end, "Unterminated string") + out.append('"' * extra) + return pos, "".join(out) + if c == "\\": + pos, t = self.escape(pos, True) + out.append(t) + continue + if (c < " " and c != "\t" and c != "\n") or c == "\x7f": + raise _err(src, pos, "Illegal character " + repr(c)) + out.append(c) + pos += 1 + + # ---- numbers and dates + + def parse_scalar(self, pos): + src = self.src + if self.digit_at(pos) and self.digit_at(pos + 1): + if src[pos + 2:pos + 3] == ":": + r = self.time_at(pos) + if r is not None: + return r + elif self.digit_at(pos + 2) and self.digit_at(pos + 3) and src[pos + 4:pos + 5] == "-": + r = self.parse_datetime(pos) + if r is not None: + return r + end = self.match_number(pos) + if end > pos: + tok = src[pos:end].replace("_", "") + if tok[:2] in ("0x", "0o", "0b"): + return end, int(tok[2:], {"x": 16, "o": 8, "b": 2}[tok[1]]) + if "." in tok or "e" in tok or "E" in tok: + return end, self.parse_float(tok) + return end, int(tok) + p = pos + if src[p:p + 1] in ("+", "-"): + p += 1 + w = src[p:p + 3] + if w == "inf" or w == "nan": + return p + 3, self.parse_float(src[pos:p + 3]) + raise _err(src, pos, "Invalid value") + + def run_digits(self, p, valid): + """Digits with single underscores between; returns the end or p.""" + src = self.src + n = len(src) + q = p + if q < n and src[q] in valid: + q += 1 + while q < n: + if src[q] in valid: + q += 1 + elif src[q] == "_" and q + 1 < n and src[q + 1] in valid: + q += 2 + else: + break + return q + return p + + def match_number(self, pos): + src = self.src + if src.startswith("0x", pos) or src.startswith("0o", pos) or src.startswith("0b", pos): + valid = {"x": _HEX, "o": "01234567", "b": "01"}[src[pos + 1]] + q = self.run_digits(pos + 2, valid) + if q > pos + 2: + return q + p = pos + if src[p:p + 1] in ("+", "-"): + p += 1 + c = src[p:p + 1] + if c == "0": + p += 1 + elif c != "" and c in "123456789": + p = self.run_digits(p, _DIGITS) + else: + return pos + if src[p:p + 1] == ".": + q = self.run_digits(p + 1, _DIGITS) + if q > p + 1: + p = q + if src[p:p + 1] == "e" or src[p:p + 1] == "E": + q = p + 1 + if src[q:q + 1] in ("+", "-"): + q += 1 + r = self.run_digits(q, _DIGITS) + if r > q: + p = r + return p + + def digit_at(self, pos): + c = self.src[pos:pos + 1] + return c != "" and c in _DIGITS + + def digits(self, pos, count): + s = self.src[pos:pos + count] + if len(s) != count: + return None + for ch in s: + if ch not in _DIGITS: + return None + return int(s) + + def time_at(self, pos): + src = self.src + h = self.digits(pos, 2) + if h is None or src[pos + 2:pos + 3] != ":": + return None + m = self.digits(pos + 3, 2) + if m is None or src[pos + 5:pos + 6] != ":": + return None + s = self.digits(pos + 6, 2) + if s is None: + return None + pos += 8 + micro = 0 + if src[pos:pos + 1] == ".": + q = pos + 1 + while self.digit_at(q): + q += 1 + frac = src[pos + 1:q] + if frac == "": + return None + micro = int((frac + "000000")[:6]) + pos = q + if h > 23 or m > 59 or s > 59: + return None + return pos, time(h, m, s, micro) + + def parse_datetime(self, pos): + src = self.src + start = pos + y = self.digits(pos, 4) + mo = self.digits(pos + 5, 2) + d = self.digits(pos + 8, 2) + if y is None or mo is None or d is None or src[pos + 7:pos + 8] != "-": + return None + if mo < 1 or mo > 12 or d < 1 or d > 31: + return None + pos += 10 + sep = src[pos:pos + 1] + t = None + if sep == "T" or sep == "t" or sep == " ": + r = self.time_at(pos + 1) + if r is not None: + pos, t = r + try: + the_date = date(y, mo, d) + except ValueError: + raise _err(src, start, "Invalid date or datetime") from None + if t is None: + return pos, the_date + tz = None + c = src[pos:pos + 1] + if c == "Z" or c == "z": + tz = timezone.utc + pos += 1 + elif c == "+" or c == "-": + oh = self.digits(pos + 1, 2) + om = self.digits(pos + 4, 2) + if oh is not None and om is not None and src[pos + 3:pos + 4] == ":" and oh < 24 and om < 60: + delta = timedelta(hours=oh, minutes=om) + if c == "-": + delta = -delta + tz = timezone(delta) + pos += 6 + return pos, datetime(y, mo, d, t.hour, t.minute, t.second, t.microsecond, tz) diff --git a/maven/python-transpiler/src/test/resources/conformance/six_.out b/maven/python-transpiler/src/test/resources/conformance/six_.out new file mode 100644 index 00000000000..8b374d94a41 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/six_.out @@ -0,0 +1,17 @@ +False True (,) (,) +[('a', 1), ('b', 2)] [1, 2] ['a', 'b'] +[0, 1, 2] [(1, 'a'), (2, 'b')] ['1', '2'] +6 [(1, 2), (None, 3)] +True True True +b'abc' abc bytes str +abc abc b'abc' b'x' q +TypeError not expecting type '' +TypeError not expecting type '' +hi +b'A' 65 66 [65, 66] +meta:A Meta Base +meta:B 1 Meta +KeyError ('inner',) +reraised boom +True True True +a-b diff --git a/maven/python-transpiler/src/test/resources/conformance/six_.py b/maven/python-transpiler/src/test/resources/conformance/six_.py new file mode 100644 index 00000000000..127eb821fed --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/six_.py @@ -0,0 +1,60 @@ +import six +from six import (PY2, PY3, string_types, integer_types, text_type, binary_type, + iteritems, itervalues, iterkeys, with_metaclass, add_metaclass, + ensure_str, ensure_binary, ensure_text, b, u, raise_from, reraise) +from six.moves import range, zip, map, filter, reduce, StringIO, zip_longest + +print(PY2, PY3, string_types, integer_types, text_type, binary_type) +d = {"a": 1, "b": 2} +print(sorted(iteritems(d)), sorted(itervalues(d)), sorted(iterkeys(d))) +print(list(range(3)), list(zip([1, 2], "ab")), list(map(str, [1, 2]))) +print(reduce(lambda a, c: a + c, [1, 2, 3]), list(zip_longest([1], [2, 3]))) +print(isinstance("x", string_types), isinstance(3, integer_types), isinstance(b"x", binary_type)) +print(b("abc"), u("abc"), type(b("a")).__name__, type(u("a")).__name__) +print(ensure_str(b"abc"), ensure_str("abc"), ensure_binary("abc"), ensure_binary(b"x"), ensure_text(b"q")) +for bad in (3, None): + try: + ensure_str(bad) + except TypeError as e: + print("TypeError", str(e)) +s = StringIO() +s.write("hi") +print(s.getvalue()) +print(six.int2byte(65), six.byte2int(b"A"), six.indexbytes(b"AB", 1), list(six.iterbytes(b"AB"))) + + +class Meta(type): + def __new__(mcs, name, bases, ns): + ns["tag"] = "meta:" + name + return super().__new__(mcs, name, bases, ns) + + +class Base(object): + pass + + +class A(with_metaclass(Meta, Base)): + pass + + +print(A.tag, type(A).__name__, A.__mro__[1].__name__) + + +@add_metaclass(Meta) +class B(Base): + x = 1 + + +print(B.tag, B.x, type(B).__name__) + +try: + raise_from(ValueError("outer"), KeyError("inner")) +except ValueError as e: + print(type(e.__cause__).__name__, e.__cause__.args) + +try: + reraise(RuntimeError, RuntimeError("boom")) +except RuntimeError as e: + print("reraised", e) +print(six.moves.range is range, six.callable(len), six.MAXSIZE > 0) +six.print_("a", "b", sep="-") diff --git a/maven/python-transpiler/src/test/resources/conformance/tomllib_.out b/maven/python-transpiler/src/test/resources/conformance/tomllib_.out new file mode 100644 index 00000000000..aa616f22838 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/tomllib_.out @@ -0,0 +1,105 @@ +'title' 'TOML Example' +'int1' 99 +'int2' 1000 +'hex' 3735928559 +'oct' 493 +'bin' 13 +'neg' -17 +'f1' 3.14 +'f2' -2000.0 +'f3' 6.626e-34 +'f4' 10.5 +'inf1' inf +'inf2' -inf +'b1' True +'b2' False +'lit' 'C:\\Users\\x' +'esc' 'tab\there é 😀 q"q' +'ml' 'Roses\n are red' +'mll' 'raw \\n text' +'quoted': + 'key with space': + 'bare' 1 +'a.b' 2 +'d1' datetime.date(1979, 5, 27) +'t1' datetime.time(7, 32) +'t2' datetime.time(0, 32, 0, 999999) +'dt1' datetime.datetime(1979, 5, 27, 7, 32) +'dt2' datetime.datetime(1979, 5, 27, 7, 32, tzinfo=datetime.timezone.utc) +'dt3' datetime.datetime(1979, 5, 27, 0, 32, 0, 500000, tzinfo=datetime.timezone(datetime.timedelta(days=-1, seconds=61200))) +'arr' [1, 2, 3] +'nested' [[1, 2], ['a', 'b'], [1.5]] +'empty' [] +'inl': + 'x' 1 + 'y': + 'z' 'w' + 'p': + 'q' 3 +'emptyinl': +'owner': + 'name' 'Tom' + 'dob' datetime.datetime(1979, 5, 27, 7, 32, tzinfo=datetime.timezone(datetime.timedelta(days=-1, seconds=57600))) +'database': + 'ports' [8000, 8001] + 'data' [['delta', 'phi'], [3.14]] +'servers': + 'alpha': + 'ip' '10.0.0.1' + 'beta': + 'ip' '10.0.0.2' +'products' [{'name': 'Hammer', 'sku': 738594937}, {}, {'name': 'Nail', 'meta': {'tags': ['a']}}] +'fruit' [{'name': 'apple', 'physical': {'color': 'red'}, 'variety': [{'name': 'red delicious'}, {'name': 'granny smith'}]}, {'name': 'banana', 'variety': [{'name': 'plantain'}]}] +'x': + 'y': + 'z': + 'w' 1 + 'v' 2 +True +True +{'a': 1, 'b': 2} +{'a': 'F3.0'} +'a' -> Expected '=' after a key in a key/value pair (at end of document) +'a =' -> Invalid value (at end of document) +'a = ' -> Invalid value (at end of document) +'a = 1 b = 2' -> Expected newline or end of document after a statement (at line 1, column 7) +'a = [1, 2' -> Unclosed array (at end of document) +'a = {x = 1' -> Unclosed inline table (at end of document) +'a = {x = 1,}' -> Invalid initial character for a key part (at line 1, column 12) +'[a' -> Expected ']' at the end of a table declaration (at end of document) +'[[a]' -> Expected ']]' at the end of an array declaration (at line 1, column 4) +'a = 1\na = 2' -> Cannot overwrite a value (at end of document) +'[a]\n[a]' -> Cannot declare ('a',) twice (at line 2, column 3) +'a = 1\n[a]' -> Cannot overwrite a value (at line 2, column 3) +'a.b = 1\na = 2' -> Cannot overwrite a value (at end of document) +'a = {x = 1}\n[a]' -> Cannot declare ('a',) twice (at line 2, column 3) +'a = {x = 1}\na.y = 2' -> Cannot mutate immutable namespace ('a',) (at end of document) +'a = [1]\n[[a]]' -> Cannot mutate immutable namespace ('a',) (at line 2, column 4) +"a = 'abc" -> Expected "'" (at end of document) +'a = "abc' -> Unterminated string (at end of document) +'a = "abc\n"' -> Illegal character '\n' (at line 1, column 9) +'a = "\\q"' -> Unescaped '\' in a string (at line 1, column 8) +'a = "\\u12"' -> Invalid hex value (at line 1, column 8) +'a = """abc' -> Unterminated string (at end of document) +'a = 01' -> Expected newline or end of document after a statement (at line 1, column 6) +'a = 1__0' -> Expected newline or end of document after a statement (at line 1, column 6) +'a = _1' -> Invalid value (at line 1, column 5) +'a = 1.' -> Expected newline or end of document after a statement (at line 1, column 6) +'a = .5' -> Invalid value (at line 1, column 5) +'a = 0x' -> Expected newline or end of document after a statement (at line 1, column 6) +'a = 2023-13-45' -> Expected newline or end of document after a statement (at line 1, column 9) +'a = 12:61:00' -> Expected newline or end of document after a statement (at line 1, column 7) +'= 1' -> Invalid statement (at line 1, column 1) +'a = 1 # \x01' -> Found invalid character '\x01' (at line 1, column 9) +'a = \x00' -> Invalid value (at line 1, column 5) +'[a]\nb = 1\n[a.b]' -> Cannot overwrite a value (at line 3, column 5) +'[a.b]\n[a]\n[a]' -> Cannot declare ('a',) twice (at line 3, column 3) +'a = tru' -> Invalid value (at line 1, column 5) +'a = +inf2' -> Expected newline or end of document after a statement (at line 1, column 9) +"x = 'a' 'b'" -> Expected newline or end of document after a statement (at line 1, column 9) +'a=1 .b' -> Expected newline or end of document after a statement (at line 1, column 5) +'@' -> Invalid statement (at line 1, column 1) +'a = """x""""""' -> Expected newline or end of document after a statement (at line 1, column 14) +TypeError Expected str object, not 'bytes' +TypeError File must be opened in binary mode, e.g. use `open('foo.toml', 'rb')` +{'a': 1, 'b': 'x'} diff --git a/maven/python-transpiler/src/test/resources/conformance/tomllib_.py b/maven/python-transpiler/src/test/resources/conformance/tomllib_.py new file mode 100644 index 00000000000..445b84e03b1 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/tomllib_.py @@ -0,0 +1,170 @@ +import tomllib + +DOC = ''' +# a comment +title = "TOML Example" # trailing +int1 = +99 +int2 = 1_000 +hex = 0xDEADbeef +oct = 0o755 +bin = 0b1101 +neg = -17 +f1 = 3.14 +f2 = -2e3 +f3 = 6.626e-34 +f4 = 1_0.5_0 +inf1 = inf +inf2 = -inf +b1 = true +b2 = false +lit = 'C:\\Users\\x' +esc = "tab\\there \\u00e9 \\U0001F600 q\\"q" +ml = """ +Roses + are \\ + red""" +mll = \'\'\' +raw \\n text\'\'\' +quoted."key with space".bare = 1 +"a.b" = 2 +d1 = 1979-05-27 +t1 = 07:32:00 +t2 = 00:32:00.999999 +dt1 = 1979-05-27T07:32:00 +dt2 = 1979-05-27 07:32:00Z +dt3 = 1979-05-27T00:32:00.5-07:00 +arr = [1, 2, 3,] +nested = [[1, 2], ["a", "b"], [ + # comment + 1.5, +]] +empty = [] +inl = { x = 1, y = { z = "w" }, p.q = 3 } +emptyinl = {} + +[owner] +name = "Tom" +dob = 1979-05-27T07:32:00-08:00 + +[database] +ports = [ 8000, 8001 ] +data = [ ["delta", "phi"], [3.14] ] + +[servers.alpha] +ip = "10.0.0.1" + +[servers.beta] +ip = "10.0.0.2" + +[[products]] +name = "Hammer" +sku = 738594937 + +[[products]] + +[[products]] +name = "Nail" +[products.meta] +tags = ["a"] + +[[fruit]] +name = "apple" +[fruit.physical] +color = "red" +[[fruit.variety]] +name = "red delicious" +[[fruit.variety]] +name = "granny smith" + +[[fruit]] +name = "banana" +[[fruit.variety]] +name = "plantain" + +[x.y.z] +w = 1 +[x] +v = 2 +''' + +data = tomllib.loads(DOC) + + +def show(d, indent=0): + for k in d: + v = d[k] + if isinstance(v, dict): + print(" " * indent + repr(k) + ":") + show(v, indent + 2) + else: + print(" " * indent + repr(k), repr(v)) + + +show(data) + +print(tomllib.loads("a = nan")["a"] != tomllib.loads("a = nan")["a"]) +print(tomllib.loads("") == {}) +print(tomllib.loads("a = 1\r\nb = 2\r\n")) +print(tomllib.loads("a = 3.0", parse_float=lambda s: "F" + s)) + +BAD = [ + "a", + "a =", + "a = ", + "a = 1 b = 2", + "a = [1, 2", + "a = {x = 1", + "a = {x = 1,}", + "[a", + "[[a]", + "a = 1\na = 2", + "[a]\n[a]", + "a = 1\n[a]", + "a.b = 1\na = 2", + "a = {x = 1}\n[a]", + "a = {x = 1}\na.y = 2", + "a = [1]\n[[a]]", + "a = 'abc", + 'a = "abc', + 'a = "abc\n"', + 'a = "\\q"', + 'a = "\\u12"', + 'a = """abc', + "a = 01", + "a = 1__0", + "a = _1", + "a = 1.", + "a = .5", + "a = 0x", + "a = 2023-13-45", + "a = 12:61:00", + "= 1", + "a = 1 # \x01", + "a = \x00", + "[a]\nb = 1\n[a.b]", + "[a.b]\n[a]\n[a]", + "a = tru", + "a = +inf2", + "x = 'a' 'b'", + "a=1 .b", + "@", + "a = \"\"\"x\"\"\"\"\"\"", +] +for s in BAD: + try: + r = tomllib.loads(s) + print("OK", repr(s), r) + except tomllib.TOMLDecodeError as e: + print(repr(s), "->", e) + +try: + tomllib.loads(b"a = 1") +except TypeError as e: + print("TypeError", e) +try: + import io + tomllib.load(io.StringIO("a = 1")) +except TypeError as e: + print("TypeError", e) +import io +print(tomllib.load(io.BytesIO(b"a = 1\nb = 'x'"))) diff --git a/maven/python-transpiler/src/test/resources/conformance/typing_extensions_.out b/maven/python-transpiler/src/test/resources/conformance/typing_extensions_.out new file mode 100644 index 00000000000..c5981314c5a --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/typing_extensions_.out @@ -0,0 +1,15 @@ +True True +2 +True False +{'title': 'x'} + count of things Doc('count of things') +True False +5 +1 DeprecationWarning use g2 +use g2 +old class +True (1, 2) +P +T +x +True False diff --git a/maven/python-transpiler/src/test/resources/conformance/typing_extensions_.py b/maven/python-transpiler/src/test/resources/conformance/typing_extensions_.py new file mode 100644 index 00000000000..7aa22f01b7c --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/typing_extensions_.py @@ -0,0 +1,77 @@ +import typing_extensions as te +from typing_extensions import (Self, Never, override, TypeAlias, ParamSpec, Concatenate, + TypeGuard, Required, NotRequired, Unpack, LiteralString, deprecated, Doc, + Annotated, Literal, Protocol, TypedDict, TypeVar, Optional, List, Dict, + Final, final, get_args, get_origin, cast, overload, NamedTuple, runtime_checkable) +import warnings + +print(te.TypeVar is TypeVar, te.List is List) + + +class Base: + def f(self) -> int: + return 1 + + +class Child(Base): + @override + def f(self) -> int: + return 2 + + +print(Child().f()) + + +def is_str(x: object) -> TypeGuard[str]: + return isinstance(x, str) + + +print(is_str("a"), is_str(1)) + + +class Movie(TypedDict): + title: str + year: NotRequired[int] + + +m: Movie = {"title": "x"} +print(m) + +A = Annotated[int, Doc("count of things")] +print(get_args(A)[0], get_args(A)[1].documentation, get_args(A)[1]) +print(Doc("a") == Doc("a"), Doc("a") == Doc("b")) + + +@deprecated("use g2") +def g(): + return 5 + + +with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + print(g()) + print(len(w), w[0].category.__name__, str(w[0].message)) +print(g.__deprecated__) + + +@deprecated("old class", category=None) +class Old: + pass + + +print(Old.__deprecated__) + +print(get_origin(List[int]) is list, get_args(Literal[1, 2])) +P = ParamSpec("P") +print(P.__name__) +T = TypeVar("T") +print(T.__name__) +print(cast(int, "x")) + + +@runtime_checkable +class HasLen(Protocol): + def __len__(self) -> int: ... + + +print(isinstance([1], HasLen), isinstance(3, HasLen)) From a4785501aad0a95872b5b19b6b83eab082691030 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:28:47 +0200 Subject: [PATCH 014/112] Python: tabulate Co-Authored-By: Claude Opus 5.5 --- .../python/site/tabulate/__init__.py | 409 ++++++++++++++++++ .../test/resources/conformance/tabulate_.out | 263 +++++++++++ .../test/resources/conformance/tabulate_.py | 33 ++ 3 files changed, 705 insertions(+) create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/tabulate/__init__.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/tabulate_.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/tabulate_.py diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/tabulate/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/tabulate/__init__.py new file mode 100644 index 00000000000..1af345b7017 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/tabulate/__init__.py @@ -0,0 +1,409 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Pretty-print tabular data as plain text, Markdown, reStructuredText or HTML.""" + +import math as _math + +__all__ = ["tabulate", "tabulate_formats", "simple_separated_format"] + +MIN_PADDING = 2 + + +class Line: + def __init__(self, begin, fill, sep, end): + self.begin = begin + self.fill = fill + self.sep = sep + self.end = end + + +class DataRow: + def __init__(self, begin, sep, end): + self.begin = begin + self.sep = sep + self.end = end + + +class TableFormat: + def __init__(self, lineabove, linebelowheader, linebetweenrows, linebelow, + headerrow, datarow, padding, with_header_hide): + self.lineabove = lineabove + self.linebelowheader = linebelowheader + self.linebetweenrows = linebetweenrows + self.linebelow = linebelow + self.headerrow = headerrow + self.datarow = datarow + self.padding = padding + self.with_header_hide = with_header_hide + + +def _pipe_segment(align, width): + if align == "left": + return ":" + "-" * (width - 1) + if align == "right" or align == "decimal": + return "-" * (width - 1) + ":" + if align == "center": + return ":" + "-" * (width - 2) + ":" + return "-" * width + + +def _pipe_line(colwidths, colaligns): + segs = [] + for i in range(len(colwidths)): + segs.append(_pipe_segment(colaligns[i], colwidths[i])) + return "|" + "|".join(segs) + "|" + + +def _html_escape(s): + return s.replace("&", "&").replace("<", "<").replace(">", ">") + + +def _html_row(tag, cells, colaligns): + out = [] + for i in range(len(cells)): + a = colaligns[i] + if a == "right" or a == "decimal": + attr = ' style="text-align: right;"' + elif a == "center": + attr = ' style="text-align: center;"' + else: + attr = "" + out.append("<" + tag + attr + ">" + _html_escape(cells[i]) + "") + return "" + "".join(out) + "" + + +_FORMATS = { + "plain": TableFormat(None, None, None, None, DataRow("", " ", ""), DataRow("", " ", ""), 0, []), + "simple": TableFormat(Line("", "-", " ", ""), Line("", "-", " ", ""), None, + Line("", "-", " ", ""), DataRow("", " ", ""), DataRow("", " ", ""), + 0, ["lineabove", "linebelow"]), + "grid": TableFormat(Line("+", "-", "+", "+"), Line("+", "=", "+", "+"), Line("+", "-", "+", "+"), + Line("+", "-", "+", "+"), DataRow("|", "|", "|"), DataRow("|", "|", "|"), 1, []), + "github": TableFormat(Line("|", "-", "|", "|"), Line("|", "-", "|", "|"), None, None, + DataRow("|", "|", "|"), DataRow("|", "|", "|"), 1, ["lineabove"]), + "pipe": TableFormat("pipe", "pipe", None, None, DataRow("|", "|", "|"), DataRow("|", "|", "|"), + 1, ["lineabove"]), + "orgtbl": TableFormat(None, Line("|", "-", "+", "|"), None, None, DataRow("|", "|", "|"), + DataRow("|", "|", "|"), 1, []), + "psql": TableFormat(Line("+", "-", "+", "+"), Line("|", "-", "+", "|"), None, + Line("+", "-", "+", "+"), DataRow("|", "|", "|"), DataRow("|", "|", "|"), 1, []), + "rst": TableFormat(Line("", "=", " ", ""), Line("", "=", " ", ""), None, + Line("", "=", " ", ""), DataRow("", " ", ""), DataRow("", " ", ""), 0, []), + "tsv": TableFormat(None, None, None, None, DataRow("", "\t", ""), DataRow("", "\t", ""), 0, []), + "html": TableFormat(None, None, None, None, None, None, 0, []), +} + +tabulate_formats = sorted(_FORMATS.keys()) + + +def simple_separated_format(separator): + return TableFormat(None, None, None, None, DataRow("", separator, ""), + DataRow("", separator, ""), 0, []) + + +def _isconvertible(kind, s): + try: + kind(s) + return True + except (ValueError, TypeError): + return False + + +def _isnumber(s): + if not _isconvertible(float, s): + return False + if isinstance(s, (str, bytes)): + f = float(s) + if _math.isinf(f) or _math.isnan(f): + return s.lower() in ("inf", "-inf", "nan") + return True + + +def _isint(s): + if type(s) is int: + return True + if isinstance(s, (str, bytes)): + return _isconvertible(int, s) + return False + + +def _isbool(s): + if type(s) is bool: + return True + return isinstance(s, str) and s in ("True", "False") + + +def _type(s, numparse): + if s is None: + return type(None) + if hasattr(s, "isoformat"): + return str + if _isbool(s): + return bool + if numparse and _isint(s): + return int + if numparse and _isnumber(s): + return float + return str + + +_RANK = {type(None): 0, bool: 1, int: 2, float: 3, str: 5} + + +def _more_generic(t1, t2): + r1 = _RANK.get(t1, 5) + r2 = _RANK.get(t2, 5) + r = r1 if r1 > r2 else r2 + for k in _RANK: + if _RANK[k] == r: + return k + return str + + +def _column_type(values, numparse): + t = bool + for v in values: + t = _more_generic(t, _type(v, numparse)) + return t + + +def _format(val, valtype, floatfmt, intfmt, missingval): + if val is None: + return missingval + if valtype is str: + return str(val) + if valtype is int: + if intfmt == "": + return str(val) + return format(int(val), intfmt) + if valtype is bool: + return str(val) + if valtype is float: + return format(float(val), floatfmt) + return str(val) + + +def _padleft(width, s): + return " " * (width - len(s)) + s if width > len(s) else s + + +def _padright(width, s): + return s + " " * (width - len(s)) if width > len(s) else s + + +def _padboth(width, s): + if width <= len(s): + return s + pad = width - len(s) + left = pad // 2 + return " " * left + s + " " * (pad - left) + + +def _afterpoint(s): + if _isnumber(s): + if _isint(s): + return -1 + pos = s.rfind(".") + if pos < 0: + pos = s.lower().rfind("e") + if pos >= 0: + return len(s) - pos - 1 + return -1 + return -1 + + +def _align_column(strings, alignment, minwidth): + if alignment == "left": + strings = [s.rstrip() for s in strings] + padfn = _padright + elif alignment == "center": + strings = [s.strip() for s in strings] + padfn = _padboth + elif alignment == "decimal": + decs = [_afterpoint(s.strip()) for s in strings] + maxdec = max(decs) if decs else 0 + strings = [s.strip() + " " * (maxdec - d) for s, d in zip(strings, decs)] + padfn = _padleft + elif alignment == "right": + strings = [s.strip() for s in strings] + padfn = _padleft + else: + return strings + maxw = max([len(s) for s in strings]) if strings else 0 + width = maxw if maxw > minwidth else minwidth + return [padfn(width, s) for s in strings] + + +def _normalize(data, headers, showindex): + index = None + if hasattr(data, "keys") and hasattr(data, "values"): + keys = list(data.keys()) + cols = [list(data[k]) for k in keys] + if isinstance(headers, str) and headers == "keys": + headers = [str(k) for k in keys] + n = max([len(c) for c in cols]) if cols else 0 + rows = [] + for i in range(n): + rows.append([c[i] if i < len(c) else None for c in cols]) + if headers is not None and not isinstance(headers, str) and hasattr(headers, "keys"): + headers = [str(headers[k]) for k in keys] + else: + items = list(data) + rows = [] + if items and hasattr(items[0], "keys"): + keys = list(items[0].keys()) + if isinstance(headers, str) and headers == "keys": + headers = [str(k) for k in keys] + elif hasattr(headers, "keys"): + keys = list(headers.keys()) + headers = [str(headers[k]) for k in keys] + elif headers: + pass + for r in items: + rows.append([r.get(k) for k in keys]) + else: + for r in items: + rows.append(list(r)) + if isinstance(headers, str) and headers == "keys": + headers = list(range(len(rows[0]))) if rows else [] + if isinstance(headers, str) and headers == "firstrow": + if rows: + headers = rows[0] + rows = rows[1:] + else: + headers = [] + headers = [str(h) for h in headers] if headers is not None else [] + if showindex is True or showindex == "always": + index = list(range(len(rows))) + elif showindex is not False and showindex != "default" and showindex != "never" and showindex is not None: + if isinstance(showindex, str): + index = None + else: + index = list(showindex) + if index is not None: + rows = [[index[i] if i < len(index) else ""] + rows[i] for i in range(len(rows))] + headers = [""] + headers if headers else headers + ncols = max([len(r) for r in rows]) if rows else len(headers) + if len(headers) > ncols: + ncols = len(headers) + if headers and len(headers) < ncols: + headers = [""] * (ncols - len(headers)) + headers + rows = [r + [None] * (ncols - len(r)) for r in rows] + return rows, headers + + +def tabulate(tabular_data, headers=(), tablefmt="simple", floatfmt="g", intfmt="", + numalign="default", stralign="default", missingval="", showindex="default", + disable_numparse=False, colalign=None): + rows, headers = _normalize(tabular_data, headers, showindex) + if isinstance(tablefmt, str): + if tablefmt not in _FORMATS: + fmt = _FORMATS["simple"] + tablefmt = "simple" + else: + fmt = _FORMATS[tablefmt] + else: + fmt = tablefmt + tablefmt = "" + numparse = not disable_numparse + ncols = len(rows[0]) if rows else len(headers) + cols = [[r[i] for r in rows] for i in range(ncols)] + coltypes = [_column_type(c, numparse) for c in cols] + scols = [] + for i in range(ncols): + ff = floatfmt[i] if isinstance(floatfmt, (list, tuple)) else floatfmt + mv = missingval[i] if isinstance(missingval, (list, tuple)) else missingval + scols.append([_format(v, coltypes[i], ff, intfmt, mv) for v in cols[i]]) + cols = scols + if numalign == "default": + numalign = "decimal" + if stralign == "default": + stralign = "left" + aligns = [] + for t in coltypes: + aligns.append(numalign if t is int or t is float else stralign) + if colalign is not None: + for i in range(len(colalign)): + if i < len(aligns): + aligns[i] = colalign[i] + min_padding = MIN_PADDING + if headers: + minwidths = [len(h) + min_padding for h in headers] + else: + minwidths = [0] * ncols + cols = [_align_column(cols[i], aligns[i], minwidths[i]) for i in range(ncols)] + if headers: + hdrs = [] + for i in range(ncols): + w = len(cols[i][0]) if cols[i] else 0 + if w < minwidths[i]: + w = minwidths[i] + a = aligns[i] + if a == "left": + hdrs.append(_padright(w, headers[i])) + elif a == "center": + hdrs.append(_padboth(w, headers[i])) + else: + hdrs.append(_padleft(w, headers[i])) + headers = hdrs + widths = [len(h) for h in headers] + # a column wider than its header (after alignment) keeps the cell width + for i in range(ncols): + if cols[i] and len(cols[i][0]) > widths[i]: + widths[i] = len(cols[i][0]) + headers[i] = (_padright if aligns[i] == "left" else _padboth if aligns[i] == "center" else _padleft)(widths[i], headers[i]) + else: + widths = [len(c[0]) if c else 0 for c in cols] + out_rows = [[cols[i][j] for i in range(ncols)] for j in range(len(rows))] + return _format_table(fmt, tablefmt, headers, out_rows, widths, aligns) + + +def _build_row(cells, row): + return (row.begin + row.sep.join(cells) + row.end).rstrip() + + +def _build_line(widths, aligns, line): + if line == "pipe": + return _pipe_line(widths, aligns) + cells = [line.fill * w for w in widths] + return (line.begin + line.sep.join(cells) + line.end).rstrip() + + +def _format_table(fmt, tablefmt, headers, rows, widths, aligns): + if tablefmt == "html": + lines = [""] + if headers: + lines.append("") + lines.append(_html_row("th", headers, aligns)) + lines.append("") + lines.append("") + for r in rows: + lines.append(_html_row("td", r, aligns)) + lines.append("") + lines.append("
") + return "\n".join(lines) + lines = [] + hidden = fmt.with_header_hide if (headers and fmt.with_header_hide) else [] + pad = fmt.padding + pw = [w + 2 * pad for w in widths] + padding = " " * pad + ph = [padding + c + padding for c in headers] + pr = [[padding + c + padding for c in r] for r in rows] + if fmt.lineabove and "lineabove" not in hidden: + lines.append(_build_line(pw, aligns, fmt.lineabove)) + if ph: + lines.append(_build_row(ph, fmt.headerrow)) + if fmt.linebelowheader and "linebelowheader" not in hidden: + lines.append(_build_line(pw, aligns, fmt.linebelowheader)) + if pr and fmt.linebetweenrows and "linebetweenrows" not in hidden: + for r in pr[:-1]: + lines.append(_build_row(r, fmt.datarow)) + lines.append(_build_line(pw, aligns, fmt.linebetweenrows)) + lines.append(_build_row(pr[-1], fmt.datarow)) + else: + for r in pr: + lines.append(_build_row(r, fmt.datarow)) + if fmt.linebelow and "linebelow" not in hidden: + lines.append(_build_line(pw, aligns, fmt.linebelow)) + return "\n".join(lines) diff --git a/maven/python-transpiler/src/test/resources/conformance/tabulate_.out b/maven/python-transpiler/src/test/resources/conformance/tabulate_.out new file mode 100644 index 00000000000..f29ed21ec62 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/tabulate_.out @@ -0,0 +1,263 @@ +plain +Planet R (km) mass (x 10^29 kg) +Sun 696000 1.9891e+09 +Earth 6371 5973.6 +Moon 1737 73.5 +Mars 3390 641.85 +Sun 696000 1.9891e+09 +Earth 6371 5973.6 +Moon 1737 73.5 +Mars 3390 641.85 + +simple +Planet R (km) mass (x 10^29 kg) +-------- -------- ------------------- +Sun 696000 1.9891e+09 +Earth 6371 5973.6 +Moon 1737 73.5 +Mars 3390 641.85 +----- ------ ------------- +Sun 696000 1.9891e+09 +Earth 6371 5973.6 +Moon 1737 73.5 +Mars 3390 641.85 +----- ------ ------------- + +grid ++----------+----------+---------------------+ +| Planet | R (km) | mass (x 10^29 kg) | ++==========+==========+=====================+ +| Sun | 696000 | 1.9891e+09 | ++----------+----------+---------------------+ +| Earth | 6371 | 5973.6 | ++----------+----------+---------------------+ +| Moon | 1737 | 73.5 | ++----------+----------+---------------------+ +| Mars | 3390 | 641.85 | ++----------+----------+---------------------+ ++-------+--------+---------------+ +| Sun | 696000 | 1.9891e+09 | ++-------+--------+---------------+ +| Earth | 6371 | 5973.6 | ++-------+--------+---------------+ +| Moon | 1737 | 73.5 | ++-------+--------+---------------+ +| Mars | 3390 | 641.85 | ++-------+--------+---------------+ + +github +| Planet | R (km) | mass (x 10^29 kg) | +|----------|----------|---------------------| +| Sun | 696000 | 1.9891e+09 | +| Earth | 6371 | 5973.6 | +| Moon | 1737 | 73.5 | +| Mars | 3390 | 641.85 | +|-------|--------|---------------| +| Sun | 696000 | 1.9891e+09 | +| Earth | 6371 | 5973.6 | +| Moon | 1737 | 73.5 | +| Mars | 3390 | 641.85 | + +pipe +| Planet | R (km) | mass (x 10^29 kg) | +|:---------|---------:|--------------------:| +| Sun | 696000 | 1.9891e+09 | +| Earth | 6371 | 5973.6 | +| Moon | 1737 | 73.5 | +| Mars | 3390 | 641.85 | +|:------|-------:|--------------:| +| Sun | 696000 | 1.9891e+09 | +| Earth | 6371 | 5973.6 | +| Moon | 1737 | 73.5 | +| Mars | 3390 | 641.85 | + +psql ++----------+----------+---------------------+ +| Planet | R (km) | mass (x 10^29 kg) | +|----------+----------+---------------------| +| Sun | 696000 | 1.9891e+09 | +| Earth | 6371 | 5973.6 | +| Moon | 1737 | 73.5 | +| Mars | 3390 | 641.85 | ++----------+----------+---------------------+ ++-------+--------+---------------+ +| Sun | 696000 | 1.9891e+09 | +| Earth | 6371 | 5973.6 | +| Moon | 1737 | 73.5 | +| Mars | 3390 | 641.85 | ++-------+--------+---------------+ + +rst +======== ======== =================== +Planet R (km) mass (x 10^29 kg) +======== ======== =================== +Sun 696000 1.9891e+09 +Earth 6371 5973.6 +Moon 1737 73.5 +Mars 3390 641.85 +======== ======== =================== +===== ====== ============= +Sun 696000 1.9891e+09 +Earth 6371 5973.6 +Moon 1737 73.5 +Mars 3390 641.85 +===== ====== ============= + +html + + + + + + + + + + +
Planet R (km) mass (x 10^29 kg)
Sun 696000 1.9891e+09
Earth 6371 5973.6
Moon 1737 73.5
Mars 3390 641.85
+ + + + + + + +
Sun 696000 1.9891e+09
Earth 63715973.6
Moon 1737 73.5
Mars 3390 641.85
+ +orgtbl +| Planet | R (km) | mass (x 10^29 kg) | +|----------+----------+---------------------| +| Sun | 696000 | 1.9891e+09 | +| Earth | 6371 | 5973.6 | +| Moon | 1737 | 73.5 | +| Mars | 3390 | 641.85 | +| Sun | 696000 | 1.9891e+09 | +| Earth | 6371 | 5973.6 | +| Moon | 1737 | 73.5 | +| Mars | 3390 | 641.85 | + +tsv +Planet R (km) mass (x 10^29 kg) +Sun 696000 1.9891e+09 +Earth 6371 5973.6 +Moon 1737 73.5 +Mars 3390 641.85 +Sun 696000 1.9891e+09 +Earth 6371 5973.6 +Moon 1737 73.5 +Mars 3390 641.85 + +Planet R (km) mass (x 10^29 kg) +-------- -------- ------------------- +Sun 696000 1989100000.00 +Earth 6371 5973.60 +Moon 1737 73.50 +Mars 3390 641.85 +Planet R (km) mass (x 10^29 kg) +-------- -------- ------------------- +Sun 696000 2.0e+09 +Earth 6371 6.0e+03 +Moon 1737 7.4e+01 +Mars 3390 6.4e+02 + Planet R (km) mass (x 10^29 kg) +-------- -------- ------------------- + Sun 696000 1.9891e+09 + Earth 6371 5973.6 + Moon 1737 73.5 + Mars 3390 641.85 ++----------+----------+---------------------+ +| Planet | R (km) | mass (x 10^29 kg) | ++==========+==========+=====================+ +| Sun | 696000 | 1.9891e+09 | ++----------+----------+---------------------+ +| Earth | 6371 | 5973.6 | ++----------+----------+---------------------+ +| Moon | 1737 | 73.5 | ++----------+----------+---------------------+ +| Mars | 3390 | 641.85 | ++----------+----------+---------------------+ + Planet R (km) mass (x 10^29 kg) +-- -------- -------- ------------------- + 0 Sun 696000 1.9891e+09 + 1 Earth 6371 5973.6 + 2 Moon 1737 73.5 + 3 Mars 3390 641.85 ++----+----------+----------+---------------------+ +| | Planet | R (km) | mass (x 10^29 kg) | +|----+----------+----------+---------------------| +| 0 | Sun | 696000 | 1.9891e+09 | +| 1 | Earth | 6371 | 5973.6 | +| 2 | Moon | 1737 | 73.5 | +| 3 | Mars | 3390 | 641.85 | ++----+----------+----------+---------------------+ + Planet R (km) mass (x 10^29 kg) +-- -------- -------- ------------------- +a Sun 696000 1.9891e+09 +b Earth 6371 5973.6 +c Moon 1737 73.5 +d Mars 3390 641.85 +- - - +1 2 3 +4 5 6 +- - - +- - - +1 ? 3 +x z +- - - + a b +--- --- + 1 x + 22 yy +| name | age | +|--------|-------| +| Al | 3 | +| Bo | 44 | +| h1 | h2 | +|-----:|------:| +| 1 | 2.5 | +| 10 | 33.25 | +k v +--- --- +a 1.5 +b 22 +c x +k v +--- --- +a 1.5 +b 22 +flag n +------ --- +True 1 +False 2 + x y +------ ---- + 1 2.55 +10.125 3 + + + + + + + +
h i
a<b&
+ only +-- ------ + 1 2 + 3 4 +a b +--- --- +| a | b | +|----:|:---:| +| 1 | 2 | ++-------+-----+ +| col | n | ++=======+=====+ +| x | 1 | ++-------+-----+ +----- ------- ----------- +1e+10 1.5e-05 1.23457e+08 +----- ------- ----------- +----- - +1,234 5 +----- - diff --git a/maven/python-transpiler/src/test/resources/conformance/tabulate_.py b/maven/python-transpiler/src/test/resources/conformance/tabulate_.py new file mode 100644 index 00000000000..972ba9eaea5 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/tabulate_.py @@ -0,0 +1,33 @@ +from tabulate import tabulate + +table = [["Sun", 696000, 1989100000], ["Earth", 6371, 5973.6], ["Moon", 1737, 73.5], ["Mars", 3390, 641.85]] +hdr = ["Planet", "R (km)", "mass (x 10^29 kg)"] +for fmt in ("plain", "simple", "grid", "github", "pipe", "psql", "rst", "html", "orgtbl", "tsv"): + print(fmt) + print(tabulate(table, headers=hdr, tablefmt=fmt)) + print(tabulate(table, tablefmt=fmt)) + print() + +print(tabulate(table, hdr, floatfmt=".2f")) +print(tabulate(table, hdr, floatfmt=".1e", numalign="right")) +print(tabulate(table, hdr, stralign="right", numalign="left")) +print(tabulate(table, hdr, stralign="center", tablefmt="grid")) +print(tabulate(table, hdr, showindex=True)) +print(tabulate(table, hdr, showindex="always", tablefmt="psql")) +print(tabulate(table, hdr, showindex=["a", "b", "c", "d"])) +print(tabulate([[1, 2, 3], [4, 5, 6]])) +print(tabulate([[1, None, 3], ["x", "", "z"]], missingval="?")) +print(tabulate([{"a": 1, "b": "x"}, {"a": 22, "b": "yy"}], headers="keys")) +print(tabulate({"name": ["Al", "Bo"], "age": [3, 44]}, headers="keys", tablefmt="github")) +print(tabulate([["h1", "h2"], [1, 2.5], [10, 33.25]], headers="firstrow", tablefmt="pipe")) +print(tabulate([["a", "1.5"], ["b", "22"], ["c", "x"]], headers=["k", "v"])) +print(tabulate([["a", "1.5"], ["b", "22"]], headers=["k", "v"], disable_numparse=True)) +print(tabulate([[True, 1], [False, 2]], headers=["flag", "n"])) +print(tabulate([[1.0, 2.55], [10.125, 3]], headers=["x", "y"])) +print(tabulate([["a Date: Tue, 27 Oct 2026 18:32:44 +0200 Subject: [PATCH 015/112] Python: idna and packaging site packages Co-Authored-By: Claude Opus 5.5 --- .../codename1/python/site/idna/__init__.py | 294 +++++++++++++++++ .../python/site/packaging/__init__.py | 4 + .../python/site/packaging/specifiers.py | 306 ++++++++++++++++++ .../codename1/python/site/packaging/utils.py | 10 + .../python/site/packaging/version.py | 291 +++++++++++++++++ .../src/test/resources/conformance/idna_.out | 47 +++ .../src/test/resources/conformance/idna_.py | 30 ++ .../test/resources/conformance/packaging_.out | 74 +++++ .../test/resources/conformance/packaging_.py | 71 ++++ 9 files changed, 1127 insertions(+) create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/idna/__init__.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/packaging/__init__.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/packaging/specifiers.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/packaging/utils.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/packaging/version.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/idna_.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/idna_.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/packaging_.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/packaging_.py diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/idna/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/idna/__init__.py new file mode 100644 index 00000000000..30956e5923e --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/idna/__init__.py @@ -0,0 +1,294 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Internationalized Domain Names in Applications (IDNA 2008): encode and decode. + +Code points are accepted when they are lowercase letters, digits or hyphens; +the full IDNA tables are not carried, so the contextual rules (joiners, bidi, +combining marks) are not enforced. +""" + +__version__ = "3.10" + +_ACE_PREFIX = "xn--" +_DOTS = (".", "\u3002", "\uff0e", "\uff61") + +_BASE = 36 +_TMIN = 1 +_TMAX = 26 +_SKEW = 38 +_DAMP = 700 +_INITIAL_BIAS = 72 +_INITIAL_N = 128 + + +class IDNAError(UnicodeError): + """Base class for IDNA failures.""" + + +class IDNABidiError(IDNAError): + """A bidi rule was violated.""" + + +class InvalidCodepoint(IDNAError): + """A code point is not allowed in a label.""" + + +class InvalidCodepointContext(IDNAError): + """A code point is not allowed in its context.""" + + +def _adapt(delta, numpoints, first): + delta = delta // _DAMP if first else delta // 2 + delta += delta // numpoints + k = 0 + while delta > ((_BASE - _TMIN) * _TMAX) // 2: + delta //= _BASE - _TMIN + k += _BASE + return k + (_BASE - _TMIN + 1) * delta // (delta + _SKEW) + + +def _digit(d): + if d < 26: + return chr(97 + d) + return chr(22 + d) + + +def punycode_encode(text): + """RFC 3492 encoding of one label (no prefix).""" + cps = [ord(c) for c in text] + out = [c for c in text if ord(c) < 128] + h = len(out) + b = h + if b > 0: + out.append("-") + n = _INITIAL_N + delta = 0 + bias = _INITIAL_BIAS + while h < len(cps): + m = min([c for c in cps if c >= n]) + delta += (m - n) * (h + 1) + n = m + for c in cps: + if c < n: + delta += 1 + if c == n: + q = delta + k = _BASE + while True: + t = _TMIN if k <= bias else (_TMAX if k >= bias + _TMAX else k - bias) + if q < t: + break + out.append(_digit(t + (q - t) % (_BASE - t))) + q = (q - t) // (_BASE - t) + k += _BASE + out.append(_digit(q)) + bias = _adapt(delta, h + 1, h == b) + delta = 0 + h += 1 + delta += 1 + n += 1 + return "".join(out) + + +def punycode_decode(text): + """RFC 3492 decoding of one label (no prefix).""" + pos = text.rfind("-") + if pos >= 0: + out = [ord(c) for c in text[:pos]] + rest = text[pos + 1:] + else: + out = [] + rest = text + n = _INITIAL_N + i = 0 + bias = _INITIAL_BIAS + idx = 0 + while idx < len(rest): + oldi = i + w = 1 + k = _BASE + while True: + if idx >= len(rest): + raise IDNAError("Invalid A-label") + c = rest[idx] + idx += 1 + o = ord(c) + if 48 <= o <= 57: + d = o - 22 + elif 65 <= o <= 90: + d = o - 65 + elif 97 <= o <= 122: + d = o - 97 + else: + raise IDNAError("Invalid A-label") + i += d * w + t = _TMIN if k <= bias else (_TMAX if k >= bias + _TMAX else k - bias) + if d < t: + break + w *= _BASE - t + k += _BASE + bias = _adapt(i - oldi, len(out) + 1, oldi == 0) + n += i // (len(out) + 1) + i = i % (len(out) + 1) + if n > 0x10FFFF: + raise IDNAError("Invalid A-label") + out.insert(i, n) + i += 1 + return "".join([chr(c) for c in out]) + + +def _is_mark(o): + """Combining marks and Indic or Thai vowel signs (no Unicode tables here).""" + if 0x0300 <= o <= 0x036F or 0x1AB0 <= o <= 0x1AFF or 0x20D0 <= o <= 0x20FF: + return True + if 0x0900 <= o <= 0x0DFF and o != 0x0964 and o != 0x0965: + return True + return 0x0E31 <= o <= 0x0E3A or 0x0E47 <= o <= 0x0E4E + + +def check_label(label): + if isinstance(label, (bytes, bytearray)): + label = label.decode("utf-8") + if len(label) == 0: + raise IDNAError("Empty Label") + if label[2:4] == "--": + raise IDNAError("Label has disallowed hyphens in 3rd and 4th position") + if label[0] == "-" or label[-1] == "-": + raise IDNAError("Label must not start or end with a hyphen") + pos = 0 + for ch in label: + pos += 1 + o = ord(ch) + if o < 128: + ok = ch == "-" or ("a" <= ch <= "z") or ("0" <= ch <= "9") + else: + ok = (ch.isalpha() or ch.isdigit()) and ch == ch.lower() and not ch.isupper() + if not ok and _is_mark(o): + ok = True + if not ok: + raise InvalidCodepoint("Codepoint U+" + format(o, "04X") + " at position " + str(pos) + + " of " + repr(label) + " not allowed") + return True + + +def valid_label_length(label): + return len(label) <= 63 + + +def valid_string_length(label, trailing_dot): + return len(label) <= (254 if trailing_dot else 253) + + +def _is_ascii(s): + for ch in s: + if ord(ch) >= 128: + return False + return True + + +def ulabel(label): + if isinstance(label, (bytes, bytearray)): + label = label.decode("ascii") + if not _is_ascii(label): + check_label(label) + return label + label = label.lower() + if label.startswith(_ACE_PREFIX): + label = label[4:] + if not label: + raise IDNAError("Malformed A-label, no Punycode eligible content found") + if label[-1] == "-": + raise IDNAError("A-label must not end with a hyphen") + else: + check_label(label) + return label + decoded = punycode_decode(label) + check_label(decoded) + return decoded + + +def alabel(label): + if isinstance(label, (bytes, bytearray)): + label = label.decode("ascii") + if _is_ascii(label): + ulabel(label) + if not valid_label_length(label): + raise IDNAError("Label too long") + return label.encode("ascii") + check_label(label) + enc = _ACE_PREFIX + punycode_encode(label) + if not valid_label_length(enc): + raise IDNAError("Label too long") + return enc.encode("ascii") + + +def uts46_remap(domain, std3_rules=True, transitional=False): + """A reduced UTS 46 mapping: case folding and the alternative dots.""" + out = domain.lower() + for d in _DOTS[1:]: + out = out.replace(d, ".") + return out + + +def _split(s): + for d in _DOTS[1:]: + s = s.replace(d, ".") + return s.split(".") + + +def encode(s, strict=False, uts46=False, std3_rules=False, transitional=False): + if not isinstance(s, str): + try: + s = str(s, "ascii") + except UnicodeDecodeError: + raise IDNAError("should pass a unicode string to the function rather than a byte string.") + if uts46: + s = uts46_remap(s, std3_rules, transitional) + trailing_dot = False + result = [] + labels = s.split(".") if strict else _split(s) + if not labels or labels == [""]: + raise IDNAError("Empty domain") + if labels[-1] == "": + del labels[-1] + trailing_dot = True + for label in labels: + s = alabel(label) + if s: + result.append(s) + else: + raise IDNAError("Empty Label") + if trailing_dot: + result.append(b"") + s = b".".join(result) + if not valid_string_length(s, trailing_dot): + raise IDNAError("Domain too long") + return s + + +def decode(s, strict=False, uts46=False, std3_rules=False): + if isinstance(s, (bytes, bytearray)): + try: + s = s.decode("ascii") + except UnicodeDecodeError: + raise IDNAError("Invalid ASCII in A-label") + if uts46: + s = uts46_remap(s, std3_rules, False) + trailing_dot = False + result = [] + labels = s.split(".") if strict else _split(s) + if not labels or labels == [""]: + raise IDNAError("Empty domain") + if labels[-1] == "": + del labels[-1] + trailing_dot = True + for label in labels: + s = ulabel(label) + if s: + result.append(s) + else: + raise IDNAError("Empty Label") + if trailing_dot: + result.append("") + return ".".join(result) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/packaging/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/packaging/__init__.py new file mode 100644 index 00000000000..24b76bca7c5 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/packaging/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Core utilities for Python packages: versions and specifiers.""" diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/packaging/specifiers.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/packaging/specifiers.py new file mode 100644 index 00000000000..d87ae7042e4 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/packaging/specifiers.py @@ -0,0 +1,306 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""PEP 440 version specifiers: Specifier and SpecifierSet.""" + +from .version import Version, InvalidVersion + +__all__ = ["InvalidSpecifier", "Specifier", "SpecifierSet"] + +_OPS = ("===", "~=", "==", "!=", "<=", ">=", "<", ">") + + +class InvalidSpecifier(ValueError): + """Raised when a specifier string is not valid.""" + + +def _to_version(v): + if isinstance(v, Version): + return v + return Version(v) + + +class Specifier: + + def __init__(self, spec="", prereleases=None): + s = spec.strip() + op = None + for o in _OPS: + if s.startswith(o): + op = o + break + if op is None: + raise InvalidSpecifier("Invalid specifier: " + repr(spec)) + ver = s[len(op):].strip() + if ver == "": + raise InvalidSpecifier("Invalid specifier: " + repr(spec)) + wildcard = ver.endswith(".*") + if op == "===": + pass + else: + base = ver[:-2] if wildcard else ver + if wildcard and op not in ("==", "!="): + raise InvalidSpecifier("Invalid specifier: " + repr(spec)) + try: + v = Version(base) + except InvalidVersion: + raise InvalidSpecifier("Invalid specifier: " + repr(spec)) from None + if wildcard and (v.local is not None or v.dev is not None): + raise InvalidSpecifier("Invalid specifier: " + repr(spec)) + if op == "~=" and len(v.release) < 2: + raise InvalidSpecifier("Invalid specifier: " + repr(spec)) + if op != "==" and op != "!=" and op != "===" and v.local is not None: + raise InvalidSpecifier("Invalid specifier: " + repr(spec)) + self._op = op + self._ver = ver + self._prereleases = prereleases + + @property + def operator(self): + return self._op + + @property + def version(self): + return self._ver + + @property + def prereleases(self): + if self._prereleases is not None: + return self._prereleases + if self._op in ("==", ">=", "<=", "~=", ">", "<"): + base = self._ver[:-2] if self._ver.endswith(".*") else self._ver + return Version(base).is_prerelease + return False + + @prereleases.setter + def prereleases(self, value): + self._prereleases = value + + def __str__(self): + return self._op + self._ver + + def __repr__(self): + return "" + + def __hash__(self): + return hash((self._op, self._canon())) + + def _canon(self): + if self._op == "===": + return self._ver + wildcard = self._ver.endswith(".*") + base = self._ver[:-2] if wildcard else self._ver + return str(Version(base)) + (".*" if wildcard else "") + + def __eq__(self, other): + if isinstance(other, str): + try: + other = Specifier(other) + except InvalidSpecifier: + return NotImplemented + if not isinstance(other, Specifier): + return NotImplemented + return self._op == other._op and self._canon() == other._canon() + + def __ne__(self, other): + r = self.__eq__(other) + if r is NotImplemented: + return r + return not r + + def __contains__(self, item): + return self.contains(item) + + def contains(self, item, prereleases=None): + if prereleases is None: + prereleases = self._prereleases + if self._op == "===": + return str(item).lower() == self._ver.lower() + try: + v = _to_version(item) + except InvalidVersion: + return False + if v.is_prerelease and prereleases is False: + return False + return self._match(v) + + def _match(self, v): + op = self._op + ver = self._ver + if op == "==" or op == "!=": + if ver.endswith(".*"): + spec = Version(ver[:-2]) + ok = self._prefix_match(v, spec) + else: + spec = Version(ver) + cand = v + if spec.local is None: + cand = Version(v.public) + ok = cand == spec + return ok if op == "==" else not ok + spec = Version(ver) + if op == "~=": + rel = list(spec.release[:-1]) + prefix = Version(("" if spec.epoch == 0 else str(spec.epoch) + "!") + + ".".join([str(x) for x in rel])) + return Version(v.public) >= spec and self._prefix_match(v, prefix) + if op == "<=": + return Version(v.public) <= spec + if op == ">=": + return Version(v.public) >= spec + if op == "<": + if not v < spec: + return False + if not spec.is_prerelease and v.is_prerelease and Version(v.base_version) == Version(spec.base_version): + return False + return True + if op == ">": + if not v > spec: + return False + if not spec.is_postrelease and v.is_postrelease and Version(v.base_version) == Version(spec.base_version): + return False + if v.local is not None and Version(v.base_version) == Version(spec.base_version): + return False + return True + return False + + @staticmethod + def _prefix_match(v, spec): + if v.epoch != spec.epoch: + return False + a = list(v.release) + b = list(spec.release) + while len(a) < len(b): + a.append(0) + return a[:len(b)] == b + + def filter(self, iterable, prereleases=None): + out = [] + for item in iterable: + if self.contains(item, prereleases=prereleases): + out.append(item) + return iter(out) + + +class SpecifierSet: + + def __init__(self, specifiers="", prereleases=None): + parsed = [] + if isinstance(specifiers, str): + for part in specifiers.split(","): + part = part.strip() + if part: + parsed.append(Specifier(part)) + else: + for sp in specifiers: + parsed.append(sp if isinstance(sp, Specifier) else Specifier(str(sp))) + self._specs = parsed + self._prereleases = prereleases + + @property + def prereleases(self): + if self._prereleases is not None: + return self._prereleases + if not self._specs: + return None + for s in self._specs: + if s.prereleases: + return True + return None + + @prereleases.setter + def prereleases(self, value): + self._prereleases = value + + def __str__(self): + return ",".join(sorted([str(s) for s in self._specs])) + + def __repr__(self): + pre = "" + if self._prereleases is not None: + pre = ", prereleases=" + repr(self._prereleases) + return "" + + def __hash__(self): + return hash(frozenset(self._specs)) + + def __and__(self, other): + if isinstance(other, str): + other = SpecifierSet(other) + if not isinstance(other, SpecifierSet): + return NotImplemented + combined = SpecifierSet() + combined._specs = list(self._specs) + list(other._specs) + if self._prereleases is None and other._prereleases is not None: + combined._prereleases = other._prereleases + else: + combined._prereleases = self._prereleases + return combined + + def __eq__(self, other): + if isinstance(other, (str, Specifier)): + other = SpecifierSet(str(other)) + if not isinstance(other, SpecifierSet): + return NotImplemented + return set(self._specs) == set(other._specs) + + def __ne__(self, other): + r = self.__eq__(other) + if r is NotImplemented: + return r + return not r + + def __len__(self): + return len(self._specs) + + def __iter__(self): + return iter(self._specs) + + def __contains__(self, item): + return self.contains(item) + + def contains(self, item, prereleases=None, installed=None): + if not self._specs: + return True + v = item + if not isinstance(v, Version): + try: + v = Version(item) + except InvalidVersion: + return False + if installed and v.is_prerelease: + item = Version(v.base_version) + v = item + if prereleases is None: + prereleases = self._prereleases + if prereleases is False and v.is_prerelease: + return False + for s in self._specs: + if not s.contains(v, prereleases=True): + return False + return True + + def filter(self, iterable, prereleases=None): + if prereleases is None: + prereleases = self.prereleases + found = [] + pres = [] + for item in iterable: + try: + v = _to_version(item) + except InvalidVersion: + continue + ok = True + for s in self._specs: + if not s.contains(v, prereleases=True): + ok = False + break + if not ok: + continue + if v.is_prerelease and not prereleases: + pres.append(item) + else: + found.append(item) + if not found and prereleases is None: + return iter(pres) + return iter(found) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/packaging/utils.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/packaging/utils.py new file mode 100644 index 00000000000..862acd123c3 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/packaging/utils.py @@ -0,0 +1,10 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Small helpers from packaging.utils.""" + +import re + + +def canonicalize_name(name): + return re.sub(r"[-_.]+", "-", name).lower() diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/packaging/version.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/packaging/version.py new file mode 100644 index 00000000000..a6eced4803a --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/packaging/version.py @@ -0,0 +1,291 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""PEP 440 versions: parsing, normalization and ordering.""" + +__all__ = ["VERSION_PATTERN", "InvalidVersion", "Version", "parse"] + +VERSION_PATTERN = r"v?(?:(?:[0-9]+)!)?[0-9]+(?:\.[0-9]+)*" + +_DIGITS = "0123456789" +_SEPS = "-_." +_PRE_WORDS = (("alpha", "a"), ("beta", "b"), ("preview", "rc"), ("pre", "rc"), + ("a", "a"), ("b", "b"), ("c", "rc"), ("rc", "rc")) +_POST_WORDS = ("post", "rev", "r") + + +class InvalidVersion(ValueError): + """Raised when a version string is not a valid PEP 440 version.""" + + +def parse(version): + return Version(version) + + +def _digits_at(s, i): + j = i + while j < len(s) and s[j] in _DIGITS: + j += 1 + return j + + +class Version: + + def __init__(self, version): + parsed = self._parse(version) + if parsed is None: + raise InvalidVersion("Invalid version: " + repr(version)) + self._epoch = parsed[0] + self._release = parsed[1] + self._pre = parsed[2] + self._post = parsed[3] + self._dev = parsed[4] + self._local = parsed[5] + self._key = self._make_key() + + @staticmethod + def _parse(text): + s = text.strip().lower() + i = 0 + n = len(s) + if s[0:1] == "v": + i = 1 + epoch = 0 + j = _digits_at(s, i) + if j > i and s[j:j + 1] == "!": + epoch = int(s[i:j]) + i = j + 1 + j = _digits_at(s, i) + if j == i: + return None + release = [int(s[i:j])] + i = j + while s[i:i + 1] == "." and _digits_at(s, i + 1) > i + 1: + j = _digits_at(s, i + 1) + release.append(int(s[i + 1:j])) + i = j + pre = None + post = None + dev = None + local = None + # pre-release + k = i + if k < n and s[k] in _SEPS: + k += 1 + for word, norm in _PRE_WORDS: + if s.startswith(word, k): + k2 = k + len(word) + if k2 < n and s[k2] in _SEPS: + k3 = k2 + 1 + else: + k3 = k2 + j = _digits_at(s, k3) + if j > k3: + pre = (norm, int(s[k3:j])) + i = j + else: + pre = (norm, 0) + i = k2 + break + # post-release + if s[i:i + 1] == "-" and _digits_at(s, i + 1) > i + 1: + j = _digits_at(s, i + 1) + post = int(s[i + 1:j]) + i = j + else: + k = i + if k < n and s[k] in _SEPS: + k += 1 + for word in _POST_WORDS: + if s.startswith(word, k): + k2 = k + len(word) + k3 = k2 + 1 if (k2 < n and s[k2] in _SEPS) else k2 + j = _digits_at(s, k3) + if j > k3: + post = int(s[k3:j]) + i = j + else: + post = 0 + i = k2 + break + # dev release + k = i + if k < n and s[k] in _SEPS: + k += 1 + if s.startswith("dev", k): + k2 = k + 3 + k3 = k2 + 1 if (k2 < n and s[k2] in _SEPS) else k2 + j = _digits_at(s, k3) + if j > k3: + dev = int(s[k3:j]) + i = j + else: + dev = 0 + i = k2 + # local + if s[i:i + 1] == "+": + rest = s[i + 1:] + if rest == "": + return None + parts = [] + cur = "" + for ch in rest: + if ch in _SEPS: + if cur == "": + return None + parts.append(cur) + cur = "" + elif ch.isalnum() and ord(ch) < 128: + cur += ch + else: + return None + if cur == "": + return None + parts.append(cur) + local = [] + for p in parts: + if _digits_at(p, 0) == len(p): + local.append(int(p)) + else: + local.append(p) + i = n + if i != n: + return None + return (epoch, tuple(release), pre, post, dev, None if local is None else tuple(local)) + + def _make_key(self): + release = list(self._release) + while len(release) > 1 and release[-1] == 0: + release.pop() + if self._pre is None and self._post is None and self._dev is not None: + pre = (0, "", 0) + elif self._pre is None: + pre = (2, "", 0) + else: + pre = (1, self._pre[0], self._pre[1]) + post = (0, 0) if self._post is None else (1, self._post) + dev = (1, 0) if self._dev is None else (0, self._dev) + if self._local is None: + local = (0,) + else: + items = [] + for p in self._local: + if isinstance(p, int): + items.append((1, p, "")) + else: + items.append((0, 0, p)) + local = (1, tuple(items)) + return (self._epoch, tuple(release), pre, post, dev, local) + + def __repr__(self): + return "" + + def __str__(self): + parts = [] + if self._epoch != 0: + parts.append(str(self._epoch) + "!") + parts.append(".".join([str(x) for x in self._release])) + if self._pre is not None: + parts.append(self._pre[0] + str(self._pre[1])) + if self._post is not None: + parts.append(".post" + str(self._post)) + if self._dev is not None: + parts.append(".dev" + str(self._dev)) + if self._local is not None: + parts.append("+" + self.local) + return "".join(parts) + + @property + def epoch(self): + return self._epoch + + @property + def release(self): + return self._release + + @property + def pre(self): + return self._pre + + @property + def post(self): + return self._post + + @property + def dev(self): + return self._dev + + @property + def local(self): + if self._local is None: + return None + return ".".join([str(x) for x in self._local]) + + @property + def public(self): + return str(self).split("+", 1)[0] + + @property + def base_version(self): + parts = [] + if self._epoch != 0: + parts.append(str(self._epoch) + "!") + parts.append(".".join([str(x) for x in self._release])) + return "".join(parts) + + @property + def is_prerelease(self): + return self._dev is not None or self._pre is not None + + @property + def is_postrelease(self): + return self._post is not None + + @property + def is_devrelease(self): + return self._dev is not None + + @property + def major(self): + return self._release[0] if len(self._release) >= 1 else 0 + + @property + def minor(self): + return self._release[1] if len(self._release) >= 2 else 0 + + @property + def micro(self): + return self._release[2] if len(self._release) >= 3 else 0 + + def __hash__(self): + return hash(self._key) + + def __lt__(self, other): + if not isinstance(other, Version): + return NotImplemented + return self._key < other._key + + def __le__(self, other): + if not isinstance(other, Version): + return NotImplemented + return self._key <= other._key + + def __eq__(self, other): + if not isinstance(other, Version): + return NotImplemented + return self._key == other._key + + def __ge__(self, other): + if not isinstance(other, Version): + return NotImplemented + return self._key >= other._key + + def __gt__(self, other): + if not isinstance(other, Version): + return NotImplemented + return self._key > other._key + + def __ne__(self, other): + if not isinstance(other, Version): + return NotImplemented + return self._key != other._key diff --git a/maven/python-transpiler/src/test/resources/conformance/idna_.out b/maven/python-transpiler/src/test/resources/conformance/idna_.out new file mode 100644 index 00000000000..a4d516bf7e8 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/idna_.out @@ -0,0 +1,47 @@ +'example.com' b'example.com' +'Example.COM' b'Example.COM' +'m\xfcnchen.de' b'xn--mnchen-3ya.de' +'b\xfccher.example' b'xn--bcher-kva.example' +'\u65e5\u672c\u8a9e.jp' b'xn--wgv71a119e.jp' +'ex--ample.com' IDNAError Label has disallowed hyphens in 3rd and 4th position +'-ab.com' IDNAError Label must not start or end with a hyphen +'ab-.com' IDNAError Label must not start or end with a hyphen +'a..b' IDNAError Empty Label +'' IDNAError Empty domain +'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa IDNAError Label too long +'B\xfccher.de' InvalidCodepoint Codepoint U+0042 at position 1 of 'Bücher' not allowed +'\u0391\u0392\u0393.gr' InvalidCodepoint Codepoint U+0391 at position 1 of 'ΑΒΓ' not allowed +'ab\u3002cd' b'ab.cd' +'xn--mnchen-3ya.de' b'xn--mnchen-3ya.de' +'xn--zzz.com' b'xn--zzz.com' +'a b.com' InvalidCodepoint Codepoint U+0020 at position 2 of 'a b' not allowed +'.com' IDNAError Empty Label +'a.' b'a.' +'\xdf.de' b'xn--zca.de' +'fa\xdf.de' b'xn--fa-hia.de' +'\u4f8b\u3048.\u30c6\u30b9\u30c8' b'xn--r8jz45g.xn--zckzah' +'a_b.com' InvalidCodepoint Codepoint U+005F at position 2 of 'a_b' not allowed +'ab.12' b'ab.12' +'1.2.3.4' b'1.2.3.4' +'\u0431\u0443\u043a\u0432\u0430.\u0440\u b'xn--80abd5a7b.xn--p1ai' +'\xe9.fr' b'xn--9ca.fr' +'caf\xe9.example.org.' b'xn--caf-dma.example.org.' +xn--mnchen-3ya.de 'm\xfcnchen.de' +xn--wgv71a119e.jp '\u65e5\u672c\u8a9e.jp' +XN--MNCHEN-3YA.de 'm\xfcnchen.de' +xn--.com IDNAError Malformed A-label, no Punycode eligible content found +abc.def 'abc.def' +xn--zzz.com '\u7ba5.com' +ab..cd IDNAError Empty Label +xn--bcher-kva.example 'b\xfccher.example' +xn--r8jz45g.xn--zckzah '\u4f8b\u3048.\u30c6\u30b9\u30c8' +xn--e1afmkfd.xn--p1ai '\u043f\u0440\u0438\u043c\u0435\u0440.\u0440\u0444' +Example.COM 'example.com' +xn--caf-dma.example.org. 'caf\xe9.example.org.' +b'xn--mnchen-3ya' 'm\xfcnchen' abc +b'xn--mnchen-3ya.de' b'xn--mnchen-3ya.de' +True True +b'abc.com' 'b\xfccher.de' +b'xn--fiq228c.com' True +b'xn--mgbh0fb.xn--kgbechtv' True +b'xn--j2bd4cyah0f.com' True diff --git a/maven/python-transpiler/src/test/resources/conformance/idna_.py b/maven/python-transpiler/src/test/resources/conformance/idna_.py new file mode 100644 index 00000000000..cb5cad68859 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/idna_.py @@ -0,0 +1,30 @@ +import idna + +for s in ["example.com", "Example.COM", "münchen.de", "bücher.example", + "日本語.jp", "ex--ample.com", "-ab.com", "ab-.com", "a..b", "", + "a" * 64 + ".com", "Bücher.de", "ΑΒΓ.gr", "ab。cd", + "xn--mnchen-3ya.de", "xn--zzz.com", "a b.com", ".com", "a.", "ß.de", + "faß.de", "例え.テスト", "a_b.com", "ab.12", "1.2.3.4", + "буква.рф", "é.fr", "café.example.org."]: + try: + print(ascii(s)[:40], idna.encode(s)) + except idna.IDNAError as e: + print(ascii(s)[:40], type(e).__name__, e) + +for s in ["xn--mnchen-3ya.de", "xn--wgv71a119e.jp", "XN--MNCHEN-3YA.de", "xn--.com", "abc.def", "xn--zzz.com", "ab..cd", "xn--bcher-kva.example", + "xn--r8jz45g.xn--zckzah", "xn--e1afmkfd.xn--p1ai", "Example.COM", "xn--caf-dma.example.org."]: + try: + print(s, ascii(idna.decode(s))) + except idna.IDNAError as e: + print(s, type(e).__name__, e) + +print(idna.alabel("münchen"), ascii(idna.ulabel("xn--mnchen-3ya")), idna.ulabel("abc")) +print(idna.encode("münchen.de", uts46=True), idna.encode("MÜNCHEN.de", uts46=True)) +print(issubclass(idna.IDNAError, UnicodeError), issubclass(idna.InvalidCodepoint, idna.IDNAError)) +print(idna.encode(b"abc.com"), ascii(idna.decode(b"xn--bcher-kva.de"))) +for host in ["中文.com", "مثال.إختبار", "हिन्दी.com"]: + try: + e = idna.encode(host) + print(e, idna.decode(e) == host) + except idna.IDNAError as ex: + print("err", type(ex).__name__) diff --git a/maven/python-transpiler/src/test/resources/conformance/packaging_.out b/maven/python-transpiler/src/test/resources/conformance/packaging_.out new file mode 100644 index 00000000000..a3fa53ecf39 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/packaging_.out @@ -0,0 +1,74 @@ +'1.0' 1.0 0 (1, 0) None None None None 1.0 1.0 False False False 1 0 0 +'1.0.0' 1.0.0 0 (1, 0, 0) None None None None 1.0.0 1.0.0 False False False 1 0 0 +'v1.2.3' 1.2.3 0 (1, 2, 3) None None None None 1.2.3 1.2.3 False False False 1 2 3 +'1!2.0' 1!2.0 1 (2, 0) None None None None 1!2.0 1!2.0 False False False 2 0 0 +'1.0a1' 1.0a1 0 (1, 0) ('a', 1) None None None 1.0a1 1.0 True False False 1 0 0 +'1.0alpha1' 1.0a1 0 (1, 0) ('a', 1) None None None 1.0a1 1.0 True False False 1 0 0 +'1.0-beta.2' 1.0b2 0 (1, 0) ('b', 2) None None None 1.0b2 1.0 True False False 1 0 0 +'1.0c3' 1.0rc3 0 (1, 0) ('rc', 3) None None None 1.0rc3 1.0 True False False 1 0 0 +'1.0rc' 1.0rc0 0 (1, 0) ('rc', 0) None None None 1.0rc0 1.0 True False False 1 0 0 +'1.0pre4' 1.0rc4 0 (1, 0) ('rc', 4) None None None 1.0rc4 1.0 True False False 1 0 0 +'1.0.post1' 1.0.post1 0 (1, 0) None 1 None None 1.0.post1 1.0 False True False 1 0 0 +'1.0-1' 1.0.post1 0 (1, 0) None 1 None None 1.0.post1 1.0 False True False 1 0 0 +'1.0.rev3' 1.0.post3 0 (1, 0) None 3 None None 1.0.post3 1.0 False True False 1 0 0 +'1.0.dev4' 1.0.dev4 0 (1, 0) None None 4 None 1.0.dev4 1.0 True False True 1 0 0 +'1.0.dev' 1.0.dev0 0 (1, 0) None None 0 None 1.0.dev0 1.0 True False True 1 0 0 +'1.0a1.post2.dev3' 1.0a1.post2.dev3 0 (1, 0) ('a', 1) 2 3 None 1.0a1.post2.dev3 1.0 True True True 1 0 0 +'1.0+local.1' 1.0+local.1 0 (1, 0) None None None local.1 1.0 1.0 False False False 1 0 0 +'1.0+Ubuntu-1_2' 1.0+ubuntu.1.2 0 (1, 0) None None None ubuntu.1.2 1.0 1.0 False False False 1 0 0 +'2.0.0.0' 2.0.0.0 0 (2, 0, 0, 0) None None None None 2.0.0.0 2.0.0.0 False False False 2 0 0 +' 3.1 ' 3.1 0 (3, 1) None None None None 3.1 3.1 False False False 3 1 0 +'1.0.POST1' 1.0.post1 0 (1, 0) None 1 None None 1.0.post1 1.0 False True False 1 0 0 +'1.0_a_1' 1.0a1 0 (1, 0) ('a', 1) None None None 1.0a1 1.0 True False False 1 0 0 +'1.0.0rc1+abc.5' 1.0.0rc1+abc.5 0 (1, 0, 0) ('rc', 1) None None abc.5 1.0.0rc1 1.0.0 True False False 1 0 0 +InvalidVersion Invalid version: '' +InvalidVersion Invalid version: 'abc' +InvalidVersion Invalid version: '1.0.' +InvalidVersion Invalid version: '1..0' +InvalidVersion Invalid version: '1.0+' +InvalidVersion Invalid version: '1.0+a..b' +InvalidVersion Invalid version: '1.0 beta' +InvalidVersion Invalid version: '1.0.0.x' +InvalidVersion Invalid version: '!1' +InvalidVersion Invalid version: '1.0-' +['1.0.dev1', '1.0a1.dev1', '1.0a1', '1.0a2', '1.0b1', '1.0rc1', '1.0', '1.0+abc', '1.0+abc.1', '1.0+5', '1.0.post1.dev1', '1.0.post1', '1.0.1', '1.1', '2.0', '1!0.5'] +True +True True True +True True True True +1.2.3 1.10 +False True +foo-bar-baz-qux +>=1.0 =1.0')> >= 1.0 False [('0.9', False), ('1.0', True), ('1.5', True), ('2.0a1', True)] +>1.0 1.0')> > 1.0 False [('1.0', False), ('1.0.post1', False), ('1.0.1', True), ('1.0+local', False)] +<2.0 < 2.0 False [('1.9', True), ('2.0', False), ('2.0a1', False), ('1.0a1', True)] +<=2.0 <= 2.0 False [('2.0', True), ('2.0.0', True), ('2.0.1', False), ('2.0+x', True)] +==1.0 == 1.0 False [('1.0', True), ('1.0.0', True), ('1.0+local', True), ('1.0.1', False)] +==1.0+local == 1.0+local False [('1.0', False), ('1.0+local', True), ('1.0+other', False)] +==1.* == 1.* False [('1.0', True), ('1.9.9', True), ('2.0', False), ('1.0a1', True), ('1', True)] +==1.2.* == 1.2.* False [('1.2', True), ('1.2.5', True), ('1.3', False)] +!=1.0 != 1.0 False [('1.0', False), ('1.0.0', False), ('1.1', True)] +!=1.* != 1.* False [('1.5', False), ('2.0', True)] +~=2.2 ~= 2.2 False [('2.1', False), ('2.2', True), ('2.9', True), ('3.0', False)] +~=1.4.5 ~= 1.4.5 False [('1.4.4', False), ('1.4.5', True), ('1.4.9', True), ('1.5.0', False)] +===1.0 === 1.0 False [('1.0', True), ('1.0.0', False)] +>=1.0a1 =1.0a1')> >= 1.0a1 True [('1.0a1', True), ('1.0a2', True), ('0.9', False)] +<1.0 < 1.0 False [('1.0a1', False), ('0.9', True)] +!=1.5,<2.0,>=1.0 =1.0')> 3 ['!=1.5', '<2.0', '>=1.0'] +['1.0', '1.6', '1.7a1'] +['1.0', '1.6'] +['1.7a1'] ['1.8'] ['1.7a1'] +True None None +True True +<2,>=1 True +True True True +True +True +InvalidSpecifier Invalid specifier: '1.0' +InvalidSpecifier Invalid specifier: '>=' +InvalidSpecifier Invalid specifier: '~=1' +InvalidSpecifier Invalid specifier: '==1.*.x' +InvalidSpecifier Invalid specifier: '>=1.0.*' +InvalidSpecifier Invalid specifier: 'abc' +InvalidSpecifier Invalid specifier: '=>1.0' +InvalidSpecifier Invalid specifier: '==1.0.dev1.*' +InvalidSpecifier Invalid specifier: 'foo' diff --git a/maven/python-transpiler/src/test/resources/conformance/packaging_.py b/maven/python-transpiler/src/test/resources/conformance/packaging_.py new file mode 100644 index 00000000000..6a0eaa1bf35 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/packaging_.py @@ -0,0 +1,71 @@ +from packaging.version import Version, parse, InvalidVersion +from packaging.specifiers import SpecifierSet, Specifier, InvalidSpecifier +from packaging.utils import canonicalize_name + +for s in ["1.0", "1.0.0", "v1.2.3", "1!2.0", "1.0a1", "1.0alpha1", "1.0-beta.2", "1.0c3", "1.0rc", "1.0pre4", + "1.0.post1", "1.0-1", "1.0.rev3", "1.0.dev4", "1.0.dev", "1.0a1.post2.dev3", "1.0+local.1", + "1.0+Ubuntu-1_2", "2.0.0.0", " 3.1 ", "1.0.POST1", "1.0_a_1", "1.0.0rc1+abc.5"]: + v = Version(s) + print(repr(s), str(v), v.epoch, v.release, v.pre, v.post, v.dev, v.local, v.public, v.base_version, + v.is_prerelease, v.is_postrelease, v.is_devrelease, v.major, v.minor, v.micro) + +for s in ["", "abc", "1.0.", "1..0", "1.0+", "1.0+a..b", "1.0 beta", "1.0.0.x", "!1", "1.0-"]: + try: + Version(s) + print("valid", repr(s)) + except InvalidVersion as e: + print("InvalidVersion", e) + +order = ["1.0.dev1", "1.0a1.dev1", "1.0a1", "1.0a2", "1.0b1", "1.0rc1", "1.0", "1.0+abc", "1.0+abc.1", + "1.0+5", "1.0.post1.dev1", "1.0.post1", "1.0.1", "1.1", "2.0", "1!0.5"] +vs = [Version(x) for x in order] +print([str(v) for v in sorted(reversed(vs))]) +print(all(vs[i] < vs[i + 1] for i in range(len(vs) - 1))) +print(Version("1.0") == Version("1.0.0"), Version("1.0") == Version("1.0.0.0.0"), hash(Version("1.0")) == hash(Version("1.0.0"))) +print(Version("1.0") != Version("1.0.1"), Version("2.0") > Version("1.9.9"), Version("1.0") >= Version("1.0"), Version("1.0a1") <= Version("1.0")) +print(parse("1.2.3"), repr(parse("1.2.3")), max(Version("1.2"), Version("1.10"))) +print(Version("1.0") == "1.0", Version("1.0") < Version("1.0.1")) +print(canonicalize_name("Foo_Bar.baz--Qux")) + +cases = [(">=1.0", ["0.9", "1.0", "1.5", "2.0a1"]), + (">1.0", ["1.0", "1.0.post1", "1.0.1", "1.0+local"]), + ("<2.0", ["1.9", "2.0", "2.0a1", "1.0a1"]), + ("<=2.0", ["2.0", "2.0.0", "2.0.1", "2.0+x"]), + ("==1.0", ["1.0", "1.0.0", "1.0+local", "1.0.1"]), + ("==1.0+local", ["1.0", "1.0+local", "1.0+other"]), + ("==1.*", ["1.0", "1.9.9", "2.0", "1.0a1", "1"]), + ("==1.2.*", ["1.2", "1.2.5", "1.3"]), + ("!=1.0", ["1.0", "1.0.0", "1.1"]), + ("!=1.*", ["1.5", "2.0"]), + ("~=2.2", ["2.1", "2.2", "2.9", "3.0"]), + ("~=1.4.5", ["1.4.4", "1.4.5", "1.4.9", "1.5.0"]), + ("===1.0", ["1.0", "1.0.0"]), + (">=1.0a1", ["1.0a1", "1.0a2", "0.9"]), + ("<1.0", ["1.0a1", "0.9"]), + ] +for spec, items in cases: + sp = Specifier(spec) + print(str(sp), repr(sp), sp.operator, sp.version, sp.prereleases, [(i, sp.contains(i)) for i in items]) + +s = SpecifierSet(">=1.0, <2.0, !=1.5") +print(s, repr(s), len(s), sorted([str(x) for x in s])) +print([v for v in ["0.9", "1.0", "1.5", "1.6", "2.0", "1.7a1"] if v in s]) +print(list(s.filter(["0.9", "1.0", "1.5", "1.6", "2.0", "1.7a1"]))) +print(list(s.filter(["1.7a1"])), list(s.filter(["1.7a1", "1.8"])), list(s.filter(["1.7a1"], prereleases=True))) +print(s.contains("1.7a1", prereleases=True), s.prereleases, SpecifierSet("").prereleases) +print(SpecifierSet("") == SpecifierSet(""), SpecifierSet(">=1,<2") == SpecifierSet("<2, >=1")) +print(SpecifierSet(">=1") & SpecifierSet("<2"), SpecifierSet(">=1.0a1").prereleases) +print(SpecifierSet("").contains("1.0"), SpecifierSet("").contains("1.0a1"), SpecifierSet("").contains("junk")) +print(Version("1.5") in SpecifierSet(">=1.0")) +print(SpecifierSet(">=1.0", prereleases=True).contains("1.1a1")) + +for bad in ["1.0", ">=", "~=1", "==1.*.x", ">=1.0.*", "abc", "=>1.0", "==1.0.dev1.*"]: + try: + Specifier(bad) + print("valid", repr(bad)) + except InvalidSpecifier as e: + print("InvalidSpecifier", e) +try: + SpecifierSet(">=1.0,foo") +except InvalidSpecifier as e: + print("InvalidSpecifier", e) From b76c77429d652242a9fc879e1ffa6f7c9084e37c Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:35:58 +0200 Subject: [PATCH 016/112] Python: Flask and FastAPI over a pure dispatch core Clean-room site packages flask, werkzeug.exceptions, fastapi, starlette (responses, requests, testclient, status) and uvicorn. Every app exposes dispatch(method, path, query, headers, body) -> (status, headers, body); app.run / uvicorn.run hand it to _cn1server.serve. Co-Authored-By: Claude Opus 5.5 --- .../codename1/python/site/fastapi/__init__.py | 21 + .../python/site/fastapi/applications.py | 339 +++++ .../codename1/python/site/fastapi/encoders.py | 63 + .../python/site/fastapi/exceptions.py | 37 + .../codename1/python/site/fastapi/params.py | 105 ++ .../python/site/fastapi/responses.py | 18 + .../codename1/python/site/fastapi/routing.py | 632 +++++++++ .../codename1/python/site/fastapi/status.py | 5 + .../python/site/fastapi/testclient.py | 5 + .../codename1/python/site/flask/__init__.py | 1220 +++++++++++++++++ .../python/site/starlette/__init__.py | 4 + .../python/site/starlette/background.py | 30 + .../python/site/starlette/datastructures.py | 238 ++++ .../python/site/starlette/exceptions.py | 41 + .../python/site/starlette/requests.py | 68 + .../python/site/starlette/responses.py | 101 ++ .../codename1/python/site/starlette/status.py | 43 + .../python/site/starlette/testclient.py | 168 +++ .../codename1/python/site/uvicorn/__init__.py | 9 + .../python/site/werkzeug/__init__.py | 4 + .../python/site/werkzeug/exceptions.py | 141 ++ .../resources/conformance/fastapi_app.out | 198 +++ .../test/resources/conformance/fastapi_app.py | 363 +++++ .../test/resources/conformance/flask_app.out | 213 +++ .../test/resources/conformance/flask_app.py | 228 +++ 25 files changed, 4294 insertions(+) create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/__init__.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/applications.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/encoders.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/exceptions.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/params.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/responses.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/routing.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/status.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/testclient.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/flask/__init__.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/__init__.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/background.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/datastructures.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/exceptions.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/requests.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/responses.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/status.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/testclient.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/uvicorn/__init__.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/werkzeug/__init__.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/werkzeug/exceptions.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/fastapi_app.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/fastapi_app.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/flask_app.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/flask_app.py diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/__init__.py new file mode 100644 index 00000000000..d12d4eed26c --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/__init__.py @@ -0,0 +1,21 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""FastAPI over a pure dispatch core: no sockets, no ASGI. + +``app.dispatch(method, path, query, headers, body)`` maps one request to +``(status, headers, body)``; ``TestClient`` and the native server both go +through it. Written from FastAPI's public documentation and behaviour. +""" +from starlette.background import BackgroundTasks +from starlette.requests import Request +from starlette.responses import Response + +from fastapi import status +from fastapi.applications import FastAPI +from fastapi.exceptions import HTTPException +from fastapi.params import Body, Cookie, Depends, File, Form, Header, Path, Query, Security +from fastapi.routing import APIRouter + +__all__ = ["FastAPI", "APIRouter", "Depends", "Query", "Path", "Body", "Header", "Cookie", "Form", + "File", "Security", "HTTPException", "Request", "Response", "BackgroundTasks", "status"] diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/applications.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/applications.py new file mode 100644 index 00000000000..ab5abfee69f --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/applications.py @@ -0,0 +1,339 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""The FastAPI application: a pure dispatch core plus OpenAPI generation.""" +import inspect +import re as _re +from urllib.parse import quote, unquote + +from pydantic import BaseModel, TypeAdapter + +from starlette.datastructures import State +from starlette.exceptions import HTTPException as StarletteHTTPException +from starlette.requests import Request +from starlette.responses import JSONResponse, PlainTextResponse, RedirectResponse, Response + +from fastapi.exceptions import RequestValidationError, ResponseValidationError +from fastapi.params import Undefined +from fastapi.routing import APIRouter, APIRoute, run_coro, run_route, _all_params, _all_body, _analyze + + +def _http_exception_handler(request, exc): + headers = getattr(exc, "headers", None) + if exc.status_code in (204, 304) or exc.status_code < 200: + return Response(status_code=exc.status_code, headers=headers) + return JSONResponse({"detail": exc.detail}, status_code=exc.status_code, headers=headers) + + +def _validation_handler(request, exc): + return JSONResponse({"detail": exc.errors()}, status_code=422) + + +class FastAPI(APIRouter): + def __init__(self, *, debug=False, title="FastAPI", summary=None, description="", version="0.1.0", + openapi_url="/openapi.json", docs_url="/docs", redoc_url="/redoc", + default_response_class=None, dependencies=None, exception_handlers=None, + lifespan=None, tags=None, root_path="", **kw): + APIRouter.__init__(self, dependencies=dependencies, default_response_class=default_response_class) + self.debug = debug + self.title = title + self.summary = summary + self.description = description + self.version = version + self.openapi_url = openapi_url + self.docs_url = docs_url + self.redoc_url = redoc_url + self.root_path = root_path + self.router = self + self.state = State() + self.extra = kw + self.dependency_overrides = {} + self.exception_handlers = {} + self.exception_handlers[StarletteHTTPException] = _http_exception_handler + self.exception_handlers[RequestValidationError] = _validation_handler + if exception_handlers: + for k, v in exception_handlers.items(): + self.exception_handlers[k] = v + self._startup_handlers = [] + self._shutdown_handlers = [] + self._lifespan = lifespan + self._lifespan_cm = None + self._openapi = None + + # -- registration + def exception_handler(self, exc_class_or_status): + def deco(f): + self.exception_handlers[exc_class_or_status] = f + return f + return deco + + def add_exception_handler(self, exc_class_or_status, handler): + self.exception_handlers[exc_class_or_status] = handler + + def on_event(self, kind): + def deco(f): + if kind == "startup": + self._startup_handlers.append(f) + else: + self._shutdown_handlers.append(f) + return f + return deco + + def include_router(self, router, **kw): + APIRouter.include_router(self, router, **kw) + self._openapi = None + + def add_api_route(self, path, endpoint, **kw): + APIRouter.add_api_route(self, path, endpoint, **kw) + self._openapi = None + + def _startup(self): + if self._lifespan is not None: + self._lifespan_cm = self._lifespan(self) + if hasattr(self._lifespan_cm, "__aenter__"): + run_coro(self._lifespan_cm.__aenter__()) + else: + self._lifespan_cm.__enter__() + for f in self._startup_handlers: + r = f() + if inspect.iscoroutine(r): + run_coro(r) + + def _shutdown(self): + for f in self._shutdown_handlers: + r = f() + if inspect.iscoroutine(r): + run_coro(r) + if self._lifespan_cm is not None: + cm = self._lifespan_cm + self._lifespan_cm = None + if hasattr(cm, "__aexit__"): + run_coro(cm.__aexit__(None, None, None)) + else: + cm.__exit__(None, None, None) + + # -- dispatch + def _find_handler(self, exc): + if isinstance(exc, StarletteHTTPException) and exc.status_code in self.exception_handlers: + return self.exception_handlers[exc.status_code] + for cls in type(exc).__mro__: + if cls in self.exception_handlers: + return self.exception_handlers[cls] + return None + + def _handle_exception(self, req, exc): + h = self._find_handler(exc) + if h is None: + raise exc + r = h(req, exc) + if inspect.iscoroutine(r): + r = run_coro(r) + return r + + def _route(self, req): + path = req._path + method = req.method + partial = None + for r in self.routes: + vals = r.match(path) + if vals is None: + continue + if method in r.methods: + req.path_params = vals + return r + if partial is None: + partial = r + if partial is not None: + raise StarletteHTTPException(405, "Method Not Allowed", + {"Allow": ", ".join(sorted(partial.methods))}) + if path.endswith("/") and len(path) > 1: + alt = path.rstrip("/") + else: + alt = path + "/" + for r in self.routes: + if r.match(alt) is not None: + host = req.headers.get("host", "testserver") + loc = "http://" + host + quote(alt) + if req._query: + loc += "?" + req._query + return RedirectResponse(loc, status_code=307) + raise StarletteHTTPException(404, "Not Found") + + def _handle(self, method, path, query, headers, body): + req = Request(method.upper(), unquote(path), query, headers, body, self) + req.app = self + try: + if self.openapi_url and req._path == self.openapi_url and req.method == "GET": + resp = JSONResponse(self.openapi()) + else: + route = self._route(req) + if isinstance(route, Response): + resp = route + else: + resp = run_route(route, req, self.default_response_class or JSONResponse) + except Exception as exc: + resp = self._handle_exception(req, exc) + if resp.background is not None: + resp.background() + return resp + + def _dispatch_for_client(self, method, path, query, headers, body, raise_exceptions): + try: + resp = self._handle(method, path, query, headers, body) + except Exception: + if raise_exceptions: + raise + resp = PlainTextResponse("Internal Server Error", status_code=500) + data = b"" if method.upper() == "HEAD" else resp.body + return resp.status_code, resp.headers.items(), data + + def dispatch(self, method, path, query, headers, body): + return self._dispatch_for_client(method, path, query, headers, body, False) + + # -- OpenAPI + def openapi(self): + if self._openapi is None: + self._openapi = self._build_openapi() + return self._openapi + + def _build_openapi(self): + info = {"title": self.title, "version": self.version} + if self.summary: + info["summary"] = self.summary + if self.description: + info["description"] = self.description + paths = {} + schemas = {} + need_validation = False + for r in self.routes: + if not r.include_in_schema: + continue + r._prepare() + item = paths.setdefault(_re.sub(":[a-z]+\\}", "}", r.path), {}) + for m in sorted(r.methods): + op = {} + if r.tags: + op["tags"] = list(r.tags) + op["summary"] = r.summary or r.name.replace("_", " ").title() + if r.description: + op["description"] = r.description + elif r.endpoint.__doc__: + op["description"] = inspect.cleandoc(r.endpoint.__doc__) + op["operationId"] = r.operation_id or (r.name + r.path.replace("{", "_").replace("}", "_") + .replace("/", "_").replace(":", "_") + "_" + m.lower()) + if r.deprecated: + op["deprecated"] = True + params = [] + allp = _all_params(r._dep) + for pre in r._pre: + allp.extend(_all_params(pre)) + for p in allp: + if p.field is not None and not p.field.include_in_schema: + continue + s = self._schema(p.tp, schemas, p) + d = {"name": p.key, "in": p.location, "required": p.required, "schema": s} + if p.field is not None and p.field.description: + d["description"] = p.field.description + params.append(d) + if params: + op["parameters"] = params + body = _all_body(r._dep) + if body: + need_validation = True + if len(body) == 1 and not (body[0].field is not None and getattr(body[0].field, "embed", None)): + bs = self._schema(body[0].tp, schemas, None) + required = body[0].required + else: + props = {} + req_names = [] + for p in body: + props[p.key] = self._schema(p.tp, schemas, p) + if p.required: + req_names.append(p.key) + name = "Body_%s_%s_%s" % (r.name, r.path.replace("/", "_").replace("{", "").replace("}", "").strip("_"), m.lower()) + sch = {"properties": props, "type": "object", "title": name.replace("_", " ")} + if req_names: + sch["required"] = req_names + schemas[name] = sch + bs = {"$ref": "#/components/schemas/" + name} + required = True + ct = "application/json" + if body[0].field is not None and getattr(body[0].field, "media_type", None): + ct = body[0].field.media_type + op["requestBody"] = {"content": {ct: {"schema": bs}}, "required": required} + status = str(r.status_code or 200) + if r._adapter is not None: + rs = self._schema(r.response_model, schemas, None) + else: + rs = {} + ct = "application/json" + rc = r.response_class + if rc is not None and getattr(rc, "media_type", None): + ct = rc.media_type + if rc is not JSONResponse and r._adapter is None: + rs = {"type": "string"} + resp = {"description": "Successful Response", "content": {ct: {"schema": rs}}} + responses = {status: resp} + for code, extra in r.responses.items(): + e = dict(extra) if isinstance(extra, dict) else {} + e.setdefault("description", "Additional Response") + responses[str(code)] = e + if allp or body: + need_validation = True + responses["422"] = {"description": "Validation Error", "content": {"application/json": { + "schema": {"$ref": "#/components/schemas/HTTPValidationError"}}}} + op["responses"] = responses + item[m.lower()] = op + if need_validation: + schemas["HTTPValidationError"] = {"properties": {"detail": {"items": {"$ref": "#/components/schemas/ValidationError"}, + "type": "array", "title": "Detail"}}, + "type": "object", "title": "HTTPValidationError"} + schemas["ValidationError"] = {"properties": { + "loc": {"items": {"anyOf": [{"type": "string"}, {"type": "integer"}]}, "type": "array", "title": "Location"}, + "msg": {"type": "string", "title": "Message"}, + "type": {"type": "string", "title": "Error Type"}, + "input": {"title": "Input"}, + "ctx": {"type": "object", "title": "Context"}}, + "type": "object", "required": ["loc", "msg", "type"], "title": "ValidationError"} + out = {"openapi": "3.1.0", "info": info, "paths": paths} + if schemas: + out["components"] = {"schemas": dict((k, schemas[k]) for k in sorted(schemas))} + return out + + def _schema(self, tp, schemas, p): + defs = {} + try: + s = dict(TypeAdapter(tp).json_schema()) + except Exception: + s = {} + defs = s.pop("$defs", {}) + for k, v in defs.items(): + schemas[k] = self._fix_refs(v) + s = self._fix_refs(s) + if p is not None and p.field is not None: + if p.field.title: + s["title"] = p.field.title + for k in ("gt", "ge", "lt", "le", "min_length", "max_length", "pattern", "multiple_of"): + v = getattr(p.field, k, None) + if v is not None: + s[{"gt": "exclusiveMinimum", "ge": "minimum", "lt": "exclusiveMaximum", "le": "maximum", + "min_length": "minLength", "max_length": "maxLength", "pattern": "pattern", + "multiple_of": "multipleOf"}[k]] = v + if not p.required and p.default is not Undefined: + s["default"] = p.default + if p is not None and "title" not in s and "$ref" not in s: + s["title"] = p.key.replace("-", " ").replace("_", " ").title() + return s + + def _fix_refs(self, v): + if isinstance(v, dict): + out = {} + for k, x in v.items(): + if k == "$ref" and isinstance(x, str): + out[k] = x.replace("#/$defs/", "#/components/schemas/") + else: + out[k] = self._fix_refs(x) + return out + if isinstance(v, list): + return [self._fix_refs(x) for x in v] + return v diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/encoders.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/encoders.py new file mode 100644 index 00000000000..4fa61f2bbcb --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/encoders.py @@ -0,0 +1,63 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""jsonable_encoder: turn endpoint results into JSON-compatible values.""" +import dataclasses +import datetime +import decimal +import enum +import uuid + +from pydantic import BaseModel + + +def jsonable_encoder(obj, include=None, exclude=None, by_alias=True, exclude_unset=False, + exclude_defaults=False, exclude_none=False, custom_encoder=None, sqlalchemy_safe=True): + if custom_encoder: + for tp, fn in custom_encoder.items(): + if isinstance(obj, tp): + return fn(obj) + if isinstance(obj, BaseModel): + return obj.model_dump(mode="json", include=include, exclude=exclude, by_alias=by_alias, + exclude_unset=exclude_unset, exclude_defaults=exclude_defaults, + exclude_none=exclude_none) + if dataclasses.is_dataclass(obj) and not isinstance(obj, type): + return jsonable_encoder(dataclasses.asdict(obj), by_alias=by_alias, exclude_none=exclude_none, + custom_encoder=custom_encoder) + if isinstance(obj, enum.Enum): + return obj.value + if obj is None or isinstance(obj, (str, int, float, bool)): + return obj + if isinstance(obj, (bytes, bytearray)): + return bytes(obj).decode("utf-8") + if isinstance(obj, decimal.Decimal): + if obj.as_tuple().exponent >= 0: + return int(obj) + return float(obj) + if isinstance(obj, uuid.UUID): + return str(obj) + if isinstance(obj, datetime.datetime) or isinstance(obj, datetime.date) or isinstance(obj, datetime.time): + return obj.isoformat() + if isinstance(obj, datetime.timedelta): + return obj.total_seconds() + if isinstance(obj, dict): + out = {} + for k, v in obj.items(): + if exclude_none and v is None: + continue + if include is not None and k not in include: + continue + if exclude is not None and k in exclude: + continue + out[jsonable_encoder(k, by_alias=by_alias, custom_encoder=custom_encoder)] = jsonable_encoder( + v, by_alias=by_alias, exclude_unset=exclude_unset, exclude_defaults=exclude_defaults, + exclude_none=exclude_none, custom_encoder=custom_encoder) + return out + if isinstance(obj, (list, tuple, set, frozenset)) or hasattr(obj, "__next__"): + return [jsonable_encoder(v, by_alias=by_alias, exclude_unset=exclude_unset, + exclude_defaults=exclude_defaults, exclude_none=exclude_none, + custom_encoder=custom_encoder) for v in obj] + if hasattr(obj, "__dict__"): + return jsonable_encoder(dict(vars(obj)), by_alias=by_alias, exclude_none=exclude_none, + custom_encoder=custom_encoder) + raise ValueError("Unable to serialize unknown type: %r" % (type(obj),)) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/exceptions.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/exceptions.py new file mode 100644 index 00000000000..ad04b36b976 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/exceptions.py @@ -0,0 +1,37 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""FastAPI's exceptions.""" +from starlette.exceptions import HTTPException as StarletteHTTPException + + +class HTTPException(StarletteHTTPException): + pass + + +class ValidationException(Exception): + def __init__(self, errors): + Exception.__init__(self, errors) + self._errors = errors + + def errors(self): + return self._errors + + +class RequestValidationError(ValidationException): + def __init__(self, errors, *, body=None): + ValidationException.__init__(self, errors) + self.body = body + + +class ResponseValidationError(ValidationException): + def __init__(self, errors, *, body=None): + ValidationException.__init__(self, errors) + self.body = body + + def __str__(self): + return "%d validation errors:\n%s" % (len(self._errors), "\n".join(" " + repr(e) for e in self._errors)) + + +class FastAPIError(RuntimeError): + pass diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/params.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/params.py new file mode 100644 index 00000000000..be3193e0567 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/params.py @@ -0,0 +1,105 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Parameter declarations: Query, Path, Header, Cookie, Body, Form, Depends.""" + + +class _Undef: + def __repr__(self): + return "PydanticUndefined" + + +Undefined = _Undef() + + +class Param: + location = "query" + + def __init__(self, default=Undefined, *, alias=None, title=None, description=None, gt=None, + ge=None, lt=None, le=None, min_length=None, max_length=None, pattern=None, + regex=None, multiple_of=None, example=None, examples=None, deprecated=None, + include_in_schema=True, default_factory=None, **kw): + if default is Ellipsis: + default = Undefined + self.default = default + self.default_factory = default_factory + self.alias = alias + self.title = title + self.description = description + self.gt = gt + self.ge = ge + self.lt = lt + self.le = le + self.min_length = min_length + self.max_length = max_length + self.pattern = pattern if pattern is not None else regex + self.multiple_of = multiple_of + self.example = example + self.examples = examples + self.deprecated = deprecated + self.include_in_schema = include_in_schema + + def constraints(self): + out = {} + for k in ("gt", "ge", "lt", "le", "min_length", "max_length", "pattern", "multiple_of"): + v = getattr(self, k) + if v is not None: + out[k] = v + return out + + def __repr__(self): + return "%s(%r)" % (type(self).__name__, self.default) + + +class Path(Param): + location = "path" + + def __init__(self, default=Undefined, **kw): + Param.__init__(self, ... if default is Undefined else default, **kw) + self.default = Undefined + + +class Query(Param): + location = "query" + + +class Header(Param): + location = "header" + + def __init__(self, default=Undefined, *, convert_underscores=True, **kw): + Param.__init__(self, default, **kw) + self.convert_underscores = convert_underscores + + +class Cookie(Param): + location = "cookie" + + +class Body(Param): + location = "body" + + def __init__(self, default=Undefined, *, embed=None, media_type="application/json", **kw): + Param.__init__(self, default, **kw) + self.embed = embed + self.media_type = media_type + + +class Form(Body): + def __init__(self, default=Undefined, **kw): + Body.__init__(self, default, media_type="application/x-www-form-urlencoded", **kw) + + +class File(Form): + pass + + +class Depends: + def __init__(self, dependency=None, *, use_cache=True): + self.dependency = dependency + self.use_cache = use_cache + + def __repr__(self): + return "Depends(%r)" % (getattr(self.dependency, "__name__", self.dependency),) + + +Security = Depends diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/responses.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/responses.py new file mode 100644 index 00000000000..f67b95d24a7 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/responses.py @@ -0,0 +1,18 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Response classes (re-exported from starlette).""" +from starlette.responses import (Response, HTMLResponse, PlainTextResponse, JSONResponse, # noqa + RedirectResponse, StreamingResponse) + + +class UJSONResponse(JSONResponse): + pass + + +class ORJSONResponse(JSONResponse): + pass + + +class FileResponse(Response): + pass diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/routing.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/routing.py new file mode 100644 index 00000000000..373848d0eef --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/routing.py @@ -0,0 +1,632 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Routes, parameter resolution, dependencies and the pure dispatch core.""" +import asyncio +import dataclasses +import inspect +import json as _json +import re as _re +import typing +from typing import Annotated, Any, Union, get_args, get_origin, get_type_hints +from urllib.parse import quote, unquote + +from pydantic import BaseModel, Field, TypeAdapter, ValidationError + +from starlette.background import BackgroundTasks +from starlette.datastructures import QueryParams +from starlette.exceptions import HTTPException as StarletteHTTPException +from starlette.requests import Request +from starlette.responses import JSONResponse, RedirectResponse, Response + +from fastapi.encoders import jsonable_encoder +from fastapi.exceptions import (HTTPException, RequestValidationError, ResponseValidationError) +from fastapi.params import Body, Cookie, Depends, Form, Header, Param, Path, Query, Undefined + +_NONE = type(None) +_EMPTY = inspect.Parameter.empty +_SEQ = (list, set, tuple, frozenset) + + +def run_coro(c): + return asyncio.run(c) + + +def _hints(call): + target = call + if isinstance(call, type): + target = call.__init__ + try: + return get_type_hints(target, include_extras=True) + except Exception: + return {} + + +def _unwrap_optional(tp): + if get_origin(tp) is Union: + args = [a for a in get_args(tp) if a is not _NONE] + if len(args) == 1: + return args[0], True + return tp, len(args) != len(get_args(tp)) + return tp, False + + +def _is_scalar(tp): + tp, _ = _unwrap_optional(tp) + org = get_origin(tp) + if org is Union: + for a in get_args(tp): + if a is not _NONE and not _is_scalar(a): + return False + return True + if org in _SEQ or org is dict: + return False + if isinstance(tp, type): + if issubclass(tp, BaseModel) or tp in _SEQ or tp is dict: + return False + if dataclasses.is_dataclass(tp): + return False + return True + + +def _is_seq(tp): + tp, _ = _unwrap_optional(tp) + return get_origin(tp) in _SEQ or tp in _SEQ + + +class _Dep: + def __init__(self, call): + self.call = call + self.params = [] # path/query/header/cookie + self.body = [] + self.deps = [] # (name, _Dep, use_cache) + self.specials = {} # name -> kind + self.is_gen = inspect.isgeneratorfunction(call) + self.single_body = False + self.form = False + + +class _P: + def __init__(self, name, tp, field, default, location): + self.name = name + self.tp = tp + self.field = field + self.default = default + self.location = location + self.is_seq = _is_seq(tp) + key = field.alias if field is not None and field.alias else None + if key is None: + key = name + if location == "header" and (field is None or getattr(field, "convert_underscores", True)): + key = name.replace("_", "-") + self.key = key + self._adapter = None + + @property + def required(self): + return self.default is Undefined and (self.field is None or self.field.default_factory is None) + + def default_value(self): + if self.field is not None and self.field.default_factory is not None: + return self.field.default_factory() + return self.default + + def adapter(self): + if self._adapter is None: + tp = self.tp + cons = self.field.constraints() if self.field is not None else {} + if cons: + tp = Annotated[tp, Field(**cons)] + self._adapter = TypeAdapter(tp) + return self._adapter + + +def _analyze(call, path_names): + dep = _Dep(call) + sig = inspect.signature(call) + hints = _hints(call) + for name, p in sig.parameters.items(): + if p.kind == inspect.Parameter.VAR_POSITIONAL or p.kind == inspect.Parameter.VAR_KEYWORD: + continue + tp = hints.get(name, p.annotation if p.annotation is not _EMPTY else Any) + default = Undefined if p.default is _EMPTY else p.default + field = None + depends = None + if get_origin(tp) is Annotated: + args = get_args(tp) + tp = args[0] + for m in args[1:]: + if isinstance(m, Depends): + depends = m + elif isinstance(m, Param): + field = m + if isinstance(default, Depends): + depends = default + default = Undefined + elif isinstance(default, Param): + field = default + default = field.default + if isinstance(tp, type) and issubclass(tp, Request): + dep.specials[name] = "request" + continue + if isinstance(tp, type) and issubclass(tp, Response): + dep.specials[name] = "response" + continue + if isinstance(tp, type) and issubclass(tp, BackgroundTasks): + dep.specials[name] = "background" + continue + if depends is not None: + sub = depends.dependency + if sub is None: + sub = tp + dep.deps.append((name, _analyze(sub, path_names), depends.use_cache)) + continue + if field is not None and field.location == "path" or (field is None and name in path_names): + dep.params.append(_P(name, tp, field, Undefined, "path")) + elif field is not None and field.location in ("query", "header", "cookie"): + dep.params.append(_P(name, tp, field, default, field.location)) + elif field is not None and field.location == "body": + dep.body.append(_P(name, tp, field, default, "body")) + if isinstance(field, Form): + dep.form = True + elif name in path_names: + dep.params.append(_P(name, tp, field, Undefined, "path")) + elif _is_scalar(tp): + dep.params.append(_P(name, tp, field, default, "query")) + else: + dep.body.append(_P(name, tp, field, default, "body")) + return dep + + +def _all_body(dep): + out = list(dep.body) + for name, sub, uc in dep.deps: + out.extend(_all_body(sub)) + return out + + +def _all_params(dep): + out = list(dep.params) + for name, sub, uc in dep.deps: + out.extend(_all_params(sub)) + return out + + +_CONSTRAINT_TYPES = ("greater_than", "greater_than_equal", "less_than", "less_than_equal", "multiple_of") +_CONV = {"str": "[^/]+", "path": ".*", "int": "[0-9]+", "float": "[0-9]+(?:\\.[0-9]+)?", + "uuid": "[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}"} + + +def _compile(path): + pat = "" + names = [] + pos = 0 + for m in _re.finditer("\\{([a-zA-Z_][a-zA-Z0-9_]*)(?::([a-z]+))?\\}", path): + pat += _re.escape(path[pos:m.start()]) + conv = m.group(2) or "str" + pat += "(" + _CONV[conv] + ")" + names.append(m.group(1)) + pos = m.end() + pat += _re.escape(path[pos:]) + return _re.compile("^" + pat + "$"), names + + +def _title(name): + return name.replace("_", " ").title() if name else "" + + +class APIRoute: + def __init__(self, path, endpoint, methods, **kw): + self.path = path + self.endpoint = endpoint + self.methods = set(m.upper() for m in methods) + self.name = kw.get("name") or endpoint.__name__ + self.regex, self.path_names = _compile(path) + self.status_code = kw.get("status_code") + self.response_model = kw.get("response_model", Undefined) + self.response_class = kw.get("response_class") + self.tags = list(kw.get("tags") or []) + self.dependencies = list(kw.get("dependencies") or []) + self.summary = kw.get("summary") + self.description = kw.get("description") + self.deprecated = kw.get("deprecated") + self.operation_id = kw.get("operation_id") + self.include_in_schema = kw.get("include_in_schema", True) + self.responses = kw.get("responses") or {} + self.exclude_unset = kw.get("response_model_exclude_unset", False) + self.exclude_defaults = kw.get("response_model_exclude_defaults", False) + self.exclude_none = kw.get("response_model_exclude_none", False) + self.by_alias = kw.get("response_model_by_alias", True) + self._dep = None + self._pre = None + self._adapter = None + self._ready = False + + def _prepare(self): + if self._ready: + return + self._dep = _analyze(self.endpoint, self.path_names) + self._pre = [_analyze(d.dependency, self.path_names) for d in self.dependencies] + rm = self.response_model + if rm is Undefined: + ann = _hints(self.endpoint).get("return", None) + rm = None + if ann is not None and ann is not _NONE and not (isinstance(ann, type) and issubclass(ann, Response)): + rm = ann + self.response_model = rm + if rm is not None: + self._adapter = TypeAdapter(rm) + body = _all_body(self._dep) + for p in self._pre: + body.extend(_all_body(p)) + self.body_params = body + self._ready = True + + def match(self, path): + m = self.regex.match(path) + if m is None: + return None + vals = {} + for i in range(len(self.path_names)): + vals[self.path_names[i]] = m.group(i + 1) + return vals + + +def _call(call, values): + r = call(**values) + if inspect.iscoroutine(r): + r = run_coro(r) + return r + + +def _prep_content(v, route): + if isinstance(v, BaseModel): + return v.model_dump(by_alias=True, exclude_unset=route.exclude_unset, + exclude_defaults=route.exclude_defaults, exclude_none=route.exclude_none) + if isinstance(v, (list, tuple, set, frozenset)): + return [_prep_content(x, route) for x in v] + if isinstance(v, dict): + return dict((k, _prep_content(x, route)) for k, x in v.items()) + return v + + +class _Ctx: + def __init__(self, route, req, body_obj): + self.route = route + self.req = req + self.cache = {} + self.cleanups = [] + self.errors = [] + self.body = body_obj # parsed body (Undefined when absent) + self.form = None + self.sub = Response() + self.sub._sub = True + self.sub.status_code = None + self.background = BackgroundTasks() + + +def _add_errors(ctx, exc, prefix, raw=None, is_body=False): + for d in exc.errors(include_url=False): + d2 = dict(d) + d2["loc"] = tuple(prefix) + tuple(d["loc"]) + # pydantic here reports the coerced number for a constraint failing on a string, and + # int_from_float for "5.5"; CPython's pydantic keeps the original string / int_parsing. + if not is_body and len(d["loc"]) == 0 and isinstance(raw, str): + if d["type"] in _CONSTRAINT_TYPES: + d2["input"] = raw + if d["type"] == "int_from_float" and isinstance(d["input"], str): + d2["type"] = "int_parsing" + if d["type"] == "model_type" and is_body: + # FastAPI validates with from_attributes=True, which words this error differently + d2["type"] = "model_attributes_type" + d2["msg"] = "Input should be a valid dictionary or object to extract fields from" + d2.pop("ctx", None) + ctx.errors.append(d2) + + +def _missing(ctx, loc): + ctx.errors.append({"type": "missing", "loc": tuple(loc), "msg": "Field required", "input": None}) + + +def _run_dep(sub, subvals, ctx): + r = sub.call(**subvals) + # duck-typed: inspect.isgeneratorfunction is not reliable on the transpiler + if sub.is_gen or (hasattr(r, "__next__") and hasattr(r, "throw") and hasattr(r, "send")): + r2 = next(r) + ctx.cleanups.append(r) + return r2 + if inspect.iscoroutine(r): + r = run_coro(r) + return r + + +def _solve(dep, ctx): + req = ctx.req + values = {} + for name, sub, use_cache in dep.deps: + key = sub.call + if use_cache and key in ctx.cache: + values[name] = ctx.cache[key] + continue + before = len(ctx.errors) + subvals = _solve(sub, ctx) + if len(ctx.errors) > before: + continue + r = _run_dep(sub, subvals, ctx) + if use_cache: + ctx.cache[key] = r + values[name] = r + for name, kind in dep.specials.items(): + if kind == "request": + values[name] = req + elif kind == "response": + values[name] = ctx.sub + else: + values[name] = ctx.background + qp = None + for p in dep.params: + loc = p.location + raw = None + present = False + if loc == "path": + raw = req.path_params.get(p.name) + present = raw is not None + elif loc == "query": + if qp is None: + qp = req.query_params + if p.is_seq: + raw = qp.getlist(p.key) + present = len(raw) > 0 + elif p.key in qp: + raw = qp.get(p.key) + present = True + elif loc == "header": + if p.is_seq: + raw = req.headers.getlist(p.key) + present = len(raw) > 0 + elif p.key in req.headers: + raw = req.headers.get(p.key) + present = True + else: + ck = req.cookies + if p.key in ck: + raw = ck[p.key] + present = True + if not present: + if p.required: + _missing(ctx, (loc, p.key)) + else: + values[p.name] = p.default_value() + continue + try: + values[p.name] = p.adapter().validate_python(raw) + except ValidationError as e: + _add_errors(ctx, e, (loc, p.key), raw) + body = dep.body + if body: + if dep.form: + for p in body: + form = ctx.form + if form is not None and p.key in form: + try: + values[p.name] = p.adapter().validate_python(form.get(p.key)) + except ValidationError as e: + _add_errors(ctx, e, ("body", p.key)) + elif p.required: + _missing(ctx, ("body", p.key)) + else: + values[p.name] = p.default_value() + else: + all_body = ctx.route.body_params + embed = len(all_body) > 1 + for p in body: + if p.field is not None and getattr(p.field, "embed", None): + embed = True + for p in body: + if not embed: + present = ctx.body is not Undefined and ctx.body is not None + val = ctx.body + loc = ("body",) + else: + present = isinstance(ctx.body, dict) and p.key in ctx.body + val = ctx.body.get(p.key) if present else None + loc = ("body", p.key) + if not present: + if p.required: + _missing(ctx, loc) + else: + values[p.name] = p.default_value() + continue + try: + values[p.name] = p.adapter().validate_python(val, from_attributes=True) + except ValidationError as e: + _add_errors(ctx, e, loc, val, True) + return values + + +def _finish_cleanups(ctx, exc): + for gen in reversed(ctx.cleanups): + try: + if exc is None: + try: + next(gen) + except StopIteration: + pass + else: + try: + gen.throw(exc) + except StopIteration: + pass + except Exception as e2: + exc = e2 + ctx.cleanups = [] + if exc is not None: + raise exc + + +def _read_body(route, req): + """Parse the request body for the route's body parameters.""" + ctx_body = Undefined + ctx_form = None + if not route.body_params: + return ctx_body, ctx_form, None + raw = req._body + ct = req.headers.get("content-type") + if route._dep.form or any(isinstance(p.field, Form) for p in route.body_params): + ctx_form = {} + if raw: + for k, v in QueryParams(raw.decode("utf-8", "replace")).items(): + ctx_form[k] = v + return ctx_body, ctx_form, None + if not ct or ct.split(";")[0].strip().lower() == "application/json" or ct.split(";")[0].strip().lower().endswith("+json"): + if raw: + try: + ctx_body = _json.loads(raw.decode("utf-8")) + except ValueError as e: + pos = getattr(e, "pos", 0) + msg = getattr(e, "msg", str(e)) + err = {"type": "json_invalid", "loc": ("body", pos), "msg": "JSON decode error", + "input": {}, "ctx": {"error": msg}} + return ctx_body, ctx_form, err + else: + ctx_body = raw + return ctx_body, ctx_form, None + + +def _serialize(route, raw): + if route._adapter is not None: + value = _prep_content(raw, route) + try: + validated = route._adapter.validate_python(value, from_attributes=True) + except ValidationError as e: + errs = [] + for d in e.errors(include_url=False): + d2 = dict(d) + d2["loc"] = ("response",) + tuple(d["loc"]) + errs.append(d2) + raise ResponseValidationError(errs, body=raw) + return route._adapter.dump_python(validated, mode="json", by_alias=route.by_alias, + exclude_unset=route.exclude_unset, + exclude_defaults=route.exclude_defaults, + exclude_none=route.exclude_none) + return jsonable_encoder(raw) + + +def run_route(route, req, default_class): + route._prepare() + body, form, err = _read_body(route, req) + if err is not None: + raise RequestValidationError([err], body=None) + ctx = _Ctx(route, req, body) + ctx.form = form + try: + for pre in route._pre: + pv = _solve(pre, ctx) + if not ctx.errors: + _run_dep(pre, pv, ctx) + values = _solve(route._dep, ctx) + if ctx.errors: + raise RequestValidationError(list(ctx.errors), body=body if body is not Undefined else None) + raw = route.endpoint(**values) + if inspect.iscoroutine(raw): + raw = run_coro(raw) + if isinstance(raw, Response): + resp = raw + else: + status = route.status_code + if ctx.sub.status_code is not None: + status = ctx.sub.status_code + if status is None: + status = 200 + content = _serialize(route, raw) + rc = route.response_class or default_class + if status in (204, 304) or status < 200: + resp = Response(status_code=status) + else: + resp = rc(content, status_code=status) + for k, v in ctx.sub.headers.items(): + if k == "content-length": + continue + resp.headers.append(k, v) + if ctx.background.tasks: + resp.background = ctx.background + _finish_cleanups(ctx, None) + except Exception as exc: + if ctx.cleanups: + _finish_cleanups(ctx, exc) + raise + return resp + + +def _verb_decorator(self, methods, path, kw): + def deco(f): + self.add_api_route(path, f, methods=methods, **kw) + return f + return deco + + +class APIRouter: + def __init__(self, *, prefix="", tags=None, dependencies=None, default_response_class=None, + responses=None, deprecated=None, include_in_schema=True, **kw): + self.prefix = prefix + self.tags = list(tags or []) + self.dependencies = list(dependencies or []) + self.default_response_class = default_response_class + self.responses = responses or {} + self.deprecated = deprecated + self.routes = [] + + def add_api_route(self, path, endpoint, *, methods=None, **kw): + kw = dict(kw) + kw["tags"] = list(self.tags) + list(kw.get("tags") or []) + kw["dependencies"] = list(self.dependencies) + list(kw.get("dependencies") or []) + if kw.get("response_class") is None: + kw["response_class"] = self.default_response_class + if kw.get("deprecated") is None: + kw["deprecated"] = self.deprecated + self.routes.append(APIRoute(self.prefix + path, endpoint, methods or ["GET"], **kw)) + + def api_route(self, path, *, methods=None, **kw): + return _verb_decorator(self, methods or ["GET"], path, kw) + + def get(self, path, **kw): + return _verb_decorator(self, ["GET"], path, kw) + + def post(self, path, **kw): + return _verb_decorator(self, ["POST"], path, kw) + + def put(self, path, **kw): + return _verb_decorator(self, ["PUT"], path, kw) + + def delete(self, path, **kw): + return _verb_decorator(self, ["DELETE"], path, kw) + + def patch(self, path, **kw): + return _verb_decorator(self, ["PATCH"], path, kw) + + def head(self, path, **kw): + return _verb_decorator(self, ["HEAD"], path, kw) + + def options(self, path, **kw): + return _verb_decorator(self, ["OPTIONS"], path, kw) + + def include_router(self, router, *, prefix="", tags=None, dependencies=None, + default_response_class=None, responses=None, deprecated=None, + include_in_schema=True, **kw): + for r in router.routes: + n = APIRoute(prefix + r.path, r.endpoint, r.methods, name=r.name, + status_code=r.status_code, response_model=r.response_model, + response_class=r.response_class or default_response_class or self.default_response_class, + tags=list(tags or []) + list(r.tags), + dependencies=list(dependencies or []) + list(r.dependencies), + summary=r.summary, description=r.description, + deprecated=r.deprecated if r.deprecated is not None else deprecated, + operation_id=r.operation_id, + include_in_schema=r.include_in_schema and include_in_schema, + responses=r.responses, + response_model_exclude_unset=r.exclude_unset, + response_model_exclude_defaults=r.exclude_defaults, + response_model_exclude_none=r.exclude_none, + response_model_by_alias=r.by_alias) + # a route registered before its response model was inferred keeps the sentinel + self.routes.append(n) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/status.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/status.py new file mode 100644 index 00000000000..3e652a3eec0 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/status.py @@ -0,0 +1,5 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""HTTP status code constants (re-exported from starlette).""" +from starlette.status import * # noqa diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/testclient.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/testclient.py new file mode 100644 index 00000000000..67a7311131d --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/testclient.py @@ -0,0 +1,5 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""TestClient (re-exported from starlette).""" +from starlette.testclient import TestClient # noqa diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/flask/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/flask/__init__.py new file mode 100644 index 00000000000..35763c8d869 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/flask/__init__.py @@ -0,0 +1,1220 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Flask over a pure dispatch core: no sockets, no WSGI. + +``app.dispatch(method, path, query, headers, body)`` maps one request to +``(status, headers, body)``; the test client and the native server both go +through it. Written from Flask's public documentation and behaviour. +""" +import json as _json +import re as _re +import uuid as _uuid +from urllib.parse import parse_qsl, quote, quote_plus, unquote, urlencode + +from werkzeug.exceptions import (HTTPException, BadRequest, NotFound, MethodNotAllowed, + UnsupportedMediaType, InternalServerError, abort, + default_exceptions) + +__all__ = ["Flask", "Blueprint", "Response", "request", "g", "jsonify", "make_response", + "abort", "redirect", "url_for", "current_app"] + +_STATUS = { + 100: "CONTINUE", 101: "SWITCHING PROTOCOLS", 200: "OK", 201: "CREATED", 202: "ACCEPTED", + 203: "NON AUTHORITATIVE INFORMATION", 204: "NO CONTENT", 205: "RESET CONTENT", + 206: "PARTIAL CONTENT", 300: "MULTIPLE CHOICES", 301: "MOVED PERMANENTLY", 302: "FOUND", + 303: "SEE OTHER", 304: "NOT MODIFIED", 307: "TEMPORARY REDIRECT", 308: "PERMANENT REDIRECT", + 400: "BAD REQUEST", 401: "UNAUTHORIZED", 403: "FORBIDDEN", 404: "NOT FOUND", + 405: "METHOD NOT ALLOWED", 406: "NOT ACCEPTABLE", 408: "REQUEST TIMEOUT", 409: "CONFLICT", + 410: "GONE", 411: "LENGTH REQUIRED", 412: "PRECONDITION FAILED", 413: "REQUEST ENTITY TOO LARGE", + 415: "UNSUPPORTED MEDIA TYPE", 418: "I'M A TEAPOT", 422: "UNPROCESSABLE ENTITY", + 429: "TOO MANY REQUESTS", 500: "INTERNAL SERVER ERROR", 501: "NOT IMPLEMENTED", + 502: "BAD GATEWAY", 503: "SERVICE UNAVAILABLE", 504: "GATEWAY TIMEOUT", +} + + +def _b(x): + if isinstance(x, (bytes, bytearray)): + return bytes(x) + return str(x).encode("utf-8") + + +class Headers: + """Case-insensitive, ordered, multi-valued header collection.""" + + def __init__(self, items=None): + self._l = [] + if items is not None: + self.extend(items) + + def extend(self, items): + if isinstance(items, Headers): + items = items._l + elif isinstance(items, dict): + items = list(items.items()) + for k, v in items: + self.add(k, v) + + def add(self, key, value): + self._l.append((key, str(value))) + + def set(self, key, value): + self.__setitem__(key, value) + + def __setitem__(self, key, value): + lk = key.lower() + out = [] + done = False + for k, v in self._l: + if k.lower() == lk: + if not done: + out.append((key, str(value))) + done = True + else: + out.append((k, v)) + if not done: + out.append((key, str(value))) + self._l = out + + def __getitem__(self, key): + lk = key.lower() + for k, v in self._l: + if k.lower() == lk: + return v + raise KeyError(key) + + def get(self, key, default=None, type=None): + try: + v = self[key] + except KeyError: + return default + if type is not None: + try: + return type(v) + except (ValueError, TypeError): + return default + return v + + def getlist(self, key): + lk = key.lower() + return [v for k, v in self._l if k.lower() == lk] + + def __contains__(self, key): + return isinstance(key, str) and key.lower() in [k.lower() for k, v in self._l] + + def __delitem__(self, key): + lk = key.lower() + self._l = [(k, v) for k, v in self._l if k.lower() != lk] + + def remove(self, key): + del self[key] + + def pop(self, key, default=None): + v = self.get(key, default) + del self[key] + return v + + def setdefault(self, key, value): + if key not in self: + self[key] = value + return self[key] + + def items(self): + return list(self._l) + + def keys(self): + return [k for k, v in self._l] + + def values(self): + return [v for k, v in self._l] + + def __iter__(self): + return iter(list(self._l)) + + def __len__(self): + return len(self._l) + + def to_wsgi_list(self): + return list(self._l) + + def __repr__(self): + return "Headers(%r)" % (self._l,) + + +class MultiDict(dict): + """dict of first values that remembers every value per key.""" + + def __init__(self, pairs=None): + dict.__init__(self) + self._all = {} + if pairs: + for k, v in pairs: + self.add(k, v) + + def add(self, key, value): + if key not in self._all: + self._all[key] = [] + dict.__setitem__(self, key, value) + self._all[key].append(value) + + def getlist(self, key, type=None): + vals = self._all.get(key, []) + if type is None: + return list(vals) + out = [] + for v in vals: + try: + out.append(type(v)) + except (ValueError, TypeError): + pass + return out + + def get(self, key, default=None, type=None): + if key not in self: + return default + v = dict.__getitem__(self, key) + if type is not None: + try: + return type(v) + except (ValueError, TypeError): + return default + return v + + def __getitem__(self, key): + if key not in self: + raise BadRequest("The browser (or proxy) sent a request that this server could not understand.") + return dict.__getitem__(self, key) + + def to_dict(self, flat=True): + if flat: + return dict(self.items()) + return dict((k, list(v)) for k, v in self._all.items()) + + def lists(self): + return [(k, list(v)) for k, v in self._all.items()] + + def __repr__(self): + return "MultiDict(%r)" % ([(k, v) for k, vs in self._all.items() for v in vs],) + + +def _parse_qs(s): + if isinstance(s, (bytes, bytearray)): + s = bytes(s).decode("utf-8", "replace") + return MultiDict(parse_qsl(s or "", keep_blank_values=True)) + + +def _parse_cookies(header): + out = {} + if not header: + return out + for part in header.split(";"): + if "=" in part: + k, v = part.split("=", 1) + k = k.strip() + v = v.strip() + if len(v) >= 2 and v[0] == '"' and v[-1] == '"': + v = v[1:-1] + if k and k not in out: + out[k] = v + return out + + +class Response: + default_status = 200 + default_mimetype = "text/html" + charset = "utf-8" + + def __init__(self, response=None, status=None, headers=None, mimetype=None, content_type=None): + self.headers = Headers(headers) if headers is not None else Headers() + if content_type is None: + if mimetype is None and "Content-Type" not in self.headers: + mimetype = self.default_mimetype + if mimetype is not None: + if mimetype.startswith("text/") or mimetype in ("application/xml", "application/javascript"): + mimetype = mimetype + "; charset=utf-8" + content_type = mimetype + if content_type is not None: + self.headers["Content-Type"] = content_type + self.status_code = self.default_status + if status is not None: + if isinstance(status, int): + self.status_code = status + else: + self.status = status + self._data = b"" + if response is not None: + if isinstance(response, (str, bytes, bytearray)): + self.set_data(response) + else: + self.set_data(b"".join([_b(c) for c in response])) + self._cookies = [] + + @property + def status(self): + s = getattr(self, "_status_text", None) + if s is not None and s[0] == self.status_code: + return s[1] + return "%d %s" % (self.status_code, _STATUS.get(self.status_code, "UNKNOWN")) + + @status.setter + def status(self, value): + if isinstance(value, int): + self.status_code = value + self._status_text = None + else: + value = str(value) + self.status_code = int(value.split(None, 1)[0]) + self._status_text = (self.status_code, value) + + def set_data(self, value): + self._data = _b(value) + self.headers["Content-Length"] = str(len(self._data)) + + def get_data(self, as_text=False): + if as_text: + return self._data.decode("utf-8", "replace") + return self._data + + data = property(get_data, set_data) + + @property + def mimetype(self): + ct = self.headers.get("Content-Type", "") + return ct.split(";")[0].strip() + + @mimetype.setter + def mimetype(self, v): + if v.startswith("text/") or v in ("application/xml", "application/javascript"): + v = v + "; charset=utf-8" + self.headers["Content-Type"] = v + + @property + def content_type(self): + return self.headers.get("Content-Type") + + @content_type.setter + def content_type(self, v): + self.headers["Content-Type"] = v + + @property + def content_length(self): + return self.headers.get("Content-Length", type=int) + + @property + def location(self): + return self.headers.get("Location") + + @property + def is_json(self): + m = self.mimetype + return m == "application/json" or (m.startswith("application/") and m.endswith("+json")) + + def get_json(self, force=False, silent=False): + if not (force or self.is_json): + return None + try: + return _json.loads(self._data.decode("utf-8")) + except ValueError: + if silent: + return None + raise BadRequest() + + @property + def json(self): + return self.get_json() + + def set_cookie(self, key, value="", max_age=None, expires=None, path="/", domain=None, + secure=False, httponly=False, samesite=None): + s = "%s=%s" % (key, quote(str(value), safe="!#$%&'()*+/:<=>?@[]^`{|}~")) + if max_age is not None: + s += "; Max-Age=%d" % int(max_age) + if domain: + s += "; Domain=" + domain + if path: + s += "; Path=" + path + if secure: + s += "; Secure" + if httponly: + s += "; HttpOnly" + if samesite: + s += "; SameSite=" + samesite + self.headers.add("Set-Cookie", s) + + def delete_cookie(self, key, path="/", domain=None): + self.set_cookie(key, "", max_age=0, path=path, domain=domain) + + def __repr__(self): + return "" % (len(self._data), self.status) + + @property + def is_redirect(self): + return self.status_code in (301, 302, 303, 307, 308) and "Location" in self.headers + + +class _Ctx: + def __init__(self, app, req): + self.app = app + self.request = req + self.g = _AppGlobals() + + +class _AppGlobals: + def get(self, name, default=None): + return self.__dict__.get(name, default) + + def pop(self, name, default=None): + return self.__dict__.pop(name, default) + + def setdefault(self, name, default=None): + return self.__dict__.setdefault(name, default) + + def __contains__(self, name): + return name in self.__dict__ + + def __iter__(self): + return iter(self.__dict__) + + +_stack = [] + + +class _Proxy: + def __init__(self, what): + object.__setattr__(self, "_what", what) + + def _get(self): + if not _stack: + raise RuntimeError("Working outside of request context.") + c = _stack[-1] + w = object.__getattribute__(self, "_what") + if w == "request": + return c.request + if w == "g": + return c.g + return c.app + + def __getattr__(self, name): + return getattr(self._get(), name) + + def __setattr__(self, name, value): + setattr(self._get(), name, value) + + def __delattr__(self, name): + delattr(self._get(), name) + + def __contains__(self, name): + return name in self._get() + + def __iter__(self): + return iter(self._get()) + + def __bool__(self): + return bool(_stack) + + def __repr__(self): + return repr(self._get()) + + +class Request: + def __init__(self, method, path, query, headers, body, app=None): + self.method = method + self.path = path + self.query_string = _b(query or "") + self.headers = Headers(headers) + self._body = bytes(body or b"") + self.view_args = None + self.endpoint = None + self.url_rule = None + self.routing_exception = None + self.scheme = "http" + self.remote_addr = "127.0.0.1" + self.environ = {} + self._args = None + self._form = None + self._json = None + self._json_done = False + + @property + def blueprint(self): + if self.endpoint and "." in self.endpoint: + return self.endpoint.rsplit(".", 1)[0] + return None + + @property + def args(self): + if self._args is None: + self._args = _parse_qs(self.query_string) + return self._args + + @property + def form(self): + if self._form is None: + if self.mimetype == "application/x-www-form-urlencoded": + self._form = _parse_qs(self._body) + else: + self._form = MultiDict() + return self._form + + @property + def values(self): + m = MultiDict() + for k, vs in self.args.lists(): + for v in vs: + m.add(k, v) + for k, vs in self.form.lists(): + for v in vs: + m.add(k, v) + return m + + @property + def files(self): + return MultiDict() + + @property + def cookies(self): + return _parse_cookies(self.headers.get("Cookie")) + + @property + def content_type(self): + return self.headers.get("Content-Type") + + @property + def mimetype(self): + ct = self.headers.get("Content-Type", "") + return ct.split(";")[0].strip().lower() + + @property + def content_length(self): + return self.headers.get("Content-Length", type=int) + + @property + def host(self): + return self.headers.get("Host", "localhost") + + @property + def host_url(self): + return "http://" + self.host + "/" + + @property + def base_url(self): + return "http://" + self.host + quote(self.path) + + @property + def url(self): + q = self.query_string.decode("utf-8", "replace") + return self.base_url + ("?" + q if q else "") + + @property + def full_path(self): + return self.path + "?" + self.query_string.decode("utf-8", "replace") + + @property + def is_json(self): + m = self.mimetype + return m == "application/json" or (m.startswith("application/") and m.endswith("+json")) + + @property + def is_secure(self): + return False + + def get_data(self, cache=True, as_text=False, parse_form_data=False): + if as_text: + return self._body.decode("utf-8", "replace") + return self._body + + @property + def data(self): + return self._body + + def get_json(self, force=False, silent=False, cache=True): + if self._json_done: + return self._json + if not (force or self.is_json): + if silent: + return None + raise UnsupportedMediaType("Did not attempt to load JSON data because the request" + " Content-Type was not 'application/json'.") + try: + val = _json.loads(self._body.decode("utf-8")) + except ValueError as e: + if silent: + return None + raise BadRequest() + self._json = val + self._json_done = True + return val + + @property + def json(self): + return self.get_json() + + def __repr__(self): + return "" % (self.url, self.method) + + +request = _Proxy("request") +g = _Proxy("g") +current_app = _Proxy("app") + + +def _to_json(obj, compact=True, sort=True): + def default(o): + if hasattr(o, "isoformat"): + return o.isoformat() + if isinstance(o, _uuid.UUID): + return str(o) + if hasattr(o, "__html__"): + return str(o.__html__()) + if hasattr(o, "model_dump"): + return o.model_dump(mode="json") + raise TypeError("Object of type %s is not JSON serializable" % type(o).__name__) + if compact: + return _json.dumps(obj, default=default, sort_keys=sort, separators=(",", ":")) + return _json.dumps(obj, default=default, sort_keys=sort, indent=2) + + +def jsonify(*args, **kwargs): + if args and kwargs: + raise TypeError("jsonify() behavior undefined when passed both args and kwargs") + if len(args) == 1: + data = args[0] + else: + data = args or kwargs + if isinstance(data, tuple): + data = list(data) + compact = not (_stack and _stack[-1].app.debug) + return Response(_to_json(data, compact) + "\n", mimetype="application/json") + + +def make_response(*args): + if not args: + return Response() + if len(args) == 1: + args = args[0] + app = _stack[-1].app if _stack else Flask.__new__(Flask) + return _make_response(app, args) + + +def _make_response(app, rv): + status = None + headers = None + if isinstance(rv, tuple): + n = len(rv) + if n == 3: + rv, status, headers = rv + elif n == 2: + if isinstance(rv[1], (Headers, dict, tuple, list)): + rv, headers = rv + else: + rv, status = rv + else: + raise TypeError("The view function did not return a valid response tuple." + " The tuple must have the form (body, status, headers)," + " (body, status), or (body, headers).") + if rv is None: + raise TypeError("The view function did not return a valid response. The function either" + " returned None or ended without a return statement.") + if not isinstance(rv, Response): + if isinstance(rv, (str, bytes, bytearray)): + rv = Response(rv) + elif isinstance(rv, (dict, list)): + rv = jsonify(rv) + elif isinstance(rv, HTTPException): + rv = rv.get_response() + elif hasattr(rv, "__iter__") or hasattr(rv, "__next__"): + rv = Response(rv) + else: + raise TypeError("The view function did not return a valid response. The return type" + " must be a string, dict, list, tuple with headers or status, Response" + " instance, or WSGI callable, but it was a %s." % type(rv).__name__) + if status is not None: + if isinstance(status, int): + rv.status_code = status + else: + rv.status = status + if headers: + if isinstance(headers, dict): + headers = headers.items() + for k, v in headers: + rv.headers[k] = v + return rv + + +def redirect(location, code=302, Response=Response): + from werkzeug.exceptions import _escape as escape + loc = str(location) + body = ('\n\nRedirecting...\n

Redirecting...

\n' + '

You should be redirected automatically to the target URL: %s. ' + 'If not, click the link.\n' % (escape(loc), escape(loc))) + r = Response(body, code, mimetype="text/html") + r.headers["Location"] = loc + return r + + +# ---- routing ------------------------------------------------------------- + +_UUID_RE = "[A-Fa-f0-9]{8}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{12}" +_CONV = { + "string": ("[^/]+", None), + "int": ("[0-9]+", int), + "float": ("[0-9]+\\.[0-9]+", float), + "path": ("[^/].*?", None), + "uuid": (_UUID_RE, _uuid.UUID), + "any": ("[^/]+", None), +} + + +class Rule: + def __init__(self, path, methods, endpoint, view, defaults=None, strict=True): + self.rule = path + self.methods = methods + self.endpoint = endpoint + self.view = view + self.defaults = defaults or {} + self.parts = [] # ("lit", text) | ("var", conv, name) + pat = "" + pos = 0 + weight = 0 + for m in _re.finditer("<(?:([a-zA-Z_][a-zA-Z0-9_]*)(?:\\(([^)]*)\\))?:)?([a-zA-Z_][a-zA-Z0-9_]*)>", path): + lit = path[pos:m.start()] + pat += _re.escape(lit) + weight += len(lit) + if lit: + self.parts.append(("lit", lit)) + conv = m.group(1) or "string" + if conv not in _CONV: + raise LookupError("the converter %r does not exist" % conv) + pat += "(" + _CONV[conv][0] + ")" + self.parts.append(("var", conv, m.group(3))) + pos = m.end() + lit = path[pos:] + pat += _re.escape(lit) + weight += len(lit) + if lit: + self.parts.append(("lit", lit)) + self.regex = _re.compile("^" + pat + "$") + self.slash = path.endswith("/") and strict + self.names = [p[2] for p in self.parts if p[0] == "var"] + self.convs = [p[1] for p in self.parts if p[0] == "var"] + self.nvars = len(self.names) + self.weight = weight + + def match(self, path): + m = self.regex.match(path) + if m is None: + return None + vals = {} + for i in range(self.nvars): + raw = m.group(i + 1) + cv = _CONV[self.convs[i]][1] + vals[self.names[i]] = cv(raw) if cv is not None else raw + for k, v in self.defaults.items(): + vals.setdefault(k, v) + return vals + + def build(self, values): + out = "" + used = set() + for p in self.parts: + if p[0] == "lit": + out += p[1] + else: + if p[2] not in values: + return None + v = values[p[2]] + used.add(p[2]) + if p[1] == "path": + out += quote(str(v), safe="/") + else: + out += quote(str(v), safe="") + return out, used + + +class _Routes: + """Decorators shared by Flask and Blueprint.""" + + def route(self, rule, **options): + def deco(f): + self.add_url_rule(rule, options.pop("endpoint", None), f, **options) + return f + return deco + + def get(self, rule, **o): + o["methods"] = ["GET"] + return self.route(rule, **o) + + def post(self, rule, **o): + o["methods"] = ["POST"] + return self.route(rule, **o) + + def put(self, rule, **o): + o["methods"] = ["PUT"] + return self.route(rule, **o) + + def delete(self, rule, **o): + o["methods"] = ["DELETE"] + return self.route(rule, **o) + + def patch(self, rule, **o): + o["methods"] = ["PATCH"] + return self.route(rule, **o) + + +class Blueprint(_Routes): + def __init__(self, name, import_name=None, url_prefix=None, **kw): + self.name = name + self.url_prefix = url_prefix + self._rules = [] + self._before = [] + self._after = [] + self._errors = [] + self._blueprints = [] + + def add_url_rule(self, rule, endpoint=None, view_func=None, **options): + if endpoint is None: + endpoint = view_func.__name__ + self._rules.append((rule, endpoint, view_func, options)) + + def before_request(self, f): + self._before.append(f) + return f + + def after_request(self, f): + self._after.append(f) + return f + + def errorhandler(self, code_or_exc): + def deco(f): + self._errors.append((code_or_exc, f)) + return f + return deco + + def register_blueprint(self, bp, **options): + self._blueprints.append((bp, options)) + + +def _join(prefix, rule): + if not prefix: + return rule + return prefix.rstrip("/") + "/" + rule.lstrip("/") if rule else prefix + + +class Flask(_Routes): + def __init__(self, import_name=None, static_folder=None, template_folder=None, **kw): + self.import_name = import_name + self.name = import_name + self.config = {"DEBUG": False, "TESTING": False} + self.debug = False + self.url_map = [] + self.view_functions = {} + self._before = [] + self._after = [] + self._teardown = [] + self._errors = {} # code -> handler + self._exc_errors = [] # (class, handler) + self._bp_before = {} + self._bp_after = {} + self._bp_errors = {} + self._sorted = None + self.json = None + + # -- registration + def add_url_rule(self, rule, endpoint=None, view_func=None, **options): + if endpoint is None: + endpoint = view_func.__name__ + methods = options.get("methods") + if methods is None: + methods = ["GET"] + methods = set(m.upper() for m in methods) + if "GET" in methods: + methods.add("HEAD") + explicit_options = "OPTIONS" in methods + methods.add("OPTIONS") + strict = options.get("strict_slashes") + r = Rule(rule, methods, endpoint, view_func, options.get("defaults"), + True if strict is None else strict) + r.auto_options = not explicit_options + self.url_map.append(r) + self.view_functions[endpoint] = view_func + self._sorted = None + + def register_blueprint(self, bp, url_prefix=None, name=None, **options): + self._register_bp(bp, url_prefix if url_prefix is not None else bp.url_prefix, name or bp.name) + + def _register_bp(self, bp, prefix, name): + for rule, ep, view, opts in bp._rules: + self.add_url_rule(_join(prefix, rule), name + "." + ep, view, **opts) + self._bp_before[name] = list(bp._before) + self._bp_after[name] = list(bp._after) + self._bp_errors[name] = list(bp._errors) + for sub, opts in bp._blueprints: + sp = opts.get("url_prefix", sub.url_prefix) + self._register_bp(sub, _join(prefix, sp or ""), name + "." + sub.name) + + def before_request(self, f): + self._before.append(f) + return f + + def after_request(self, f): + self._after.append(f) + return f + + def teardown_request(self, f): + self._teardown.append(f) + return f + + def errorhandler(self, code_or_exc): + def deco(f): + self.register_error_handler(code_or_exc, f) + return f + return deco + + def register_error_handler(self, code_or_exc, f): + if isinstance(code_or_exc, int): + self._errors[code_or_exc] = f + else: + self._exc_errors.append((code_or_exc, f)) + + # -- helpers + def make_response(self, rv): + return _make_response(self, rv) + + def test_client(self, **kw): + return FlaskClient(self) + + def test_request_context(self, path="/", method="GET", **kw): + return _TestContext(self, path, method) + + def app_context(self): + return _TestContext(self, "/", "GET") + + def url_for(self, endpoint, **values): + return _url_for(self, endpoint, values) + + def run(self, host="127.0.0.1", port=5000, debug=None, **kw): + import _cn1server + if debug is not None: + self.debug = bool(debug) + _cn1server.serve(self.dispatch, host, port) + + # -- request handling + def _rules_sorted(self): + if self._sorted is None: + self._sorted = sorted(self.url_map, key=lambda r: (r.nvars, -r.weight)) + return self._sorted + + def _route(self, req): + path = req.path + allowed = set() + redirect_rule = None + for r in self._rules_sorted(): + vals = r.match(path) + if vals is not None: + if req.method in r.methods: + return r, vals, None + allowed |= r.methods + elif r.slash and not path.endswith("/"): + if r.match(path + "/") is not None and redirect_rule is None: + if req.method in r.methods: + redirect_rule = r + else: + allowed |= r.methods + if redirect_rule is not None: + return None, None, ("redirect", redirect_rule) + if allowed: + return None, None, ("405", sorted(allowed)) + return None, None, ("404", None) + + def _find_handler(self, exc, bp): + names = [] + if bp: + parts = bp.split(".") + for i in range(len(parts), 0, -1): + names.append(".".join(parts[:i])) + for scope in names: + h = self._scoped_handler(self._bp_errors.get(scope, []), exc) + if h is not None: + return h + h = self._scoped_handler([(c, f) for c, f in self._errors.items()] + self._exc_errors, exc) + return h + + def _scoped_handler(self, entries, exc): + best = None + for key, f in entries: + if isinstance(key, int): + if isinstance(exc, HTTPException) and exc.code == key: + return f + elif isinstance(exc, key): + if best is None or issubclass(key, best[0]): + best = (key, f) + return best[1] if best else None + + def _handle_exception(self, exc, bp): + if isinstance(exc, HTTPException): + h = self._find_handler(exc, bp) + if h is None: + return exc.get_response() + return _make_response(self, h(exc)) + h = self._find_handler(exc, bp) + if h is not None: + return _make_response(self, h(exc)) + if self.config.get("PROPAGATE_EXCEPTIONS") or self.config.get("TESTING"): + raise exc + ise = InternalServerError() + ise.original_exception = exc + h = self._find_handler(ise, bp) + if h is not None: + return _make_response(self, h(ise)) + return ise.get_response() + + def full_dispatch(self, req): + bp = None + try: + rule, vals, fail = self._route(req) + rv = None + chain = [] + if fail is None: + req.url_rule = rule + req.endpoint = rule.endpoint + req.view_args = vals + bp = req.blueprint + if bp: + parts = bp.split(".") + for i in range(1, len(parts) + 1): + chain.append(".".join(parts[:i])) + for f in self._before: + rv = f() + if rv is not None: + break + if rv is None: + for sc in chain: + for f in self._bp_before.get(sc, []): + rv = f() + if rv is not None: + break + if rv is not None: + break + if rv is None and fail is not None: + kind, info = fail + if kind == "redirect": + loc = "http://" + req.host + quote(req.path) + "/" + q = req.query_string.decode("utf-8", "replace") + if q: + loc += "?" + q + return self._finish(redirect(loc, 308), None, []) + if kind == "405": + raise MethodNotAllowed(info) + raise NotFound() + if rv is None: + if req.method == "OPTIONS" and rule.auto_options: + resp = Response() + resp.headers["Allow"] = ", ".join(sorted(self._allowed_for(req.path))) + resp.headers["Content-Length"] = "0" + return self._finish(resp, bp, chain) + rv = rule.view(**vals) + resp = _make_response(self, rv) + except Exception as e: + resp = self._handle_exception(e, bp) + chain = [] + return self._finish(resp, bp, chain if bp else []) + + def _allowed_for(self, path): + out = set() + for r in self.url_map: + if r.match(path) is not None: + out |= r.methods + return out + + def _finish(self, resp, bp, chain): + try: + for sc in reversed(chain): + for f in reversed(self._bp_after.get(sc, [])): + r2 = f(resp) + if r2 is not None: + resp = r2 + for f in reversed(self._after): + r2 = f(resp) + if r2 is not None: + resp = r2 + except Exception as e: + resp = self._handle_exception(e, bp) + return resp + + def dispatch(self, method, path, query, headers, body): + method = method.upper() + path = unquote(path) + req = Request(method, path, query, headers, body, self) + ctx = _Ctx(self, req) + _stack.append(ctx) + try: + real = "GET" if method == "HEAD" else method + req.method = method + if method == "HEAD": + req.method = "HEAD" + resp = self.full_dispatch(req) + for f in self._teardown: + try: + f(None) + except Exception: + pass + finally: + _stack.pop() + if method == "HEAD": + hdrs = resp.headers.items() + return resp.status_code, hdrs, b"" + if "Content-Length" not in resp.headers and resp.status_code not in (204, 304): + resp.headers["Content-Length"] = str(len(resp._data)) + return resp.status_code, resp.headers.items(), resp._data + + +class _TestContext: + def __init__(self, app, path, method): + self.ctx = _Ctx(app, Request(method, path, "", [], b"", app)) + + def __enter__(self): + _stack.append(self.ctx) + return self + + def __exit__(self, *a): + _stack.pop() + return False + + +def _url_for(app, endpoint, values): + external = values.pop("_external", False) + values.pop("_scheme", None) + anchor = values.pop("_anchor", None) + values.pop("_method", None) + if endpoint.startswith("."): + cur = _stack[-1].request.blueprint if _stack else None + endpoint = (cur + endpoint) if cur else endpoint[1:] + best = None + for r in app.url_map: + if r.endpoint != endpoint: + continue + b = r.build(values) + if b is None: + continue + if best is None or len(b[1]) > len(best[1]): + best = b + if best is None: + raise BuildError(endpoint, values) + url, used = best + rest = [(k, v) for k, v in values.items() if k not in used and v is not None] + if rest: + url += "?" + urlencode(rest) + if anchor: + url += "#" + quote(str(anchor)) + if external: + host = _stack[-1].request.host if _stack else "localhost" + url = "http://" + host + url + return url + + +class BuildError(Exception): + def __init__(self, endpoint, values): + Exception.__init__(self, "Could not build url for endpoint %r with values %r." % (endpoint, sorted(values))) + + +def url_for(endpoint, **values): + if not _stack: + raise RuntimeError("Attempted to generate a URL without the application context being pushed.") + return _url_for(_stack[-1].app, endpoint, values) + + +# ---- test client --------------------------------------------------------- + +class FlaskClient: + def __init__(self, app): + self.application = app + self._cookies = {} + + def get(self, *a, **k): + return self.open(*a, method="GET", **k) + + def post(self, *a, **k): + return self.open(*a, method="POST", **k) + + def put(self, *a, **k): + return self.open(*a, method="PUT", **k) + + def delete(self, *a, **k): + return self.open(*a, method="DELETE", **k) + + def patch(self, *a, **k): + return self.open(*a, method="PATCH", **k) + + def head(self, *a, **k): + return self.open(*a, method="HEAD", **k) + + def options(self, *a, **k): + return self.open(*a, method="OPTIONS", **k) + + def set_cookie(self, key, value="", **kw): + self._cookies[key] = value + + def open(self, path="/", method="GET", json=None, data=None, query_string=None, headers=None, + content_type=None, follow_redirects=False, **kw): + query = "" + if "?" in path: + path, query = path.split("?", 1) + if "://" in path: + path = "/" + path.split("://", 1)[1].split("/", 1)[-1] if "/" in path.split("://", 1)[1] else "/" + if query_string is not None: + if isinstance(query_string, dict): + query_string = urlencode(query_string, doseq=True) + query = query_string + hdrs = Headers([("Host", "localhost")]) + if headers is not None: + for k, v in (headers.items() if hasattr(headers, "items") else headers): + hdrs[k] = v + body = b"" + if json is not None: + body = _to_json(json, True).encode("utf-8") + content_type = content_type or "application/json" + elif data is not None: + if isinstance(data, dict): + body = urlencode(data, doseq=True).encode("utf-8") + content_type = content_type or "application/x-www-form-urlencoded" + else: + body = _b(data) + if content_type: + hdrs["Content-Type"] = content_type + if body or method in ("POST", "PUT", "PATCH"): + hdrs["Content-Length"] = str(len(body)) + if self._cookies and "Cookie" not in hdrs: + hdrs["Cookie"] = "; ".join(k + "=" + v for k, v in self._cookies.items()) + status, rh, rb = self.application.dispatch(method, path, query, hdrs.items(), body) + resp = TestResponse(rb, status, rh) + for k, v in resp.headers.items(): + if k.lower() == "set-cookie": + kv = v.split(";", 1)[0] + if "=" in kv: + ck, cv = kv.split("=", 1) + if "max-age=0" in v.lower(): + self._cookies.pop(ck, None) + else: + self._cookies[ck] = cv + if follow_redirects and resp.status_code in (301, 302, 303, 307, 308): + loc = resp.headers.get("Location") + if "://" in loc: + loc = "/" + loc.split("://", 1)[1].split("/", 1)[1] if "/" in loc.split("://", 1)[1] else "/" + return self.open(loc, method="GET" if resp.status_code in (301, 302, 303) else method, + follow_redirects=True) + return resp + + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + +class TestResponse(Response): + __test__ = False + + def __init__(self, body, status, headers): + Response.__init__(self) + self.headers = Headers(headers) + self._data = bytes(body) + self.status_code = status diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/__init__.py new file mode 100644 index 00000000000..cd09c2c2aaf --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Clean-room subset of starlette: the pieces FastAPI is built from.""" diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/background.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/background.py new file mode 100644 index 00000000000..a37bc83c8fa --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/background.py @@ -0,0 +1,30 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Background tasks, run after the response has been produced.""" +import asyncio +import inspect + + +class BackgroundTask: + def __init__(self, func, *args, **kwargs): + self.func = func + self.args = args + self.kwargs = kwargs + + def __call__(self): + r = self.func(*self.args, **self.kwargs) + if inspect.iscoroutine(r): + asyncio.run(r) + + +class BackgroundTasks(BackgroundTask): + def __init__(self, tasks=None): + self.tasks = list(tasks) if tasks else [] + + def add_task(self, func, *args, **kwargs): + self.tasks.append(BackgroundTask(func, *args, **kwargs)) + + def __call__(self): + for t in self.tasks: + t() diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/datastructures.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/datastructures.py new file mode 100644 index 00000000000..f87ee22bae7 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/datastructures.py @@ -0,0 +1,238 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Headers, query parameters and URLs.""" +from urllib.parse import parse_qsl, quote, urlencode + + +class Headers: + """Case-insensitive, multi-valued, lower-cased header mapping.""" + + def __init__(self, headers=None): + self._list = [] + if headers is not None: + items = headers.items() if isinstance(headers, (dict, Headers)) else headers + for k, v in items: + self._list.append((str(k).lower(), str(v))) + + @property + def raw(self): + return [(k.encode("latin-1"), v.encode("latin-1")) for k, v in self._list] + + def get(self, key, default=None): + lk = key.lower() + for k, v in self._list: + if k == lk: + return v + return default + + def getlist(self, key): + lk = key.lower() + return [v for k, v in self._list if k == lk] + + def __getitem__(self, key): + lk = key.lower() + for k, v in self._list: + if k == lk: + return v + raise KeyError(key) + + def __contains__(self, key): + lk = str(key).lower() + for k, v in self._list: + if k == lk: + return True + return False + + def keys(self): + return [k for k, v in self._list] + + def values(self): + return [v for k, v in self._list] + + def items(self): + return list(self._list) + + def multi_items(self): + return list(self._list) + + def __iter__(self): + return iter(self.keys()) + + def __len__(self): + return len(self._list) + + def __eq__(self, other): + if isinstance(other, Headers): + return sorted(self._list) == sorted(other._list) + if isinstance(other, dict): + return sorted(self._list) == sorted((k.lower(), v) for k, v in other.items()) + return False + + def __repr__(self): + return "Headers(%r)" % (dict(self._list),) + + +class MutableHeaders(Headers): + def __setitem__(self, key, value): + lk = key.lower() + out = [] + done = False + for k, v in self._list: + if k == lk: + if not done: + out.append((lk, str(value))) + done = True + else: + out.append((k, v)) + if not done: + out.append((lk, str(value))) + self._list = out + + def __delitem__(self, key): + lk = key.lower() + self._list = [(k, v) for k, v in self._list if k != lk] + + def append(self, key, value): + self._list.append((key.lower(), str(value))) + + def setdefault(self, key, value): + if key not in self: + self[key] = value + return self[key] + + def update(self, other): + items = other.items() if hasattr(other, "items") else other + for k, v in items: + self[k] = v + + +class QueryParams: + """Immutable multi-dict of decoded query parameters; the last value wins on get.""" + + def __init__(self, query=""): + if isinstance(query, (bytes, bytearray)): + query = bytes(query).decode("latin-1") + if isinstance(query, str): + self._list = list(parse_qsl(query, keep_blank_values=True)) + elif hasattr(query, "multi_items"): + self._list = list(query.multi_items()) + elif isinstance(query, dict): + self._list = list(query.items()) + else: + self._list = list(query) + self._dict = {} + for k, v in self._list: + self._dict[k] = v + + def get(self, key, default=None): + return self._dict.get(key, default) + + def getlist(self, key): + return [v for k, v in self._list if k == key] + + def __getitem__(self, key): + return self._dict[key] + + def __contains__(self, key): + return key in self._dict + + def keys(self): + return list(self._dict.keys()) + + def values(self): + return list(self._dict.values()) + + def items(self): + return list(self._dict.items()) + + def multi_items(self): + return list(self._list) + + def __iter__(self): + return iter(list(self._dict.keys())) + + def __len__(self): + return len(self._dict) + + def __str__(self): + return urlencode(self._list) + + def __repr__(self): + return "QueryParams(%r)" % (str(self),) + + +class URL: + def __init__(self, url="", scheme="http", host="testserver", path="/", query=""): + if url: + rest = url + self.scheme = "http" + if "://" in rest: + self.scheme, rest = rest.split("://", 1) + if "/" in rest: + self.netloc, rest = rest.split("/", 1) + rest = "/" + rest + else: + self.netloc, rest = rest, "/" + else: + self.netloc = "" + self.fragment = "" + if "#" in rest: + rest, self.fragment = rest.split("#", 1) + self.query = "" + if "?" in rest: + rest, self.query = rest.split("?", 1) + self.path = rest + else: + self.scheme = scheme + self.netloc = host + self.path = path + self.query = query + self.fragment = "" + + @property + def hostname(self): + return self.netloc.split(":")[0] + + @property + def port(self): + if ":" in self.netloc: + return int(self.netloc.split(":")[1]) + return None + + def __str__(self): + s = "" + if self.netloc: + s = self.scheme + "://" + self.netloc + s += quote(self.path, safe="/%:@!$&'()*+,;=-._~") + if self.query: + s += "?" + self.query + if self.fragment: + s += "#" + self.fragment + return s + + def __repr__(self): + return "URL(%r)" % (str(self),) + + def __eq__(self, other): + return str(self) == str(other) + + def __hash__(self): + return hash(str(self)) + + +class State: + def __init__(self, state=None): + self.__dict__["_state"] = dict(state or {}) + + def __setattr__(self, key, value): + self._state[key] = value + + def __getattr__(self, key): + try: + return self.__dict__["_state"][key] + except KeyError: + raise AttributeError("'State' object has no attribute '%s'" % key) + + def __delattr__(self, key): + del self._state[key] diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/exceptions.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/exceptions.py new file mode 100644 index 00000000000..ae60dcd28b3 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/exceptions.py @@ -0,0 +1,41 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""HTTP exceptions.""" + +_PHRASES = { + 200: "OK", 201: "Created", 202: "Accepted", 204: "No Content", 301: "Moved Permanently", + 302: "Found", 303: "See Other", 304: "Not Modified", 307: "Temporary Redirect", + 308: "Permanent Redirect", 400: "Bad Request", 401: "Unauthorized", 402: "Payment Required", + 403: "Forbidden", 404: "Not Found", 405: "Method Not Allowed", 406: "Not Acceptable", + 408: "Request Timeout", 409: "Conflict", 410: "Gone", 411: "Length Required", + 412: "Precondition Failed", 413: "Content Too Large", 415: "Unsupported Media Type", + 418: "I'm a Teapot", 422: "Unprocessable Content", 429: "Too Many Requests", + 500: "Internal Server Error", 501: "Not Implemented", 502: "Bad Gateway", + 503: "Service Unavailable", 504: "Gateway Timeout", +} + + +def phrase(code): + return _PHRASES.get(code, "") + + +class HTTPException(Exception): + def __init__(self, status_code, detail=None, headers=None): + if detail is None: + detail = phrase(status_code) + self.status_code = status_code + self.detail = detail + self.headers = headers + + def __str__(self): + return "%d: %s" % (self.status_code, self.detail) + + def __repr__(self): + return "%s(status_code=%r, detail=%r)" % (type(self).__name__, self.status_code, self.detail) + + +class WebSocketException(Exception): + def __init__(self, code, reason=None): + self.code = code + self.reason = reason or "" diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/requests.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/requests.py new file mode 100644 index 00000000000..423481b7eab --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/requests.py @@ -0,0 +1,68 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""The request object handed to endpoints.""" +import json as _json + +from starlette.datastructures import Headers, QueryParams, URL, State + + +def _parse_cookies(header): + out = {} + if not header: + return out + for part in header.split(";"): + if "=" in part: + k, v = part.split("=", 1) + k = k.strip() + v = v.strip() + if len(v) >= 2 and v[0] == '"' and v[-1] == '"': + v = v[1:-1] + if k and k not in out: + out[k] = v + return out + + +class HTTPConnection: + def __init__(self, method, path, query, headers, body, app=None): + self.method = method + self._path = path + self._query = query or "" + self.headers = Headers(headers) + self._body = bytes(body or b"") + self.app = app + self.path_params = {} + self.state = State() + self.scope = {"type": "http", "method": method, "path": path} + self.client = ("testclient", 50000) + + @property + def url(self): + host = self.headers.get("host", "testserver") + return URL("http://" + host + self._path + ("?" + self._query if self._query else "")) + + @property + def base_url(self): + return URL("http://" + self.headers.get("host", "testserver") + "/") + + @property + def query_params(self): + return QueryParams(self._query) + + @property + def cookies(self): + return _parse_cookies(self.headers.get("cookie")) + + +class Request(HTTPConnection): + async def body(self): + return self._body + + async def json(self): + return _json.loads(self._body.decode("utf-8")) + + async def form(self): + return QueryParams(self._body.decode("utf-8")) + + async def is_disconnected(self): + return False diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/responses.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/responses.py new file mode 100644 index 00000000000..8e4a054d70a --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/responses.py @@ -0,0 +1,101 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Response classes.""" +import json as _json +from urllib.parse import quote + +from starlette.datastructures import MutableHeaders + + +class Response: + media_type = None + charset = "utf-8" + + def __init__(self, content=None, status_code=200, headers=None, media_type=None, background=None): + self.status_code = status_code + if media_type is not None: + self.media_type = media_type + self.background = background + self.body = self.render(content) + self.headers = MutableHeaders() + self._init_headers(headers) + + def render(self, content): + if content is None: + return b"" + if isinstance(content, (bytes, bytearray)): + return bytes(content) + return str(content).encode(self.charset) + + def _init_headers(self, headers): + user = headers or {} + items = user.items() if hasattr(user, "items") else user + have_len = False + have_type = False + for k, v in items: + self.headers.append(k, v) + if k.lower() == "content-length": + have_len = True + if k.lower() == "content-type": + have_type = True + body = self.body + if body is not None and not have_len and not (self.status_code < 200 or self.status_code in (204, 304)): + self.headers.append("content-length", str(len(body))) + ct = self.media_type + if ct is not None and not have_type: + if ct.startswith("text/") and "charset=" not in ct.lower(): + ct += "; charset=" + self.charset + self.headers.append("content-type", ct) + + def set_cookie(self, key, value="", max_age=None, expires=None, path="/", domain=None, + secure=False, httponly=False, samesite="lax"): + s = "%s=%s" % (key, quote(str(value), safe="!#$%&'()*+/:<=>?@[]^`{|}~")) + if max_age is not None: + s += "; Max-Age=%d" % int(max_age) + if path is not None: + s += "; Path=" + path + if domain is not None: + s += "; Domain=" + domain + if secure: + s += "; Secure" + if httponly: + s += "; HttpOnly" + if samesite is not None: + s += "; SameSite=" + samesite + self.headers.append("set-cookie", s) + + def delete_cookie(self, key, path="/", domain=None, secure=False, httponly=False, samesite="lax"): + self.set_cookie(key, "", max_age=0, path=path, domain=domain, secure=secure, + httponly=httponly, samesite=samesite) + + +class HTMLResponse(Response): + media_type = "text/html" + + +class PlainTextResponse(Response): + media_type = "text/plain" + + +class JSONResponse(Response): + media_type = "application/json" + + def render(self, content): + return _json.dumps(content, ensure_ascii=False, allow_nan=False, indent=None, + separators=(",", ":")).encode("utf-8") + + +class RedirectResponse(Response): + def __init__(self, url, status_code=307, headers=None, background=None): + Response.__init__(self, content=b"", status_code=status_code, headers=headers, + background=background) + self.headers["location"] = quote(str(url), safe=":/%#?=@[]!$&'()*+,;") + + +class StreamingResponse(Response): + def __init__(self, content, status_code=200, headers=None, media_type=None, background=None): + if hasattr(content, "__iter__") and not isinstance(content, (str, bytes)): + content = b"".join([c if isinstance(c, (bytes, bytearray)) else str(c).encode("utf-8") + for c in content]) + Response.__init__(self, content, status_code, headers, media_type, background) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/status.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/status.py new file mode 100644 index 00000000000..25ac9362a3f --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/status.py @@ -0,0 +1,43 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""HTTP status code constants.""" + +HTTP_100_CONTINUE = 100 +HTTP_101_SWITCHING_PROTOCOLS = 101 +HTTP_200_OK = 200 +HTTP_201_CREATED = 201 +HTTP_202_ACCEPTED = 202 +HTTP_203_NON_AUTHORITATIVE_INFORMATION = 203 +HTTP_204_NO_CONTENT = 204 +HTTP_205_RESET_CONTENT = 205 +HTTP_206_PARTIAL_CONTENT = 206 +HTTP_300_MULTIPLE_CHOICES = 300 +HTTP_301_MOVED_PERMANENTLY = 301 +HTTP_302_FOUND = 302 +HTTP_303_SEE_OTHER = 303 +HTTP_304_NOT_MODIFIED = 304 +HTTP_307_TEMPORARY_REDIRECT = 307 +HTTP_308_PERMANENT_REDIRECT = 308 +HTTP_400_BAD_REQUEST = 400 +HTTP_401_UNAUTHORIZED = 401 +HTTP_402_PAYMENT_REQUIRED = 402 +HTTP_403_FORBIDDEN = 403 +HTTP_404_NOT_FOUND = 404 +HTTP_405_METHOD_NOT_ALLOWED = 405 +HTTP_406_NOT_ACCEPTABLE = 406 +HTTP_408_REQUEST_TIMEOUT = 408 +HTTP_409_CONFLICT = 409 +HTTP_410_GONE = 410 +HTTP_411_LENGTH_REQUIRED = 411 +HTTP_412_PRECONDITION_FAILED = 412 +HTTP_413_REQUEST_ENTITY_TOO_LARGE = 413 +HTTP_415_UNSUPPORTED_MEDIA_TYPE = 415 +HTTP_418_IM_A_TEAPOT = 418 +HTTP_422_UNPROCESSABLE_ENTITY = 422 +HTTP_429_TOO_MANY_REQUESTS = 429 +HTTP_500_INTERNAL_SERVER_ERROR = 500 +HTTP_501_NOT_IMPLEMENTED = 501 +HTTP_502_BAD_GATEWAY = 502 +HTTP_503_SERVICE_UNAVAILABLE = 503 +HTTP_504_GATEWAY_TIMEOUT = 504 diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/testclient.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/testclient.py new file mode 100644 index 00000000000..1e874af6f1b --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/testclient.py @@ -0,0 +1,168 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""A test client that calls the app's pure dispatch core.""" +import json as _json +from urllib.parse import urlencode, urljoin + +from starlette.datastructures import Headers, MutableHeaders, URL +from starlette.exceptions import phrase + + +class TestResponse: + __test__ = False + + def __init__(self, status_code, headers, content, url): + self.status_code = status_code + self.headers = Headers(headers) + self.content = content + self.url = url + self.history = [] + self.encoding = "utf-8" + + @property + def text(self): + return self.content.decode("utf-8", "replace") + + @property + def reason_phrase(self): + return phrase(self.status_code) + + @property + def is_success(self): + return 200 <= self.status_code < 300 + + @property + def is_redirect(self): + return self.status_code in (301, 302, 303, 307, 308) and "location" in self.headers + + @property + def is_error(self): + return self.status_code >= 400 + + def json(self, **kw): + return _json.loads(self.content.decode("utf-8")) + + def raise_for_status(self): + if self.status_code >= 400: + raise RuntimeError("HTTP %d for url %s" % (self.status_code, self.url)) + return self + + def __repr__(self): + return "" % (self.status_code, self.reason_phrase) + + +class TestClient: + __test__ = False + + def __init__(self, app, base_url="http://testserver", raise_server_exceptions=True, + follow_redirects=True, headers=None, cookies=None, **kw): + self.app = app + self.base_url = base_url + self.raise_server_exceptions = raise_server_exceptions + self.follow_redirects = follow_redirects + self.headers = MutableHeaders({"accept": "*/*", "accept-encoding": "gzip, deflate", + "connection": "keep-alive", "user-agent": "testclient"}) + if headers: + self.headers.update(headers) + self.cookies = dict(cookies or {}) + self._entered = False + + def __enter__(self): + if hasattr(self.app, "_startup"): + self.app._startup() + self._entered = True + return self + + def __exit__(self, *a): + if hasattr(self.app, "_shutdown"): + self.app._shutdown() + return False + + def get(self, url, **kw): + return self.request("GET", url, **kw) + + def head(self, url, **kw): + return self.request("HEAD", url, **kw) + + def options(self, url, **kw): + return self.request("OPTIONS", url, **kw) + + def post(self, url, **kw): + return self.request("POST", url, **kw) + + def put(self, url, **kw): + return self.request("PUT", url, **kw) + + def patch(self, url, **kw): + return self.request("PATCH", url, **kw) + + def delete(self, url, **kw): + return self.request("DELETE", url, **kw) + + def request(self, method, url, params=None, headers=None, json=None, data=None, content=None, + cookies=None, follow_redirects=None, **kw): + if follow_redirects is None: + follow_redirects = self.follow_redirects + hist = [] + for _ in range(21): + resp = self._send(method, url, params, headers, json, data, content, cookies) + if follow_redirects and resp.is_redirect: + hist.append(resp) + url = urljoin(str(resp.url), resp.headers["location"]) + if resp.status_code in (301, 302, 303) and method != "HEAD": + method = "GET" + json = data = content = None + params = None + continue + resp.history = hist + return resp + raise RuntimeError("too many redirects") + + def _send(self, method, url, params, headers, json, data, content, cookies): + if "://" not in url: + url = urljoin(self.base_url.rstrip("/") + "/", url.lstrip("/") if not url.startswith("/") else url[1:]) + u = URL(url) + query = u.query + if params: + extra = urlencode(params, doseq=True) if isinstance(params, dict) else str(params) + query = extra if not query else query + "&" + extra + hdrs = MutableHeaders(self.headers.items()) + hdrs["host"] = u.netloc + if headers: + hdrs.update(headers) + body = b"" + if json is not None: + body = _json.dumps(json, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + hdrs["content-type"] = "application/json" + elif data is not None: + if isinstance(data, dict): + body = urlencode(data, doseq=True).encode("utf-8") + hdrs["content-type"] = "application/x-www-form-urlencoded" + elif isinstance(data, (bytes, str)): + body = data.encode("utf-8") if isinstance(data, str) else data + elif content is not None: + body = content.encode("utf-8") if isinstance(content, str) else bytes(content) + if body or method in ("POST", "PUT", "PATCH"): + hdrs["content-length"] = str(len(body)) + jar = dict(self.cookies) + if cookies: + jar.update(cookies) + if jar and "cookie" not in hdrs: + hdrs["cookie"] = "; ".join(k + "=" + v for k, v in jar.items()) + try: + status, rh, rb = self.app._dispatch_for_client(method, u.path, query, hdrs.items(), body, + self.raise_server_exceptions) + except Exception: + raise + resp = TestResponse(status, rh, rb, url if not query else url.split("?")[0] + "?" + query) + for k, v in resp.headers.multi_items(): + if k == "set-cookie": + kv = v.split(";", 1)[0] + if "=" in kv: + ck, cv = kv.split("=", 1) + if "max-age=0" in v.lower(): + self.cookies.pop(ck, None) + else: + self.cookies[ck] = cv + return resp diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/uvicorn/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/uvicorn/__init__.py new file mode 100644 index 00000000000..da44a319336 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/uvicorn/__init__.py @@ -0,0 +1,9 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Clean-room subset of uvicorn: run an app through the platform server.""" + + +def run(app, host="127.0.0.1", port=8000, **kw): + import _cn1server + _cn1server.serve(app.dispatch, host, port) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/werkzeug/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/werkzeug/__init__.py new file mode 100644 index 00000000000..c8d8e738269 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/werkzeug/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Clean-room subset of werkzeug (exceptions only).""" diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/werkzeug/exceptions.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/werkzeug/exceptions.py new file mode 100644 index 00000000000..c48e612d6d3 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/werkzeug/exceptions.py @@ -0,0 +1,141 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Clean-room subset of werkzeug: HTTP exceptions with werkzeug's default pages.""" + + +def _escape(s): + """markupsafe-style escaping (single quote is the decimal reference).""" + return (str(s).replace("&", "&").replace("<", "<").replace(">", ">") + .replace('"', """).replace("'", "'")) + +_DESCRIPTIONS = { + 301: ("Moved Permanently", "The resource has moved permanently."), + 302: ("Found", "The resource has moved temporarily."), + 304: ("Not Modified", ""), + 307: ("Temporary Redirect", "The resource has moved temporarily."), + 308: ("Permanent Redirect", "The resource has moved permanently."), + 400: ("Bad Request", "The browser (or proxy) sent a request that this server could not understand."), + 401: ("Unauthorized", "The server could not verify that you are authorized to access the URL requested. You either supplied the wrong credentials (e.g. a bad password), or your browser doesn't understand how to supply the credentials required."), + 403: ("Forbidden", "You don't have the permission to access the requested resource. It is either read-protected or not readable by the server."), + 404: ("Not Found", "The requested URL was not found on the server. If you entered the URL manually please check your spelling and try again."), + 405: ("Method Not Allowed", "The method is not allowed for the requested URL."), + 406: ("Not Acceptable", "The resource identified by the request is only capable of generating response entities which have content characteristics not acceptable according to the accept headers sent in the request."), + 408: ("Request Timeout", "The server closed the network connection because the browser didn't finish the request within the specified time."), + 409: ("Conflict", "A conflict happened while processing the request. The resource might have been modified while the request was being processed."), + 410: ("Gone", "The requested URL is no longer available on this server and there is no forwarding address. If you followed a link from a foreign page, please contact the author of this page."), + 411: ("Length Required", "A request with this method requires a valid Content-Length header."), + 412: ("Precondition Failed", "The precondition on the request for the URL failed positive evaluation."), + 413: ("Request Entity Too Large", "The data value transmitted exceeds the capacity limit."), + 415: ("Unsupported Media Type", "The server does not support the media type transmitted in the request."), + 418: ("I'm a teapot", "This server is a teapot, not a coffee machine"), + 422: ("Unprocessable Entity", "The request was well-formed but was unable to be followed due to semantic errors."), + 429: ("Too Many Requests", "This user has exceeded an allotted request count. Try again later."), + 500: ("Internal Server Error", "The server encountered an internal error and was unable to complete your request. Either the server is overloaded or there is an error in the application."), + 501: ("Not Implemented", "The server does not support the action requested by the browser."), + 502: ("Bad Gateway", "The proxy server received an invalid response from an upstream server."), + 503: ("Service Unavailable", "The server is temporarily unable to service your request due to maintenance downtime or capacity problems. Please try again later."), + 504: ("Gateway Timeout", "The connection to an upstream server timed out."), +} + + +class HTTPException(Exception): + code = None + description = None + + def __init__(self, description=None, response=None): + Exception.__init__(self) + if description is not None: + self.description = description + self.response = response + + @property + def name(self): + e = _DESCRIPTIONS.get(self.code) + return e[0] if e else "Unknown Error" + + def _desc(self): + d = self.description + if d is None: + e = _DESCRIPTIONS.get(self.code) + d = e[1] if e else "" + return d + + def get_description(self): + return "

" + _escape(self._desc()).replace("\n", "
\n") + "

" + + def get_body(self): + return ("\n\n%d %s\n

%s

\n%s\n" + % (self.code, self.name, self.name, self.get_description())) + + def get_headers(self): + return [("Content-Type", "text/html; charset=utf-8")] + + def get_response(self): + if self.response is not None: + return self.response + from flask import Response + return Response(self.get_body(), self.code, self.get_headers()) + + def __str__(self): + return "%d %s: %s" % (self.code, self.name, self._desc()) + + def __repr__(self): + return "<%s '%d: %s'>" % (type(self).__name__, self.code, self.name) + + +def _make(name, code): + return type(name, (HTTPException,), {"code": code}) + + +BadRequest = _make("BadRequest", 400) +Unauthorized = _make("Unauthorized", 401) +Forbidden = _make("Forbidden", 403) +NotFound = _make("NotFound", 404) +class MethodNotAllowed(HTTPException): + code = 405 + valid_methods = None + + def __init__(self, valid_methods=None, description=None, response=None): + HTTPException.__init__(self, description, response) + self.valid_methods = valid_methods + + def get_headers(self): + h = HTTPException.get_headers(self) + if self.valid_methods: + h.append(("Allow", ", ".join(self.valid_methods))) + return h + + +NotAcceptable = _make("NotAcceptable", 406) +RequestTimeout = _make("RequestTimeout", 408) +Conflict = _make("Conflict", 409) +Gone = _make("Gone", 410) +LengthRequired = _make("LengthRequired", 411) +PreconditionFailed = _make("PreconditionFailed", 412) +RequestEntityTooLarge = _make("RequestEntityTooLarge", 413) +UnsupportedMediaType = _make("UnsupportedMediaType", 415) +ImATeapot = _make("ImATeapot", 418) +UnprocessableEntity = _make("UnprocessableEntity", 422) +TooManyRequests = _make("TooManyRequests", 429) +InternalServerError = _make("InternalServerError", 500) +NotImplemented = _make("NotImplemented", 501) +BadGateway = _make("BadGateway", 502) +ServiceUnavailable = _make("ServiceUnavailable", 503) +GatewayTimeout = _make("GatewayTimeout", 504) + +default_exceptions = {} +for _c in (BadRequest, Unauthorized, Forbidden, NotFound, MethodNotAllowed, NotAcceptable, RequestTimeout, + Conflict, Gone, LengthRequired, PreconditionFailed, RequestEntityTooLarge, UnsupportedMediaType, + ImATeapot, UnprocessableEntity, TooManyRequests, InternalServerError, NotImplemented, BadGateway, + ServiceUnavailable, GatewayTimeout): + default_exceptions[_c.code] = _c + + +def abort(status, *args, **kwargs): + if isinstance(status, Exception): + raise status + cls = default_exceptions.get(status) + if cls is None: + raise LookupError("no exception for %r" % (status,)) + raise cls(*args, **kwargs) diff --git a/maven/python-transpiler/src/test/resources/conformance/fastapi_app.out b/maven/python-transpiler/src/test/resources/conformance/fastapi_app.out new file mode 100644 index 00000000000..996fb02afab --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/fastapi_app.out @@ -0,0 +1,198 @@ +200 +content-type: application/json +{"hello":"world"} +-- +200 +{"item_id":5,"q":"abc"} +-- +422 +{"detail":[{"type":"int_parsing","loc":["path","item_id"],"msg":"Input should be a valid integer, unable to parse string as an integer","input":"abc"}]} +-- +422 +{"detail":[{"type":"int_parsing","loc":["path","item_id"],"msg":"Input should be a valid integer, unable to parse string as an integer","input":"5.5"}]} +-- +200 +{"x":2.0} +-- +200 +{"color":"red","value":"red"} +-- +422 +{"detail":[{"type":"enum","loc":["path","c"],"msg":"Input should be 'red' or 'green'","input":"blue","ctx":{"expected":"'red' or 'green'"}}]} +-- +200 +{"path":"a/b/c.txt"} +-- +200 +{"q":"x","limit":3,"flag":true,"tags":["a","b"]} +-- +422 +{"detail":[{"type":"missing","loc":["query","q"],"msg":"Field required","input":null}]} +-- +422 +{"detail":[{"type":"int_parsing","loc":["query","limit"],"msg":"Input should be a valid integer, unable to parse string as an integer","input":"abc"},{"type":"bool_parsing","loc":["query","flag"],"msg":"Input should be a valid boolean, unable to interpret input","input":"maybe"}]} +-- +422 +{"detail":[{"type":"string_too_short","loc":["query","name"],"msg":"String should have at least 3 characters","input":"ab","ctx":{"min_length":3}},{"type":"greater_than_equal","loc":["query","n"],"msg":"Input should be greater than or equal to 1","input":"0","ctx":{"ge":1}}]} +-- +422 +{"detail":[{"type":"string_too_long","loc":["query","name"],"msg":"String should have at most 5 characters","input":"abcdefg","ctx":{"max_length":5}},{"type":"less_than_equal","loc":["query","n"],"msg":"Input should be less than or equal to 10","input":"11","ctx":{"le":10}}]} +-- +200 +{"name":"abc","n":1} +-- +422 +{"detail":[{"type":"greater_than","loc":["path","pid"],"msg":"Input should be greater than 0","input":"0","ctx":{"gt":0}}]} +-- +200 +{"pid":3,"q":"zz"} +-- +422 +{"detail":[{"type":"missing","loc":["header","x-api-key"],"msg":"Field required","input":null}]} +-- +200 +{"k":"k","c":"cc"} +-- +200 +{"x_token":"tok","ua":"ua1"} +-- +201 +{"name":"n","price":2.0,"tags":["a"],"address":{"city":"X","zip":null}} +-- +422 +{"detail":[{"type":"missing","loc":["body","price"],"msg":"Field required","input":{"name":"n"}}]} +-- +422 +{"detail":[{"type":"string_type","loc":["body","name"],"msg":"Input should be a valid string","input":5},{"type":"float_parsing","loc":["body","price"],"msg":"Input should be a valid number, unable to parse string as a number","input":"abc"},{"type":"missing","loc":["body","address","city"],"msg":"Field required","input":{}}]} +-- +422 +{"detail":[{"type":"model_attributes_type","loc":["body"],"msg":"Input should be a valid dictionary or object to extract fields from","input":[1]}]} +-- +422 +{"detail":[{"type":"missing","loc":["body"],"msg":"Field required","input":null}]} +-- +422 +{"detail":[{"type":"json_invalid","loc":["body",0],"msg":"JSON decode error","input":{},"ctx":{"error":"Expecting value"}}]} +-- +200 +{"item_id":7,"name":"z","notify":true,"city":"C"} +-- +422 +{"detail":[{"type":"int_parsing","loc":["path","item_id"],"msg":"Input should be a valid integer, unable to parse string as an integer","input":"x"}]} +-- +201 +{"username":"u","age":18} +-- +422 +{"detail":[{"type":"greater_than_equal","loc":["body","age"],"msg":"Input should be greater than or equal to 0","input":-1,"ctx":{"ge":0}}]} +-- +200 +{"name":"e","importance":5} +-- +200 +{"name":"e","importance":1} +-- +422 +{"detail":[{"type":"missing","loc":["body","item"],"msg":"Field required","input":null}]} +-- +200 +{"sum":6} +-- +422 +{"detail":[{"type":"int_parsing","loc":["body",1],"msg":"Input should be a valid integer, unable to parse string as an integer","input":"x"}]} +-- +200 +{"commons":{"skip":2,"limit":100},"token":"t"} +-- +['common', 'nested'] +200 +{"r":"res","log":["open","handler"]} +-- +['open', 'handler', 'close'] +200 +{"a":1} +-- +403 +x-err: 1 +{"detail":"custom detail"} +-- +400 +{"detail":{"a":1}} +-- +418 +{"message":"oops bob"} +-- +202 +x-a: b +content-type: application/json +{"a":1} +-- +200 +content-type: text/plain; charset=utf-8 +plain text +-- +200 +content-type: text/html; charset=utf-8 +

hi

+-- +204 + +-- +307 +location: / + +-- +200 +{"async":true} +-- +200 +{"d":"async-dep"} +-- +200 +{"method":"GET","path":"/req","q":{"a":"1","b":"2"},"ua":"testclient"} +-- +200 +{"name":"n","price":1.5} +-- +200 +[{"username":"a","age":1},{"username":"b","age":18}] +-- +200 +{"tid":3} +-- +201 +{"created":"t"} +-- +200 +"pong" +-- +['router_dep'] +404 +{"detail":"Not Found"} +-- +405 +allow: GET +{"detail":"Method Not Allowed"} +-- +405 +allow: GET +{"detail":"Method Not Allowed"} +-- +405 +allow: POST +{"detail":"Method Not Allowed"} +-- +200 +{"item_id":1,"q":null} +-- +307 +location: http://testserver/sub/ping + +-- +400 +{"custom":true,"n":1} +-- +{'title': 'Demo', 'version': '1.2.3'} +['/', '/async', '/async-dep', '/cls', '/color/{c}', '/constrained', '/deps', '/embed', '/files/{file_path}', '/float/{x}', '/headers', '/html', '/http-error-dict', '/http-error/{code}', '/items', '/items/{item_id}', '/json-response', '/list', '/model-list', '/myerror', '/needhdr', '/nocontent', '/pathc/{pid}', '/plain', '/redirect', '/req', '/search', '/sub/ping', '/unset', '/users', '/v1/things', '/v1/things/{tid}', '/yield'] +['Address', 'Body_embed_embed_post', 'Color', 'HTTPValidationError', 'Item', 'UserIn', 'UserOut', 'ValidationError'] +[{'name': 'item_id', 'in': 'path', 'required': True, 'schema': {'type': 'integer', 'title': 'Item Id'}}, {'name': 'q', 'in': 'query', 'required': False, 'schema': {'anyOf': [{'type': 'string'}, {'type': 'null'}], 'title': 'Q'}}] diff --git a/maven/python-transpiler/src/test/resources/conformance/fastapi_app.py b/maven/python-transpiler/src/test/resources/conformance/fastapi_app.py new file mode 100644 index 00000000000..c29dd7f1189 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/fastapi_app.py @@ -0,0 +1,363 @@ +from enum import Enum +from typing import Annotated, List, Optional + +from fastapi import (APIRouter, Body, Depends, FastAPI, Header, HTTPException, Path, Query, + Request, Response) +from fastapi.exceptions import RequestValidationError +from fastapi.responses import HTMLResponse, JSONResponse, PlainTextResponse, RedirectResponse +from fastapi.testclient import TestClient +from pydantic import BaseModel, Field + +app = FastAPI(title="Demo", version="1.2.3") +log = [] + + +class Color(str, Enum): + red = "red" + green = "green" + + +class Address(BaseModel): + city: str + zip: Optional[str] = None + + +class Item(BaseModel): + name: str + price: float + tags: List[str] = [] + address: Optional[Address] = None + + +class UserIn(BaseModel): + username: str + password: str + age: int = Field(default=18, ge=0) + + +class UserOut(BaseModel): + username: str + age: int + + +@app.get("/") +def root(): + return {"hello": "world"} + + +@app.get("/items/{item_id}") +def read_item(item_id: int, q: Optional[str] = None): + return {"item_id": item_id, "q": q} + + +@app.get("/float/{x}") +def read_float(x: float): + return {"x": x} + + +@app.get("/color/{c}") +def read_color(c: Color): + return {"color": c, "value": c.value} + + +@app.get("/files/{file_path:path}") +def read_file(file_path: str): + return {"path": file_path} + + +@app.get("/search") +def search(q: str, limit: int = 10, flag: bool = False, tags: List[str] = Query(default=[])): + return {"q": q, "limit": limit, "flag": flag, "tags": tags} + + +@app.get("/constrained") +def constrained(name: str = Query(min_length=3, max_length=5), n: int = Query(default=1, ge=1, le=10)): + return {"name": name, "n": n} + + +@app.get("/pathc/{pid}") +def pathc(pid: int = Path(gt=0, description="positive"), q: Annotated[Optional[str], Query(alias="query")] = None): + return {"pid": pid, "q": q} + + +@app.get("/headers") +def headers(x_token: Optional[str] = Header(default=None), user_agent: Optional[str] = Header(default=None)): + return {"x_token": x_token, "ua": user_agent} + + +@app.post("/items", status_code=201) +def create_item(item: Item): + return item + + +@app.put("/items/{item_id}") +def update_item(item_id: int, item: Item, notify: bool = False): + return {"item_id": item_id, "name": item.name, "notify": notify, + "city": item.address.city if item.address else None} + + +@app.post("/users", response_model=UserOut, status_code=201) +def create_user(user: UserIn): + return user + + +@app.post("/embed") +def embed(item: Item = Body(embed=True), importance: int = Body(default=1)): + return {"name": item.name, "importance": importance} + + +@app.post("/list") +def body_list(values: List[int]): + return {"sum": sum(values)} + + +def common(skip: int = 0, limit: int = 100): + log.append("common") + return {"skip": skip, "limit": limit} + + +def nested(commons: dict = Depends(common), token: Optional[str] = Header(default=None)): + log.append("nested") + return {"commons": commons, "token": token} + + +@app.get("/deps") +def deps(n: dict = Depends(nested)): + return n + + +def resource(): + log.append("open") + try: + yield "res" + finally: + log.append("close") + + +@app.get("/yield") +def use_yield(r: str = Depends(resource)): + log.append("handler") + return {"r": r, "log": list(log)} + + +class Fancy: + def __init__(self, a: int = 1): + self.a = a + + +@app.get("/cls") +def use_class(f: Fancy = Depends()): + return {"a": f.a} + + +@app.get("/http-error/{code}") +def http_error(code: int): + raise HTTPException(status_code=code, detail="custom detail", headers={"X-Err": "1"}) + + +@app.get("/http-error-dict") +def http_error_dict(): + raise HTTPException(status_code=400, detail={"a": 1}) + + +class MyError(Exception): + def __init__(self, name): + self.name = name + + +@app.exception_handler(MyError) +async def my_error_handler(request: Request, exc: MyError): + return JSONResponse(status_code=418, content={"message": "oops " + exc.name}) + + +@app.get("/myerror") +def myerror(): + raise MyError("bob") + + +@app.get("/json-response") +def json_response(): + return JSONResponse({"a": 1}, status_code=202, headers={"X-A": "b"}) + + +@app.get("/plain", response_class=PlainTextResponse) +def plain(): + return "plain text" + + +@app.get("/html", response_class=HTMLResponse) +def html(): + return "

hi

" + + +@app.get("/nocontent", status_code=204) +def nocontent(): + return Response(status_code=204) + + +@app.get("/redirect") +def redirect(): + return RedirectResponse("/", status_code=307) + + +@app.get("/async") +async def async_endpoint(): + return {"async": True} + + +async def async_dep(): + return "async-dep" + + +@app.get("/async-dep") +async def use_async_dep(d: str = Depends(async_dep)): + return {"d": d} + + +@app.get("/req") +def use_request(request: Request): + return {"method": request.method, "path": request.url.path, "q": dict(request.query_params), + "ua": request.headers.get("user-agent")} + + +@app.get("/unset", response_model=Item, response_model_exclude_unset=True) +def unset(): + return {"name": "n", "price": 1.5} + + +@app.get("/model-list", response_model=List[UserOut]) +def model_list(): + return [{"username": "a", "age": 1, "password": "x"}, UserIn(username="b", password="y")] + + +@app.get("/needhdr") +def needhdr(x_api_key: str = Header(), c: Optional[str] = Header(default=None, alias="x-c")): + return {"k": x_api_key, "c": c} + + +router = APIRouter(prefix="/v1", tags=["v1"]) + + +def router_dep(): + log.append("router_dep") + + +@router.get("/things/{tid}") +def get_thing(tid: int): + return {"tid": tid} + + +@router.post("/things", status_code=201) +def post_thing(item: Item): + return {"created": item.name} + + +app.include_router(router) +sub = APIRouter() + + +@sub.get("/ping") +def ping(): + return "pong" + + +app.include_router(sub, prefix="/sub", dependencies=[Depends(router_dep)]) + +client = TestClient(app) + + +def show(r, *hdrs): + print(r.status_code) + for h in hdrs: + v = r.headers.get(h) + if v is not None: + print(h + ":", v) + print(r.text) + print("--") + + +show(client.get("/"), "content-type") +show(client.get("/items/5?q=abc")) +show(client.get("/items/abc")) +show(client.get("/items/5.5")) +show(client.get("/float/2")) +show(client.get("/color/red")) +show(client.get("/color/blue")) +show(client.get("/files/a/b/c.txt")) +show(client.get("/search?q=x&limit=3&flag=true&tags=a&tags=b")) +show(client.get("/search")) +show(client.get("/search?q=x&limit=abc&flag=maybe")) +show(client.get("/constrained?name=ab&n=0")) +show(client.get("/constrained?name=abcdefg&n=11")) +show(client.get("/constrained?name=abc")) +show(client.get("/pathc/0")) +show(client.get("/pathc/3?query=zz")) +show(client.get("/needhdr")) +show(client.get("/needhdr", headers={"X-Api-Key": "k", "x-c": "cc"})) +show(client.get("/headers", headers={"X-Token": "tok", "User-Agent": "ua1"})) +show(client.post("/items", json={"name": "n", "price": 2, "tags": ["a"], "address": {"city": "X"}})) +show(client.post("/items", json={"name": "n"})) +show(client.post("/items", json={"name": 5, "price": "abc", "address": {}})) +show(client.post("/items", json=[1])) +show(client.post("/items")) +show(client.post("/items", content="not json", headers={"content-type": "application/json"})) +show(client.put("/items/7?notify=1", json={"name": "z", "price": 1, "address": {"city": "C"}})) +show(client.put("/items/x", json={"name": "z", "price": 1})) +show(client.post("/users", json={"username": "u", "password": "secret"})) +show(client.post("/users", json={"username": "u", "password": "secret", "age": -1})) +show(client.post("/embed", json={"item": {"name": "e", "price": 1}, "importance": 5})) +show(client.post("/embed", json={"item": {"name": "e", "price": 1}})) +show(client.post("/embed", json={"name": "e", "price": 1})) +show(client.post("/list", json=[1, 2, 3])) +show(client.post("/list", json=[1, "x"])) +show(client.get("/deps?skip=2", headers={"token": "t"})) +print(log) +log.clear() +show(client.get("/yield")) +print(log) +show(client.get("/cls")) +show(client.get("/http-error/403"), "x-err") +show(client.get("/http-error-dict")) +show(client.get("/myerror")) +show(client.get("/json-response"), "x-a", "content-type") +show(client.get("/plain"), "content-type") +show(client.get("/html"), "content-type") +show(client.get("/nocontent")) +show(client.get("/redirect", follow_redirects=False), "location") +show(client.get("/async")) +show(client.get("/async-dep")) +show(client.get("/req?a=1&b=2")) +show(client.get("/unset")) +show(client.get("/model-list")) +show(client.get("/v1/things/3")) +show(client.post("/v1/things", json={"name": "t", "price": 1})) +log.clear() +show(client.get("/sub/ping")) +print(log) +show(client.get("/nope")) +show(client.post("/"), "allow") +show(client.delete("/items/1"), "allow") +show(client.get("/items"), "allow") +show(client.get("/items/1/"), "location") +show(client.get("/sub/ping/", follow_redirects=False), "location") + + +app2 = FastAPI() + + +@app2.exception_handler(RequestValidationError) +async def validation_handler(request: Request, exc: RequestValidationError): + return JSONResponse(status_code=400, content={"custom": True, "n": len(exc.errors())}) + + +@app2.get("/x/{i}") +def x(i: int): + return i + + +show(TestClient(app2).get("/x/abc")) +schema = app.openapi() +print(schema["info"]) +print(sorted(schema["paths"])) +print(sorted(schema["components"]["schemas"])) +print(schema["paths"]["/items/{item_id}"]["get"]["parameters"]) diff --git a/maven/python-transpiler/src/test/resources/conformance/flask_app.out b/maven/python-transpiler/src/test/resources/conformance/flask_app.out new file mode 100644 index 00000000000..3899b3f01c4 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/flask_app.out @@ -0,0 +1,213 @@ +200 200 OK +content-type: text/html; charset=utf-8 +content-length: 13 +Hello, World! +-- +200 200 OK +User alice +-- +200 200 OK +content-type: application/json +{"id":42,"type":"int"} +-- +404 404 NOT FOUND + + +404 Not Found +

Not Found

+

The requested URL was not found on the server. If you entered the URL manually please check your spelling and try again.

+-- +200 200 OK +{"price":3.5} +-- +200 200 OK +file=a/b/c.txt +-- +200 200 OK +4 +-- +200 200 OK +{"args":{"page":"3","tag":"a","x":" y"},"method":"GET","page":3,"path":"/echo","tags":["a","b"]} +-- +200 200 OK +{"args":{"k":"v"},"method":"POST","page":1,"path":"/echo","tags":[]} +-- +200 200 OK +{"a":"1","b":"two words"} +-- +200 200 OK +{"got":{"x":[1,2],"y":"z"},"silent":true} +-- +400 400 BAD REQUEST + + +400 Bad Request +

Bad Request

+

The browser (or proxy) sent a request that this server could not understand.

+-- +415 415 UNSUPPORTED MEDIA TYPE + + +415 Unsupported Media Type +

Unsupported Media Type

+

Did not attempt to load JSON data because the request Content-Type was not 'application/json'.

+-- +200 200 OK +bytes +-- +200 200 OK +{"cookie":"zz","ct":null,"ua":"ua","x":"t1"} +-- +308 308 PERMANENT REDIRECT +location: http://localhost/slash/ + + +Redirecting... +

Redirecting...

+

You should be redirected automatically to the target URL: http://localhost/slash/. If not, click the link. +-- +200 200 OK +slash +-- +404 404 NOT FOUND + + +404 Not Found +

Not Found

+

The requested URL was not found on the server. If you entered the URL manually please check your spelling and try again.

+-- +308 308 PERMANENT REDIRECT +location: http://localhost/slash/?a=1 + + +Redirecting... +

Redirecting...

+

You should be redirected automatically to the target URL: http://localhost/slash/?a=1. If not, click the link. +-- +201 201 CREATED +created +-- +202 202 ACCEPTED +x-extra: yes +content-type: application/json +{"a":1} +-- +203 203 NON AUTHORITATIVE INFORMATION +x-foo: bar +content-type: text/plain; charset=utf-8 +set-cookie: sid=abc; Path=/ +custom +-- +200 200 OK +[1,2,{"a":null}] +-- +404 404 NOT FOUND + + +404 Not Found +

Not Found

+

The requested URL was not found on the server. If you entered the URL manually please check your spelling and try again.

+-- +403 403 FORBIDDEN + + +403 Forbidden +

Forbidden

+

You don't have the permission to access the requested resource. It is either read-protected or not readable by the server.

+-- +418 418 I'M A TEAPOT +I am a teapot: 418 +-- +400 400 BAD REQUEST + + +400 Bad Request +

Bad Request

+

The browser (or proxy) sent a request that this server could not understand.

+-- +500 500 INTERNAL SERVER ERROR + + +500 Internal Server Error +

Internal Server Error

+

The server encountered an internal error and was unable to complete your request. Either the server is overloaded or there is an error in the application.

+-- +500 500 INTERNAL SERVER ERROR + + +500 Internal Server Error +

Internal Server Error

+

The server encountered an internal error and was unable to complete your request. Either the server is overloaded or there is an error in the application.

+-- +400 400 BAD REQUEST +{"error":"bad value"} +-- +404 404 NOT FOUND + + +404 Not Found +

Not Found

+

The requested URL was not found on the server. If you entered the URL manually please check your spelling and try again.

+-- +404 404 NOT FOUND +content-type: application/json +{"error":"not found","path":"/api/nowhere"} +-- +405 405 METHOD NOT ALLOWED +allow: GET, HEAD, OPTIONS + + +405 Method Not Allowed +

Method Not Allowed

+

The method is not allowed for the requested URL.

+-- +405 405 METHOD NOT ALLOWED +allow: GET, HEAD, OPTIONS, POST, PUT + + +405 Method Not Allowed +

Method Not Allowed

+

The method is not allowed for the requested URL.

+-- +200 200 OK +allow: GET, HEAD, OPTIONS, POST, PUT + +-- +200 200 OK +content-type: text/html; charset=utf-8 + +-- +302 302 FOUND +location: /user/bob + + +Redirecting... +

Redirecting...

+

You should be redirected automatically to the target URL: /user/bob. If not, click the link. +-- +200 200 OK +User bob +-- +403 403 FORBIDDEN +x-seen: /blocked +blocked +-- +200 200 OK +x-seen: / +Hello, World! +-- +200 200 OK +{"endpoint":"api.thing","n":7,"url":"/api/v1/things/8"} +-- +200 200 OK +pong +-- +405 405 METHOD NOT ALLOWED +allow: OPTIONS, POST + + +405 Method Not Allowed +

Method Not Allowed

+

The method is not allowed for the requested URL.

+-- +/item/5 /echo?a=1&b=x+y /files/p/q diff --git a/maven/python-transpiler/src/test/resources/conformance/flask_app.py b/maven/python-transpiler/src/test/resources/conformance/flask_app.py new file mode 100644 index 00000000000..5bf3ade57c1 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/flask_app.py @@ -0,0 +1,228 @@ +from flask import (Flask, Blueprint, request, jsonify, make_response, abort, redirect, + url_for, g, Response) + +app = Flask(__name__) + + +@app.route("/") +def index(): + return "Hello, World!" + + +@app.route("/user/") +def user(name): + return "User " + name + + +@app.route("/item/") +def item(item_id): + return {"id": item_id, "type": type(item_id).__name__} + + +@app.route("/price/") +def price(p): + return jsonify(price=p) + + +@app.route("/files/") +def files(fp): + return "file=" + fp + + +@app.route("/uid/") +def uid(u): + return str(u.version) + + +@app.route("/echo", methods=["GET", "POST", "PUT"]) +def echo(): + return jsonify(method=request.method, path=request.path, args=request.args.to_dict(), + tags=request.args.getlist("tag"), page=request.args.get("page", 1, type=int)) + + +@app.post("/form") +def form(): + return jsonify(request.form.to_dict()) + + +@app.post("/json") +def js(): + data = request.get_json() + return jsonify(got=data, silent=request.get_json(silent=True) is not None) + + +@app.route("/raw", methods=["POST"]) +def raw(): + return request.data + + +@app.route("/hdr") +def hdr(): + return jsonify(ua=request.headers.get("user-agent"), x=request.headers["X-Token"], + cookie=request.cookies.get("sid"), ct=request.headers.get("Content-Type")) + + +@app.route("/slash/") +def slash(): + return "slash" + + +@app.route("/noslash") +def noslash(): + return "noslash" + + +@app.route("/tuple") +def tup(): + return "created", 201 + + +@app.route("/tuple3") +def tup3(): + return {"a": 1}, 202, {"X-Extra": "yes"} + + +@app.route("/resp") +def resp(): + r = make_response("custom", 203) + r.headers["X-Foo"] = "bar" + r.set_cookie("sid", "abc") + r.mimetype = "text/plain" + return r + + +@app.route("/lst") +def lst(): + return [1, 2, {"a": None}] + + +@app.route("/abort/") +def do_abort(code): + abort(code) + + +@app.route("/boom") +def boom(): + raise RuntimeError("boom") + + +@app.route("/redir") +def redir(): + return redirect(url_for("user", name="bob")) + + +@app.errorhandler(418) +def teapot(e): + return "I am a teapot: " + str(e.code), 418 + + +@app.errorhandler(ValueError) +def value_error(e): + return jsonify(error=str(e)), 400 + + +@app.route("/value") +def value(): + raise ValueError("bad value") + + +@app.errorhandler(404) +def not_found(e): + if request.path.startswith("/api"): + return jsonify(error="not found", path=request.path), 404 + return e + + +@app.before_request +def before(): + g.seen = request.path + if request.path == "/blocked": + return "blocked", 403 + + +@app.after_request +def after(resp): + resp.headers["X-Seen"] = g.seen + return resp + + +@app.route("/blocked") +def blocked(): + return "never" + + +bp = Blueprint("api", __name__, url_prefix="/api/v1") + + +@bp.route("/things/") +def thing(n): + return jsonify(n=n, endpoint=request.endpoint, url=url_for(".thing", n=n + 1)) + + +@bp.route("/ping", methods=["POST"]) +def ping(): + return "pong" + + +app.register_blueprint(bp) + + +def show(r, *hdrs): + print(r.status_code, r.status) + for h in hdrs: + v = r.headers.get(h) + if v is not None: + if h == "allow": + v = ", ".join(sorted(v.split(", "))) + print(h + ":", v) + print(r.get_data(as_text=True).rstrip("\n")) + print("--") + + +c = app.test_client() +show(c.get("/"), "content-type", "content-length") +show(c.get("/user/alice")) +show(c.get("/item/42"), "content-type") +show(c.get("/item/abc")) +show(c.get("/price/3.5")) +show(c.get("/files/a/b/c.txt")) +show(c.get("/uid/12345678-1234-4234-8234-123456789abc")) +show(c.get("/echo?page=3&tag=a&tag=b&x=%20y")) +show(c.post("/echo", query_string={"k": "v"})) +show(c.post("/form", data={"a": "1", "b": "two words"})) +show(c.post("/json", json={"x": [1, 2], "y": "z"})) +show(c.post("/json", data="not json", content_type="application/json")) +show(c.post("/json", data="x", content_type="text/plain")) +show(c.post("/raw", data=b"bytes")) +c.set_cookie("sid", "zz") +show(c.get("/hdr", headers={"X-Token": "t1", "User-Agent": "ua"})) +show(c.get("/slash"), "location") +show(c.get("/slash/")) +show(c.get("/noslash/")) +show(c.get("/slash?a=1"), "location") +show(c.get("/tuple")) +show(c.get("/tuple3"), "x-extra", "content-type") +show(c.get("/resp"), "x-foo", "content-type", "set-cookie") +show(c.get("/lst")) +show(c.get("/abort/404")) +show(c.get("/abort/403")) +show(c.get("/abort/418")) +show(c.get("/abort/400")) +show(c.get("/abort/500")) +show(c.get("/boom")) +show(c.get("/value")) +show(c.get("/nowhere")) +show(c.get("/api/nowhere"), "content-type") +show(c.post("/"), "allow") +show(c.delete("/echo"), "allow") +show(c.open("/echo", method="OPTIONS"), "allow") +show(c.head("/"), "content-type") +show(c.get("/redir"), "location") +show(c.get("/redir", follow_redirects=True)) +show(c.get("/blocked"), "x-seen") +show(c.get("/", headers={}), "x-seen") +show(c.get("/api/v1/things/7")) +show(c.post("/api/v1/ping")) +show(c.get("/api/v1/ping"), "allow") +with app.test_request_context("/"): + print(url_for("item", item_id=5), url_for("echo", a=1, b="x y"), url_for("files", fp="p/q")) From 9c6462d1f37fd5b2770053c44d7e4c543a8665e0 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:38:44 +0200 Subject: [PATCH 017/112] Python: co_flags marks generator, coroutine and async generator functions inspect.isgeneratorfunction/iscoroutinefunction/isasyncgenfunction now answer from __code__ as CPython's do; FastAPI's yield dependencies use it. Co-Authored-By: Claude Opus 5.5 --- .../com/codename1/python/runtime/PyCode.java | 2 +- .../codename1/python/runtime/PyFunction.java | 9 +++++++++ .../python/transpiler/codegen/Emitter.java | 5 +++++ .../codename1/python/site/fastapi/routing.py | 3 +-- .../com/codename1/python/stdlib/inspect.py | 4 ++++ .../test/resources/conformance/introspect.out | 5 +++++ .../test/resources/conformance/introspect.py | 20 +++++++++++++++++++ 7 files changed, 45 insertions(+), 3 deletions(-) diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyCode.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyCode.java index 62066460c81..154c45c88b0 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyCode.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyCode.java @@ -80,7 +80,7 @@ public Object getAttrOrNull(String attr) { case "co_qualname": return f.getAttrOrNull("__qualname__"); case "co_flags": - return Py.of((long) ((f.varArgs ? 0x04 : 0) | (f.varKw ? 0x08 : 0))); + return Py.of((long) ((f.varArgs ? 0x04 : 0) | (f.varKw ? 0x08 : 0) | f.kindFlags)); case "co_filename": return ""; default: diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyFunction.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyFunction.java index 924436f4e7a..2c11a404b57 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyFunction.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyFunction.java @@ -306,12 +306,21 @@ public static Object annotation(Body0 value, Object fallback) { } } + int kindFlags; + /// Sets `__qualname__` where it is not just the name. public PyFunction qual(String q) { qualname = q; return this; } + /// Marks a generator, coroutine or async generator function with the + /// `co_flags` bit CPython gives it (0x20, 0x80, 0x200). + public PyFunction kind(int flags) { + kindFlags = flags; + return this; + } + private String qn() { return qualname != null ? qualname : name; } diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Emitter.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Emitter.java index bedb0ad10c7..6591605468b 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Emitter.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Emitter.java @@ -1862,6 +1862,11 @@ private String function(String name, Arguments args, Scope fs, List body, if (!q.equals(name)) { r.append(".qual(").append(javaString(q)).append(")"); } + if (generator) { + // co_flags' CO_GENERATOR, CO_COROUTINE or CO_ASYNC_GENERATOR, + // which inspect reads. + r.append(".kind(").append(async && fs.yields ? 0x200 : async ? 0x80 : 0x20).append(")"); + } return r.toString(); } diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/routing.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/routing.py index 373848d0eef..e9f7cdf1270 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/routing.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/routing.py @@ -330,8 +330,7 @@ def _missing(ctx, loc): def _run_dep(sub, subvals, ctx): r = sub.call(**subvals) - # duck-typed: inspect.isgeneratorfunction is not reliable on the transpiler - if sub.is_gen or (hasattr(r, "__next__") and hasattr(r, "throw") and hasattr(r, "send")): + if sub.is_gen: r2 = next(r) ctx.cleanups.append(r) return r2 diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/inspect.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/inspect.py index 456726ee808..e0353e32b09 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/inspect.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/inspect.py @@ -310,6 +310,10 @@ def isgeneratorfunction(obj): return bool(getattr(obj, "__code__", None) and obj.__code__.co_flags & 0x20) +def isasyncgenfunction(obj): + return bool(getattr(obj, "__code__", None) and obj.__code__.co_flags & 0x200) + + def isawaitable(obj): return hasattr(obj, "__await__") diff --git a/maven/python-transpiler/src/test/resources/conformance/introspect.out b/maven/python-transpiler/src/test/resources/conformance/introspect.out index c39b738fe2b..580e02a00b5 100644 --- a/maven/python-transpiler/src/test/resources/conformance/introspect.out +++ b/maven/python-transpiler/src/test/resources/conformance/introspect.out @@ -8,3 +8,8 @@ True True __main__ frozen q frozen q +True False False +False True False +False False True +False False False +False False False diff --git a/maven/python-transpiler/src/test/resources/conformance/introspect.py b/maven/python-transpiler/src/test/resources/conformance/introspect.py index de9f0269239..7f2687c6379 100644 --- a/maven/python-transpiler/src/test/resources/conformance/introspect.py +++ b/maven/python-transpiler/src/test/resources/conformance/introspect.py @@ -42,3 +42,23 @@ def __delattr__(self, name): delattr(z, "q") except AttributeError as e: print(e) + + +def _gen(): + try: + yield 1 + finally: + pass + + +async def _coro(): + return 1 + + +async def _agen(): + yield 1 + + +for _f in (_gen, _coro, _agen, len, lambda: 0): + print(inspect.isgeneratorfunction(_f), inspect.iscoroutinefunction(_f), + inspect.isasyncgenfunction(_f) if hasattr(inspect, "isasyncgenfunction") else None) From 04f235d259a0bb9eaf74804a0ccf79c21b7d0f00 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:50:40 +0200 Subject: [PATCH 018/112] Python: Flask sample, CPython/PyPy comparison and developer guide chapter Co-Authored-By: Claude Opus 5.5 --- docs/developer-guide/Python-Interop.asciidoc | 231 ++++++++++++++++++ docs/developer-guide/developer-guide.asciidoc | 2 + .../samples/flask-todo/src/main.py | 56 +++++ scripts/python-compare/README.md | 29 +++ scripts/python-compare/RESULTS.md | 43 ++++ scripts/python-compare/compare.py | 212 ++++++++++++++++ scripts/python-compare/hello.py | 1 + 7 files changed, 574 insertions(+) create mode 100644 docs/developer-guide/Python-Interop.asciidoc create mode 100644 scripts/python-backend/samples/flask-todo/src/main.py create mode 100644 scripts/python-compare/README.md create mode 100644 scripts/python-compare/RESULTS.md create mode 100644 scripts/python-compare/compare.py create mode 100644 scripts/python-compare/hello.py diff --git a/docs/developer-guide/Python-Interop.asciidoc b/docs/developer-guide/Python-Interop.asciidoc new file mode 100644 index 00000000000..4ea77a2c4bf --- /dev/null +++ b/docs/developer-guide/Python-Interop.asciidoc @@ -0,0 +1,231 @@ +== Writing applications and backends in Python + +Python source builds as a Codename One application. The build translates each +module to Java, compiles it with the rest of the project, and from there it +goes wherever Java goes: iOS, Android, the JavaScript port, the desktop ports, +the simulator, and native server binaries built on the Codename One backend. + +No Python interpreter ships with the application. Each module becomes a Java +class, each function a method, and a small runtime supplies Python's object +model, its built-in types and the standard library modules applications use. +Annotated code is compiled to unboxed Java arithmetic, so a typed numeric loop +runs at the speed of the equivalent Java. + +Two things follow from compiling ahead of time rather than interpreting: + +* The program is known in full at build time. A module imported by a + name computed while the program runs, `exec`, `eval` and `compile` of a + string, frame introspection and C extension modules are build errors that + name the file and the line. None of them reaches a device as a hidden failure. +* Packages that CPython implements in C -- `numpy`, `pydantic`'s core, + `yaml`'s scanner -- are provided as Codename One implementations of the same + public API. Pure Python packages are translated like your own code. + +=== Adding Python to a project + +Put the sources in `common/src/main/python`, or import an existing Python +project into a Codename One application: + +[source,bash] +---- +mvn cn1:import-python-project -Dsource=/path/to/my-python-app +---- + +The import copies the `.py` files, adds the Python runtime dependency and the +`transpile-python` goal to the common module, records the entry module and +makes the application's main class run it. Build and run the project as usual +afterwards; the translation runs in `generate-sources`. Gradle projects get the +same step as the `transpilePython` task. + +The goal reads these properties: + +[cols="1,3",options="header"] +|=== +| Property | Meaning +| `cn1.python.sourceDir` | The Python sources, `src/main/python` by default. +| `cn1.python.mainModule` | The module run as `+__main__+` when the application starts. +| `cn1.python.outputDir` | Where the generated Java goes. +| `cn1.python.skip` | `true` skips the step. +|=== + +=== Calling Codename One and Java from Python + +Any class on the project's class path is importable under the `java.` prefix: + +[source,python] +---- +from java.com.codename1.ui import Form, Button, Label, TextField, Dialog +from java.com.codename1.ui.layouts import BoxLayout + + +def main(): + form = Form("Hello Python", BoxLayout.y()) + name = TextField("", "Your name") + out = Label("Hello, world!") + + def on_greet(event): + out.setText(f"Hello, {name.getText().strip()}!") + form.revalidate() + + button = Button("Greet") + button.addActionListener(on_greet) + form.add(name).add(button).add(out) + form.show() + + +main() +---- + +A Python function or lambda passed where Java expects a listener or another +single-method interface becomes an implementation of that interface. Overloads +are chosen at build time when the argument types are known, and otherwise by +the runtime from the values passed. + +Values cross the boundary as follows: + +[cols="1,1",options="header"] +|=== +| Python | Java +| `int` | `long`, or `BigInteger` past 64 bits +| `float` | `double` +| `bool` | `boolean` +| `str` | `String` +| `bytes` | `byte[]` +| `None` | `null` +| `list`, `dict` | `java.util.List`, `java.util.Map` +|=== + +Java collections behave as their Python counterparts in Python code: `len`, +indexing, `in` and iteration work on an `ArrayList` or a `HashMap`, and a +Java exception arrives as a Python exception whose message names the Java +type. + +To call into Python from Java, import the module through the runtime and call +its attributes: + +[source,java] +---- +PythonModules.registerAll(); +PyModule pricing = Py.importModule("pricing"); +Object total = Py.call(Py.getattr(pricing, "total"), cart); +---- + +=== Libraries + +The standard library modules applications use most are included: `json`, `re`, +`datetime`, `decimal`, `fractions`, `collections`, `itertools`, `functools`, +`dataclasses`, `enum`, `typing`, `inspect`, `asyncio`, `logging`, `pathlib`, +`os.path`, `csv`, `struct`, `hashlib`, `base64`, `uuid`, `random`, +`statistics`, `heapq`, `bisect`, `textwrap`, `string`, `tomllib` and others. + +These third party packages are provided, each tested against the output of +the real package: + +[cols="1,3",options="header"] +|=== +| Package | Notes +| `requests`, `httpx` | Over Codename One's network stack. +| `pydantic` | Models, fields, validators, `TypeAdapter`, JSON round trips. +| `attrs`, `python-dateutil`, `PyYAML` | +| `numpy` | `ndarray` of `float64`, `int64` and `bool` with broadcasting, + views, fancy indexing, ufuncs, reductions, sorting and `numpy.linalg`. The + kernels are Java loops over primitive arrays. `numpy.random` draws from + Python's generator, so its streams differ from NumPy's. +| `flask`, `fastapi`, `uvicorn` | See <>. +| `typing_extensions`, `six`, `tabulate`, `idna`, `packaging` | +|=== + +Import other pure Python packages by copying their sources into the source +directory; they translate like the application's own modules. + +[[python-backends]] +=== Python backends + +A Flask or FastAPI application builds into a single native server executable. +The application starts the server the way it always does: + +[source,python] +---- +from flask import Flask, jsonify + +app = Flask(__name__) + + +@app.get("/fib/") +def fib(n): + a, b = 0, 1 + for _ in range(n): + a, b = b, a + b + return jsonify({"n": n, "fib": str(a)}) + + +if __name__ == "__main__": + app.run(host="0.0.0.0", port=8080) +---- + +`app.run()` and `uvicorn.run(app)` are lowered onto the Codename One backend's +HTTP server, so the same file still runs under CPython for development: + +[source,bash] +---- +scripts/python-backend/build.sh scripts/python-backend/samples/flask-todo ./todo-server +./todo-server +---- + +Routing, converters, JSON bodies, `abort`, error handlers and the default +error pages of Flask, and FastAPI's path, query, body and header parameters, +dependencies, `response_model` and OpenAPI generation are supported. Flask +sessions, templates, file uploads and streaming responses aren't, nor are +FastAPI WebSockets and middleware. + +=== Compared with CPython and PyPy + +The same programs, run on one Apple silicon Mac, with +`scripts/python-compare/compare.py`. Codename One rows are native binaries. + +Startup, a program that prints one line: + +[cols="1,1,1",options="header"] +|=== +| | Wall time (ms) | Peak memory (MB) +| CPython 3.14 | 21.2 | 14.4 +| PyPy 3.11 | 20.4 | 29.2 +| Codename One | 7.6 | 8.7 +|=== + +CPU kernels, fastest round in milliseconds: + +[cols="1,1,1,1",options="header"] +|=== +| Kernel | CPython 3.14 | PyPy 3.11 | Codename One +| nbody | 785.9 | 63.7 | 90.1 +| richards | 653.9 | 40.3 | 44.7 +| fannkuch | 2691.2 | 612.5 | 731.8 +| spectral_norm | 734.5 | 11.7 | 11.1 +| dicts_strings | 397.5 | 145.0 | 211.2 +|=== + +PyPy's JIT and Codename One's ahead of time compilation land close to each +other, from twice to sixty times as fast as CPython. Codename One gets there +without a warm-up and with a third of PyPy's memory on the numeric kernels. + +The Flask sample above, measured with `ab -n 20000 -c 8`: + +[cols="1,1,1,1",options="header"] +|=== +| | First response (ms) | Requests per second | Deployed size (MB) +| CPython 3.14 | 97 | 687 | 82.0 +| PyPy 3.11 | 200 | 674 | 137.7 +| Codename One | 61 | 16,332 | 22.7 +|=== + +The deployed size is the interpreter's installation plus Flask and its +dependencies, against the one executable. CPython and PyPy serve with +Werkzeug's threaded server, which is what `app.run()` starts; a production +CPython deployment behind a WSGI server with several worker processes serves +more, using a full interpreter's memory per worker. The Codename One server +idles at 27 MB and its heap grew to 395 MB under this load, where it stayed. + +Where an application needs CPython instead: packages with C extensions that +have no Codename One implementation (for example `pandas`, `scipy` or `torch`), +and code that builds source text and runs it with `exec`. diff --git a/docs/developer-guide/developer-guide.asciidoc b/docs/developer-guide/developer-guide.asciidoc index 1c4157334ae..81aeef4b512 100644 --- a/docs/developer-guide/developer-guide.asciidoc +++ b/docs/developer-guide/developer-guide.asciidoc @@ -278,6 +278,8 @@ include::Android-Interop.asciidoc[] include::Unity-Interop.asciidoc[] +include::Python-Interop.asciidoc[] + include::Desktop-Interop.asciidoc[] include::Working-with-Mac-OS-X.asciidoc[] diff --git a/scripts/python-backend/samples/flask-todo/src/main.py b/scripts/python-backend/samples/flask-todo/src/main.py new file mode 100644 index 00000000000..0a59bb2649b --- /dev/null +++ b/scripts/python-backend/samples/flask-todo/src/main.py @@ -0,0 +1,56 @@ +# A small JSON API in plain Flask. Built by scripts/python-backend/build.sh +# into one native executable that serves through the backend's HTTP server; +# `python3 main.py` with Flask installed runs the same file unchanged. +from flask import Flask, jsonify, request, abort + +app = Flask(__name__) +todos = {} +next_id = [1] + + +@app.get("/todos") +def list_todos(): + done = request.args.get("done") + items = list(todos.values()) + if done is not None: + items = [t for t in items if t["done"] == (done == "true")] + return jsonify(items) + + +@app.post("/todos") +def create_todo(): + body = request.get_json() + if not body or "title" not in body: + abort(400) + todo = {"id": next_id[0], "title": body["title"], "done": False} + todos[todo["id"]] = todo + next_id[0] += 1 + return jsonify(todo), 201 + + +@app.put("/todos/") +def update_todo(todo_id): + todo = todos.get(todo_id) + if todo is None: + abort(404) + todo.update(request.get_json() or {}) + return jsonify(todo) + + +@app.delete("/todos/") +def delete_todo(todo_id): + if todos.pop(todo_id, None) is None: + abort(404) + return "", 204 + + +@app.get("/fib/") +def fib(n): + a, b = 0, 1 + for _ in range(n): + a, b = b, a + b + return jsonify({"n": n, "fib": str(a)}) + + +if __name__ == "__main__": + app.run(host="127.0.0.1", port=8080) diff --git a/scripts/python-compare/README.md b/scripts/python-compare/README.md new file mode 100644 index 00000000000..590e64b70c5 --- /dev/null +++ b/scripts/python-compare/README.md @@ -0,0 +1,29 @@ +# Codename One Python vs CPython, PyPy and GraalPy + +`compare.py` runs the same programs on each runtime and writes a Markdown +table; `RESULTS.md` is its last output. Every cell is a measurement on the +machine that ran it. + +Rows: + +- **Startup**: `hello.py`, best of ten. +- **CPU kernels**: `scripts/python-bench/kernels`, the fastest round. +- **Flask JSON API**: `scripts/python-backend/samples/flask-todo` under `ab`. +- **What you deploy**: interpreter install plus Flask, against one binary. + +Build the Codename One binaries first: + +```bash +for k in nbody richards fannkuch spectral_norm dicts_strings; do + scripts/python-bench/build_parparvm.sh $k +done +PY_SOURCES=scripts/python-compare scripts/python-bench/build_parparvm.sh hello +scripts/python-backend/build.sh scripts/python-backend/samples/flask-todo /tmp/flask-todo +python3 scripts/python-compare/compare.py \ + --cpython --pypy [--graalpy ] \ + --cn1-kernels scripts/python-bench/target --cn1-hello scripts/python-bench/target/hello/hello \ + --cn1-server /tmp/flask-todo --out scripts/python-compare/RESULTS.md +``` + +GraalPy is an optional arm (`--graalpy`). It is not in `RESULTS.md` because +it was not installed on the machine that produced it. diff --git a/scripts/python-compare/RESULTS.md b/scripts/python-compare/RESULTS.md new file mode 100644 index 00000000000..6b1cce63a22 --- /dev/null +++ b/scripts/python-compare/RESULTS.md @@ -0,0 +1,43 @@ +# Codename One Python vs CPython, PyPy + +Measured on Darwin arm64, 2026-10-27. Codename One rows are native binaries (Python -> Java -> ParparVM C -> clang -O3). + +- CPython: `python3.14 (venv with Flask)` (3.14.5 cpython) +- PyPy: `~/.pyenv/versions/pypy3.11-7.3.19/bin/pypy3` (3.11.11 pypy) + +## Startup: `print('hello')` + +| | wall ms (best of 10) | peak RSS MB | +|---|---:|---:| +| CPython | 21.2 | 14.4 | +| PyPy | 20.4 | 29.2 | +| Codename One | 7.6 | 8.7 | + +## CPU kernels (fastest of the kernel's rounds, ms; peak RSS MB) + +| kernel | CPython ms | CPython MB | PyPy ms | PyPy MB | Codename One ms | Codename One MB | +|---|---:|---:|---:|---:|---:|---:| +| nbody | 785.9 | 15.5 | 63.7 | 32.6 | 90.1 | 10.3 | +| richards | 653.9 | 15.3 | 40.3 | 34.0 | 44.7 | 10.6 | +| fannkuch | 2691.2 | 14.6 | 612.5 | 32.8 | 731.8 | 10.2 | +| spectral_norm | 734.5 | 14.8 | 11.7 | 32.2 | 11.1 | 10.3 | +| dicts_strings | 397.5 | 62.2 | 145.0 | 96.7 | 211.2 | 128.9 | + +## Flask JSON API (`scripts/python-backend/samples/flask-todo`) + +`ab -n 20000 -c 8 /fib/30` after a 2000-request warm-up. CPython, PyPy and GraalPy serve with `app.run()` (Werkzeug's threaded server), Codename One with the same call, which it lowers onto the native backend HTTP server. + +| | first response ms | idle RSS MB | RSS after load MB | req/s | p99 ms | +|---|---:|---:|---:|---:|---:| +| CPython | 97 | 39.3 | 24.5 | 687 | 16 | +| PyPy | 200 | 64.9 | 86.5 | 674 | 18 | +| Codename One | 61 | 26.7 | 390.4 | 16332 | 1 | + +## What you deploy + +| | size MB | what | +|---|---:|---| +| CPython | 82.0 | interpreter install (`/opt/homebrew/opt/python@3.14/Frameworks/Python.framework/Versions/3.14`) + Flask and its dependencies (5.2) | +| PyPy | 137.7 | interpreter install (`~/.pyenv/versions/pypy3.11-7.3.19`) + Flask and its dependencies (4.9) | +| Codename One | 22.7 | one self-contained executable, Flask included | + diff --git a/scripts/python-compare/compare.py b/scripts/python-compare/compare.py new file mode 100644 index 00000000000..3d579546939 --- /dev/null +++ b/scripts/python-compare/compare.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python3 +# Measures Codename One's native Python against CPython and PyPy (and GraalPy +# when one is given) on the same programs, on this machine, and writes the +# numbers as Markdown. Nothing here is estimated: every cell is a run. +# +# compare.py --cn1-kernels DIR --cn1-server BIN [--pypy P] [--graalpy G] +# [--cpython C] [--out report.md] +# +# --cn1-kernels is scripts/python-bench/target (build_parparvm.sh output, +# one / binary each); --cn1-server is the flask-todo sample +# built by scripts/python-backend/build.sh. CPython, PyPy and GraalPy need +# Flask installed for the server rows (a row is skipped without it). +import argparse +import os +import re +import shutil +import subprocess +import sys +import time +import urllib.request + +HERE = os.path.dirname(os.path.abspath(__file__)) +REPO = os.path.dirname(os.path.dirname(HERE)) +KERNELS = os.path.join(REPO, "scripts", "python-bench", "kernels") +SERVER_APP = os.path.join(REPO, "scripts", "python-backend", "samples", "flask-todo", "src", "main.py") +KERNEL_NAMES = ["nbody", "richards", "fannkuch", "spectral_norm", "dicts_strings"] +PORT = 8080 + + +def timed(cmd, cwd=None): + """Runs cmd under /usr/bin/time -l; answers (stdout, wall s, peak RSS MB).""" + t0 = time.perf_counter() + p = subprocess.run(["/usr/bin/time", "-l"] + cmd, cwd=cwd, capture_output=True, text=True) + wall = time.perf_counter() - t0 + if p.returncode != 0: + raise RuntimeError("%s failed: %s" % (cmd, p.stderr[-500:])) + m = re.search(r"(\d+)\s+maximum resident set size", p.stderr) + return p.stdout, wall, int(m.group(1)) / (1024.0 * 1024.0) + + +def du_mb(path): + total = 0 + if os.path.isfile(path): + return os.path.getsize(path) / 1e6 + for root, _, files in os.walk(path): + for f in files: + fp = os.path.join(root, f) + if not os.path.islink(fp): + total += os.path.getsize(fp) + return total / 1e6 + + +def interpreter_home(exe): + out = subprocess.run([exe, "-c", "import sys; print(sys.base_prefix)"], + capture_output=True, text=True).stdout.strip() + return out + + +def has_flask(exe): + return subprocess.run([exe, "-c", "import flask"], capture_output=True).returncode == 0 + + +def flask_size(exe): + code = ("import os, flask, werkzeug, jinja2, markupsafe, itsdangerous, click, blinker\n" + "print('\\n'.join(os.path.dirname(m.__file__) for m in " + "(flask, werkzeug, jinja2, markupsafe, itsdangerous, click, blinker)))") + out = subprocess.run([exe, "-c", code], capture_output=True, text=True).stdout.split() + return sum(du_mb(d) for d in out) + + +def kernel_rows(arms, cn1_dir): + rows = [] + for k in KERNEL_NAMES: + row = {"kernel": k} + for name, exe in arms: + out, wall, rss = timed([exe, os.path.join(KERNELS, k + ".py")], cwd=KERNELS) + row[name] = (best_ms(out), rss) + out, wall, rss = timed([os.path.join(cn1_dir, k, k)]) + row["Codename One"] = (best_ms(out), rss) + rows.append(row) + return rows + + +def best_ms(out): + m = re.search(r"^BENCH \S+ (\d+)", out, re.M) + return int(m.group(1)) / 1000.0 + + +def wait_up(url, deadline): + while time.perf_counter() < deadline: + try: + urllib.request.urlopen(url, timeout=0.5).read() + return True + except Exception: + time.sleep(0.005) + return False + + +def rss_of(pid): + out = subprocess.run(["ps", "-o", "rss=", "-p", str(pid)], capture_output=True, text=True).stdout + return int(out.strip() or 0) / 1024.0 + + +def server_row(cmd, env=None): + url = "http://127.0.0.1:%d/todos" % PORT + t0 = time.perf_counter() + p = subprocess.Popen(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, env=env) + try: + if not wait_up(url, t0 + 60): + raise RuntimeError("server did not start: %s" % cmd) + first = time.perf_counter() - t0 + idle = rss_of(p.pid) + # Warm up, then the measured run; one connection at a time per + # client, keep-alive off, the same for every server. + ab = ["ab", "-q", "-n", "20000", "-c", "8", "http://127.0.0.1:%d/fib/30" % PORT] + subprocess.run(ab[:2] + ["-n", "2000"] + ab[4:], capture_output=True) + out = subprocess.run(ab, capture_output=True, text=True).stdout + rps = float(re.search(r"Requests per second:\s+([\d.]+)", out).group(1)) + p99 = re.search(r"\s99%\s+(\d+)", out) + loaded = rss_of(p.pid) + return first * 1000.0, idle, loaded, rps, int(p99.group(1)) if p99 else None + finally: + p.terminate() + try: + p.wait(5) + except subprocess.TimeoutExpired: + p.kill() + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--cpython", default=shutil.which("python3")) + ap.add_argument("--pypy") + ap.add_argument("--graalpy") + ap.add_argument("--cn1-kernels", required=True) + ap.add_argument("--cn1-hello", help="native binary of hello.py, for the startup row") + ap.add_argument("--cn1-server", required=True) + ap.add_argument("--out") + a = ap.parse_args() + arms = [("CPython", a.cpython)] + if a.pypy: + arms.append(("PyPy", a.pypy)) + if a.graalpy: + arms.append(("GraalPy", a.graalpy)) + names = [n for n, _ in arms] + ["Codename One"] + lines = [] + w = lines.append + w("# Codename One Python vs %s\n" % ", ".join(n for n, _ in arms)) + w("Measured on %s, %s. Codename One rows are native binaries " + "(Python -> Java -> ParparVM C -> clang -O3).\n" % ( + subprocess.run(["uname", "-sm"], capture_output=True, text=True).stdout.strip(), + time.strftime("%Y-%m-%d"))) + for n, exe in arms: + v = subprocess.run([exe, "-c", "import sys; print(sys.version.split()[0], sys.implementation.name)"], + capture_output=True, text=True).stdout.strip() + w("- %s: `%s` (%s)" % (n, exe, v)) + w("") + + hello = os.path.join(HERE, "hello.py") + w("## Startup: `print('hello')`\n") + w("| | wall ms (best of 10) | peak RSS MB |") + w("|---|---:|---:|") + startup = [(n, [exe, hello]) for n, exe in arms] + if a.cn1_hello: + startup.append(("Codename One", [a.cn1_hello])) + for n, cmd in startup: + runs = [timed(cmd) for _ in range(10)] + w("| %s | %.1f | %.1f |" % (n, min(r[1] for r in runs) * 1000, min(r[2] for r in runs))) + w("") + + w("## CPU kernels (fastest of the kernel's rounds, ms; peak RSS MB)\n") + w("| kernel | " + " | ".join("%s ms | %s MB" % (n, n) for n in names) + " |") + w("|---|" + "---:|---:|" * len(names)) + for r in kernel_rows(arms, a.cn1_kernels): + w("| %s | " % r["kernel"] + " | ".join("%.1f | %.1f" % r[n] for n in names) + " |") + w("") + + w("## Flask JSON API (`scripts/python-backend/samples/flask-todo`)\n") + w("`ab -n 20000 -c 8 /fib/30` after a 2000-request warm-up. CPython, PyPy and " + "GraalPy serve with `app.run()` (Werkzeug's threaded server), Codename One " + "with the same call, which it lowers onto the native backend HTTP server.\n") + w("| | first response ms | idle RSS MB | RSS after load MB | req/s | p99 ms |") + w("|---|---:|---:|---:|---:|---:|") + for n, exe in arms: + if not has_flask(exe): + w("| %s | (Flask not installed) | | | | |" % n) + continue + r = server_row([exe, SERVER_APP]) + w("| %s | %.0f | %.1f | %.1f | %.0f | %s |" % ((n,) + r)) + r = server_row([a.cn1_server]) + w("| Codename One | %.0f | %.1f | %.1f | %.0f | %s |" % (r[0], r[1], r[2], r[3], r[4])) + w("") + + w("## What you deploy\n") + w("| | size MB | what |") + w("|---|---:|---|") + for n, exe in arms: + home = interpreter_home(exe) + extra = flask_size(exe) if has_flask(exe) else 0.0 + w("| %s | %.1f | interpreter install (`%s`) + Flask and its dependencies (%.1f) |" + % (n, du_mb(home) + extra, home, extra)) + w("| Codename One | %.1f | one self-contained executable, Flask included |" % du_mb(a.cn1_server)) + w("") + text = "\n".join(lines) + if a.out: + with open(a.out, "w") as f: + f.write(text + "\n") + print(text) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/python-compare/hello.py b/scripts/python-compare/hello.py new file mode 100644 index 00000000000..11b15b1a458 --- /dev/null +++ b/scripts/python-compare/hello.py @@ -0,0 +1 @@ +print("hello") From 1e33bef37336612d59ccf72b0cc2e6472ce0d5a3 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:56:42 +0200 Subject: [PATCH 019/112] Python: runtime and transpiler at zero SpotBugs, PMD and Checkstyle findings Co-Authored-By: Claude Opus 5.5 --- maven/python-runtime/spotbugs-exclude.xml | 13 +++++++ .../com/codename1/python/runtime/Cn1Http.java | 31 ++++++++++------ .../codename1/python/runtime/JavaBridge.java | 5 ++- .../com/codename1/python/runtime/NumPy.java | 4 +-- .../python/runtime/PyGenericAlias.java | 2 +- .../python/runtime/PyJavaObject.java | 25 ++++++++----- .../com/codename1/python/runtime/PyUnion.java | 8 ++--- maven/python-transpiler/spotbugs-exclude.xml | 11 ++++++ .../transpiler/api/PythonTranspiler.java | 36 +++++++++---------- .../transpiler/codegen/ClassPathReader.java | 10 +++--- .../codegen/JavaInteropGenerator.java | 2 +- 11 files changed, 92 insertions(+), 55 deletions(-) diff --git a/maven/python-runtime/spotbugs-exclude.xml b/maven/python-runtime/spotbugs-exclude.xml index d78931a00c0..93b4c09145e 100644 --- a/maven/python-runtime/spotbugs-exclude.xml +++ b/maven/python-runtime/spotbugs-exclude.xml @@ -90,4 +90,17 @@ + + + + + + + + + + diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Cn1Http.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Cn1Http.java index 3cf8d343a3f..d74587e7992 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Cn1Http.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Cn1Http.java @@ -61,17 +61,7 @@ public Response send(String method, String url, String[] headers, final byte[] b r.addRequestHeader(headers[i], headers[i + 1]); } if (body != null) { - r.setRequestBody(new Data() { - @Override - public void appendTo(OutputStream output) throws IOException { - output.write(body); - } - - @Override - public long getSize() { - return body.length; - } - }); + r.setRequestBody(new Body(body)); } NetworkManager.getInstance().addToQueueAndWait(r); if (r.failure != null && r.getResponseCode() <= 0) { @@ -88,6 +78,25 @@ public long getSize() { data == null ? new byte[0] : data); } + /// A request body held in memory. + private static final class Body implements Data { + private final byte[] bytes; + + Body(byte[] bytes) { + this.bytes = bytes; + } + + @Override + public void appendTo(OutputStream output) throws IOException { + output.write(bytes); + } + + @Override + public long getSize() { + return bytes.length; + } + } + /// Keeps what Python needs: every header as received, and the failure /// instead of the default error dialog. private static final class Request extends ConnectionRequest { diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/JavaBridge.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/JavaBridge.java index 96d8b547511..1457a954667 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/JavaBridge.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/JavaBridge.java @@ -164,9 +164,8 @@ public static Object toJavaMap(Object v) { if (v instanceof PyDict) { PyDict d = (PyDict) v; java.util.LinkedHashMap r = new java.util.LinkedHashMap(); - Object[] keys = d.keyArray(); - for (int i = 0; i < keys.length; i++) { - r.put(toJava(keys[i]), toJava(d.getItem(keys[i]))); + for (Object k : d.keyArray()) { + r.put(toJava(k), toJava(d.getItem(k))); } return r; } diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/NumPy.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/NumPy.java index 15bbe280ae7..5a7d0cdefeb 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/NumPy.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/NumPy.java @@ -129,7 +129,7 @@ private static int count(int[] shape) { } private static boolean isFloatScalar(Object v) { - return v instanceof Double || Py.typeOf(v) == Py.FLOAT_TYPE; + return v instanceof Double || Py.typeOf(v) == Py.FLOAT_TYPE; // NOPMD CompareObjectsWithEquals - identity is meant } private static void setScalar(NdBuf b, int i, Object v) { @@ -216,7 +216,7 @@ private static void assign(int[] shape, NdBuf dst, int doff, int[] dst2, NdBuf s int[] idx = new int[shape.length]; int[] pos = {doff, soff}; int[][] st = {dst2, sst}; - if (dst == src) { + if (dst == src) { // NOPMD CompareObjectsWithEquals - identity is meant // Overlapping views: read everything before writing anything. NdBuf tmp = copy(src.kind, shape, src, soff, sst); src = tmp; diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyGenericAlias.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyGenericAlias.java index 18e16dc1c01..e3e2be29efd 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyGenericAlias.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyGenericAlias.java @@ -77,7 +77,7 @@ public Object eq(Object o) { return null; } PyGenericAlias g = (PyGenericAlias) o; - return Py.bool(g.origin == origin && Py.eqBool(g.args, args)); + return Py.bool(g.origin == origin && Py.eqBool(g.args, args)); // NOPMD CompareObjectsWithEquals - identity is meant } @Override diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyJavaObject.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyJavaObject.java index f61ec83761d..2b9a079601c 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyJavaObject.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyJavaObject.java @@ -107,8 +107,8 @@ public boolean contains(Object x) { @Override public Object iter() { final java.util.Iterator it; - if (value instanceof java.lang.Iterable) { - it = ((java.lang.Iterable) value).iterator(); + if (value instanceof Iterable) { + it = ((Iterable) value).iterator(); } else if (value instanceof java.util.Map) { it = ((java.util.Map) value).keySet().iterator(); } else if (value instanceof java.util.Iterator) { @@ -116,12 +116,21 @@ public Object iter() { } else { return super.iter(); } - return new PyJavaIter(new PyJavaIter.Source() { - @Override - public Object next() { - return it.hasNext() ? JavaBridge.toPy(it.next()) : null; - } - }); + return new PyJavaIter(new IteratorSource(it)); + } + + /// Python iteration over a Java iterator. + private static final class IteratorSource implements PyJavaIter.Source { + private final java.util.Iterator it; + + IteratorSource(java.util.Iterator it) { + this.it = it; + } + + @Override + public Object next() { + return it.hasNext() ? JavaBridge.toPy(it.next()) : null; + } } @Override diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyUnion.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyUnion.java index fb8c9cbcff1..3837b4f860e 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyUnion.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyUnion.java @@ -54,9 +54,9 @@ private static void add(List out, Object t) { } return; } - Object v = t == Py.NONE ? Py.NONE_TYPE : t; + Object v = t == Py.NONE ? Py.NONE_TYPE : t; // NOPMD CompareObjectsWithEquals - identity is meant for (Object o : out) { - if (o == v || Py.eqBool(o, v)) { + if (o == v || Py.eqBool(o, v)) { // NOPMD CompareObjectsWithEquals - identity is meant return; } } @@ -65,10 +65,10 @@ private static void add(List out, Object t) { /// How a type reads inside an alias: `int`, `mod.Cls`, `None`. static String typeRepr(Object t) { - if (t == Py.NONE || t == Py.NONE_TYPE) { + if (t == Py.NONE || t == Py.NONE_TYPE) { // NOPMD CompareObjectsWithEquals - identity is meant return "None"; } - if (t == Py.ELLIPSIS) { + if (t == Py.ELLIPSIS) { // NOPMD CompareObjectsWithEquals - identity is meant return "..."; } if (t instanceof PyType) { diff --git a/maven/python-transpiler/spotbugs-exclude.xml b/maven/python-transpiler/spotbugs-exclude.xml index fffa5af9893..8fef78dc488 100644 --- a/maven/python-transpiler/spotbugs-exclude.xml +++ b/maven/python-transpiler/spotbugs-exclude.xml @@ -23,4 +23,15 @@ + + + + + + + + + + diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java index ce71d430ee9..4bc21e10a31 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java @@ -103,7 +103,7 @@ public static Map transpile(Map sources, String try (com.codename1.python.transpiler.codegen.ClassPathReader cp = new com.codename1.python.transpiler.codegen.ClassPathReader(classPath)) { interop = com.codename1.python.transpiler.codegen.JavaInteropGenerator.generate(trees, cp, PACKAGE); - } catch (java.io.IOException e) { + } catch (IOException e) { throw new IllegalStateException("cannot read the class path: " + e.getMessage(), e); } } @@ -122,7 +122,7 @@ public static Map transpile(Map sources, String try (com.codename1.python.transpiler.codegen.ClassPathReader cp = new com.codename1.python.transpiler.codegen.ClassPathReader(classPath)) { server = cp.find("com/codename1/backend/HttpServer") != null; - } catch (java.io.IOException e) { + } catch (IOException e) { throw new IllegalStateException("cannot read the class path: " + e.getMessage(), e); } } @@ -186,7 +186,7 @@ private static Set addStdlib(Map sources, Map imported = new ArrayList(); String own = ""; for (Map.Entry e : trees.entrySet()) { - if (e.getValue() == tree) { + if (e.getValue() == tree) { // NOPMD CompareObjectsWithEquals - identity is meant own = e.getKey(); break; } @@ -337,26 +337,24 @@ private static List> bodiesOf(Node.Stmt s) { } private static String resource(String path) { - InputStream in = PythonTranspiler.class.getResourceAsStream(STDLIB + path); - if (in == null) { - in = PythonTranspiler.class.getResourceAsStream(SITE + path); - } - if (in == null) { + String s = read(STDLIB + path); + return s != null ? s : read(SITE + path); + } + + private static String read(String resource) { + InputStream found = PythonTranspiler.class.getResourceAsStream(resource); // NOPMD CloseResource - closed by the try below + if (found == null) { return null; } - try { - try { - ByteArrayOutputStream b = new ByteArrayOutputStream(); - byte[] buf = new byte[8192]; - for (int n = in.read(buf); n > 0; n = in.read(buf)) { - b.write(buf, 0, n); - } - return new String(b.toByteArray(), StandardCharsets.UTF_8); - } finally { - in.close(); + try (InputStream in = found) { + ByteArrayOutputStream b = new ByteArrayOutputStream(); + byte[] buf = new byte[8192]; + for (int n = in.read(buf); n > 0; n = in.read(buf)) { + b.write(buf, 0, n); } + return new String(b.toByteArray(), StandardCharsets.UTF_8); } catch (IOException e) { - throw new IllegalStateException("cannot read the standard library module " + path, e); + throw new IllegalStateException("cannot read the standard library module " + resource, e); } } diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/ClassPathReader.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/ClassPathReader.java index ae10e2121ae..6b81a7c7f63 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/ClassPathReader.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/ClassPathReader.java @@ -72,16 +72,14 @@ public boolean is(int flag) { /// One class. public static final class ClassInfo { - public final String name; public final int access; public final String superName; public final List interfaces; public final List fields; public final List methods; - ClassInfo(String name, int access, String superName, List interfaces, List fields, + ClassInfo(int access, String superName, List interfaces, List fields, List methods) { - this.name = name; this.access = access; this.superName = superName; this.interfaces = interfaces; @@ -152,7 +150,7 @@ private byte[] fromRoots(String path) throws IOException { return Files.readAllBytes(f.toPath()); } } else { - ZipFile z = (ZipFile) r; + ZipFile z = (ZipFile) r; // NOPMD CloseResource - a class path root, closed by close() ZipEntry e = z.getEntry(path); if (e != null) { try (InputStream in = z.getInputStream(e)) { @@ -236,7 +234,7 @@ private static ClassInfo parse(byte[] b) throws IOException { } } int access = in.readUnsignedShort(); - String name = utf[classIdx[in.readUnsignedShort()]]; + in.readUnsignedShort(); // this_class: the caller already knows the name int sup = in.readUnsignedShort(); String superName = sup == 0 ? null : utf[classIdx[sup]]; int ni = in.readUnsignedShort(); @@ -246,7 +244,7 @@ private static ClassInfo parse(byte[] b) throws IOException { } List fields = members(in, utf); List methods = members(in, utf); - return new ClassInfo(name, access, superName, interfaces, fields, methods); + return new ClassInfo(access, superName, interfaces, fields, methods); } private static List members(DataInputStream in, String[] utf) throws IOException { diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/JavaInteropGenerator.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/JavaInteropGenerator.java index da959a2af28..65765001537 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/JavaInteropGenerator.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/JavaInteropGenerator.java @@ -67,7 +67,7 @@ public final class JavaInteropGenerator { /// Every public class whose instances the bridge dispatches on. private final Set universe = new LinkedHashSet(); private final Map adapters = new LinkedHashMap(); - private final StringBuilder adapterCode = new StringBuilder(); + private final StringBuilder adapterCode = new StringBuilder(); // NOPMD AvoidStringBufferField - the generator's output private JavaInteropGenerator(ClassPathReader cp) { this.cp = cp; From 8ef35f8bfd08e003b7a98d49234a3b4aa76963ad Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Tue, 27 Oct 2026 19:23:25 +0200 Subject: [PATCH 020/112] Python: typed math, slots classes, annotated locals and per-call-site helper returns CommonWorkloads ported to Python (scripts/python-bench/common) with a Java hot-spot variant, and a comparison across CPython, PyPy, GraalPy and Codename One with peak RSS per workload. Co-Authored-By: Claude Opus 5.5 --- .../codename1/python/runtime/Builtins.java | 2 +- .../com/codename1/python/runtime/PyMath.java | 127 ++++++++ .../codename1/python/runtime/StdModules.java | 25 +- .../com/codename1/python/runtime/Typed.java | 11 + .../python/transpiler/analyze/Desugar.java | 17 +- .../python/transpiler/codegen/Infer.java | 131 +++++++++ .../python/transpiler/codegen/Layouts.java | 4 +- .../python/transpiler/codegen/TypedEmit.java | 93 +++++- scripts/python-bench/build_parparvm.sh | 14 +- scripts/python-bench/common/common_bench.py | 41 +++ .../python-bench/common/common_workloads.py | 270 ++++++++++++++++++ scripts/python-bench/common/java_hotspots.py | 19 ++ scripts/python-compare/common_compare.py | 103 +++++++ 13 files changed, 831 insertions(+), 26 deletions(-) create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/PyMath.java create mode 100644 scripts/python-bench/common/common_bench.py create mode 100644 scripts/python-bench/common/common_workloads.py create mode 100644 scripts/python-bench/common/java_hotspots.py create mode 100644 scripts/python-compare/common_compare.py diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Builtins.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Builtins.java index c14381a415b..5b3bbbded2e 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Builtins.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Builtins.java @@ -927,7 +927,7 @@ private static Object next(Object[] a) { return v; } - private static Object ord(Object x) { + static Object ord(Object x) { if (!(x instanceof String) || PyStr.codePoints((String) x) != 1) { throw Py.typeError("ord() expected a character, but string of length " + (x instanceof String ? PyStr.codePoints((String) x) : 0) + " found"); diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyMath.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyMath.java new file mode 100644 index 00000000000..9fcb7dc111c --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyMath.java @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +import com.codename1.util.MathUtil; + +/// The `math` functions over doubles, with CPython's domain and range +/// errors. The `math` module and typed code (which calls these directly +/// when the arguments are known numbers) share them, so both raise alike. +public final class PyMath { + private PyMath() { + } + + /// `ValueError: math domain error`. + private static PyException domain() { + return Py.valueError("math domain error"); + } + + private static double finite(double x, double r) { + if (Double.isInfinite(r) && !Double.isInfinite(x) && !Double.isNaN(x)) { + throw Py.error(Py.OVERFLOW_ERROR, "math range error"); + } + return r; + } + + public static double sqrt(double x) { + if (x < 0) { + throw domain(); + } + return Math.sqrt(x); + } + + public static double sin(double x) { + if (Double.isInfinite(x)) { + throw domain(); + } + return Math.sin(x); + } + + public static double cos(double x) { + if (Double.isInfinite(x)) { + throw domain(); + } + return Math.cos(x); + } + + public static double tan(double x) { + if (Double.isInfinite(x)) { + throw domain(); + } + return Math.tan(x); + } + + public static double asin(double x) { + if (x < -1 || x > 1) { + throw domain(); + } + return MathUtil.asin(x); + } + + public static double acos(double x) { + if (x < -1 || x > 1) { + throw domain(); + } + return MathUtil.acos(x); + } + + public static double atan(double x) { + return MathUtil.atan(x); + } + + public static double atan2(double y, double x) { + return MathUtil.atan2(y, x); + } + + public static double exp(double x) { + return finite(x, MathUtil.exp(x)); + } + + public static double fabs(double x) { + return Math.abs(x); + } + + public static double log(double x) { + if (x <= 0) { + throw domain(); + } + return MathUtil.log(x); + } + + public static double log10(double x) { + if (x <= 0) { + throw domain(); + } + return MathUtil.log10(x); + } + + /// C's fmod, which is Java's `%` on doubles; CPython rejects a zero + /// divisor and an infinite dividend. + public static double fmod(double x, double y) { + if (y == 0 && !Double.isNaN(x) || Double.isInfinite(x) && !Double.isNaN(y)) { + throw domain(); + } + return x % y; + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java index 78778e611a7..6523e579a18 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java @@ -326,20 +326,14 @@ private static PyModule math() { m.setAttr("tau", Py.ofFloat(2 * Math.PI)); m.setAttr("inf", Py.ofFloat(Double.POSITIVE_INFINITY)); m.setAttr("nan", Py.ofFloat(Double.NaN)); - m.setAttr("sqrt", PyBuiltin.positional("sqrt", a -> { - double x = Py.asDouble(a[0]); - if (x < 0) { - throw Py.valueError("math domain error"); - } - return Py.ofFloat(Math.sqrt(x)); - })); - fn1(m, "sin", Math::sin); - fn1(m, "cos", Math::cos); - fn1(m, "tan", Math::tan); - fn1(m, "asin", com.codename1.util.MathUtil::asin); - fn1(m, "acos", com.codename1.util.MathUtil::acos); + fn1(m, "sqrt", PyMath::sqrt); + fn1(m, "sin", PyMath::sin); + fn1(m, "cos", PyMath::cos); + fn1(m, "tan", PyMath::tan); + fn1(m, "asin", PyMath::asin); + fn1(m, "acos", PyMath::acos); fn1(m, "atan", com.codename1.util.MathUtil::atan); - fn1(m, "exp", com.codename1.util.MathUtil::exp); + fn1(m, "exp", PyMath::exp); fn1(m, "fabs", Math::abs); fn1(m, "degrees", Math::toDegrees); fn1(m, "radians", Math::toRadians); @@ -366,8 +360,7 @@ private static PyModule math() { } return Py.ofFloat(r); })); - m.setAttr("log10", PyBuiltin.positional("log10", - a -> Py.ofFloat(com.codename1.util.MathUtil.log10(Py.asDouble(a[0]))))); + fn1(m, "log10", PyMath::log10); m.setAttr("log2", PyBuiltin.positional("log2", a -> Py.ofFloat(com.codename1.util.MathUtil.log(Py.asDouble(a[0])) / com.codename1.util.MathUtil.log(2)))); m.setAttr("floor", PyBuiltin.positional("floor", a -> a[0] instanceof PyInt ? a[0] @@ -386,7 +379,7 @@ private static PyModule math() { double s = Py.asDouble(a[1]); return Py.ofFloat(s < 0 || s == 0 && 1 / s < 0 ? -x : x); })); - m.setAttr("fmod", PyBuiltin.positional("fmod", a -> Py.ofFloat(Py.asDouble(a[0]) % Py.asDouble(a[1])))); + m.setAttr("fmod", PyBuiltin.positional("fmod", a -> Py.ofFloat(PyMath.fmod(Py.asDouble(a[0]), Py.asDouble(a[1]))))); m.setAttr("gcd", PyBuiltin.positional("gcd", a -> { com.codename1.util.BigInteger g = com.codename1.util.BigInteger.ZERO; for (int i = 0; i < a.length; i++) { diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Typed.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Typed.java index 0154c5f5d91..14369782c36 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Typed.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Typed.java @@ -317,6 +317,17 @@ public static Object append(PyList l, Object v) { return Py.NONE; } + /// `l.extend(other)` for two lists. + public static Object extend(PyList l, PyList other) { + nzl(l).extend(nzl(other)); + return Py.NONE; + } + + /// `ord(s)`: TypeError unless `s` is one character. + public static Object ord(String s) { + return Builtins.ord(s); + } + public static boolean truthy(String s) { return s != null && s.length() != 0; } diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/analyze/Desugar.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/analyze/Desugar.java index a2f7f2ab4ae..606b854d694 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/analyze/Desugar.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/analyze/Desugar.java @@ -23,6 +23,7 @@ */ package com.codename1.python.transpiler.analyze; +import com.codename1.python.transpiler.ast.Node.AnnAssign; import com.codename1.python.transpiler.ast.Node.Assign; import com.codename1.python.transpiler.ast.Node.Call; import com.codename1.python.transpiler.ast.Node.ClassDef; @@ -94,6 +95,8 @@ /// ``` public final class Desugar { private int counter; + /// Whether the block being rewritten is a function's own scope. + private boolean inFunction; private Desugar() { } @@ -113,6 +116,12 @@ private void block(List body) { i += r.size() - 1; } else if (s instanceof Try && ((Try) s).star) { body.set(i, exceptStar((Try) s)); + } else if (inFunction && s instanceof AnnAssign && ((AnnAssign) s).value != null) { + // CPython never evaluates the annotation of a function's + // local (PEP 526), so `x: int = v` there is `x = v`; as a + // plain assignment it also takes the typed path. + AnnAssign a = (AnnAssign) s; + body.set(i, assign(a.target, a.value, a)); } } } @@ -138,9 +147,15 @@ private void children(Stmt s) { block(t.orelse); block(t.finalbody); } else if (s instanceof FunctionDef) { + boolean saved = inFunction; + inFunction = true; block(((FunctionDef) s).body); + inFunction = saved; } else if (s instanceof ClassDef) { + boolean saved = inFunction; + inFunction = false; block(((ClassDef) s).body); + inFunction = saved; } else if (s instanceof Match) { for (MatchCase c : ((Match) s).cases) { block(c.body); @@ -383,7 +398,7 @@ private static Name name(String id, com.codename1.python.transpiler.ast.Node at) return new Name(id).at(at.line, at.col); } - private static Stmt assign(Name target, Expr value, com.codename1.python.transpiler.ast.Node at) { + private static Stmt assign(Expr target, Expr value, com.codename1.python.transpiler.ast.Node at) { return new Assign(Collections.singletonList(target), value).at(at.line, at.col); } } diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Infer.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Infer.java index 9beab7ab71b..1b88a6bec12 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Infer.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Infer.java @@ -90,6 +90,29 @@ Ty param(String n) { /// Stable global name -> the function or class it is bound to, once. final Map globalFns = new HashMap(); final Map globalClasses = new HashMap(); + /// The type a call of an expression function (see `exprBody`) has at + /// that call site, from the argument types there; where it is narrower + /// than the function's own return type, typed code relabels the result. + final Map siteRet = new HashMap(); + /// Globals bound once, by `import math`: calls of its functions on + /// numbers are typed (see `MATH_FNS`) and run as `PyMath` directly. + final java.util.Set mathNames = new HashSet(); + /// `math` functions typed code calls, with their arity. + static final Map MATH_FNS = new HashMap(); + /// `math` constants, all floats. + static final java.util.Set MATH_CONSTS = new HashSet(); + + static { + String[] unary = {"sqrt", "sin", "cos", "tan", "asin", "acos", "atan", "exp", "fabs", "log", "log10"}; + for (String f : unary) { + MATH_FNS.put(f, Integer.valueOf(1)); + } + MATH_FNS.put("fmod", Integer.valueOf(2)); + MATH_FNS.put("atan2", Integer.valueOf(2)); + for (String c : new String[] {"pi", "e", "tau", "inf", "nan"}) { + MATH_CONSTS.add(c); + } + } final Map globals = new HashMap(); /// Field name -> type, per declaring class. final Map> fields = new LinkedHashMap>(); @@ -155,6 +178,13 @@ private void collect(List body) { if (k != null && stable(((ClassDef) s).name)) { globalClasses.put(((ClassDef) s).name, k); } + } else if (s instanceof Import) { + for (Alias a : ((Import) s).names) { + String n = a.asname != null ? a.asname : a.name; + if ("math".equals(a.name) && stable(n)) { + mathNames.add(n); + } + } } } markOpen(body); @@ -163,6 +193,82 @@ private void collect(List body) { } } + /// Whether `e` is the `math` module, through a global only `import math` + /// binds. + boolean isMath(Expr e, Scope sc) { + return e instanceof Name && mathNames.contains(((Name) e).id) + && sc.resolve(((Name) e).id) == Scope.Binding.GLOBAL; + } + + /// Whether `c` is a `math` function typed code runs, on the right number + /// of plain positional arguments. + boolean isMathCall(Call c, Scope sc) { + if (!(c.func instanceof Attribute) || !c.keywords.isEmpty() || anyStarred(c.args)) { + return false; + } + Attribute a = (Attribute) c.func; + Integer n = MATH_FNS.get(a.attr); + return n != null && n.intValue() == c.args.size() && isMath(a.value, sc); + } + + /// The return type of expression function `f` called with `args`, or + /// null. Helpers such as `def i32(v): return ((v + B) & M) - B` are + /// shared by every caller, and one caller passing an untyped value + /// makes the parameter, and so the result, untyped for all of them. The + /// body is plain arithmetic on its parameters and global constants, so + /// its type at one call follows from that call's arguments alone. + private Ty siteType(Fn f, List args) { + Expr body = exprBody(f.def); + if (body == null) { + return null; + } + for (Ty t : args) { + if (t.opt || t.k != Ty.K.INT && t.k != Ty.K.FLOAT && t.k != Ty.K.BOOL) { + return null; + } + } + Fn site = new Fn(f.def, null); + for (int i = 0; i < args.size(); i++) { + site.params[i] = args.get(i); + site.locals.put(site.names.get(i), args.get(i)); + } + Ty t = type(body, site, f.def.scope); + return !t.opt && (t.k == Ty.K.INT || t.k == Ty.K.FLOAT || t.k == Ty.K.BOOL) ? t : null; + } + + /// The returned expression of a function whose body is only `return + /// ` (after an optional docstring), the arithmetic made of + /// operators, constants, its parameters and global names; else null. + static Expr exprBody(FunctionDef d) { + List b = d.body; + int i = 0; + if (b.size() == 2 && b.get(0) instanceof ExprStmt && ((ExprStmt) b.get(0)).value instanceof Constant) { + i = 1; + } + if (b.size() != i + 1 || !(b.get(i) instanceof Return) || ((Return) b.get(i)).value == null) { + return null; + } + Expr e = ((Return) b.get(i)).value; + return arithmetic(e, d.scope) ? e : null; + } + + private static boolean arithmetic(Expr e, Scope sc) { + if (e instanceof Constant) { + return true; + } + if (e instanceof Name) { + Scope.Binding r = sc.resolve(((Name) e).id); + return r == Scope.Binding.GLOBAL || sc.params.contains(((Name) e).id); + } + if (e instanceof BinOp) { + return arithmetic(((BinOp) e).left, sc) && arithmetic(((BinOp) e).right, sc); + } + if (e instanceof UnaryOp) { + return arithmetic(((UnaryOp) e).operand, sc); + } + return false; + } + private boolean stable(String n) { Integer c = moduleBindings.get(n); return c != null && c.intValue() == 1; @@ -1082,6 +1188,9 @@ Ty type(Expr e, Fn fn, Scope sc) { } if (e instanceof Attribute) { Attribute a = (Attribute) e; + if (MATH_CONSTS.contains(a.attr) && isMath(a.value, sc)) { + return Ty.FLOAT; + } if (a.value instanceof Name && globalClasses.containsKey(((Name) a.value).id) && sc.resolve(((Name) a.value).id) == Scope.Binding.GLOBAL) { // `Cls.attr` as a value: a method taken off the class runs @@ -1373,6 +1482,12 @@ private Ty call(Call c, Fn fn, Scope sc) { if (f != null) { callFn(f, args, 0, simple && args.size() <= f.params.length && args.size() >= f.params.length - f.def.args.defaults.size()); + Ty site = simple && args.size() == f.params.length ? siteType(f, args) : null; + if (site != null) { + siteRet.put(c, site); + return site; + } + siteRet.remove(c); return f.ret; } if (open.contains(k)) { @@ -1403,6 +1518,14 @@ private Ty call(Call c, Fn fn, Scope sc) { return Ty.inst(k, false); } } + if (isMathCall(c, sc)) { + boolean numbers = true; + for (Expr x : c.args) { + Ty t = type(x, fn, sc); + numbers &= (t.k == Ty.K.INT || t.k == Ty.K.FLOAT) && !t.opt; + } + return numbers ? Ty.FLOAT : Ty.DYN; + } if (c.func instanceof Attribute) { Attribute a = (Attribute) c.func; if (a.value instanceof Name && globalClasses.containsKey(((Name) a.value).id) @@ -1492,6 +1615,14 @@ private Ty call(Call c, Fn fn, Scope sc) { && (args.get(0).k == Ty.K.INT || args.get(0).k == Ty.K.FLOAT) && !args.get(0).opt) { return Ty.FLOAT; } + if (simple && "str".equals(n) && args.size() == 1 && !args.get(0).opt + && (args.get(0).k == Ty.K.INT || args.get(0).k == Ty.K.FLOAT + || args.get(0).k == Ty.K.STR || args.get(0).k == Ty.K.BOOL)) { + return Ty.STR; + } + if (simple && "ord".equals(n) && args.size() == 1 && args.get(0).k == Ty.K.STR && !args.get(0).opt) { + return Ty.INT; + } if (simple && "abs".equals(n) && args.size() == 1 && !args.get(0).opt && (args.get(0).k == Ty.K.INT || args.get(0).k == Ty.K.FLOAT)) { return args.get(0); diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Layouts.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Layouts.java index 0c527bd6b1f..50d98b7e8ce 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Layouts.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Layouts.java @@ -86,10 +86,12 @@ static final class Cls { static { // A class that customises attribute access itself is left alone. + // `__slots__` is not on the list: it fixes the attribute set, which + // is what a layout is, and the generic instance does not enforce it + // either, so the two paths agree. FORBIDDEN.add("__setattr__"); FORBIDDEN.add("__getattribute__"); FORBIDDEN.add("__delattr__"); - FORBIDDEN.add("__slots__"); FORBIDDEN.add("__dict__"); FORBIDDEN.add("__class__"); FORBIDDEN.add("__new__"); diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/TypedEmit.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/TypedEmit.java index 637b3811005..4e60cb83c5d 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/TypedEmit.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/TypedEmit.java @@ -1410,6 +1410,9 @@ E expr(Expr e) { } if (e instanceof Attribute) { Attribute a = (Attribute) e; + if (Infer.MATH_CONSTS.contains(a.attr) && in.isMath(a.value, sc)) { + return new E(mathConstant(a.attr), Ty.FLOAT); + } E r = expr(a.value); Ty ft = fieldTy(r, a.attr); String p = path(a); @@ -1506,7 +1509,7 @@ E expr(Expr e) { E l = expr(s.value); E i = expr(s.index); if (l.t.k != Ty.K.LIST || !(i.t.k == Ty.K.INT || i.t.k == Ty.K.BOOL)) { - throw new Unsupported("subscript of " + l.t); + throw new Unsupported("subscript of " + l.t + " by " + i.t); } Ty el = in.elem(l.t); if (el.k == Ty.K.BOT) { @@ -1867,6 +1870,20 @@ private E hoistInt(Expr e) { int ind = preInd; indent(pre, ind); pre.append(jt(t)).append(' ').append(r).append(";\n"); + if (!f.toString().contains("break " + fast + ";")) { + // Nothing in the fast path can leave it (no operand that may be + // a big int, no operation that may overflow): the generic + // fallback would be unreachable, which javac rejects. + for (String line : f.toString().split("\n")) { + if (!line.isEmpty()) { + indent(pre, ind); + pre.append(line).append('\n'); + } + } + indent(pre, ind); + pre.append(r).append(" = ").append(v).append(";\n"); + return new E(r, t); + } indent(pre, ind); pre.append(done).append(": {\n"); indent(pre, ind + 1); @@ -2342,10 +2359,9 @@ private E call0(Call c) { List a = args(c); calls.add(f); String lcall = longCall(f, a); - if (lcall != null) { - return result(f, lcall); - } - return result(f, typed.get(f) + "(" + argList(f, 0, a) + ")"); + E r = lcall != null ? result(f, lcall) : result(f, typed.get(f) + "(" + argList(f, 0, a) + ")"); + Ty site = in.siteRet.get(c); + return site == null || site.k == r.t.k ? r : relabel(r, site); } Layouts.Cls k = in.globalClasses.get(n); if (k != null) { @@ -2375,6 +2391,17 @@ private E call0(Call c) { } throw new Unsupported("call of " + n); } + if (in.isMathCall(c, sc)) { + StringBuilder sb = new StringBuilder("PyMath.").append(((Attribute) c.func).attr).append('('); + for (int i = 0; i < c.args.size(); i++) { + E x = expr(c.args.get(i)); + if (x.t.opt || x.t.k != Ty.K.INT && x.t.k != Ty.K.FLOAT) { + throw new Unsupported("math of " + x.t); + } + sb.append(i == 0 ? "" : ", ").append(toD(x)); + } + return new E(sb.append(')').toString(), Ty.FLOAT); + } if (c.func instanceof Attribute) { Attribute a = (Attribute) c.func; if (a.value instanceof Name && in.globalClasses.containsKey(((Name) a.value).id) @@ -2427,6 +2454,34 @@ private E call0(Call c) { throw new Unsupported("call"); } + /// A call's result known, at this call site, to be `t` (`Infer.siteRet`) + /// though the function returns something wider. + private E relabel(E r, Ty t) { + switch (t.k) { + case INT: + return new E(r.t.k == Ty.K.DYN ? r.c : box(r), Ty.INT); + case FLOAT: + return new E("Typed.asFloat(" + box(r) + ")", Ty.FLOAT); + default: + return new E("Typed.asBool(" + box(r) + ")", Ty.BOOL); + } + } + + private static String mathConstant(String n) { + switch (n) { + case "pi": + return "Math.PI"; + case "e": + return "Math.E"; + case "tau": + return "(2 * Math.PI)"; + case "inf": + return "Double.POSITIVE_INFINITY"; + default: + return "Double.NaN"; + } + } + private E construct(Layouts.Cls k, List args) { Layouts.Cls o = Layouts.resolver(k, "__init__"); if (o == null) { @@ -2459,6 +2514,27 @@ private E builtin(String n, List a) { return new E(toD(a.get(0)), Ty.FLOAT); } break; + case "str": + if (a.size() == 1 && !a.get(0).t.opt) { + E x = a.get(0); + switch (x.t.k) { + case STR: + return x; + case BOOL: + return new E("((" + x.c + ") ? \"True\" : \"False\")", Ty.STR); + case INT: + case FLOAT: + return new E("Py.toStr(" + box(x) + ")", Ty.STR); + default: + break; + } + } + break; + case "ord": + if (a.size() == 1 && a.get(0).t.k == Ty.K.STR && !a.get(0).t.opt) { + return new E("Typed.ord(" + a.get(0).c + ")", Ty.INT); + } + break; case "abs": if (a.size() == 1 && a.get(0).t.k == Ty.K.FLOAT) { return new E("Math.abs(" + a.get(0).c + ")", Ty.FLOAT); @@ -2470,7 +2546,7 @@ private E builtin(String n, List a) { default: break; } - throw new Unsupported("builtin " + n); + throw new Unsupported("builtin " + n + (a.isEmpty() ? "" : " of " + a.get(0).t)); } private E listMethod(E l, String m, List a) { @@ -2480,6 +2556,11 @@ private E listMethod(E l, String m, List a) { return new E("Typed.append(" + l.c + ", " + box(a.get(0)) + ")", Ty.NONE); } break; + case "extend": + if (a.size() == 1 && a.get(0).t.k == Ty.K.LIST && !a.get(0).t.opt) { + return new E("Typed.extend(" + l.c + ", " + a.get(0).c + ")", Ty.NONE); + } + break; case "pop": if (a.size() <= 1) { Ty el = in.elem(l.t); diff --git a/scripts/python-bench/build_parparvm.sh b/scripts/python-bench/build_parparvm.sh index c616dbf3828..492e98e4d79 100755 --- a/scripts/python-bench/build_parparvm.sh +++ b/scripts/python-bench/build_parparvm.sh @@ -12,6 +12,10 @@ # JAVA_HOME=$JAVA17_HOME mvn -Dmaven.repo.local=$(pwd)/../.m2-repo -pl python-runtime install -DskipTests # mvn -Dmaven.repo.local=$(pwd)/../.m2-repo -f ../vm/pom.xml -pl JavaAPI,ByteCodeTranslator -am install -DskipTests # +# PY_EXTRA_FILES names more modules the program imports (space separated), +# and PY_JAVA_SOURCES a directory of Java sources the Python calls into -- +# the hot spots an application moves to Java (see common/java_hotspots.py). +# # Output: scripts/python-bench/target// (and the translated # sources beside it). Needs JAVA17_HOME (javac for the generated code, which is # Java 17) and JDK_8_HOME or JAVA_HOME (the translator), and clang. @@ -41,8 +45,15 @@ mkdir -p "$W/src" "$W/gen/com/pybench" "$W/jclasses" "$W/javaapi" "$W/out" SRC_DIR="${PY_SOURCES:-$HERE/kernels}" cp "$SRC_DIR/$K.py" "$W/src/" [ -f "$HERE/kernels/_harness.py" ] && cp "$HERE/kernels/_harness.py" "$W/src/" +for f in ${PY_EXTRA_FILES:-}; do cp "$f" "$W/src/"; done +APPCP="" +if [ -n "${PY_JAVA_SOURCES:-}" ]; then + mkdir -p "$W/appclasses" + "$J17/bin/javac" -nowarn --release 8 -d "$W/appclasses" $(find "$PY_JAVA_SOURCES" -name '*.java') + APPCP=":$W/appclasses" +fi # Java imports resolve against JavaAPI, the class library the binary links. -"$J17/bin/java" -cp "$TRANSPILER" com.codename1.python.transpiler.api.Main -cp "$JAVAAPI:$CORE" "$W/src" "$W/gen" "$K" +"$J17/bin/java" -cp "$TRANSPILER" com.codename1.python.transpiler.api.Main -cp "$JAVAAPI:$CORE$APPCP" "$W/src" "$W/gen" "$K" cat > "$W/gen/com/pybench/PyBench.java" < [python|java] [warmup] [measured] +# +# Prints `BENCH ms= checksum=`. `java` runs the same +# workload through java_hotspots.py, the Java implementation called from +# Python, which only a Codename One build can import. +import sys +import time + +import common_workloads + + +def main(): + name = sys.argv[1] + mode = sys.argv[2] if len(sys.argv) > 2 else "python" + warmup = int(sys.argv[3]) if len(sys.argv) > 3 else 1 + measured = int(sys.argv[4]) if len(sys.argv) > 4 else 3 + if mode == "java": + import java_hotspots + work = java_hotspots.WORKLOADS[name] + else: + work = common_workloads.WORKLOADS[name] + checksum = None + for _ in range(warmup): + checksum = work() + best = None + for _ in range(measured): + t0 = time.perf_counter() + r = work() + dt = time.perf_counter() - t0 + if checksum is not None and r != checksum: + raise RuntimeError("rounds disagree: %r != %r" % (r, checksum)) + checksum = r + if best is None or dt < best: + best = dt + print("BENCH %s ms=%.1f checksum=%d" % (name, best * 1000.0, checksum)) + + +main() diff --git a/scripts/python-bench/common/common_workloads.py b/scripts/python-bench/common/common_workloads.py new file mode 100644 index 00000000000..51778ac086e --- /dev/null +++ b/scripts/python-bench/common/common_workloads.py @@ -0,0 +1,270 @@ +# Python port of com.bench.CommonWorkloads (vm/benchmarks/common), the +# workloads every Codename One port and the Dart port already run. +# +# Like the Dart port it reproduces Java's semantics rather than idiomatic +# Python, so the checksums are the contract: each workload returns exactly +# the value the Java method returns. +# +# * Java `int` is 32-bit and `long` 64-bit, and both wrap. Python's int does +# not, so every expression that can overflow is folded back with a mask. +# * Java `>>>` shifts the unsigned bit pattern: mask first, then shift. +# * Java `%` on doubles is C's fmod, not Python's floor modulo. +# * Java's String.hashCode is reimplemented. +# +# The code is plain, type-annotated Python and runs unchanged on CPython, +# PyPy, GraalPy and Codename One. +import math +import struct + +M32 = 0xFFFFFFFF +M64 = 0xFFFFFFFFFFFFFFFF + + +def i32(v: int) -> int: + return ((v + 0x80000000) & M32) - 0x80000000 + + +def i64(v: int) -> int: + return ((v + 0x8000000000000000) & M64) - 0x8000000000000000 + + +# ---- 1. integer arithmetic: dependent ALU chain ---- +def int_arithmetic() -> int: + a: int = 0x12345678 + b: int = i32(0x9E3779B9) + checksum: int = 0 + for _ in range(40000000): + a = i32((a * 1103515245 + 12345) ^ ((b & M32) >> 3)) + b = i32((b + a) * 5 - (a << 7)) + checksum += (a ^ b) & 0xFFFF + return i64(checksum + a + b) + + +# ---- 2. long (64-bit) arithmetic: dependent chain ---- +def long_arithmetic() -> int: + a: int = 0x0123456789ABCDEF + b: int = -0x123456789 + checksum: int = 0 + for _ in range(30000000): + a = i64((a * 6364136223846793005 + 1442695040888963407) ^ ((b & M64) >> 7)) + b = i64((b ^ (a << 13)) + ((a & M64) >> 11)) + checksum += (a + b) & 0xFF + return i64(checksum + a + b) + + +# ---- 3. floating point + transcendental ---- +def math_transcendental() -> int: + acc: float = 1.0 + x: float = 0.5 + for i in range(8000000): + x = x + 0.000001 * (i & 1023) + acc += math.sqrt(x) + math.sin(x) * math.cos(x) - math.sqrt(math.fmod(acc, 1000.0) + 1.0) + if acc > 1e12 or acc < -1e12: + acc = math.fmod(acc, 1000.0) + return struct.unpack(">q", struct.pack(">d", acc))[0] + + +# ---- 4. sequential array fill + reduce ---- +# The arrays are allocated on first use and kept, as Java's static ones are +# when the class is first touched, so each workload's memory is its own. +seq_arr = [] + + +def array_sequential() -> int: + arr = seq_arr + if not arr: + arr.extend([0] * 8000000) + n: int = len(arr) + checksum: int = 0 + seed: int = i32(0x9E3779B9) + for i in range(n): + seed = i32(seed * 1103515245 + 12345) + arr[i] = seed + for p in range(4): + s: int = 0 + for i in range(n): + s += arr[i] + checksum ^= s + p + return checksum + + +# ---- 5. random-access gather ---- +rand_arr = [] + + +def array_random() -> int: + arr = rand_arr + if not arr: + arr.extend([0] * 4000000) + n: int = len(arr) + for i in range(n): + arr[i] = i32(i * 2654435761) if ((i * 2654435761) >> 8) > 0 else i + checksum: int = 0 + idx: int = 12345 + for _ in range(20000000): + v: int = arr[(idx & 0x7FFFFFFF) % n] + checksum += v + idx = i32(v ^ (idx * 31 + 7)) + return checksum + + +# ---- 6. object allocation + GC churn ---- +class Node: + __slots__ = ("v", "next") + + def __init__(self, v: int, nxt): + self.v = v + self.next = nxt + + +def object_allocation() -> int: + checksum: int = 0 + head = None + for i in range(8000000): + head = Node(i, head) + if (i & 511) == 0: + p = head + steps: int = 0 + while p is not None and steps < 48: + checksum += p.v + p = p.next + steps += 1 + head = None + return checksum + + +# ---- non-escaping value object ---- +class Vec: + __slots__ = ("x", "y") + + def __init__(self, x: int, y: int): + self.x = x + self.y = y + + def get_x(self) -> int: + return self.x + + def get_y(self) -> int: + return self.y + + +def value_escape() -> int: + total: int = 0 + for i in range(8000000): + v = Vec(i, i * 2) + total = (total + v.get_x() + v.get_y()) & 0x3FFFFFFF + return total + + +# ---- 7. hash map churn ---- +def hash_map_churn() -> int: + m = {} + checksum: int = 0 + window: int = 50000 + for i in range(3000000): + key: int = i & 0x3FFFF + prev = m.get(key) + m[key] = i if prev is None else i32(prev + i) + if prev is not None: + checksum += prev + if len(m) > window: + m.clear() + return checksum + len(m) + + +# ---- 8. string building + hashing ---- +def java_hash(s: str) -> int: + h: int = 0 + for c in s: + h = (h * 31 + ord(c)) & M32 + return h - 0x100000000 if h > 0x7FFFFFFF else h + + +sb_ring = [""] * 256 + + +def string_building() -> int: + ring = sb_ring + checksum: int = 0 + for i in range(400000): + ring[i & 255] = "item-" + str(i) + "-" + str(i32(i * 31) ^ 0x55AA) + ":" + ( + "even" if (i & 1) == 0 else "odd") + if (i & 255) == 255: + for j in range(256): + s = ring[j] + checksum += java_hash(s) + len(s) + return checksum + + +# ---- 9. recursion ---- +def fib(n: int) -> int: + if n < 2: + return n + return fib(n - 1) + fib(n - 2) + + +def recursion() -> int: + checksum: int = 0 + for i in range(3): + checksum += fib(35 + (i & 1)) + return checksum + + +# ---- 10. quicksort ---- +sort_arr = [] + + +def quicksort(a, lo: int, hi: int) -> None: + while lo < hi: + pivot: int = a[(lo + hi) >> 1] + i: int = lo + j: int = hi + while i <= j: + while a[i] < pivot: + i += 1 + while a[j] > pivot: + j -= 1 + if i <= j: + a[i], a[j] = a[j], a[i] + i += 1 + j -= 1 + if j - lo < hi - i: + quicksort(a, lo, j) + lo = i + else: + quicksort(a, i, hi) + hi = j + + +def quicksort_bench() -> int: + a = sort_arr + if not a: + a.extend([0] * 1500000) + n: int = len(a) + seed: int = i32(0xCAFEBABE) + for i in range(n): + seed = i32(seed * 1103515245 + 12345) + a[i] = seed + quicksort(a, 0, n - 1) + checksum: int = 0 + for i in range(0, n, 997): + checksum += a[i] * (i + 1) + for i in range(1, n): + if a[i - 1] > a[i]: + checksum ^= 0xDEADBEEF + return checksum + + +WORKLOADS = { + "intArithmetic": int_arithmetic, + "longArithmetic": long_arithmetic, + "mathTranscendental": math_transcendental, + "arraySequential": array_sequential, + "arrayRandom": array_random, + "objectAllocation": object_allocation, + "valueEscape": value_escape, + "hashMapChurn": hash_map_churn, + "stringBuilding": string_building, + "recursion": recursion, + "quicksort": quicksort_bench, +} diff --git a/scripts/python-bench/common/java_hotspots.py b/scripts/python-bench/common/java_hotspots.py new file mode 100644 index 00000000000..c7d10bb06f9 --- /dev/null +++ b/scripts/python-bench/common/java_hotspots.py @@ -0,0 +1,19 @@ +# The same workloads with the hot spot written in Java: each entry calls +# com.bench.CommonWorkloads (vm/benchmarks/common), the class every Codename +# One port benchmarks, straight from Python. This is how an application +# tunes a measured hot spot -- the Python around it stays as it is. +from java.com.bench import CommonWorkloads + +WORKLOADS = { + "intArithmetic": CommonWorkloads.intArithmetic, + "longArithmetic": CommonWorkloads.longArithmetic, + "mathTranscendental": CommonWorkloads.mathTranscendental, + "arraySequential": CommonWorkloads.arraySequential, + "arrayRandom": CommonWorkloads.arrayRandom, + "objectAllocation": CommonWorkloads.objectAllocation, + "valueEscape": CommonWorkloads.valueEscape, + "hashMapChurn": CommonWorkloads.hashMapChurn, + "stringBuilding": CommonWorkloads.stringBuilding, + "recursion": CommonWorkloads.recursion, + "quicksort": CommonWorkloads.quicksortBench, +} diff --git a/scripts/python-compare/common_compare.py b/scripts/python-compare/common_compare.py new file mode 100644 index 00000000000..359f1c97099 --- /dev/null +++ b/scripts/python-compare/common_compare.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +# Runs vm/benchmarks' CommonWorkloads, ported to Python +# (scripts/python-bench/common), on every Python runtime given, plus the +# Codename One build twice: the Python as written, and the same program with +# the hot spot in Java. Each workload runs in a process of its own, so every +# row has its own peak RSS. Checksums are checked against Java's. +# +# common_compare.py --cn1 --cpython P [--pypy P] +# [--graalpy P] [--json out.json] [--md out.md] +import argparse +import json +import os +import re +import subprocess +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +COMMON = os.path.join(os.path.dirname(HERE), "python-bench", "common") + +# com.bench.CommonWorkloads on HotSpot: the contract every arm must meet. +EXPECTED = { + "intArithmetic": 1307491170054, + "longArithmetic": 6887886960473257608, + "mathTranscendental": 4729652805076374709, + "arraySequential": 0, + "arrayRandom": -2288487891715278, + "objectAllocation": 2999790376128, + "valueEscape": 1026483456, + "hashMapChurn": 49941, + "stringBuilding": -609121604069, + "recursion": 33385282, + "quicksort": 809667393311589960, +} + + +def run(cmd, timeout): + p = subprocess.run(["/usr/bin/time", "-l"] + cmd, capture_output=True, text=True, + cwd=COMMON, timeout=timeout) + if p.returncode != 0: + raise RuntimeError("%s: %s" % (cmd, p.stderr[-400:])) + m = re.search(r"BENCH (\S+) ms=([\d.]+) checksum=(-?\d+)", p.stdout) + rss = int(re.search(r"(\d+)\s+maximum resident set size", p.stderr).group(1)) / 1048576.0 + return float(m.group(2)), int(m.group(3)), rss + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--cn1", required=True) + ap.add_argument("--cpython", required=True) + ap.add_argument("--pypy") + ap.add_argument("--graalpy") + ap.add_argument("--warmup", type=int, default=1) + ap.add_argument("--measured", type=int, default=3) + ap.add_argument("--json") + ap.add_argument("--md") + ap.add_argument("--only", help="comma separated workloads") + a = ap.parse_args() + script = os.path.join(COMMON, "common_bench.py") + arms = [("CPython", lambda w: [a.cpython, script, w])] + if a.pypy: + arms.append(("PyPy", lambda w: [a.pypy, script, w])) + if a.graalpy: + arms.append(("GraalPy", lambda w: [a.graalpy, script, w])) + arms.append(("Codename One", lambda w: [a.cn1, w, "python"])) + arms.append(("Codename One, hot spot in Java", lambda w: [a.cn1, w, "java"])) + reps = [str(a.warmup), str(a.measured)] + names = [w for w in EXPECTED if not a.only or w in a.only.split(",")] + results = {} + for w in names: + results[w] = {} + for arm, cmd in arms: + ms, checksum, rss = run(cmd(w) + _reps(arm, reps), 3600) + if checksum != EXPECTED[w]: + raise RuntimeError("%s on %s: checksum %d, Java's is %d" % (w, arm, checksum, EXPECTED[w])) + results[w][arm] = {"ms": ms, "rss_mb": rss} + print("%-20s %-30s %10.1f ms %8.1f MB" % (w, arm, ms, rss), flush=True) + out = {"arms": [n for n, _ in arms], "results": results} + if a.json: + with open(a.json, "w") as f: + json.dump(out, f, indent=1) + if a.md: + with open(a.md, "w") as f: + f.write(markdown(out)) + + +def _reps(arm, reps): + # common_bench.py takes ; the Python + # interpreters are given the mode here, the native binary already has it. + return reps if arm.startswith("Codename") else ["python"] + reps + + +def markdown(out): + arms = out["arms"] + lines = ["| workload | " + " | ".join("%s ms | MB" % n for n in arms) + " |", + "|---|" + "---:|---:|" * len(arms)] + for w, row in out["results"].items(): + lines.append("| %s | " % w + " | ".join( + "%.1f | %.1f" % (row[n]["ms"], row[n]["rss_mb"]) for n in arms) + " |") + return "\n".join(lines) + "\n" + + +if __name__ == "__main__": + sys.exit(main()) From a6aa20abb9694cd597c211219bfbe9bf7bda8aef Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Tue, 27 Oct 2026 19:33:20 +0200 Subject: [PATCH 021/112] Python: GraalPy in every comparison, CommonWorkloads with RSS, performance chapter The developer guide's Python chapter compares CPython, PyPy, GraalPy and Codename One on startup, kernels, Flask and the CommonWorkloads suite, with peak memory in every table, two charts, and a section on moving a measured hot spot into Java. Co-Authored-By: Claude Opus 5.5 --- docs/developer-guide/Python-Interop.asciidoc | 188 +++++++++++-- docs/developer-guide/img/python-perf-rss.svg | 145 ++++++++++ docs/developer-guide/img/python-perf-time.svg | 147 ++++++++++ scripts/python-compare/README.md | 23 +- scripts/python-compare/RESULTS.md | 48 +++- scripts/python-compare/common-results.json | 253 ++++++++++++++++++ scripts/python-compare/render_charts.py | 131 +++++++++ 7 files changed, 895 insertions(+), 40 deletions(-) create mode 100644 docs/developer-guide/img/python-perf-rss.svg create mode 100644 docs/developer-guide/img/python-perf-time.svg create mode 100644 scripts/python-compare/common-results.json create mode 100644 scripts/python-compare/render_charts.py diff --git a/docs/developer-guide/Python-Interop.asciidoc b/docs/developer-guide/Python-Interop.asciidoc index 4ea77a2c4bf..21694c51d9e 100644 --- a/docs/developer-guide/Python-Interop.asciidoc +++ b/docs/developer-guide/Python-Interop.asciidoc @@ -178,54 +178,194 @@ dependencies, `response_model` and OpenAPI generation are supported. Flask sessions, templates, file uploads and streaming responses aren't, nor are FastAPI WebSockets and middleware. -=== Compared with CPython and PyPy +=== Compared with CPython, PyPy and GraalPy The same programs, run on one Apple silicon Mac, with -`scripts/python-compare/compare.py`. Codename One rows are native binaries. +`scripts/python-compare/compare.py`. Codename One rows are native binaries; +GraalPy is the Community edition, 25.4, on its own JVM. Startup, a program that prints one line: [cols="1,1,1",options="header"] |=== | | Wall time (ms) | Peak memory (MB) -| CPython 3.14 | 21.2 | 14.4 -| PyPy 3.11 | 20.4 | 29.2 -| Codename One | 7.6 | 8.7 +| CPython 3.14 | 20.3 | 14.3 +| PyPy 3.11 | 18.2 | 29.2 +| GraalPy 3.13 | 34.9 | 116.6 +| Codename One | 5.9 | 8.7 |=== -CPU kernels, fastest round in milliseconds: +CPU kernels, fastest round in milliseconds, then the process's peak memory in +MB: -[cols="1,1,1,1",options="header"] +[cols="2,1,1,1,1,1,1,1,1",options="header"] |=== -| Kernel | CPython 3.14 | PyPy 3.11 | Codename One -| nbody | 785.9 | 63.7 | 90.1 -| richards | 653.9 | 40.3 | 44.7 -| fannkuch | 2691.2 | 612.5 | 731.8 -| spectral_norm | 734.5 | 11.7 | 11.1 -| dicts_strings | 397.5 | 145.0 | 211.2 +| Kernel 2+| CPython 3.14 2+| PyPy 3.11 2+| GraalPy 3.13 2+| Codename One +| nbody | 776.0 | 15 | 62.7 | 32 | 192.5 | 234 | 89.2 | 10 +| richards | 634.8 | 16 | 39.9 | 34 | 146.5 | 260 | 44.7 | 11 +| fannkuch | 2652.3 | 15 | 604.7 | 33 | 541.2 | 244 | 723.4 | 10 +| spectral_norm | 729.4 | 15 | 12.0 | 32 | 7.4 | 248 | 11.1 | 10 +| dicts_strings | 404.2 | 63 | 141.5 | 87 | 274.4 | 383 | 208.6 | 127 |=== -PyPy's JIT and Codename One's ahead of time compilation land close to each -other, from twice to sixty times as fast as CPython. Codename One gets there -without a warm-up and with a third of PyPy's memory on the numeric kernels. +The two JITs and Codename One's ahead of time compilation land within a small +factor of each other, from three to sixty times as fast as CPython. PyPy is +the fastest on most kernels and GraalPy on two of them; Codename One gets +close without a warm-up and in a third of PyPy's memory and a twentieth of +GraalPy's on the numeric kernels. The Flask sample above, measured with `ab -n 20000 -c 8`: -[cols="1,1,1,1",options="header"] +[cols="1,1,1,1,1,1",options="header"] |=== -| | First response (ms) | Requests per second | Deployed size (MB) -| CPython 3.14 | 97 | 687 | 82.0 -| PyPy 3.11 | 200 | 674 | 137.7 -| Codename One | 61 | 16,332 | 22.7 +| | First response (ms) | Requests per second | Idle memory (MB) | Memory after load (MB) | Deployed size (MB) +| CPython 3.14 | 85 | 685 | 39.0 | 24.5 | 82.0 +| PyPy 3.11 | 196 | 688 | 65.3 | 88.7 | 137.7 +| GraalPy 3.13 | 609 | 561 | 325.7 | 931.0 | 343.1 +| Codename One | 58 | 13,564 | 26.8 | 423.4 | 22.7 |=== The deployed size is the interpreter's installation plus Flask and its -dependencies, against the one executable. CPython and PyPy serve with +dependencies, against the one executable. CPython, PyPy and GraalPy serve with Werkzeug's threaded server, which is what `app.run()` starts; a production CPython deployment behind a WSGI server with several worker processes serves -more, using a full interpreter's memory per worker. The Codename One server -idles at 27 MB and its heap grew to 395 MB under this load, where it stayed. +more, using a full interpreter's memory per worker. The Codename One server's +heap grows under load and stays at that size, which is the memory after load +column; it isn't a leak, and it doesn't grow further on longer runs. Where an application needs CPython instead: packages with C extensions that have no Codename One implementation (for example `pandas`, `scipy` or `torch`), and code that builds source text and runs it with `exec`. + +=== Performance and Java hot spots + +The comparisons above are Python programs written for Python. The workloads +below are the ones every Codename One port is measured with, +`com.bench.CommonWorkloads`, ported line by line to Python in +`scripts/python-bench/common`. Each one returns a checksum, and every runtime +has to return the checksum Java returns, so all five columns compute the same +thing. The Python port reproduces Java's 32 and 64-bit wrap-around with masks, +since a Python `int` never overflows. + +Five ways to run them, each workload in a process of its own so that the peak +memory column belongs to that workload alone +(`scripts/python-compare/common_compare.py`): + +* CPython 3.14, PyPy 3.11 and GraalPy 3.13 (Community, 25.4) running the + Python source. +* Codename One, the same Python compiled to a native executable. +* Codename One with the hot spot in Java: the same executable, where the + Python calls the Java method for the workload instead of its own loop. + +image::img/python-perf-time.svg["Run time of each CommonWorkloads benchmark on CPython, PyPy, GraalPy and Codename One, with and without a Java hot spot"] + +image::img/python-perf-rss.svg["Peak memory of each CommonWorkloads benchmark on CPython, PyPy, GraalPy and Codename One, with and without a Java hot spot"] + +[cols="2,1,1,1,1,1,1,1,1,1,1",options="header"] +|=== +| Workload 2+| CPython (ms, MB) 2+| PyPy 2+| GraalPy 2+| Codename One 2+| Hot spot in Java +| `intArithmetic` | 15125 | 15 | 142 | 32 | 114 | 353 | 1754 | 86 | 65 | 11 +| `longArithmetic` | 11839 | 15 | 5410 | 42 | 11762 | 259 | 26379 | 135 | 41 | 11 +| `mathTranscendental` | 1024 | 15 | 192 | 32 | 292 | 343 | 2016 | 93 | 181 | 11 +| `arraySequential` | 1778 | 382 | 34 | 154 | 26 | 331 | 282 | 273 | 10 | 42 +| `arrayRandom` | 4662 | 199 | 308 | 93 | 252 | 248 | 550 | 105 | 240 | 27 +| `objectAllocation` | 528 | 15 | 19 | 63 | 62 | 344 | 68 | 133 | 17 | 192 +| `valueEscape` | 805 | 15 | 6.3 | 32 | 59 | 331 | 114 | 91 | 5.0 | 11 +| `hashMapChurn` | 161 | 26 | 44 | 78 | 98 | 243 | 127 | 127 | 21 | 30 +| `stringBuilding` | 670 | 15 | 30 | 40 | 71 | 257 | 695 | 86 | 14 | 86 +| `recursion` | 2228 | 15 | 182 | 57 | 193 | 280 | 355 | 11 | 88 | 11 +| `quicksort` | 1780 | 84 | 171 | 58 | 160 | 268 | 410 | 114 | 89 | 17 +|=== + +What the numbers say: + +* **Python as written.** Compiled ahead of time, Codename One runs seven of + these eleven workloads 4 to 9 times faster than CPython, without a warm-up + and in less memory than GraalPy. PyPy and GraalPy are faster still on most of + them: a JIT watches the program run and specializes the loop for the values + it sees, which a compiler that runs before the program can't do. +* **Where Codename One is slowest.** Arithmetic that leaves 64 bits + (`longArithmetic` masks the result of a multiplication that needs 128) runs + on arbitrary precision integers, and code the compiler can't type, such as + the string loop in `stringBuilding` and the `struct` call in + `mathTranscendental`, runs on the generic object path. Those are the long + blue bars, and the workloads where CPython keeps up or wins. +* **Hot spot in Java.** Moving the measured hot spot into a Java method made + each of these workloads the fastest of the five runs, and usually the + smallest in memory as well because the loop then runs on unboxed `int`, + `long` and `double` values and arrays. + +==== Tuning a hot spot in Java + +A profile usually shows a handful of functions that dominate the run time. +Any of them can be rewritten as a Java method in the same project and called +from Python like any other Java class. The rest of the program stays Python. + +The integer kernel in Python: + +[source,python] +---- +M32 = 0xFFFFFFFF + +def i32(v: int) -> int: + return ((v + 0x80000000) & M32) - 0x80000000 + +def int_arithmetic() -> int: + a: int = 0x12345678 + b: int = i32(0x9E3779B9) + checksum: int = 0 + for _ in range(40000000): + a = i32((a * 1103515245 + 12345) ^ ((b & M32) >> 3)) + b = i32((b + a) * 5 - (a << 7)) + checksum += (a ^ b) & 0xFFFF + return checksum + a + b +---- + +The same function in Java, under `common/src/main/java`: + +[source,java] +---- +package com.bench; + +public final class CommonWorkloads { + public static long intArithmetic() { + int a = 0x12345678; + int b = 0x9E3779B9; + long checksum = 0; + for (int i = 0; i < 40000000; i++) { + a = (a * 1103515245 + 12345) ^ (b >>> 3); + b = (b + a) * 5 - (a << 7); + checksum += (a ^ b) & 0xFFFF; + } + return checksum + a + b; + } +} +---- + +And the Python that calls it: + +[source,python] +---- +from java.com.bench import CommonWorkloads + +checksum = CommonWorkloads.intArithmetic() +---- + +The Java `int` wraps by itself, so the masks disappear and the loop compiles +to a few machine instructions. In the run above that took the workload from +1.75 seconds to 65 milliseconds, faster than PyPy and GraalPy, in 11 MB. + +Guidelines for choosing what to move: + +* Measure first. Only a loop that dominates a profile is worth a second + language. +* Move whole loops, not single operations. Every call across the boundary + converts its arguments, so a Java method called once per element gains + little; one called once per batch gains the most. +* Pass primitive values and Java arrays where possible. A Python `list` passed + to Java arrives as a `java.util.List` of boxed values. +* Annotate the Python that stays. Codename One compiles functions whose + parameters and locals have known types (`int`, `float`, `bool`, `str`, + lists of them and classes with `__slots__`) to typed Java code. The rest + takes the generic path, which is the one that CPython-like timings come + from. diff --git a/docs/developer-guide/img/python-perf-rss.svg b/docs/developer-guide/img/python-perf-rss.svg new file mode 100644 index 00000000000..98df6a70331 --- /dev/null +++ b/docs/developer-guide/img/python-perf-rss.svg @@ -0,0 +1,145 @@ + + +Peak memory per workload (lower is better) + +CPython + +PyPy + +GraalPy + +Codename One + +Codename One, hot spot in Java + +0 MB + +100 MB + +200 MB + +300 MB + +400 MB +intArithmetic + +15 + +32 + +353 + +86 + +11 +longArithmetic + +15 + +42 + +259 + +135 + +11 +mathTranscendental + +15 + +32 + +343 + +93 + +11 +arraySequential + +382 + +154 + +331 + +273 + +42 +arrayRandom + +199 + +93 + +248 + +105 + +27 +objectAllocation + +15 + +63 + +344 + +133 + +192 +valueEscape + +15 + +32 + +331 + +91 + +11 +hashMapChurn + +26 + +78 + +243 + +127 + +30 +stringBuilding + +15 + +40 + +257 + +86 + +86 +recursion + +15 + +57 + +280 + +11 + +11 +quicksort + +84 + +58 + +268 + +114 + +17 + diff --git a/docs/developer-guide/img/python-perf-time.svg b/docs/developer-guide/img/python-perf-time.svg new file mode 100644 index 00000000000..4bfabebcc0e --- /dev/null +++ b/docs/developer-guide/img/python-perf-time.svg @@ -0,0 +1,147 @@ + + +Run time per workload (lower is better, log scale) + +CPython + +PyPy + +GraalPy + +Codename One + +Codename One, hot spot in Java + +1 ms + +10 ms + +100 ms + +1000 ms + +10k ms + +100k ms +intArithmetic + +15125 + +142 + +114 + +1754 + +65 +longArithmetic + +11839 + +5410 + +11762 + +26379 + +41 +mathTranscendental + +1024 + +192 + +292 + +2016 + +181 +arraySequential + +1778 + +34 + +26 + +282 + +10 +arrayRandom + +4662 + +308 + +252 + +550 + +240 +objectAllocation + +528 + +19 + +62 + +68 + +17 +valueEscape + +805 + +6.3 + +59 + +114 + +5.0 +hashMapChurn + +161 + +44 + +98 + +127 + +21 +stringBuilding + +670 + +30 + +71 + +695 + +14 +recursion + +2228 + +182 + +193 + +355 + +88 +quicksort + +1780 + +171 + +160 + +410 + +89 + diff --git a/scripts/python-compare/README.md b/scripts/python-compare/README.md index 590e64b70c5..9b2ad54d0da 100644 --- a/scripts/python-compare/README.md +++ b/scripts/python-compare/README.md @@ -25,5 +25,24 @@ python3 scripts/python-compare/compare.py \ --cn1-server /tmp/flask-todo --out scripts/python-compare/RESULTS.md ``` -GraalPy is an optional arm (`--graalpy`). It is not in `RESULTS.md` because -it was not installed on the machine that produced it. +GraalPy is an optional arm (`--graalpy`); `RESULTS.md` includes it. Install +Flask into it with `graalpy -m ensurepip` and `graalpy -m pip install flask`. + +## CommonWorkloads + +`common_compare.py` runs `com.bench.CommonWorkloads`, the benchmark every +Codename One port runs, ported to Python in `scripts/python-bench/common`. +Each workload runs in its own process, so the peak RSS is that workload's, +and every checksum must equal Java's. Codename One runs twice: the Python as +written, and the same executable calling the Java method for the hot spot. + +```bash +C=scripts/python-bench/common +PY_SOURCES=$C PY_EXTRA_FILES="$C/common_workloads.py $C/java_hotspots.py" \ + PY_JAVA_SOURCES=vm/benchmarks/common/src/main/java \ + scripts/python-bench/build_parparvm.sh common_bench +python3 scripts/python-compare/common_compare.py \ + --cn1 scripts/python-bench/target/common_bench/common_bench \ + --cpython --pypy --graalpy --json common.json +python3 scripts/python-compare/render_charts.py common.json --out docs/developer-guide/img +``` diff --git a/scripts/python-compare/RESULTS.md b/scripts/python-compare/RESULTS.md index 6b1cce63a22..04e9f40b2ae 100644 --- a/scripts/python-compare/RESULTS.md +++ b/scripts/python-compare/RESULTS.md @@ -1,27 +1,29 @@ -# Codename One Python vs CPython, PyPy +# Codename One Python vs CPython, PyPy, GraalPy Measured on Darwin arm64, 2026-10-27. Codename One rows are native binaries (Python -> Java -> ParparVM C -> clang -O3). - CPython: `python3.14 (venv with Flask)` (3.14.5 cpython) - PyPy: `~/.pyenv/versions/pypy3.11-7.3.19/bin/pypy3` (3.11.11 pypy) +- GraalPy: `graalpy-community3.13-25.4.4.1.1-macos-aarch64/bin/graalpy` (3.13.14 graalpy) ## Startup: `print('hello')` | | wall ms (best of 10) | peak RSS MB | |---|---:|---:| -| CPython | 21.2 | 14.4 | -| PyPy | 20.4 | 29.2 | -| Codename One | 7.6 | 8.7 | +| CPython | 20.3 | 14.3 | +| PyPy | 18.2 | 29.2 | +| GraalPy | 34.9 | 116.6 | +| Codename One | 5.9 | 8.7 | ## CPU kernels (fastest of the kernel's rounds, ms; peak RSS MB) -| kernel | CPython ms | CPython MB | PyPy ms | PyPy MB | Codename One ms | Codename One MB | -|---|---:|---:|---:|---:|---:|---:| -| nbody | 785.9 | 15.5 | 63.7 | 32.6 | 90.1 | 10.3 | -| richards | 653.9 | 15.3 | 40.3 | 34.0 | 44.7 | 10.6 | -| fannkuch | 2691.2 | 14.6 | 612.5 | 32.8 | 731.8 | 10.2 | -| spectral_norm | 734.5 | 14.8 | 11.7 | 32.2 | 11.1 | 10.3 | -| dicts_strings | 397.5 | 62.2 | 145.0 | 96.7 | 211.2 | 128.9 | +| kernel | CPython ms | CPython MB | PyPy ms | PyPy MB | GraalPy ms | GraalPy MB | Codename One ms | Codename One MB | +|---|---:|---:|---:|---:|---:|---:|---:|---:| +| nbody | 776.0 | 14.9 | 62.7 | 32.3 | 192.5 | 233.6 | 89.2 | 10.2 | +| richards | 634.8 | 15.5 | 39.9 | 33.8 | 146.5 | 260.1 | 44.7 | 10.6 | +| fannkuch | 2652.3 | 14.5 | 604.7 | 32.8 | 541.2 | 244.0 | 723.4 | 10.4 | +| spectral_norm | 729.4 | 14.6 | 12.0 | 32.2 | 7.4 | 248.3 | 11.1 | 10.2 | +| dicts_strings | 404.2 | 62.6 | 141.5 | 87.2 | 274.4 | 382.8 | 208.6 | 126.5 | ## Flask JSON API (`scripts/python-backend/samples/flask-todo`) @@ -29,9 +31,10 @@ Measured on Darwin arm64, 2026-10-27. Codename One rows are native binaries (Pyt | | first response ms | idle RSS MB | RSS after load MB | req/s | p99 ms | |---|---:|---:|---:|---:|---:| -| CPython | 97 | 39.3 | 24.5 | 687 | 16 | -| PyPy | 200 | 64.9 | 86.5 | 674 | 18 | -| Codename One | 61 | 26.7 | 390.4 | 16332 | 1 | +| CPython | 85 | 39.0 | 24.5 | 685 | 18 | +| PyPy | 196 | 65.3 | 88.7 | 688 | 17 | +| GraalPy | 609 | 325.7 | 931.0 | 561 | 44 | +| Codename One | 58 | 26.8 | 423.4 | 13564 | 2 | ## What you deploy @@ -39,5 +42,22 @@ Measured on Darwin arm64, 2026-10-27. Codename One rows are native binaries (Pyt |---|---:|---| | CPython | 82.0 | interpreter install (`/opt/homebrew/opt/python@3.14/Frameworks/Python.framework/Versions/3.14`) + Flask and its dependencies (5.2) | | PyPy | 137.7 | interpreter install (`~/.pyenv/versions/pypy3.11-7.3.19`) + Flask and its dependencies (4.9) | +| GraalPy | 343.1 | interpreter install (`graalpy-community3.13-25.4.4.1.1-macos-aarch64`) + Flask and its dependencies (11.2) | | Codename One | 22.7 | one self-contained executable, Flask included | + +## CommonWorkloads (common_compare.py; ms, peak RSS MB) + +| workload | CPython ms | MB | PyPy ms | MB | GraalPy ms | MB | Codename One ms | MB | Codename One, hot spot in Java ms | MB | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| intArithmetic | 15124.8 | 15.0 | 141.8 | 32.1 | 114.2 | 353.0 | 1753.8 | 85.6 | 65.2 | 11.4 | +| longArithmetic | 11839.2 | 14.8 | 5410.0 | 41.6 | 11761.8 | 259.0 | 26378.6 | 135.3 | 40.6 | 11.3 | +| mathTranscendental | 1024.5 | 15.0 | 191.6 | 31.8 | 292.2 | 342.9 | 2015.7 | 93.0 | 180.8 | 11.4 | +| arraySequential | 1778.4 | 382.2 | 33.7 | 154.0 | 26.2 | 331.3 | 282.4 | 273.1 | 10.5 | 41.8 | +| arrayRandom | 4662.5 | 198.6 | 308.2 | 93.1 | 252.1 | 248.1 | 549.5 | 104.6 | 239.8 | 26.6 | +| objectAllocation | 527.8 | 15.1 | 19.1 | 63.3 | 62.0 | 344.0 | 68.2 | 133.1 | 16.6 | 191.7 | +| valueEscape | 805.3 | 14.8 | 6.3 | 32.0 | 59.1 | 330.7 | 114.2 | 91.0 | 5.0 | 11.5 | +| hashMapChurn | 160.9 | 25.5 | 43.6 | 77.6 | 97.8 | 243.5 | 127.3 | 126.6 | 20.8 | 30.3 | +| stringBuilding | 670.0 | 14.9 | 30.5 | 40.1 | 70.7 | 256.9 | 695.2 | 86.5 | 14.4 | 86.4 | +| recursion | 2227.7 | 14.8 | 181.9 | 57.2 | 193.3 | 280.4 | 355.1 | 11.0 | 87.6 | 11.3 | +| quicksort | 1779.8 | 83.6 | 171.4 | 58.0 | 160.0 | 267.6 | 410.4 | 113.5 | 89.3 | 17.2 | diff --git a/scripts/python-compare/common-results.json b/scripts/python-compare/common-results.json new file mode 100644 index 00000000000..237a5d31a41 --- /dev/null +++ b/scripts/python-compare/common-results.json @@ -0,0 +1,253 @@ +{ + "arms": [ + "CPython", + "PyPy", + "GraalPy", + "Codename One", + "Codename One, hot spot in Java" + ], + "results": { + "intArithmetic": { + "CPython": { + "ms": 15124.8, + "rss_mb": 14.96875 + }, + "PyPy": { + "ms": 141.8, + "rss_mb": 32.125 + }, + "GraalPy": { + "ms": 114.2, + "rss_mb": 352.953125 + }, + "Codename One": { + "ms": 1753.8, + "rss_mb": 85.5625 + }, + "Codename One, hot spot in Java": { + "ms": 65.2, + "rss_mb": 11.375 + } + }, + "longArithmetic": { + "CPython": { + "ms": 11839.2, + "rss_mb": 14.796875 + }, + "PyPy": { + "ms": 5410.0, + "rss_mb": 41.578125 + }, + "GraalPy": { + "ms": 11761.8, + "rss_mb": 259.015625 + }, + "Codename One": { + "ms": 26378.6, + "rss_mb": 135.328125 + }, + "Codename One, hot spot in Java": { + "ms": 40.6, + "rss_mb": 11.3125 + } + }, + "mathTranscendental": { + "CPython": { + "ms": 1024.5, + "rss_mb": 14.953125 + }, + "PyPy": { + "ms": 191.6, + "rss_mb": 31.765625 + }, + "GraalPy": { + "ms": 292.2, + "rss_mb": 342.9375 + }, + "Codename One": { + "ms": 2015.7, + "rss_mb": 93.0 + }, + "Codename One, hot spot in Java": { + "ms": 180.8, + "rss_mb": 11.40625 + } + }, + "arraySequential": { + "CPython": { + "ms": 1778.4, + "rss_mb": 382.203125 + }, + "PyPy": { + "ms": 33.7, + "rss_mb": 154.046875 + }, + "GraalPy": { + "ms": 26.2, + "rss_mb": 331.28125 + }, + "Codename One": { + "ms": 282.4, + "rss_mb": 273.09375 + }, + "Codename One, hot spot in Java": { + "ms": 10.5, + "rss_mb": 41.796875 + } + }, + "arrayRandom": { + "CPython": { + "ms": 4662.5, + "rss_mb": 198.609375 + }, + "PyPy": { + "ms": 308.2, + "rss_mb": 93.078125 + }, + "GraalPy": { + "ms": 252.1, + "rss_mb": 248.125 + }, + "Codename One": { + "ms": 549.5, + "rss_mb": 104.59375 + }, + "Codename One, hot spot in Java": { + "ms": 239.8, + "rss_mb": 26.625 + } + }, + "objectAllocation": { + "CPython": { + "ms": 527.8, + "rss_mb": 15.078125 + }, + "PyPy": { + "ms": 19.1, + "rss_mb": 63.28125 + }, + "GraalPy": { + "ms": 62.0, + "rss_mb": 344.03125 + }, + "Codename One": { + "ms": 68.2, + "rss_mb": 133.0625 + }, + "Codename One, hot spot in Java": { + "ms": 16.6, + "rss_mb": 191.6875 + } + }, + "valueEscape": { + "CPython": { + "ms": 805.3, + "rss_mb": 14.78125 + }, + "PyPy": { + "ms": 6.3, + "rss_mb": 32.03125 + }, + "GraalPy": { + "ms": 59.1, + "rss_mb": 330.71875 + }, + "Codename One": { + "ms": 114.2, + "rss_mb": 90.984375 + }, + "Codename One, hot spot in Java": { + "ms": 5.0, + "rss_mb": 11.453125 + } + }, + "hashMapChurn": { + "CPython": { + "ms": 160.9, + "rss_mb": 25.546875 + }, + "PyPy": { + "ms": 43.6, + "rss_mb": 77.625 + }, + "GraalPy": { + "ms": 97.8, + "rss_mb": 243.46875 + }, + "Codename One": { + "ms": 127.3, + "rss_mb": 126.5625 + }, + "Codename One, hot spot in Java": { + "ms": 20.8, + "rss_mb": 30.265625 + } + }, + "stringBuilding": { + "CPython": { + "ms": 670.0, + "rss_mb": 14.921875 + }, + "PyPy": { + "ms": 30.5, + "rss_mb": 40.078125 + }, + "GraalPy": { + "ms": 70.7, + "rss_mb": 256.90625 + }, + "Codename One": { + "ms": 695.2, + "rss_mb": 86.5 + }, + "Codename One, hot spot in Java": { + "ms": 14.4, + "rss_mb": 86.421875 + } + }, + "recursion": { + "CPython": { + "ms": 2227.7, + "rss_mb": 14.796875 + }, + "PyPy": { + "ms": 181.9, + "rss_mb": 57.15625 + }, + "GraalPy": { + "ms": 193.3, + "rss_mb": 280.40625 + }, + "Codename One": { + "ms": 355.1, + "rss_mb": 10.96875 + }, + "Codename One, hot spot in Java": { + "ms": 87.6, + "rss_mb": 11.34375 + } + }, + "quicksort": { + "CPython": { + "ms": 1779.8, + "rss_mb": 83.609375 + }, + "PyPy": { + "ms": 171.4, + "rss_mb": 57.96875 + }, + "GraalPy": { + "ms": 160.0, + "rss_mb": 267.625 + }, + "Codename One": { + "ms": 410.4, + "rss_mb": 113.53125 + }, + "Codename One, hot spot in Java": { + "ms": 89.3, + "rss_mb": 17.171875 + } + } + } +} \ No newline at end of file diff --git a/scripts/python-compare/render_charts.py b/scripts/python-compare/render_charts.py new file mode 100644 index 00000000000..35232028841 --- /dev/null +++ b/scripts/python-compare/render_charts.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# This code is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License version 2 only, as +# published by the Free Software Foundation. Codename One designates this +# particular file as subject to the "Classpath" exception as provided +# by Oracle in the LICENSE file that accompanied this code. +# +# This code is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# version 2 for more details (a copy is included in the LICENSE file that +# accompanied this code). +# +# You should have received a copy of the GNU General Public License version +# 2 along with this work; if not, write to the Free Software Foundation, +# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Please contact Codename One through http://www.codenameone.com/ if you +# need additional information or have any questions. +"""Renders the developer guide's Python performance charts. + + render_charts.py common.json --out docs/developer-guide/img + +The input is what common_compare.py --json writes. Two SVGs come out: +python-perf-time.svg (milliseconds, log scale, because the runtimes differ by +two orders of magnitude) and python-perf-rss.svg (peak RSS, linear). The SVGs +are self-contained so the website and the PDF build render them alike. +""" + +import argparse +import json +import math +import os + +COLORS = { + "CPython": "#9aa0a6", + "PyPy": "#f9ab00", + "GraalPy": "#a142f4", + "Codename One": "#1a73e8", + "Codename One, hot spot in Java": "#0b8043", +} +TEXT = "#202124" +MUTED = "#5f6368" +WIDTH = 760 +LEFT = 150 +RIGHT = 80 +BAR = 9 +ROW_GAP = 14 + + +def esc(s): + return s.replace("&", "&").replace("<", "<").replace(">", ">") + + +def chart(data, key, title, unit, log): + arms = data["arms"] + rows = list(data["results"].items()) + plot = WIDTH - LEFT - RIGHT + values = [r[a][key] for _, r in rows for a in arms] + hi = max(values) + lo = min(v for v in values if v > 0) + if log: + lo_e = math.floor(math.log10(lo)) + hi_e = math.ceil(math.log10(hi)) + + def x(v): + return plot * (math.log10(max(v, 10 ** lo_e)) - lo_e) / (hi_e - lo_e) + ticks = [10 ** e for e in range(lo_e, hi_e + 1)] + else: + step = 10 ** math.floor(math.log10(hi)) + top = math.ceil(hi / step) * step + + def x(v): + return plot * v / top + ticks = [step * i for i in range(int(top / step) + 1)] + legend_h = 22 * ((len(arms) + 2) // 3) + top_y = 40 + legend_h + group = len(arms) * BAR + ROW_GAP + height = top_y + group * len(rows) + 30 + o = ['' % (WIDTH, height, WIDTH, height), + '', + '%s' + % (LEFT, TEXT, esc(title))] + for i, a in enumerate(arms): + lx = LEFT + (i % 3) * 200 + ly = 36 + (i // 3) * 22 + o.append('' % (lx, ly, COLORS.get(a, MUTED))) + o.append('%s' % (lx + 18, ly + 11, TEXT, esc(a))) + bottom = top_y + group * len(rows) + for t in ticks: + tx = LEFT + x(t) + o.append('' % (tx, top_y - 4, tx, bottom)) + label = ("%g" % t) if t < 10000 else ("%gk" % (t / 1000)) + o.append('%s %s' + % (tx, bottom + 16, MUTED, label, unit)) + y = top_y + for name, r in rows: + o.append('%s' + % (LEFT - 8, y + len(arms) * BAR / 2 + 4, TEXT, esc(name))) + for a in arms: + v = r[a][key] + w = max(x(v), 1) + o.append('' + % (LEFT, y, w, BAR - 1, COLORS.get(a, MUTED))) + o.append('%s' + % (LEFT + w + 4, y + BAR - 2, MUTED, ("%.0f" % v) if v >= 10 else ("%.1f" % v))) + y += BAR + y += ROW_GAP + o.append("") + return "\n".join(o) + "\n" + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("json") + ap.add_argument("--out", required=True) + a = ap.parse_args() + with open(a.json) as f: + data = json.load(f) + with open(os.path.join(a.out, "python-perf-time.svg"), "w") as f: + f.write(chart(data, "ms", "Run time per workload (lower is better, log scale)", "ms", True)) + with open(os.path.join(a.out, "python-perf-rss.svg"), "w") as f: + f.write(chart(data, "rss_mb", "Peak memory per workload (lower is better)", "MB", False)) + + +if __name__ == "__main__": + main() From 448ee65d97a92f1374527151451912de32f4dc60 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Tue, 27 Oct 2026 19:34:22 +0200 Subject: [PATCH 022/112] Python: drop fully qualified Set in Infer (PMD) Co-Authored-By: Claude Opus 5.5 --- .../java/com/codename1/python/transpiler/codegen/Infer.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Infer.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Infer.java index 1b88a6bec12..c0570e1d883 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Infer.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Infer.java @@ -96,11 +96,11 @@ Ty param(String n) { final Map siteRet = new HashMap(); /// Globals bound once, by `import math`: calls of its functions on /// numbers are typed (see `MATH_FNS`) and run as `PyMath` directly. - final java.util.Set mathNames = new HashSet(); + final Set mathNames = new HashSet(); /// `math` functions typed code calls, with their arity. static final Map MATH_FNS = new HashMap(); /// `math` constants, all floats. - static final java.util.Set MATH_CONSTS = new HashSet(); + static final Set MATH_CONSTS = new HashSet(); static { String[] unary = {"sqrt", "sin", "cos", "tan", "asin", "acos", "atan", "exp", "fabs", "log", "log10"}; From 8d787024ed0f3256c40698e45e521c15c38e852d Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Tue, 27 Oct 2026 19:57:48 +0200 Subject: [PATCH 023/112] Python: hold provably bounded ints in Java longs IntRanges runs interval analysis over each typed function: loop counters are bounded by their range, accumulators by trip count times their per-pass growth, and masked values by the mask. A local whose every value fits in 64 bits becomes a Java long, and an int expression whose result fits is computed with wrapping long arithmetic, which equals the exact result however large the intermediates are. Expression helpers such as i32/i64 are inlined, module int constants are folded, and >> of a value masked into [0, 2^64) becomes >>>. Builtins without a typed lowering (print, sorted, ...) are now called generically from typed code instead of leaving the whole function untyped. Co-Authored-By: Claude Opus 5.5 --- .../com/codename1/python/runtime/Typed.java | 12 + .../python/transpiler/codegen/Emitter.java | 17 +- .../python/transpiler/codegen/Infer.java | 18 + .../python/transpiler/codegen/IntRanges.java | 618 ++++++++++++++++++ .../python/transpiler/codegen/TypedEmit.java | 320 ++++++++- .../test/resources/conformance/int_ranges.out | 33 + .../test/resources/conformance/int_ranges.py | 92 +++ 7 files changed, 1098 insertions(+), 12 deletions(-) create mode 100644 maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/IntRanges.java create mode 100644 maven/python-transpiler/src/test/resources/conformance/int_ranges.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/int_ranges.py diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Typed.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Typed.java index 14369782c36..4e0f2a61091 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Typed.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Typed.java @@ -79,6 +79,18 @@ public static boolean asBool(Object o) { throw mismatch(o); } + /// An int the compiler proved fits in 64 bits, as a `long`. Ints in + /// that range are always held as a `Long`. + public static long toLong(Object o) { + if (o instanceof Long) { + return ((Long) o).longValue(); + } + if (o instanceof Boolean) { + return ((Boolean) o).booleanValue() ? 1L : 0L; + } + throw mismatch(o); + } + public static double asFloat(Object o) { if (o instanceof Double) { return ((Double) o).doubleValue(); diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Emitter.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Emitter.java index 6591605468b..7faad717012 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Emitter.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Emitter.java @@ -389,13 +389,18 @@ private String load(Name nm) { if (moduleScope.hasStarImport) { return "dyn(" + javaString(n) + ")"; } - String f = builtins.get(n); - if (f == null) { - f = "B" + builtins.size() + "_" + mangle(n); - builtins.put(n, f); - } - return "b(" + f + ", " + javaString(n) + ")"; + return builtinRef(n); + } + } + + /// The builtin `n`, read through its lazily bound field. + String builtinRef(String n) { + String f = builtins.get(n); + if (f == null) { + f = "B" + builtins.size() + "_" + mangle(n); + builtins.put(n, f); } + return "b(" + f + ", " + javaString(n) + ")"; } void storeName(String n, String value, StringBuilder out, int ind) { diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Infer.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Infer.java index c0570e1d883..76699402d3f 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Infer.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Infer.java @@ -152,6 +152,7 @@ static Infer run(List body, Layouts layouts, Scope moduleScope) { private void collect(List body) { countBindings(body); + intConstants(body); for (Layouts.Cls k : layouts.classes.values()) { fields.put(k, new LinkedHashMap()); for (String f : k.fields) { @@ -269,6 +270,23 @@ private static boolean arithmetic(Expr e, Scope sc) { return false; } + /// Module globals bound once, at the top level, to an int literal (`M32 = + /// 0xFFFFFFFF`): their value is known wherever a function reads them. + final Map intConsts = new HashMap(); + + private void intConstants(List body) { + for (Stmt s : body) { + if (s instanceof Assign && ((Assign) s).targets.size() == 1 + && ((Assign) s).targets.get(0) instanceof Name) { + String n = ((Name) ((Assign) s).targets.get(0)).id; + java.math.BigInteger v = IntRanges.literal(((Assign) s).value); + if (v != null && stable(n)) { + intConsts.put(n, v); + } + } + } + } + private boolean stable(String n) { Integer c = moduleBindings.get(n); return c != null && c.intValue() == 1; diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/IntRanges.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/IntRanges.java new file mode 100644 index 00000000000..7b99b05bea7 --- /dev/null +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/IntRanges.java @@ -0,0 +1,618 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.transpiler.codegen; + +import com.codename1.python.transpiler.ast.Node.*; +import com.codename1.python.transpiler.ast.Scope; +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/// The values each int local of a typed function can hold, as intervals. +/// +/// A Python int never overflows, so typed code has kept every int boxed +/// and checked each operation for overflow. Most ints in real code are +/// bounded, by construction: a loop counter by its `range`, an accumulator +/// by its trip count, and a value masked with `& 0xFFFFFFFF` by the mask. +/// The masking idioms that emulate fixed-width arithmetic (`((v + 2**31) & +/// M32) - 2**31`) bound their result whatever goes in. A local whose every +/// value fits in 64 bits is held in a Java `long`, and an expression whose +/// result fits is computed with wrapping `long` arithmetic: when the true +/// result is known to fit, the result modulo 2^64 is the result, however +/// large the intermediate values were. +/// +/// The analysis is abstract interpretation over the statements, with loops +/// run to a fixpoint. A local that still grows after a few passes over a +/// `range` loop of known trip count, by the same amount each pass, is an +/// accumulator: its bound is the trip count times that amount. Anything +/// else that grows has no bound. +final class IntRanges { + static final BigInteger LONG_MIN = BigInteger.valueOf(Long.MIN_VALUE); + static final BigInteger LONG_MAX = BigInteger.valueOf(Long.MAX_VALUE); + private static final BigInteger TWO64 = BigInteger.ONE.shiftLeft(64); + private static final int PASSES = 4; + + /// An interval; a null bound is unbounded. + static final class Iv { + static final Iv TOP = new Iv(null, null); + final BigInteger lo; + final BigInteger hi; + + Iv(BigInteger lo, BigInteger hi) { + this.lo = lo; + this.hi = hi; + } + + static Iv of(long v) { + BigInteger b = BigInteger.valueOf(v); + return new Iv(b, b); + } + + boolean fitsLong() { + return lo != null && hi != null && lo.compareTo(LONG_MIN) >= 0 && hi.compareTo(LONG_MAX) <= 0; + } + + /// Within [0, 2^64): a value whose low 64 bits, read unsigned, are it. + boolean fitsUnsigned64() { + return lo != null && hi != null && lo.signum() >= 0 && hi.compareTo(TWO64) < 0; + } + + Iv join(Iv o) { + if (o == null) { + return this; + } + return new Iv(lo == null || o.lo == null ? null : lo.min(o.lo), + hi == null || o.hi == null ? null : hi.max(o.hi)); + } + + boolean same(Iv o) { + return o != null && eq(lo, o.lo) && eq(hi, o.hi); + } + + private static boolean eq(BigInteger a, BigInteger b) { + return a == null ? b == null : a.equals(b); + } + } + + private final Infer in; + private final Infer.Fn fn; + private final Scope sc; + /// The int locals considered: typed INT, not parameters. + private final Set ints = new HashSet(); + /// Every value each local ever holds. + private final Map seen = new HashMap(); + /// Parameters of an expression function being inlined, bound to the + /// argument's interval. + private final List> bound = new ArrayList>(); + + IntRanges(Infer in, Infer.Fn fn) { + this.in = in; + this.fn = fn; + this.sc = fn.def.scope; + for (Map.Entry e : fn.locals.entrySet()) { + if (e.getValue().k == Ty.K.INT && !e.getValue().opt && !sc.params.contains(e.getKey())) { + ints.add(e.getKey()); + } + } + if (!ints.isEmpty()) { + Map env = new HashMap(); + run(fn.def.body, env, new Loop()); + } + } + + /// The locals held in a Java `long`. + Set longLocals() { + Set r = new HashSet(); + for (String n : ints) { + Iv v = seen.get(n); + if (v != null && v.fitsLong()) { + r.add(n); + } + } + return r; + } + + /// The int literal `e`, or null: a constant, or a negated one. + static BigInteger literal(Expr e) { + if (e instanceof Constant && ((Constant) e).kind == Constant.Kind.INT) { + return parse(((Constant) e).text); + } + if (e instanceof UnaryOp && "-".equals(((UnaryOp) e).op)) { + BigInteger v = literal(((UnaryOp) e).operand); + return v == null ? null : v.negate(); + } + return null; + } + + static BigInteger parse(String text) { + String t = text.replace("_", "").toLowerCase(java.util.Locale.ROOT); + try { + if (t.startsWith("0x")) { + return new BigInteger(t.substring(2), 16); + } + if (t.startsWith("0o")) { + return new BigInteger(t.substring(2), 8); + } + if (t.startsWith("0b")) { + return new BigInteger(t.substring(2), 2); + } + return new BigInteger(t); + } catch (NumberFormatException e) { + return null; + } + } + + // ---- statements ------------------------------------------------------ + + /// The states that leave a loop body early. + private static final class Loop { + Map brk; + Map cont; + } + + private void store(String n, Iv v, Map env) { + if (!ints.contains(n)) { + return; + } + env.put(n, v); + Iv s = seen.get(n); + seen.put(n, s == null ? v : s.join(v)); + } + + private void top(Set names, Map env) { + for (String n : names) { + store(n, Iv.TOP, env); + } + } + + private static Map join(Map a, Map b) { + if (a == null) { + return b == null ? null : new HashMap(b); + } + if (b == null) { + return new HashMap(a); + } + Map r = new HashMap(a); + for (Map.Entry e : b.entrySet()) { + Iv x = r.get(e.getKey()); + r.put(e.getKey(), x == null ? e.getValue() : x.join(e.getValue())); + } + return r; + } + + /// Runs `body` from `env`; answers the state at its normal end, or null + /// when it cannot end normally. + private Map run(List body, Map env, Loop loop) { + Map cur = env; + for (Stmt s : body) { + if (cur == null) { + return null; + } + cur = stmt(s, cur, loop); + } + return cur; + } + + private Map stmt(Stmt s, Map env, Loop loop) { + walrus(s, env); + if (s instanceof Assign) { + Assign a = (Assign) s; + Iv v = eval(a.value, env); + for (Expr t : a.targets) { + if (t instanceof Name) { + store(((Name) t).id, v, env); + } else { + top(TypedEmit.assigned(Collections.singletonList(new Assign( + Collections.singletonList(t), a.value))), env); + } + } + return env; + } + if (s instanceof AugAssign) { + AugAssign a = (AugAssign) s; + if (a.target instanceof Name) { + store(((Name) a.target).id, eval(new BinOp(a.target, a.op, a.value), env), env); + } + return env; + } + if (s instanceof If) { + If i = (If) s; + return join(run(i.body, new HashMap(env), loop), + run(i.orelse, new HashMap(env), loop)); + } + if (s instanceof For) { + return forLoop((For) s, env); + } + if (s instanceof While) { + While w = (While) s; + Map out = loop(w.body, env, null, null, null); + return run(w.orelse, out, loop); + } + if (s instanceof Return || s instanceof Raise) { + return null; + } + if (s instanceof Break) { + loop.brk = join(loop.brk, env); + return null; + } + if (s instanceof Continue) { + loop.cont = join(loop.cont, env); + return null; + } + if (s instanceof ExprStmt || s instanceof Pass) { + return env; + } + // Anything else (try, with, match, a nested def): what it stores is + // unbounded, and it may leave the loop in ways not followed here. + top(TypedEmit.assigned(Collections.singletonList(s)), env); + return env; + } + + private void walrus(Object o, Map env) { + if (o instanceof NamedExpr) { + store(((NamedExpr) o).target.id, Iv.TOP, env); + } + if (!(o instanceof Stmt) || o instanceof ExprStmt || o instanceof Assign || o instanceof AugAssign + || o instanceof Return) { + Infer.children(o, x -> walrus(x, env)); + } + } + + private Map forLoop(For f, Map env) { + String target = f.target instanceof Name ? ((Name) f.target).id : null; + Iv tv = Iv.TOP; + BigInteger trips = null; + if (in.rangeCall(f.iter, sc)) { + List a = ((Call) f.iter).args; + Iv start = a.size() == 1 ? Iv.of(0) : eval(a.get(0), env); + Iv stop = eval(a.get(a.size() == 1 ? 0 : 1), env); + BigInteger step = a.size() == 3 ? literal(a.get(2)) : BigInteger.ONE; + if (step != null && step.signum() != 0 && start.lo != null && start.hi != null && stop.lo != null + && stop.hi != null) { + if (step.signum() > 0) { + tv = new Iv(start.lo, stop.hi.subtract(BigInteger.ONE).max(start.lo)); + trips = stop.hi.subtract(start.lo).max(BigInteger.ZERO); + } else { + tv = new Iv(stop.lo.add(BigInteger.ONE).min(start.hi), start.hi); + trips = start.hi.subtract(stop.lo).max(BigInteger.ZERO); + } + trips = trips.add(step.abs()).subtract(BigInteger.ONE).divide(step.abs()); + } + } else if (target != null) { + top(Collections.singleton(target), env); + } else { + top(TypedEmit.assigned(Collections.singletonList(f)), env); + } + Map out = loop(f.body, env, target, tv, trips); + return run(f.orelse, out, new Loop()); + } + + /// A loop over `body`, entered from `entry`, that runs at most `trips` + /// times (null: unknown). Answers the state after it. + /// + /// Passes run until the state at the head of the loop stops changing. + /// From the third pass on, a local that grew by the same interval in the + /// last two passes, in a loop of known trip count, is widened to its + /// value after `trips - 1` passes, and one more pass must confirm that + /// it grows by that same interval and nothing else moves. Whatever fails + /// that is unbounded. + private Map loop(List body, Map entry, String target, Iv tv, BigInteger trips) { + Map head = new HashMap(entry); + Map exit = new HashMap(entry); + Map prevDelta = new HashMap(); + Map widened = new HashMap(); + for (int pass = 0;; pass++) { + Map in0 = new HashMap(head); + if (target != null) { + store(target, tv, in0); + } + Loop l = new Loop(); + Map out = join(run(body, in0, l), l.cont); + exit = join(join(exit, out), l.brk); + Map delta = new HashMap(); + Set growing = new HashSet(); + if (out != null) { + for (Map.Entry e : out.entrySet()) { + String n = e.getKey(); + Iv h = head.get(n); + Iv j = h == null ? e.getValue() : h.join(e.getValue()); + Iv w = widened.get(n); + if (w != null) { + // Widened: it may grow by its delta, past the head. + Iv d = diff(e.getValue(), h); + if (d == null || !d.same(w)) { + growing.add(n); + delta.put(n, null); + } + } else if (!j.same(h)) { + growing.add(n); + delta.put(n, diff(e.getValue(), h)); + } + } + } + if (growing.isEmpty()) { + return exit; + } + Map next = join(head, out); + boolean widen = pass >= 2 && pass < PASSES; + for (String n : growing) { + Iv d = delta.get(n); + Iv e0 = entry.get(n); + if (widen && !widened.containsKey(n) && trips != null && trips.signum() > 0 && d != null + && d.same(prevDelta.get(n)) && e0 != null && e0.lo != null && e0.hi != null) { + BigInteger k = trips.subtract(BigInteger.ONE); + next.put(n, new Iv(e0.lo.add(d.lo.min(BigInteger.ZERO).multiply(k)), + e0.hi.add(d.hi.max(BigInteger.ZERO).multiply(k)))); + widened.put(n, d); + } else if (pass >= 2) { + next.put(n, Iv.TOP); + widened.remove(n); + } + } + prevDelta = delta; + head = next; + } + } + + /// How much `b` moved to become `a`, per bound; null when either is + /// unbounded there. + private static Iv diff(Iv a, Iv b) { + if (a == null || b == null || a.lo == null || a.hi == null || b.lo == null || b.hi == null) { + return null; + } + return new Iv(a.lo.subtract(b.lo), a.hi.subtract(b.hi)); + } + + // ---- expressions ----------------------------------------------------- + + /// The interval of int expression `e` in state `env`, using what each + /// local has been anywhere when `env` is null. TOP for anything that is + /// not provably an int built from ints the analysis knows. + Iv eval(Expr e, Map env) { + if (e instanceof Constant) { + Constant c = (Constant) e; + if (c.kind == Constant.Kind.INT) { + BigInteger v = parse(c.text); + return v == null ? Iv.TOP : new Iv(v, v); + } + return Iv.TOP; + } + if (e instanceof Name) { + return name(((Name) e).id, env); + } + if (e instanceof UnaryOp) { + UnaryOp u = (UnaryOp) e; + Iv o = eval(u.operand, env); + switch (u.op) { + case "+": + return o; + case "-": + return new Iv(o.hi == null ? null : o.hi.negate(), o.lo == null ? null : o.lo.negate()); + case "~": + return new Iv(o.hi == null ? null : o.hi.not(), o.lo == null ? null : o.lo.not()); + default: + return Iv.TOP; + } + } + if (e instanceof BinOp) { + BinOp b = (BinOp) e; + return binop(b.op, eval(b.left, env), eval(b.right, env), b.right); + } + if (e instanceof IfExp) { + IfExp x = (IfExp) e; + return eval(x.body, env).join(eval(x.orelse, env)); + } + if (e instanceof Call) { + return call((Call) e, env); + } + return Iv.TOP; + } + + private Iv name(String n, Map env) { + for (int i = bound.size() - 1; i >= 0; i--) { + Iv v = bound.get(i).get(n); + if (v != null) { + return v; + } + if (i == bound.size() - 1) { + // An inlined body sees its parameters and the globals only. + BigInteger c = in.intConsts.get(n); + return c == null ? Iv.TOP : new Iv(c, c); + } + } + switch (sc.resolve(n)) { + case LOCAL: { + if (!ints.contains(n)) { + return Iv.TOP; + } + Iv v = env == null ? seen.get(n) : env.get(n); + return v == null ? Iv.TOP : v; + } + case GLOBAL: { + BigInteger c = in.intConsts.get(n); + return c == null ? Iv.TOP : new Iv(c, c); + } + default: + return Iv.TOP; + } + } + + /// The expression function `c` calls (see `Infer.exprBody`), or null. + Infer.Fn inlinable(Call c) { + if (!(c.func instanceof Name) || !c.keywords.isEmpty()) { + return null; + } + String n = ((Name) c.func).id; + Scope s = bound.isEmpty() ? sc : null; + if (s != null && s.resolve(n) != Scope.Binding.GLOBAL) { + return null; + } + Infer.Fn f = in.globalFns.get(n); + if (f == null || Infer.exprBody(f.def) == null || f.names.size() != c.args.size() + || !f.def.args.defaults.isEmpty() || !f.def.decorators.isEmpty() || bound.size() > 4) { + return null; + } + for (Expr a : c.args) { + if (a instanceof Starred) { + return null; + } + } + return f; + } + + private Iv call(Call c, Map env) { + Infer.Fn f = inlinable(c); + if (f != null) { + Map b = new HashMap(); + for (int i = 0; i < c.args.size(); i++) { + b.put(f.names.get(i), eval(c.args.get(i), env)); + } + bound.add(b); + try { + return eval(Infer.exprBody(f.def), env); + } finally { + bound.remove(bound.size() - 1); + } + } + return Iv.TOP; + } + + static Iv binop(String op, Iv l, Iv r, Expr right) { + switch (op) { + case "+": + return new Iv(add(l.lo, r.lo), add(l.hi, r.hi)); + case "-": + return new Iv(l.lo == null || r.hi == null ? null : l.lo.subtract(r.hi), + l.hi == null || r.lo == null ? null : l.hi.subtract(r.lo)); + case "*": + return mul(l, r); + case "&": + return and(l, r); + case "|": + case "^": + return bits(l, r); + case "<<": { + BigInteger c = literal(right); + if (c == null || c.signum() < 0 || c.bitLength() > 7 || l.lo == null || l.hi == null) { + return Iv.TOP; + } + return new Iv(l.lo.shiftLeft(c.intValue()), l.hi.shiftLeft(c.intValue())); + } + case ">>": { + BigInteger c = literal(right); + if (c == null || c.signum() < 0 || c.bitLength() > 7) { + return Iv.TOP; + } + return new Iv(l.lo == null ? null : l.lo.shiftRight(c.intValue()), + l.hi == null ? null : l.hi.shiftRight(c.intValue())); + } + case "%": + if (r.lo != null && r.lo.signum() > 0) { + return new Iv(BigInteger.ZERO, r.hi == null ? null : r.hi.subtract(BigInteger.ONE)); + } + return Iv.TOP; + case "//": + if (r.lo != null && r.lo.signum() > 0 && l.lo != null && l.hi != null) { + BigInteger lo = l.lo.signum() >= 0 ? l.lo.divide(r.hi == null ? l.lo.add(BigInteger.ONE) : r.hi) + : floorDiv(l.lo, r.lo); + return new Iv(lo, l.hi.signum() >= 0 ? l.hi.divide(r.lo) : floorDiv(l.hi, r.hi == null ? r.lo : r.hi)); + } + return Iv.TOP; + default: + return Iv.TOP; + } + } + + private static BigInteger floorDiv(BigInteger a, BigInteger b) { + BigInteger[] qr = a.divideAndRemainder(b); + return qr[1].signum() != 0 && (qr[1].signum() != b.signum()) ? qr[0].subtract(BigInteger.ONE) : qr[0]; + } + + private static BigInteger add(BigInteger a, BigInteger b) { + return a == null || b == null ? null : a.add(b); + } + + private static Iv mul(Iv l, Iv r) { + if (l.lo == null || l.hi == null || r.lo == null || r.hi == null) { + return Iv.TOP; + } + BigInteger a = l.lo.multiply(r.lo); + BigInteger b = l.lo.multiply(r.hi); + BigInteger c = l.hi.multiply(r.lo); + BigInteger d = l.hi.multiply(r.hi); + return new Iv(a.min(b).min(c).min(d), a.max(b).max(c).max(d)); + } + + /// `&`: with a non-negative side the result lies in [0, its maximum], + /// whatever the other side is. + private static Iv and(Iv l, Iv r) { + boolean ln = l.lo != null && l.lo.signum() >= 0; + boolean rn = r.lo != null && r.lo.signum() >= 0; + if (ln && rn) { + return new Iv(BigInteger.ZERO, l.hi == null ? r.hi : r.hi == null ? l.hi : l.hi.min(r.hi)); + } + if (ln) { + return new Iv(BigInteger.ZERO, l.hi); + } + if (rn) { + return new Iv(BigInteger.ZERO, r.hi); + } + return bits(l, r); + } + + /// `|`, `^` (and `&` of two possibly negative values): within the + /// two's complement width of the wider operand. + private static Iv bits(Iv l, Iv r) { + if (l.lo == null || l.hi == null || r.lo == null || r.hi == null) { + return Iv.TOP; + } + int w = Math.max(Math.max(l.lo.bitLength(), l.hi.bitLength()), Math.max(r.lo.bitLength(), r.hi.bitLength())); + BigInteger lim = BigInteger.ONE.shiftLeft(w); + if (l.lo.signum() >= 0 && r.lo.signum() >= 0) { + return new Iv(BigInteger.ZERO, lim.subtract(BigInteger.ONE)); + } + return new Iv(lim.negate(), lim.subtract(BigInteger.ONE)); + } + + // ---- inlined calls, for the emitter --------------------------------- + + /// Binds the parameters of an expression function the emitter inlines, + /// until the matching `pop`. + void push(Map params) { + bound.add(params); + } + + void pop() { + bound.remove(bound.size() - 1); + } + + boolean inlining() { + return !bound.isEmpty(); + } +} diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/TypedEmit.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/TypedEmit.java index 4e60cb83c5d..8236083b944 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/TypedEmit.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/TypedEmit.java @@ -110,6 +110,16 @@ static final class E { /// Shadowed locals with no Object variable at all: read boxed only on a /// slow path. private Set longOnly = new HashSet(); + /// The int ranges of the function being emitted, and the locals they + /// put in a Java `long` (see `IntRanges`). + private IntRanges ranges; + private Set longLocals = new HashSet(); + /// The Java code of each parameter of the expression functions being + /// inlined, innermost last, and whether it is the exact value (true) or + /// only the value modulo 2^64. + private final List> inlined = new ArrayList>(); + /// An expression `expr` must lower without trying `wrapped` first. + private Expr plainExpr; /// Locals whose previous list a new one may be written into (see /// ListRecycling). private Set recycled = new HashSet(); @@ -560,6 +570,8 @@ private String function(Infer.Fn f) { nonNull = new HashSet(); shadow = new HashMap(); longOnly = new HashSet(); + ranges = new IntRanges(in, f); + longLocals = ranges.longLocals(); if (!f.def.scope.cells.isEmpty() || f.def.scope.hasYield || f.def.isAsync || !f.def.decorators.isEmpty() || f.def.args.vararg != null || f.def.args.kwarg != null || !f.def.args.kwonly.isEmpty()) { throw new Unsupported("signature"); @@ -590,7 +602,10 @@ private String function(Infer.Fn f) { } b.append(") {\n"); for (Map.Entry e : f.locals.entrySet()) { - if (declared.add(e.getKey())) { + if (longLocals.contains(e.getKey())) { + declared.add(e.getKey()); + b.append(" long ").append(var(e.getKey())).append(" = 0L;\n"); + } else if (declared.add(e.getKey())) { b.append(" ").append(jt(e.getValue())).append(' ').append(var(e.getKey())).append(" = ") .append(zero(e.getValue())).append(";\n"); } @@ -906,8 +921,12 @@ private void forLoop(For f, StringBuilder b, int ind) { .append("L) {\n"); String tn = ((Name) f.target).id; boolean counter = vt.k == Ty.K.INT && !inBody.contains(tn); - boolean unboxed = counter && !longOnly.contains(tn) && !readOutside(tn, f); - if (unboxed) { + boolean unboxed = counter && !longOnly.contains(tn) && !readOutside(tn, f) + && !longLocals.contains(tn); + if (longLocals.contains(tn)) { + indent(b, ind + 2); + b.append(v).append(" = ").append(i).append(";\n"); + } else if (unboxed) { // Read nowhere but in this body: no boxed copy at all. longOnly.add(tn); } else if (longOnly.contains(tn)) { @@ -982,6 +1001,12 @@ private void store(Expr t, E v, StringBuilder b, int ind) { switch (sc.resolve(n)) { case LOCAL: indent(b, ind); + if (longLocals.contains(n)) { + // The analysis proved every value it holds fits. + b.append(var(n)).append(" = ").append(v.lc != null ? v.lc : "Typed.toLong(" + box(v) + ")") + .append(";\n"); + return; + } b.append(var(n)).append(" = ").append(conv(v, localType(n))).append(";\n"); forgetPathsOf(n); if (longOnly.contains(n)) { @@ -1081,6 +1106,10 @@ private void store(Expr t, E v, StringBuilder b, int ind) { private void augAssign(AugAssign a, StringBuilder b, int ind) { Expr t = a.target; + if (t instanceof Name && longLocals.contains(((Name) t).id)) { + store(t, expr(new BinOp(t, a.op, a.value)), b, ind); + return; + } if (t instanceof Name) { E cur0 = expr(t); E r = binop(a.op, cur0, expr(a.value), a.value); @@ -1262,7 +1291,7 @@ private String noneTest(Compare c) { } /// Every local `body` may store into. - private static Set assigned(List body) { + static Set assigned(List body) { List out = new ArrayList(); for (Stmt s : body) { collectStores(s, out); @@ -1378,6 +1407,13 @@ String testExpr(Expr e) { } E expr(Expr e) { + if (e != plainExpr && (e instanceof BinOp || e instanceof UnaryOp || e instanceof IfExp //NOPMD CompareObjectsWithEquals + || e instanceof Call && ranges.inlinable((Call) e) != null)) { + E w = wrapped(e); + if (w != null) { + return w; + } + } if (e instanceof Constant) { Constant c = (Constant) e; switch (c.kind) { @@ -1601,6 +1637,9 @@ private E name(Name nm) { if (sc.deleted.contains(n)) { throw new Unsupported("deleted " + n); } + if (longLocals.contains(n)) { + return new E("Long.valueOf(" + var(n) + ")", Ty.INT, var(n)); + } String l = shadow.get(n); if (l != null) { return new E(longOnly.contains(n) ? "Long.valueOf(" + l + ")" : var(n), t, l); @@ -2387,7 +2426,13 @@ private E call0(Call c) { throw new Unsupported("list of " + x.t); } if (b == Scope.Binding.BUILTIN && !em.moduleScopeHasStarImport()) { - return builtin(n, args(c)); + if (TYPED_BUILTINS.contains(n) && c.keywords.isEmpty()) { + E r = builtin(n, args(c)); + if (r != null) { + return r; + } + } + return genericCall(em.builtinRef(n), c); } throw new Unsupported("call of " + n); } @@ -2467,6 +2512,239 @@ private E relabel(E r, Ty t) { } } + // ---- int arithmetic in a Java long ----------------------------------- + + /// `e` as a `long`, when `IntRanges` proves its value fits in one; or + /// null. The result is then computed modulo 2^64 (`mod64`), which is + /// the exact value however large the intermediate values are. + private E wrapped(Expr e) { + IntRanges.Iv r = ranges.eval(e, null); + if (!r.fitsLong()) { + return null; + } + String c = mod64(e); + if (c == null) { + return null; + } + if (pre != null && !plain(c)) { + String t = fresh(); + indent(pre, preInd); + pre.append("long ").append(t).append(" = ").append(c).append(";\n"); + c = t; + } + return new E("Long.valueOf(" + c + ")", Ty.INT, c); + } + + /// Whether `e`, an int expression, provably fits in a `long`. + private boolean fits(Expr e) { + return ranges.eval(e, null).fitsLong(); + } + + /// Java `long` code for int expression `e` modulo 2^64, or null. Only + /// `+ - * & | ^ ~ <<` commute with that reduction; anything else + /// (`>>`, `//`, `%`) needs the exact value of its operands. + private String mod64(Expr e) { + if (e instanceof Constant) { + if (((Constant) e).kind != Constant.Kind.INT) { + return null; + } + java.math.BigInteger v = IntRanges.parse(((Constant) e).text); + return v == null ? null : lit(v); + } + if (e instanceof Name) { + return nameLong((Name) e); + } + if (e instanceof UnaryOp) { + UnaryOp u = (UnaryOp) e; + String o = mod64(u.operand); + if (o == null) { + return null; + } + switch (u.op) { + case "+": + return o; + case "-": + return "(-" + o + ")"; + case "~": + return "(~" + o + ")"; + default: + return null; + } + } + if (e instanceof BinOp) { + return binLong((BinOp) e); + } + if (e instanceof Call) { + Infer.Fn f = ranges.inlinable((Call) e); + if (f != null) { + return inline(f, (Call) e); + } + } + if (e instanceof IfExp && inlined.isEmpty()) { + IfExp x = (IfExp) e; + String a = mod64(x.body); + String b = mod64(x.orelse); + return a == null || b == null ? null : "(" + testExpr(x.test) + " ? " + a + " : " + b + ")"; + } + return generic(e); + } + + /// An int expression the long arithmetic cannot follow (a subscript, a + /// call) whose value is known to fit: computed as before and unboxed. + private String generic(Expr e) { + if (!inlined.isEmpty() || !fits(e)) { + return null; + } + Expr saved = plainExpr; + plainExpr = e; + try { + E v = expr(e); + if (v.t.k != Ty.K.INT) { + return null; + } + return v.lc != null ? v.lc : "Typed.toLong(" + box(v) + ")"; + } finally { + plainExpr = saved; + } + } + + private static String lit(java.math.BigInteger v) { + long l = v.longValue(); + return l < 0 ? "(" + l + "L)" : l + "L"; + } + + private String nameLong(Name nm) { + String n = nm.id; + if (!inlined.isEmpty()) { + String[] p = inlined.get(inlined.size() - 1).get(n); + if (p != null) { + return p[0]; + } + java.math.BigInteger c = in.intConsts.get(n); + return c == null ? null : lit(c); + } + if (subst != null && subst.containsKey(n)) { + // Inside an out-of-line slow path, where locals are parameters. + E v = subst.get(n); + return v.t.k != Ty.K.INT ? null : v.lc != null ? v.lc : "Typed.toLong(" + box(v) + ")"; + } + switch (sc.resolve(n)) { + case LOCAL: + if (longLocals.contains(n)) { + if (!sc.params.contains(n) && !da.safe.contains(nm)) { + return null; + } + return var(n); + } + return generic(nm); + case GLOBAL: { + java.math.BigInteger c = in.intConsts.get(n); + return c != null ? lit(c) : generic(nm); + } + default: + return null; + } + } + + /// Whether `e` is known exactly, not only modulo 2^64, by `mod64`. + private boolean exact(Expr e) { + if (!inlined.isEmpty() && e instanceof Name) { + String[] p = inlined.get(inlined.size() - 1).get(((Name) e).id); + if (p != null) { + return "exact".equals(p[1]); + } + } + return fits(e); + } + + private String binLong(BinOp b) { + switch (b.op) { + case "+": + case "-": + case "*": + case "&": + case "|": + case "^": { + String l = mod64(b.left); + String r = l == null ? null : mod64(b.right); + return r == null ? null : "(" + l + " " + b.op + " " + r + ")"; + } + case "<<": { + java.math.BigInteger c = IntRanges.literal(b.right); + if (c == null || c.signum() < 0 || c.bitLength() > 7) { + return null; + } + String l = mod64(b.left); + if (l == null) { + return null; + } + return c.intValue() >= 64 ? "0L" : "(" + l + " << " + c.intValue() + ")"; + } + case ">>": { + java.math.BigInteger c = IntRanges.literal(b.right); + if (c == null || c.signum() < 0 || c.bitLength() > 7) { + return null; + } + int k = c.intValue(); + if (exact(b.left)) { + String l = mod64(b.left); + return l == null ? null : "(" + l + " >> " + Math.min(k, 63) + ")"; + } + if (k >= 1 && ranges.eval(b.left, null).fitsUnsigned64()) { + // In [0, 2^64): its 64 bits read unsigned are the value. + String l = mod64(b.left); + return l == null ? null : k >= 64 ? "0L" : "(" + l + " >>> " + k + ")"; + } + return null; + } + case "%": + case "//": { + IntRanges.Iv d = ranges.eval(b.right, null); + boolean nonZero = d.lo != null && d.lo.signum() > 0 || d.hi != null && d.hi.signum() < 0; + if (!nonZero || !exact(b.left) || !exact(b.right)) { + return null; + } + String l = mod64(b.left); + String r = l == null ? null : mod64(b.right); + if (r == null) { + return null; + } + return ("%".equals(b.op) ? "Math.floorMod(" : "Math.floorDiv(") + l + ", " + r + ")"; + } + default: + return null; + } + } + + /// The body of expression function `f` in place of the call `c`. + private String inline(Infer.Fn f, Call c) { + Map p = new HashMap(); + Map pr = new HashMap(); + for (int i = 0; i < c.args.size(); i++) { + Expr a = c.args.get(i); + String code = mod64(a); + if (code == null) { + return null; + } + if (pre != null && !plain(code)) { + String t = fresh(); + indent(pre, preInd); + pre.append("long ").append(t).append(" = ").append(code).append(";\n"); + code = t; + } + p.put(f.names.get(i), new String[] {code, exact(a) ? "exact" : "mod"}); + pr.put(f.names.get(i), ranges.eval(a, null)); + } + inlined.add(p); + ranges.push(pr); + try { + return mod64(Infer.exprBody(f.def)); + } finally { + ranges.pop(); + inlined.remove(inlined.size() - 1); + } + } + private static String mathConstant(String n) { switch (n) { case "pi": @@ -2546,7 +2824,37 @@ private E builtin(String n, List a) { default: break; } - throw new Unsupported("builtin " + n + (a.isEmpty() ? "" : " of " + a.get(0).t)); + return null; + } + + /// The builtins `builtin` lowers; any other is called as Python calls it. + private static final Set TYPED_BUILTINS = new HashSet(Arrays.asList("len", "float", "str", "ord", + "abs")); + + /// A call of `f` with the arguments of `c` boxed, keywords included: + /// what the generic code does, with an untyped result. Reached for + /// builtins that have no typed lowering (`print`, `sorted`, ...), so one + /// such call no longer leaves the whole function untyped. + private E genericCall(String f, Call c) { + StringBuilder a = new StringBuilder(); + for (Expr x : c.args) { + if (x instanceof Starred) { + throw new Unsupported("starred argument"); + } + a.append(a.length() == 0 ? "" : ", ").append(box(expr(x))); + } + if (c.keywords.isEmpty()) { + return new E("Py.call(" + f + (a.length() == 0 ? "" : ", " + a) + ")", Ty.DYN); + } + StringBuilder names = new StringBuilder(); + for (Keyword kw : c.keywords) { + if (kw.arg == null) { + throw new Unsupported("** argument"); + } + a.append(a.length() == 0 ? "" : ", ").append(box(expr(kw.value))); + names.append(names.length() == 0 ? "" : ", ").append(Emitter.javaString(kw.arg)); + } + return new E("Py.callKw(" + f + ", new Object[] {" + a + "}, new String[] {" + names + "})", Ty.DYN); } private E listMethod(E l, String m, List a) { diff --git a/maven/python-transpiler/src/test/resources/conformance/int_ranges.out b/maven/python-transpiler/src/test/resources/conformance/int_ranges.out new file mode 100644 index 00000000000..7f64919637f --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/int_ranges.out @@ -0,0 +1,33 @@ +wrap32 1078119050 -1252290967 +wrap64 108445073120740926 9056328395920085615 +bounded sum 3276617296 +grows past 64 bits 644221900915014163795576412207026590877634402451 +doubling 1180591620717411303423 +down -19999900000 +while 99999999999999999999999999999999999999999999999999 +-7 -3 2 -1 -4 -56 7 6 -7 -3 4 +-7 -1 7 0 -4 -56 7 6 -7 -1 6 +-7 2 -4 1 -4 -56 7 6 0 -5 -5 +-7 5 -2 3 -4 -56 7 6 1 -3 -4 +-4 -3 1 -1 -2 -32 4 3 -4 -3 1 +-4 -1 4 0 -2 -32 4 3 -4 -1 3 +-4 2 -2 0 -2 -32 4 3 0 -2 -2 +-4 5 -1 1 -2 -32 4 3 4 -3 -7 +-1 -3 0 -1 -1 -8 1 0 -3 -1 2 +-1 -1 1 0 -1 -8 1 0 -1 -1 0 +-1 2 -1 1 -1 -8 1 0 2 -1 -3 +-1 5 -1 4 -1 -8 1 0 5 -1 -6 +2 -3 -1 -1 1 16 -2 -3 0 -1 -1 +2 -1 -2 0 1 16 -2 -3 2 -1 -3 +2 2 1 0 1 16 -2 -3 2 2 0 +2 5 0 2 1 16 -2 -3 0 7 7 +5 -3 -2 -1 2 40 -5 -6 5 -3 -8 +5 -1 -5 0 2 40 -5 -6 5 -1 -6 +5 2 2 1 2 40 -5 -6 0 7 7 +5 5 1 0 2 40 -5 -6 5 5 0 +15 9223372030681936357 -1543209863 -1 0 +660917953 1786349750302592705 5918522610932723073 -660917953 +223293718787824088 -1675506475 8 +ifexp -5 +len 10 +poly 645 diff --git a/maven/python-transpiler/src/test/resources/conformance/int_ranges.py b/maven/python-transpiler/src/test/resources/conformance/int_ranges.py new file mode 100644 index 00000000000..634aff5edd6 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/int_ranges.py @@ -0,0 +1,92 @@ +# Ints that typed code holds in a Java long (bounded by a mask, a range or a +# trip count) and ints that must not be: every result here has to equal +# CPython's, including the ones that leave 64 bits. +M32 = 0xFFFFFFFF +M64 = 0xFFFFFFFFFFFFFFFF + + +def i32(v: int) -> int: + return ((v + 0x80000000) & M32) - 0x80000000 + + +def i64(v: int) -> int: + return ((v + 0x8000000000000000) & M64) - 0x8000000000000000 + + +def wraps() -> None: + a: int = 0x12345678 + b: int = i32(0x9E3779B9) + for _ in range(1000): + a = i32((a * 1103515245 + 12345) ^ ((b & M32) >> 3)) + b = i32((b + a) * 5 - (a << 7)) + print("wrap32", a, b) + x: int = 0x0123456789ABCDEF + y: int = -0x123456789 + for _ in range(1000): + x = i64((x * 6364136223846793005 + 1442695040888963407) ^ ((y & M64) >> 7)) + y = i64((y ^ (x << 13)) + ((x & M64) >> 11)) + print("wrap64", x, y) + + +def accumulators() -> None: + s: int = 0 + for i in range(100000): + s += (i * 7919) & 0xFFFF + print("bounded sum", s) + big: int = 1 + for i in range(100): + big = big * 3 + i + print("grows past 64 bits", big) + t: int = 0 + for i in range(70): + t = t * 2 + 1 + print("doubling", t) + d: int = 0 + for i in range(200000): + d -= i + print("down", d) + n: int = 0 + k: int = 0 + while k < 50: + n = n * 10 + 9 + k += 1 + print("while", n) + + +def signed_ops() -> None: + for a in range(-7, 8, 3): + for b in (-3, -1, 2, 5): + print(a, b, a // b, a % b, a >> 1, a << 3, -a, ~a, a & b, a | b, a ^ b) + v: int = -12345678901 + print((v & M64) >> 60, (v & M64) >> 1, v >> 3, v >> 70, (v & M32) >> 31) + + +def helpers_with_big_args() -> None: + v: int = 1 + for i in range(80): + v = v * 5 + print(i32(v), i64(v), i64(v * v), i32(-v)) + w: int = i64(v) >> 3 + print(w, i32(w // 7), w % 11) + + +def flags_and_mixed() -> None: + c: int = 0 + for i in range(10): + c += i if i % 2 == 0 else -i + print("ifexp", c) + s: int = 0 + for i in range(5): + s += len("x" * i) + print("len", s) + t: int = 0 + for i in range(1, 6): + t = (t + i) * i + print("poly", t) + + +wraps() +accumulators() +signed_ops() +helpers_with_big_args() +flags_and_mixed() From 664222076cb5752824a7fe2aed4173ac36c3200c Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Tue, 27 Oct 2026 21:23:24 +0200 Subject: [PATCH 024/112] Python: scalar-replace non-escaping objects, typed str loops, int % and //, lowered loop tests Locals that only ever hold a freshly constructed object whose __init__ copies its arguments into fields, and are only read through fields or getters, become one local per field. while tests lower their arithmetic and compares like any statement. for c in str walks code points, with ord(c) the code point itself. l.extend([v] * n) fills in place. Lone surrogates index and repr as CPython does. Co-Authored-By: Claude Opus 5.5 --- .../com/codename1/python/runtime/PyList.java | 45 + .../com/codename1/python/runtime/PyStr.java | 18 +- .../com/codename1/python/runtime/Typed.java | 83 + .../python/transpiler/codegen/Emitter.java | 8 + .../python/transpiler/codegen/Infer.java | 25 + .../python/transpiler/codegen/IntRanges.java | 13 + .../python/transpiler/codegen/TypedEmit.java | 1688 ++++++++++++++--- .../test/resources/conformance/int_divmod.out | 50 + .../test/resources/conformance/int_divmod.py | 23 + .../test/resources/conformance/int_pairs.out | 15 + .../test/resources/conformance/int_pairs.py | 102 + .../resources/conformance/scalar_objects.out | 8 + .../resources/conformance/scalar_objects.py | 70 + .../test/resources/conformance/str_loops.out | 8 + .../test/resources/conformance/str_loops.py | 46 + 15 files changed, 1890 insertions(+), 312 deletions(-) create mode 100644 maven/python-transpiler/src/test/resources/conformance/int_divmod.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/int_divmod.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/int_pairs.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/int_pairs.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/scalar_objects.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/scalar_objects.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/str_loops.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/str_loops.py diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyList.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyList.java index eb46b173b1b..f82064a836f 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyList.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyList.java @@ -203,6 +203,14 @@ public void extend(Object iterable) { if (iterable instanceof PyList) { PyList o = (PyList) iterable; int n = o.size; + if (o.ints != null && intsOrEmpty()) { + // One copy, into an array of the exact size when this list + // was empty: no element by element growth. + long[] l = intRoom(n); + System.arraycopy(o.ints, 0, l, size, n); + size += n; + return; + } for (int i = 0; i < n; i++) { append(o.get(i)); } @@ -214,6 +222,43 @@ public void extend(Object iterable) { } } + /// Whether ints may go into the unboxed `ints` storage. + private boolean intsOrEmpty() { + return ints != null || floats == null && size == 0 && items == NO_ITEMS; //NOPMD CompareObjectsWithEquals + } + + /// `ints`, with room for `n` more after `size`. + private long[] intRoom(int n) { + long[] l = ints; + if (l == null || l.length - size < n) { + long[] g = new long[l == null ? n : Math.max(size + n, size * 2 + 4)]; + if (l != null) { + System.arraycopy(l, 0, g, 0, size); + } + ints = g; + l = g; + } + return l; + } + + /// `extend([v] * n)`, without the intermediate list. + public void extendRepeat(Object v, long n) { + if (n <= 0) { + return; + } + if (v instanceof Long && n <= Integer.MAX_VALUE - size && intsOrEmpty()) { + long x = ((Long) v).longValue(); + long[] l = intRoom((int) n); + int end = size + (int) n; + for (int i = size; i < end; i++) { + l[i] = x; + } + size = end; + return; + } + extend(new PyList(new Object[] {v}).mul(Long.valueOf(n))); + } + public void insert(long index, Object v) { objects(); if (index < 0) { diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyStr.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyStr.java index c3edd344244..198bb82f08c 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyStr.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyStr.java @@ -113,11 +113,18 @@ static int offset(String s, int cp) { } int off = 0; for (int i = 0; i < cp; i++) { - off += Character.isHighSurrogate(s.charAt(off)) && off + 1 < s.length() ? 2 : 1; + off += pairAt(s, off) ? 2 : 1; } return off; } + /// Whether a surrogate pair, one code point, starts at offset `i`; a + /// lone surrogate is a code point of its own. + static boolean pairAt(String s, int i) { + return i + 1 < s.length() && Character.isHighSurrogate(s.charAt(i)) + && Character.isLowSurrogate(s.charAt(i + 1)); + } + int offset(int cp) { return offset(value, cp); } @@ -129,7 +136,7 @@ static int cpIndex(String s, int off) { } int cp = 0; for (int i = 0; i < off; i++) { - if (!Character.isLowSurrogate(s.charAt(i)) || i == 0) { + if (i == 0 || !pairAt(s, i - 1)) { cp++; } } @@ -192,9 +199,14 @@ public static String repr(String s) { sb.append("\\r"); } else if (c == '\t') { sb.append("\\t"); - } else if (c < 0x20 || c == 0x7f) { + } else if (c < 0x20 || c >= 0x7f && c <= 0xa0 || c == 0xad) { + // C0 and C1 controls and the two unprintable Latin-1 spaces. sb.append("\\x"); hex(sb, c, 2); + } else if (c >= 0xD800 && c < 0xE000 && !pairAt(s, i) && (i == 0 || !pairAt(s, i - 1))) { + // A lone surrogate is not printable. + sb.append('\\').append('u'); + hex(sb, c, 4); } else { sb.append(c); } diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Typed.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Typed.java index 4e0f2a61091..5c41158563f 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Typed.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Typed.java @@ -91,6 +91,83 @@ public static long toLong(Object o) { throw mismatch(o); } + private static final String[] CHARS = new String[256]; + + /// The one-character string of code point `cp`, as iterating a string + /// yields it: shared below 256, so a loop over Latin-1 text allocates + /// nothing. + public static String ch(int cp) { + if (cp < 256) { + String s = CHARS[cp]; + if (s == null) { + s = String.valueOf((char) cp); + CHARS[cp] = s; + } + return s; + } + return new String(Character.toChars(cp)); + } + + /// `len(s)`: code points, which Java's UTF-16 length overcounts by one + /// per surrogate pair. + public static long slen(String s) { + int n = s.length(); + long r = n; + for (int i = 0; i < n; i++) { + char c = s.charAt(i); + if (c >= 0xD800 && c < 0xDC00 && i + 1 < n) { + char d = s.charAt(i + 1); + if (d >= 0xDC00 && d < 0xE000) { + r--; + i++; + } + } + } + return r; + } + + /// Int `o` modulo 2^64: its low 64 bits, as two's complement keeps them. + public static long low64(Object o) { + if (o instanceof PyInt) { + return ((PyInt) o).toBig().longValue(); + } + return toLong(o); + } + + /// The int a typed function returned in place of its `long` result: + /// a big int, a bool, or a `Long.MIN_VALUE` (the marker itself). + private static final ThreadLocal RETURNED = new ThreadLocal(); + + /// Returns int `o` from a typed function whose result is a `long`: the + /// marker, with `o` left for the caller's `take`. + public static long retBig(Object o) { + RETURNED.set(o); + return Long.MIN_VALUE; + } + + /// `retBig` for an int that may be a `Long` that fits. + public static long retObj(Object o) { + if (o instanceof Long) { + long v = ((Long) o).longValue(); + if (v != Long.MIN_VALUE) { + return v; + } + } + return retBig(o); + } + + /// The int behind a `Long.MIN_VALUE` a typed function returned. + public static Object take() { + Object o = RETURNED.get(); + RETURNED.set(null); + return o; + } + + /// A typed function's `long` result as an object. + public static Object unret(long v) { + return v != Long.MIN_VALUE ? Long.valueOf(v) : take(); + } + public static double asFloat(Object o) { if (o instanceof Double) { return ((Double) o).doubleValue(); @@ -335,6 +412,12 @@ public static Object extend(PyList l, PyList other) { return Py.NONE; } + /// `l.extend([v] * n)`. + public static Object extendRepeat(PyList l, Object v, Object n) { + nzl(l).extendRepeat(v, Py.index(n)); + return Py.NONE; + } + /// `ord(s)`: TypeError unless `s` is one character. public static Object ord(String s) { return Builtins.ord(s); diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Emitter.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Emitter.java index 7faad717012..024f7a13485 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Emitter.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Emitter.java @@ -287,6 +287,10 @@ public String emit(Node.Module m) { + " private static void noop(Object v) {\n" + " }\n"; + Scope moduleScope() { + return moduleScope; + } + boolean moduleScopeHasStarImport() { return moduleScope.hasStarImport; } @@ -3039,6 +3043,10 @@ private static String dparams(int n) { /// A fresh inline cache for one attribute access. Never shared between two /// accesses, unlike `hoist`: each site caches what ITS receivers look like. + String attrSite(String attr) { + return site(attr); + } + private String site(String attr) { String f = "S" + (++sites); fields.append(" private static final AttrSite ").append(f).append(" = new AttrSite(") diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Infer.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Infer.java index 76699402d3f..fe8c392889f 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Infer.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Infer.java @@ -97,6 +97,8 @@ Ty param(String n) { /// Globals bound once, by `import math`: calls of its functions on /// numbers are typed (see `MATH_FNS`) and run as `PyMath` directly. final Set mathNames = new HashSet(); + /// The globals only `import struct` binds. + final Set structNames = new HashSet(); /// `math` functions typed code calls, with their arity. static final Map MATH_FNS = new HashMap(); /// `math` constants, all floats. @@ -185,6 +187,9 @@ private void collect(List body) { if ("math".equals(a.name) && stable(n)) { mathNames.add(n); } + if ("struct".equals(a.name) && stable(n)) { + structNames.add(n); + } } } } @@ -194,6 +199,12 @@ private void collect(List body) { } } + /// Whether `e` is the `struct` module, as `isMath` is for `math`. + boolean isStruct(Expr e, Scope sc) { + return e instanceof Name && structNames.contains(((Name) e).id) + && sc.resolve(((Name) e).id) == Scope.Binding.GLOBAL; + } + /// Whether `e` is the `math` module, through a global only `import math` /// binds. boolean isMath(Expr e, Scope sc) { @@ -975,6 +986,14 @@ Ty iterElem(Expr it, Fn fn, Scope sc) { if (t.k == Ty.K.LIST) { return elem(t); } + if (t.k == Ty.K.STR && !t.opt) { + // A str yields one-character strs. + return Ty.STR; + } + if (t.k == Ty.K.BOT) { + // Nothing reaches the iterable yet; a later pass decides. + return Ty.BOT; + } escape(t); return Ty.DYN; } @@ -1638,6 +1657,12 @@ private Ty call(Call c, Fn fn, Scope sc) { || args.get(0).k == Ty.K.STR || args.get(0).k == Ty.K.BOOL)) { return Ty.STR; } + if (simple && args.size() == 1 && args.get(0).k == Ty.K.BOT + && ("ord".equals(n) || "abs".equals(n) || "float".equals(n) || "str".equals(n))) { + // Nothing reaches the argument yet: a later pass decides, + // where DYN now would stay DYN for good. + return Ty.BOT; + } if (simple && "ord".equals(n) && args.size() == 1 && args.get(0).k == Ty.K.STR && !args.get(0).opt) { return Ty.INT; } diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/IntRanges.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/IntRanges.java index 7b99b05bea7..c14c5e54694 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/IntRanges.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/IntRanges.java @@ -487,6 +487,19 @@ Infer.Fn inlinable(Call c) { } private Iv call(Call c, Map env) { + if (c.func instanceof Name && c.args.size() == 1 && c.keywords.isEmpty() && !(c.args.get(0) instanceof Starred) + && sc.resolve(((Name) c.func).id) == Scope.Binding.BUILTIN) { + switch (((Name) c.func).id) { + case "ord": + // A code point, or an exception. + return new Iv(BigInteger.ZERO, BigInteger.valueOf(0x10FFFF)); + case "len": + // OverflowError past sys.maxsize. + return new Iv(BigInteger.ZERO, BigInteger.valueOf(Long.MAX_VALUE)); + default: + break; + } + } Infer.Fn f = inlinable(c); if (f != null) { Map b = new HashMap(); diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/TypedEmit.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/TypedEmit.java index 8236083b944..2b60fbc0d1a 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/TypedEmit.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/TypedEmit.java @@ -30,6 +30,7 @@ import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; +import java.util.IdentityHashMap; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; @@ -69,18 +70,35 @@ static final class E { /// The same int as a Java `long` expression, cheap and free of side /// effects, when one is at hand (a loop counter, a constant); or null. final String lc; + /// An int held as a pair (see `dual`): a Java `long` and the int + /// object that replaces it when it is not null (a big int, or a + /// bool). Both plain Java names; or both null. + final String ll; + final String bg; E(String c, Ty t) { this(c, t, null); } E(String c, Ty t, String lc) { + this(c, t, lc, null, null); + } + + E(String c, Ty t, String lc, String ll, String bg) { this.c = c; this.t = t; this.lc = lc; + this.ll = ll; + this.bg = bg; } } + /// An int pair `l`/`b` as an expression: the boxed value, built only + /// when something needs the object. + static E dual(String l, String b) { + return new E("(" + b + " == null ? Long.valueOf(" + l + ") : " + b + ")", Ty.INT, null, l, b); + } + private final Emitter em; final Infer in; private final Layouts layouts; @@ -107,9 +125,6 @@ static final class E { private Set nonNull = new HashSet(); /// Int locals whose value a Java long holds right now, and that long. private Map shadow = new HashMap(); - /// Shadowed locals with no Object variable at all: read boxed only on a - /// slow path. - private Set longOnly = new HashSet(); /// The int ranges of the function being emitted, and the locals they /// put in a Java `long` (see `IntRanges`). private IntRanges ranges; @@ -123,18 +138,12 @@ static final class E { /// Locals whose previous list a new one may be written into (see /// ListRecycling). private Set recycled = new HashSet(); - /// What `name` answers for a local instead, inside an outlined slow path. - private Map subst; + /// Already evaluated operands of hoisted arithmetic (see `hoistInt`): + /// what `expr` answers for each instead. + private Map leafSubst = new IdentityHashMap(); /// The outlined slow paths of hoisted arithmetic. private final StringBuilder slow = new StringBuilder(); //NOPMD AvoidStringBufferField - the generator's output private int slowCount; - /// The function being emitted is its `TL_` variant: its unassigned int - /// parameters arrive as longs. - private boolean longParams; - /// Typed functions some call site wants a `TL_` variant of. - private final Set wantLong = new LinkedHashSet(); - private final Map longBodies = new LinkedHashMap(); - TypedEmit(Emitter em, Infer in, Layouts layouts) { this.em = em; this.in = in; @@ -157,33 +166,97 @@ static final class E { /// if-expression branch): nothing there may be hoisted to run always. private int condDepth; + /// Methods and `__init__`s are pure on the same terms, but for + /// `__init__` storing into the fields of the object it initializes: + /// a new object nothing else can see yet. private void findPure() { for (Map.Entry e : in.globalFns.entrySet()) { pure.add(e.getValue()); } + for (Infer.Fn f : in.fns.values()) { + if (f.cls != null && !f.def.name.equals("__init__")) { + pure.add(f); + } else if (f.cls != null) { + pureInits.add(f); + } + } boolean changed = true; while (changed) { changed = false; for (Infer.Fn f : new ArrayList(pure)) { - for (Stmt st : f.def.body) { - if (!pureNode(st)) { - pure.remove(f); + if (!pureBody(f)) { + pure.remove(f); + changed = true; + } + } + for (Infer.Fn f : new ArrayList(pureInits)) { + initSelf = f.names.isEmpty() ? null : f.names.get(0); + try { + if (!pureBody(f)) { + pureInits.remove(f); changed = true; - break; } + } finally { + initSelf = null; + } + } + } + } + + private final Set pureInits = new HashSet(); + /// The `self` of the `__init__` `findPure` is checking, or null. + private String initSelf; + + private boolean pureBody(Infer.Fn f) { + if (f.def.args.vararg != null || f.def.args.kwarg != null || !f.def.decorators.isEmpty() + || f.def.scope.hasYield || f.def.isAsync) { + return false; + } + for (Stmt st : f.def.body) { + if (!pureNode(st)) { + return false; + } + } + return true; + } + + /// Methods of the built-in types that change their object: a call of + /// one by name is never taken for a pure method of a class. + private static final Set MUTATORS = new HashSet(Arrays.asList("append", "extend", "insert", + "pop", "remove", "clear", "sort", "reverse", "update", "setdefault", "popitem", "add", "discard", + "write", "send", "close", "put", "acquire", "release", "set", "throw", "__setitem__", "__delitem__", + "__iadd__", "__init__")); + + /// Whether every method named `m` of every class is pure. + private boolean pureMethod(String m) { + if (MUTATORS.contains(m)) { + return false; + } + boolean any = false; + for (Infer.Fn f : in.fns.values()) { + if (f.cls != null && f.def.name.equals(m)) { + if (!pure.contains(f)) { + return false; } + any = true; } } + return any; } private boolean pureNode(Object o) { + if (!(o instanceof com.codename1.python.transpiler.ast.Node)) { + // An operator or a name in a node's list of strings. + return true; + } if (o instanceof Call) { if (!pureCall((Call) o, null)) { return false; } } else if (o instanceof Assign) { for (Expr t : ((Assign) o).targets) { - if (!localTarget(t)) { + if (!localTarget(t) && !(initSelf != null && t instanceof Attribute + && ((Attribute) t).value instanceof Name && ((Name) ((Attribute) t).value).id.equals(initSelf))) { return false; } } @@ -233,7 +306,17 @@ private static boolean localTarget(Expr t) { /// Whether `c` calls a pure function; `scope` resolves its name (null: /// the module's view, for `findPure`). private boolean pureCall(Call c, Scope scope) { - if (!(c.func instanceof Name) || !c.keywords.isEmpty()) { + if (!c.keywords.isEmpty()) { + return false; + } + if (c.func instanceof Attribute) { + Attribute a = (Attribute) c.func; + if (in.isMath(a.value, scope == null ? em.moduleScope() : scope)) { + return true; + } + return pureMethod(a.attr); + } + if (!(c.func instanceof Name)) { return false; } String n = ((Name) c.func).id; @@ -242,6 +325,15 @@ private boolean pureCall(Call c, Scope scope) { if (f != null && (b == null || b == Scope.Binding.GLOBAL)) { return pure.contains(f); } + Layouts.Cls k = in.globalClasses.get(n); + if (k != null && (b == null || b == Scope.Binding.GLOBAL)) { + Layouts.Cls o = Layouts.resolver(k, "__init__"); + if (o == null) { + return true; + } + FunctionDef init = o.typable.get("__init__"); + return init != null && pureInits.contains(in.fns.get(init)); + } return (b == null || b == Scope.Binding.BUILTIN) && PURE_BUILTINS.contains(n) && !em.moduleScopeHasStarImport(); } @@ -352,6 +444,7 @@ private Set liveGroups(Set ok) { private static boolean representable(Ty t) { switch (t.k) { + case INT: case BOOL: case FLOAT: case STR: @@ -398,8 +491,11 @@ static boolean objectRet(Ty t) { return "Object".equals(jt(t)); } + /// The Java return type of typed `f`. An int returns as a `long`; one + /// that is not a long returns `Long.MIN_VALUE` and leaves the object + /// in `Typed.retBig`, for the caller to `Typed.take`. static String retJ(Infer.Fn f) { - return jt(f.ret); + return f.ret.k == Ty.K.INT ? "long" : jt(f.ret); } private static String zero(Ty t) { @@ -521,6 +617,34 @@ private String var(String n) { return "v_" + Emitter.mangle(n); } + /// The object half of int local `n` (see `dual`). + private static String big(String n) { + return "w_" + Emitter.mangle(n); + } + + /// A Java parameter of type `t`: an int is two, its long and its object. + static String param(Ty t, String name, String bigName) { + return t.k == Ty.K.INT ? "long " + name + ", Object " + bigName : jt(t) + " " + name; + } + + /// Whether `n` is an int local held as a pair. + private boolean pairLocal(String n) { + return sc.resolve(n) == Scope.Binding.LOCAL && localType(n).k == Ty.K.INT && !longLocals.contains(n); + } + + /// The long and object halves of int (or bool) `e`, for a pair: the + /// object may be a `Long` here when nothing better is at hand, which + /// the receiving end normalizes. + private String[] parts(E e) { + if (e.lc != null) { + return new String[] {e.lc, "null"}; + } + if (e.ll != null) { + return new String[] {e.ll, e.bg}; + } + return new String[] {"0L", box(e)}; + } + private Ty localType(String n) { Ty t = cur.locals.get(n); return t == null ? Ty.BOT : t; @@ -545,22 +669,6 @@ private static void countReads(Object o, String n, For skip, int[] c) { Infer.children(o, x -> countReads(x, n, skip, c)); } - /// Whether parameter `i` of `f` can arrive as a long: an int nothing in - /// the body stores into. - private static boolean longParam(Infer.Fn f, int i, Set stored) { - return f.params[i].k == Ty.K.INT && !stored.contains(f.names.get(i)); - } - - private static boolean hasLongParam(Infer.Fn f) { - Set stored = assigned(f.def.body); - for (int i = 0; i < f.names.size(); i++) { - if (longParam(f, i, stored)) { - return true; - } - } - return false; - } - private String function(Infer.Fn f) { cur = f; sc = f.def.scope; @@ -569,9 +677,11 @@ private String function(Infer.Fn f) { tmp = 0; nonNull = new HashSet(); shadow = new HashMap(); - longOnly = new HashSet(); + codePoints.clear(); ranges = new IntRanges(in, f); longLocals = ranges.longLocals(); + // A parameter arrives as a pair, whatever its range. + longLocals.removeAll(f.names); if (!f.def.scope.cells.isEmpty() || f.def.scope.hasYield || f.def.isAsync || !f.def.decorators.isEmpty() || f.def.args.vararg != null || f.def.args.kwarg != null || !f.def.args.kwonly.isEmpty()) { throw new Unsupported("signature"); @@ -584,35 +694,47 @@ private String function(Infer.Fn f) { da = new DefAssign(); da.function(f.def.body, sc.params); recycled = ListRecycling.find(f.def.body, f.names, sc); + scalars = findScalars(f); StringBuilder b = new StringBuilder(); - String fname = longParams ? "TL" + typed.get(f).substring(2) : typed.get(f); - b.append(" static ").append(retJ(f)).append(' ').append(fname).append('('); + b.append(" static ").append(retJ(f)).append(' ').append(typed.get(f)).append('('); Set declared = new HashSet(f.names); - Set stored = assigned(f.def.body); + StringBuilder norm = new StringBuilder(); for (int i = 0; i < f.names.size(); i++) { String n = f.names.get(i); - if (longParams && longParam(f, i, stored)) { - String q = "q_" + Emitter.mangle(n); - b.append(i == 0 ? "" : ", ").append("long ").append(q); - shadow.put(n, q); - longOnly.add(n); - } else { - b.append(i == 0 ? "" : ", ").append(jt(f.params[i])).append(' ').append(var(n)); + b.append(i == 0 ? "" : ", ").append(param(f.params[i], var(n), big(n))); + if (f.params[i].k == Ty.K.INT) { + // A caller holding only the object passes it whole. + norm.append(" if (").append(big(n)).append(" instanceof Long) {\n ").append(var(n)) + .append(" = ((Long) ").append(big(n)).append(").longValue();\n ").append(big(n)) + .append(" = null;\n }\n"); } } - b.append(") {\n"); + b.append(") {\n").append(norm); for (Map.Entry e : f.locals.entrySet()) { if (longLocals.contains(e.getKey())) { + if (declared.contains(e.getKey())) { + continue; + } declared.add(e.getKey()); b.append(" long ").append(var(e.getKey())).append(" = 0L;\n"); } else if (declared.add(e.getKey())) { - b.append(" ").append(jt(e.getValue())).append(' ').append(var(e.getKey())).append(" = ") - .append(zero(e.getValue())).append(";\n"); + if (e.getValue().k == Ty.K.INT) { + b.append(" long ").append(var(e.getKey())).append(" = 0L;\n Object ") + .append(big(e.getKey())).append(" = null;\n"); + } else { + b.append(" ").append(jt(e.getValue())).append(' ').append(var(e.getKey())).append(" = ") + .append(zero(e.getValue())).append(";\n"); + } } } + scalarDecls(b); boolean abrupt = block(f.def.body, b, 2); if (!abrupt) { - if (objectRet(f.ret)) { + if (f.ret.k == Ty.K.INT) { + // Never reached (the function would not return an int if + // it could fall off its end), but javac cannot tell. + b.append(" return Typed.retBig(Py.NONE);\n"); + } else if (objectRet(f.ret)) { b.append(" return Py.NONE;\n"); } else if (f.ret.nullable()) { b.append(" return null;\n"); @@ -703,6 +825,11 @@ private boolean stmt0(Stmt s, StringBuilder b, int ind) { } return false; } + if (s instanceof Assign && ((Assign) s).targets.size() == 1 && ((Assign) s).targets.get(0) instanceof Name + && scalars.containsKey(((Name) ((Assign) s).targets.get(0)).id)) { + scalarStore(((Name) ((Assign) s).targets.get(0)).id, (Call) ((Assign) s).value, b, ind); + return false; + } if (s instanceof Assign && ((Assign) s).targets.size() == 1 && sliceOf(((Assign) s).targets.get(0)) && sliceOf(((Assign) s).value)) { // `o[a:b] = v[d:e]`: copied without the intermediate list, the @@ -757,11 +884,7 @@ && localType(((Name) ((Assign) s).targets.get(0)).id).k == Ty.K.LIST if (v instanceof Starred) { throw new Unsupported("starred"); } - E e = expr(v); - String t = fresh(); - indent(b, ind); - b.append(jt(e.t)).append(' ').append(t).append(" = ").append(e.c).append(";\n"); - tv.add(new E(t, e.t)); + tv.add(spillTo(expr(v), b, ind)); } for (int i = 0; i < ts.size(); i++) { store(ts.get(i), tv.get(i), b, ind); @@ -795,6 +918,8 @@ && localType(((Name) ((Assign) s).targets.get(0)).id).k == Ty.K.LIST } else { throw new Unsupported("return None as " + cur.ret); } + } else if (cur.ret.k == Ty.K.INT) { + returnInt(expr(v), b, ind); } else { b.append("return ").append(conv(expr(v), cur.ret)).append(";\n"); } @@ -842,8 +967,40 @@ && localType(((Name) ((Assign) s).targets.get(0)).id).k == Ty.K.LIST shadow.keySet().removeAll(assigned(w.body)); forgetPaths(); Set inv = new HashSet(nonNull); + String test = "true"; + StringBuilder head = null; + if (!forever) { + // The test may lower its int arithmetic and compares to + // statements (see `hoistInt`); they then run at the top of + // every iteration, before a test that leaves the loop. + StringBuilder savedPre = pre; + int savedInd = preInd; + boolean savedPure = stmtPure; + boolean lower = !hasWalrus(w.test); + pre = lower ? new StringBuilder() : null; + preInd = ind + 1; + stmtPure = lower && callsArePure(w.test); + try { + test = testExpr(w.test); + head = pre; + } finally { + pre = savedPre; + preInd = savedInd; + stmtPure = savedPure; + } + } indent(b, ind); - b.append("while (").append(forever ? "true" : testExpr(w.test)).append(") {\n"); + if (head != null && head.length() > 0) { + b.append("while (true) {\n").append(head); + indent(b, ind + 1); + b.append("if (!").append(test).append(") {\n"); + indent(b, ind + 2); + b.append("break;\n"); + indent(b, ind + 1); + b.append("}\n"); + } else { + b.append("while (").append(test).append(") {\n"); + } nonNull.addAll(whenTrue(w.test)); block(w.body, b, ind + 1); nonNull = inv; @@ -871,6 +1028,121 @@ && localType(((Name) ((Assign) s).targets.get(0)).id).k == Ty.K.LIST throw new Unsupported(s.getClass().getSimpleName()); } + /// `return e` from a function returning an int (see `retJ`); the + /// statement is already indented. + private void returnInt(E e, StringBuilder b, int ind) { + if (e.lc != null && literal(e.lc) && !e.lc.contains("-9223372036854775808")) { + b.append("return ").append(e.lc).append(";\n"); + return; + } + if (e.lc != null) { + String t = fresh(); + b.append("long ").append(t).append(" = ").append(e.lc).append(";\n"); + indent(b, ind); + b.append("return ").append(t).append(" != Long.MIN_VALUE ? ").append(t) + .append(" : Typed.retBig(Long.valueOf(").append(t).append("));\n"); + return; + } + if (e.ll != null) { + b.append("return ").append(e.bg).append(" == null && ").append(e.ll).append(" != Long.MIN_VALUE ? ") + .append(e.ll).append(" : Typed.retBig(").append(e.c).append(");\n"); + return; + } + b.append("return Typed.retObj(").append(box(e)).append(");\n"); + } + + /// A `range` bound as a Java long. + private String bound(E e) { + if (e.lc != null) { + return e.lc; + } + if (e.ll != null) { + return "(" + e.bg + " == null ? " + e.ll + " : Typed.bound(" + e.bg + "))"; + } + return "Typed.bound(" + box(e) + ")"; + } + + /// Loop targets over a str whose only uses are `ord(c)`, to the Java + /// variable holding the code point. + private final Map codePoints = new HashMap(); + + /// `for c in s`: the code points of `s`, one string each only when + /// something reads `c` other than `ord(c)`. + private void strLoop(For f, E it, Set inBody, StringBuilder b, int ind) { + String tn = ((Name) f.target).id; + Ty vt = localType(tn); + String s = fresh(); + String n = fresh(); + String i = fresh(); + String cp = fresh(); + indent(b, ind); + b.append("{\n"); + indent(b, ind + 1); + b.append("String ").append(s).append(" = ").append(it.c).append(";\n"); + indent(b, ind + 1); + b.append("int ").append(n).append(" = ").append(s).append(".length();\n"); + indent(b, ind + 1); + b.append("for (int ").append(i).append(" = 0; ").append(i).append(" < ").append(n).append(";) {\n"); + indent(b, ind + 2); + b.append("int ").append(cp).append(" = ").append(s).append(".charAt(").append(i).append("++);\n"); + indent(b, ind + 2); + // A surrogate pair is one character to Python. + b.append("if (").append(cp).append(" >= 0xD800 && ").append(cp).append(" < 0xDC00 && ").append(i).append(" < ") + .append(n).append(" && ").append(s).append(".charAt(").append(i).append(") >= 0xDC00 && ").append(s) + .append(".charAt(").append(i).append(") < 0xE000) {\n"); + indent(b, ind + 3); + b.append(cp).append(" = ((").append(cp).append(" - 0xD800) << 10) + (").append(s).append(".charAt(").append(i) + .append("++) - 0xDC00) + 0x10000;\n"); + indent(b, ind + 2); + b.append("}\n"); + boolean ordOnly = !inBody.contains(tn) && !readOutside(tn, f) && onlyOrd(f.body, tn); + if (!ordOnly) { + E c = new E("Typed.ch(" + cp + ")", Ty.STR); + indent(b, ind + 2); + if (vt.k == Ty.K.STR) { + b.append(var(tn)).append(" = ").append(c.c).append(";\n"); + } else { + b.append(var(tn)).append(" = ").append(conv(c, vt)).append(";\n"); + } + if (vt.k == Ty.K.STR || vt.k == Ty.K.DYN) { + nonNull.add(tn); + } + } + if (!inBody.contains(tn)) { + codePoints.put(tn, cp); + } + block(f.body, b, ind + 2); + codePoints.remove(tn); + indent(b, ind + 1); + b.append("}\n"); + indent(b, ind); + b.append("}\n"); + } + + /// Whether every read of `n` in `o` is the argument of the builtin `ord`. + private boolean onlyOrd(Object o, String n) { + if (ordOf(o, n)) { + return true; + } + if (o instanceof Name && ((Name) o).id.equals(n)) { + return false; + } + boolean[] ok = {true}; + Infer.children(o, x -> ok[0] &= onlyOrd(x, n)); + return ok[0]; + } + + /// Whether `o` is `ord(n)`, with `ord` the builtin. + private boolean ordOf(Object o, String n) { + if (!(o instanceof Call)) { + return false; + } + Call c = (Call) o; + return c.func instanceof Name && "ord".equals(((Name) c.func).id) && c.keywords.isEmpty() + && c.args.size() == 1 && c.args.get(0) instanceof Name && ((Name) c.args.get(0)).id.equals(n) + && sc.resolve("ord") == Scope.Binding.BUILTIN && !em.moduleScopeHasStarImport(); + } + private void forLoop(For f, StringBuilder b, int ind) { if (!f.orelse.isEmpty()) { throw new Unsupported("for-else"); @@ -893,10 +1165,10 @@ private void forLoop(For f, StringBuilder b, int ind) { String stop; long step = 1; if (a.size() == 1) { - stop = "Typed.bound(" + box(expr(a.get(0))) + ")"; + stop = bound(expr(a.get(0))); } else { - start = "Typed.bound(" + box(expr(a.get(0))) + ")"; - stop = "Typed.bound(" + box(expr(a.get(1))) + ")"; + start = bound(expr(a.get(0))); + stop = bound(expr(a.get(1))); if (a.size() == 3) { Long c = intConstant(a.get(2)); if (c == null || c.longValue() == 0) { @@ -921,18 +1193,14 @@ private void forLoop(For f, StringBuilder b, int ind) { .append("L) {\n"); String tn = ((Name) f.target).id; boolean counter = vt.k == Ty.K.INT && !inBody.contains(tn); - boolean unboxed = counter && !longOnly.contains(tn) && !readOutside(tn, f) - && !longLocals.contains(tn); + indent(b, ind + 2); if (longLocals.contains(tn)) { - indent(b, ind + 2); b.append(v).append(" = ").append(i).append(";\n"); - } else if (unboxed) { - // Read nowhere but in this body: no boxed copy at all. - longOnly.add(tn); - } else if (longOnly.contains(tn)) { - throw new Unsupported("long-only " + tn + " reused"); - } else { + } else if (vt.k == Ty.K.INT) { + b.append(v).append(" = ").append(i).append(";\n"); indent(b, ind + 2); + b.append(big(tn)).append(" = null;\n"); + } else { b.append(v).append(" = ").append(conv(new E("Long.valueOf(" + i + ")", Ty.INT), vt)).append(";\n"); } if (counter) { @@ -940,9 +1208,6 @@ private void forLoop(For f, StringBuilder b, int ind) { } block(f.body, b, ind + 2); shadow.remove(tn); - if (unboxed) { - longOnly.remove(tn); - } indent(b, ind + 1); b.append("}\n"); indent(b, ind); @@ -950,6 +1215,10 @@ private void forLoop(For f, StringBuilder b, int ind) { return; } E it = expr(f.iter); + if (it.t.k == Ty.K.STR && !it.t.opt) { + strLoop(f, it, inBody, b, ind); + return; + } if (it.t.k != Ty.K.LIST) { throw new Unsupported("for over " + it.t); } @@ -966,7 +1235,13 @@ private void forLoop(For f, StringBuilder b, int ind) { indent(b, ind + 2); E item = el.k == Ty.K.FLOAT ? new E("Typed.atF(" + l + ", " + k + ")", Ty.FLOAT) : new E(unbox(l + ".get(" + k + ")", el), el); - b.append(v).append(" = ").append(conv(item, vt)).append(";\n"); + if (vt.k == Ty.K.INT && !longLocals.contains(((Name) f.target).id)) { + storePair(v, big(((Name) f.target).id), item, b, ind + 2); + } else if (vt.k == Ty.K.INT) { + b.append(v).append(" = Typed.toLong(").append(box(item)).append(");\n"); + } else { + b.append(v).append(" = ").append(conv(item, vt)).append(";\n"); + } block(f.body, b, ind + 2); indent(b, ind + 1); b.append("}\n"); @@ -976,11 +1251,10 @@ private void forLoop(For f, StringBuilder b, int ind) { private static Long intConstant(Expr e) { if (e instanceof Constant && ((Constant) e).kind == Constant.Kind.INT) { - try { - return Long.valueOf(Long.parseLong(((Constant) e).text.replace("_", ""))); - } catch (NumberFormatException x) { - return null; - } + // Any spelling: hex, octal and binary constants are as common + // as decimal ones in integer code. + java.math.BigInteger v = IntRanges.parse(((Constant) e).text); + return v == null || v.bitLength() > 63 ? null : Long.valueOf(v.longValue()); } if (e instanceof UnaryOp && "-".equals(((UnaryOp) e).op)) { Long v = intConstant(((UnaryOp) e).operand); @@ -995,6 +1269,11 @@ private static boolean plain(String c) { return c.matches("[A-Za-z_][A-Za-z0-9_]*"); } + /// Whether `c` is a Java long literal. + private static boolean literal(String c) { + return c.matches("\\(?-?[0-9]+L\\)?"); + } + private void store(Expr t, E v, StringBuilder b, int ind) { if (t instanceof Name) { String n = ((Name) t).id; @@ -1003,15 +1282,16 @@ private void store(Expr t, E v, StringBuilder b, int ind) { indent(b, ind); if (longLocals.contains(n)) { // The analysis proved every value it holds fits. - b.append(var(n)).append(" = ").append(v.lc != null ? v.lc : "Typed.toLong(" + box(v) + ")") + b.append(var(n)).append(" = ").append(fitting(v)) .append(";\n"); return; } - b.append(var(n)).append(" = ").append(conv(v, localType(n))).append(";\n"); - forgetPathsOf(n); - if (longOnly.contains(n)) { - throw new Unsupported("store to a long-only " + n); + if (localType(n).k == Ty.K.INT) { + storePair(var(n), big(n), v, b, ind); + } else { + b.append(var(n)).append(" = ").append(conv(v, localType(n))).append(";\n"); } + forgetPathsOf(n); shadow.remove(n); if (v.t.k == localType(n).k && !v.t.opt && v.t.k != Ty.K.NONE) { nonNull.add(n); @@ -1030,6 +1310,24 @@ private void store(Expr t, E v, StringBuilder b, int ind) { if (t instanceof Attribute) { Attribute a = (Attribute) t; E r = expr(a.value); + if (fieldTy(r, a.attr).k == Ty.K.INT && typedField(owner(r, a.attr), a.attr) != null) { + if (!plain(r.c) && !(v.ll != null || v.lc != null && plain(v.lc) || plain(v.c))) { + // Python evaluates the value before the receiver. + String tv = fresh(); + String tb = fresh(); + indent(b, ind); + b.append("long ").append(tv).append(" = 0L;\n"); + indent(b, ind); + b.append("Object ").append(tb).append(" = null;\n"); + indent(b, ind); + storePair(tv, tb, v, b, ind); + v = dual(tv, tb); + } + indent(b, ind); + storeIntField(r, a.attr, v, b, ind); + forgetAttr(a.attr); + return; + } if (!plain(v.c) && !plain(r.c)) { // Python evaluates the value before the receiver. String tv = fresh(); @@ -1061,9 +1359,15 @@ private void store(Expr t, E v, StringBuilder b, int ind) { E l = expr(s.value); E i = expr(s.index); if (l.t.k != Ty.K.LIST || !(i.t.k == Ty.K.INT || i.t.k == Ty.K.BOOL)) { - throw new Unsupported("subscript store on " + l.t); + if (!plain(v.c) && v.ll == null && !(v.lc != null && (plain(v.lc) || literal(v.lc)))) { + v = spillTo(v, b, ind); + } + indent(b, ind); + b.append("Py.setitem(").append(box(l)).append(", ").append(box(i)).append(", ").append(box(v)) + .append(");\n"); + return; } - if (!plain(v.c)) { + if (!plain(v.c) && v.ll == null && !(v.lc != null && (plain(v.lc) || literal(v.lc)))) { String tv = fresh(); indent(b, ind); b.append(jt(v.t)).append(' ').append(tv).append(" = ").append(v.c).append(";\n"); @@ -1071,6 +1375,22 @@ private void store(Expr t, E v, StringBuilder b, int ind) { } Ty el = in.elem(l.t); indent(b, ind); + String[] ix = index(i); + if (el.k == Ty.K.INT && ix != null && plain(l.c)) { + String[] vp = parts(v); + b.append("if (").append(ix[1]).append(inRange(l, ix[0])).append(" && ").append(l.c) + .append(".ints != null").append("null".equals(vp[1]) ? "" : " && " + vp[1] + " == null") + .append(") {\n"); + indent(b, ind + 1); + b.append(l.c).append(".ints[(int) ").append(ix[0]).append("] = ").append(vp[0]).append(";\n"); + indent(b, ind); + b.append("} else {\n"); + indent(b, ind + 1); + b.append("h_set(").append(l.c).append(", ").append(box(i)).append(", ").append(box(v)).append(");\n"); + indent(b, ind); + b.append("}\n"); + return; + } if (i.lc != null && plain(l.c)) { String in = inRange(l, i.lc); if (el.k == Ty.K.FLOAT) { @@ -1104,12 +1424,60 @@ private void store(Expr t, E v, StringBuilder b, int ind) { throw new Unsupported("store target " + t.getClass().getSimpleName()); } + /// `l`/`o = v` for an int pair, the line already indented: the object + /// half holds no `Long`. + private void storePair(String l, String o, E v, StringBuilder b, int ind) { + if (v.lc != null) { + b.append(l).append(" = ").append(v.lc).append(";\n"); + indent(b, ind); + b.append(o).append(" = null;\n"); + return; + } + if (v.ll != null) { + if (!v.ll.equals(l)) { + b.append(l).append(" = ").append(v.ll).append(";\n"); + indent(b, ind); + } + b.append(o).append(" = ").append(v.bg).append(";\n"); + return; + } + String t = fresh(); + b.append("Object ").append(t).append(" = ").append(box(v)).append(";\n"); + indent(b, ind); + b.append("if (").append(t).append(" instanceof Long) {\n"); + indent(b, ind + 1); + b.append(l).append(" = ((Long) ").append(t).append(").longValue();\n"); + indent(b, ind + 1); + b.append(o).append(" = null;\n"); + indent(b, ind); + b.append("} else {\n"); + indent(b, ind + 1); + b.append(o).append(" = ").append(t).append(";\n"); + indent(b, ind); + b.append("}\n"); + } + private void augAssign(AugAssign a, StringBuilder b, int ind) { Expr t = a.target; if (t instanceof Name && longLocals.contains(((Name) t).id)) { store(t, expr(new BinOp(t, a.op, a.value)), b, ind); return; } + if (t instanceof Name && (localType(((Name) t).id).k == Ty.K.INT || localType(((Name) t).id).k == Ty.K.FLOAT) + && sc.resolve(((Name) t).id) == Scope.Binding.LOCAL) { + store(t, expr(new BinOp(t, a.op, a.value)), b, ind); + return; + } + if (t instanceof Attribute && ((Attribute) t).value instanceof Name + && sc.resolve(((Name) ((Attribute) t).value).id) == Scope.Binding.LOCAL) { + // The receiver is a local: reading it again is reading it once. + E r0 = expr(((Attribute) t).value); + if (r0.t.k == Ty.K.INST && fieldTy(r0, ((Attribute) t).attr).k == Ty.K.INT + && typedField(owner(r0, ((Attribute) t).attr), ((Attribute) t).attr) != null) { + store(t, expr(new BinOp(t, a.op, a.value)), b, ind); + return; + } + } if (t instanceof Name) { E cur0 = expr(t); E r = binop(a.op, cur0, expr(a.value), a.value); @@ -1126,6 +1494,12 @@ private void augAssign(AugAssign a, StringBuilder b, int ind) { b.append(jt(r.t)).append(' ').append(rv).append(" = ").append(r.c).append(";\n"); } E recv = new E(rv, r.t); + if (fieldTy(recv, at.attr).k == Ty.K.INT && typedField(owner(recv, at.attr), at.attr) != null) { + E v = binop(a.op, intField(recv, at.attr), expr(a.value), a.value); + indent(b, ind); + storeIntField(recv, at.attr, v, b, ind); + return; + } String ref = fieldRef(recv, at.attr); E old = new E(ref, fieldTy(recv, at.attr)); E v = binop(a.op, old, expr(a.value), a.value); @@ -1375,6 +1749,44 @@ private String fieldRef(E r, String attr) { return recv + ".f_" + Emitter.mangle(attr); } + /// Reads of typed int fields through a helper, where the receiver is + /// an expression that may not be evaluated twice nor moved. + private final Set fieldBoxers = new TreeSet(); + + /// Typed int field `attr` of `r`, as a pair when the receiver is (or + /// may be put in) a local. + private E intField(E r, String attr) { + Layouts.Cls o = owner(r, attr); + String m = Emitter.mangle(attr); + String recv = nz(r, attr); + if (!plain(recv) && pre != null && stmtPure && condDepth == 0) { + String t = fresh(); + indent(pre, preInd); + pre.append(o.javaName).append(' ').append(t).append(" = ").append(recv).append(";\n"); + recv = t; + } + if (plain(recv)) { + return dual(recv + ".t_" + m, recv + ".b_" + m); + } + fieldBoxers.add(o.javaName + " " + m); + return new E("fb_" + o.javaName + "_" + m + "(" + recv + ")", Ty.INT); + } + + /// Stores `v` into typed int field `attr` of `r`; Python evaluates the + /// value first. + private void storeIntField(E r, String attr, E v, StringBuilder b, int ind) { + String m = Emitter.mangle(attr); + Layouts.Cls o = owner(r, attr); + String recv = nz(r, attr); + if (!plain(recv)) { + String t = fresh(); + b.append(o.javaName).append(' ').append(t).append(" = ").append(recv).append(";\n"); + indent(b, ind); + recv = t; + } + storePair(recv + ".t_" + m, recv + ".b_" + m, v, b, ind); + } + // ---- expressions ----------------------------------------------------- /// `e` as a Java boolean, for a test: `and`, `or` and `not` need no @@ -1407,6 +1819,10 @@ String testExpr(Expr e) { } E expr(Expr e) { + E sub = leafSubst.get(e); + if (sub != null) { + return sub; + } if (e != plainExpr && (e instanceof BinOp || e instanceof UnaryOp || e instanceof IfExp //NOPMD CompareObjectsWithEquals || e instanceof Call && ranges.inlinable((Call) e) != null)) { E w = wrapped(e); @@ -1449,14 +1865,26 @@ E expr(Expr e) { if (Infer.MATH_CONSTS.contains(a.attr) && in.isMath(a.value, sc)) { return new E(mathConstant(a.attr), Ty.FLOAT); } + if (a.value instanceof Name && scalars.containsKey(((Name) a.value).id)) { + return scalarRead(((Name) a.value).id, a.attr); + } E r = expr(a.value); + if (r.t.k != Ty.K.INST) { + return new E(em.attrSite(a.attr) + ".get(" + box(r) + ")", Ty.DYN); + } Ty ft = fieldTy(r, a.attr); + if (ft.k == Ty.K.INT && typedField(owner(r, a.attr), a.attr) != null) { + return intField(r, a.attr); + } String p = path(a); return new E(fieldRef(r, a.attr), p != null && nonNull.contains(p) ? ft.nonNone() : ft); } if (e instanceof BinOp) { - if (pre != null && (pureInt(e) || pureFloat(e))) { - return hoistInt(e); + if (pre != null && arith(e)) { + E h = hoistInt(e); + if (h != null) { + return h; + } } BinOp b = (BinOp) e; return binop(b.op, expr(b.left), expr(b.right), b.right); @@ -1504,8 +1932,13 @@ E expr(Expr e) { return new E(sb.append(')').toString(), Ty.BOOL); } if (e instanceof Compare) { - if (pre != null && intCompare((Compare) e)) { - return hoistInt(e); + Compare cm = (Compare) e; + if (pre != null && cm.ops.size() == 1 && ORDERS.contains(cm.ops.get(0)) + && (arith(cm.left) || arith(cm.comparators.get(0)) || numeric(cm))) { + E h = hoistInt(e); + if (h != null) { + return h; + } } return compare((Compare) e); } @@ -1542,10 +1975,14 @@ E expr(Expr e) { return new E("Typed.asList(Py.getslice(" + l.c + ", " + sliceParts((Slice) s.index) + "))", l.t.nonNone()); } + E bits = structCast(s); + if (bits != null) { + return bits; + } E l = expr(s.value); E i = expr(s.index); if (l.t.k != Ty.K.LIST || !(i.t.k == Ty.K.INT || i.t.k == Ty.K.BOOL)) { - throw new Unsupported("subscript of " + l.t + " by " + i.t); + return new E("Py.getitem(" + box(l) + ", " + box(i) + ")", Ty.DYN); } Ty el = in.elem(l.t); if (el.k == Ty.K.BOT) { @@ -1553,6 +1990,30 @@ E expr(Expr e) { } String slow = el.k == Ty.K.FLOAT ? "h_getF(" + l.c + ", " + box(i) + ")" : "h_get(" + l.c + ", " + box(i) + ")"; String fast = slow; + String[] ix = index(i); + if (el.k == Ty.K.INT && ix != null && plain(l.c) && pre != null && stmtPure && condDepth == 0) { + // From the list's longs into a pair, before the statement. + String r = fresh(); + String rb = fresh(); + indent(pre, preInd); + pre.append("long ").append(r).append(";\n"); + indent(pre, preInd); + pre.append("Object ").append(rb).append(" = null;\n"); + indent(pre, preInd); + pre.append("if (").append(ix[1]).append(inRange(l, ix[0])).append(" && ").append(l.c) + .append(".ints != null) {\n"); + indent(pre, preInd + 1); + pre.append(r).append(" = ").append(l.c).append(".ints[(int) ").append(ix[0]).append("];\n"); + indent(pre, preInd); + pre.append("} else {\n"); + indent(pre, preInd + 1); + pre.append(r).append(" = 0L;\n"); + indent(pre, preInd + 1); + storePair(r, rb, new E(slow, Ty.INT), pre, preInd + 1); + indent(pre, preInd); + pre.append("}\n"); + return dual(r, rb); + } if (i.lc != null && plain(l.c) && pre != null && stmtPure && condDepth == 0) { // Read into a local before the statement: no call in it can // change the list, and a Java local, unlike an operand of a @@ -1620,14 +2081,23 @@ E expr(Expr e) { if (e instanceof Call) { return call((Call) e); } + if (e instanceof DictExpr) { + DictExpr d = (DictExpr) e; + StringBuilder sb = new StringBuilder("Py.dict("); + for (int i = 0; i < d.keys.size(); i++) { + if (d.keys.get(i) == null) { + throw new Unsupported("dict spread"); + } + sb.append(i == 0 ? "" : ", ").append(box(expr(d.keys.get(i)))).append(", ") + .append(box(expr(d.values.get(i)))); + } + return new E(sb.append(')').toString(), Ty.DYN); + } throw new Unsupported(e.getClass().getSimpleName()); } private E name(Name nm) { String n = nm.id; - if (subst != null && subst.containsKey(n)) { - return subst.get(n); - } switch (sc.resolve(n)) { case LOCAL: { if (!sc.params.contains(n) && !da.safe.contains(nm)) { @@ -1642,7 +2112,10 @@ private E name(Name nm) { } String l = shadow.get(n); if (l != null) { - return new E(longOnly.contains(n) ? "Long.valueOf(" + l + ")" : var(n), t, l); + return new E("Long.valueOf(" + l + ")", t, l); + } + if (t.k == Ty.K.INT) { + return dual(var(n), big(n)); } return new E(var(n), nonNull.contains(n) ? t.nonNone() : t); } @@ -1757,159 +2230,239 @@ private static String pyOp(String op) { // ---- inline int arithmetic ------------------------------------------- - /// Whether `e` is int arithmetic over int locals and constants that - /// cannot raise: evaluating it early, or twice, changes nothing. - private boolean pureInt(Expr e) { - if (e instanceof Constant) { - Long c = intConstant(e); - return c != null && Math.abs(c.longValue()) < (1L << 60); - } - if (e instanceof Name) { - String n = ((Name) e).id; - return sc.resolve(n) == Scope.Binding.LOCAL && localType(n).k == Ty.K.INT; - } + /// Whether `e` is an inner node of hoisted arithmetic: an operation + /// on ints or floats that `longOf`/`doubleOf` spell out. Anything else + /// under one is a leaf, evaluated once, in order, before it. + private static boolean arith(Expr e) { if (e instanceof UnaryOp) { - UnaryOp u = (UnaryOp) e; - return "-".equals(u.op) && pureInt(u.operand); + return "-".equals(((UnaryOp) e).op); } - if (e instanceof BinOp) { - BinOp b = (BinOp) e; - switch (b.op) { - case "+": - case "-": - case "*": - case "&": - case "|": - case "^": - return pureInt(b.left) && pureInt(b.right); - case ">>": { - Long c = intConstant(b.right); - return c != null && c.longValue() >= 0 && c.longValue() < 64 && pureInt(b.left); - } - default: - return false; - } + if (!(e instanceof BinOp)) { + return false; } - return false; - } - - /// Float arithmetic over float and int locals and constants, as - /// `pureInt` is for ints; every step here is IEEE arithmetic, as in - /// Python, except a division by zero, which takes the generic path. - private boolean pureFloat(Expr e) { - if (e instanceof Constant) { - return ((Constant) e).kind == Constant.Kind.FLOAT && !((Constant) e).text.matches(".*[jJ]"); - } - if (e instanceof Name) { - String n = ((Name) e).id; - return sc.resolve(n) == Scope.Binding.LOCAL && localType(n).k == Ty.K.FLOAT; + BinOp b = (BinOp) e; + switch (b.op) { + case "+": + case "-": + case "*": + case "/": + case "%": + case "//": + case "&": + case "|": + case "^": + return true; + case ">>": { + Long c = intConstant(b.right); + return c != null && c.longValue() >= 0 && c.longValue() < 64; + } + default: + return false; } - if (e instanceof UnaryOp) { - UnaryOp u = (UnaryOp) e; - return "-".equals(u.op) && pureFloat(u.operand); + } + + /// The leaves of arithmetic `e` in evaluation order, or null when one + /// may not run before the statement: only a local or a constant may, + /// unless every call in the statement is pure and none of it runs + /// conditionally. + private List leaves(Expr e) { + List out = new ArrayList(); + return collectLeaves(e, out) ? out : null; + } + + private boolean collectLeaves(Expr e, List out) { + if (e instanceof Compare) { + Compare c = (Compare) e; + return collectLeaves(c.left, out) && collectLeaves(c.comparators.get(0), out); } - if (e instanceof BinOp) { - BinOp b = (BinOp) e; - switch (b.op) { - case "+": - case "-": - case "*": - return pureFloat(b.left) && (pureFloat(b.right) || pureInt(b.right)) - || pureInt(b.left) && pureFloat(b.right); - case "/": - return (pureFloat(b.left) || pureInt(b.left)) && (pureFloat(b.right) || pureInt(b.right)); - default: - return false; + if (arith(e)) { + if (e instanceof UnaryOp) { + return collectLeaves(((UnaryOp) e).operand, out); } + return collectLeaves(((BinOp) e).left, out) && collectLeaves(((BinOp) e).right, out); } - return false; + boolean quiet = e instanceof Constant || e instanceof Name && sc.resolve(((Name) e).id) == Scope.Binding.LOCAL; + if (!quiet && !(stmtPure && condDepth == 0)) { + return false; + } + out.add(e); + return true; } - private boolean intCompare(Compare c) { - if (c.ops.size() != 1) { - return false; + /// 'i' (int), 'f' (float) or 0 (neither) for arithmetic `e` over the + /// evaluated leaves. + private char kindOf(Expr e) { + E v = leafSubst.get(e); + if (v != null) { + switch (v.t.k) { + case INT: + case BOOL: + return 'i'; + case FLOAT: + return 'f'; + default: + return 0; + } } - switch (c.ops.get(0)) { - case "<": - case "<=": - case ">": - case ">=": - case "==": - case "!=": - return pureInt(c.left) && pureInt(c.comparators.get(0)); + if (e instanceof UnaryOp) { + return kindOf(((UnaryOp) e).operand); + } + BinOp b = (BinOp) e; + char l = kindOf(b.left); + char r = kindOf(b.right); + if (l == 0 || r == 0) { + return 0; + } + switch (b.op) { + case "/": + return 'f'; + case "+": + case "-": + case "*": + return l == 'f' || r == 'f' ? 'f' : 'i'; default: - return false; + return l == 'i' && r == 'i' ? 'i' : 0; } } - /// `e` computed on Java longs in a block before the statement, falling - /// back to the generic operations when an operand is a big int or a - /// step overflows. ParparVM inlines no call, not even to a helper of - /// the same class, so each operation is spelled out here. + /// `e`, int or float arithmetic (or a comparison of it), computed on + /// Java longs and doubles in a block before the statement, falling back + /// to the generic operations when an operand is a big int or a step + /// overflows; or null when its leaves may not move there. ParparVM + /// inlines no call, not even to a helper of the same class, so each + /// operation is spelled out here. private E hoistInt(Expr e) { + List ls = leaves(e); + if (ls == null) { + return null; + } + Map savedLeaves = leafSubst; + leafSubst = new IdentityHashMap(savedLeaves); + try { + for (Expr x : ls) { + leafSubst.put(x, spill(expr(x))); + } + char k; + if (e instanceof Compare) { + Compare c = (Compare) e; + char a = kindOf(c.left); + char b = kindOf(c.comparators.get(0)); + k = a == 0 || b == 0 ? 0 : a == 'f' || b == 'f' ? 'f' : 'i'; + } else { + k = kindOf(e); + } + if (k == 0) { + return generic0(e); + } + return fastPath(e, ls, k); + } finally { + leafSubst = savedLeaves; + } + } + + /// `e` by the generic operations, over the leaves already evaluated. + private E generic0(Expr e) { StringBuilder saved = pre; - String generic; - Ty t; + pre = null; + try { + return e instanceof Compare ? compare((Compare) e) : expr(e); + } finally { + pre = saved; + } + } + + /// `v` in plain Java locals, so it may be read twice: declared in `pre`. + private E spill(E v) { + if (v.ll != null || plain(v.c) && v.lc == null || v.lc != null && (plain(v.lc) || literal(v.lc))) { + return v; + } + return spillTo(v, pre, preInd); + } + + /// `v` copied into fresh Java locals declared in `into`. + private E spillTo(E v, StringBuilder into, int ind) { + String t = fresh(); + indent(into, ind); + if (v.lc != null) { + into.append("long ").append(t).append(" = ").append(v.lc).append(";\n"); + return new E("Long.valueOf(" + t + ")", v.t, t); + } + if (v.t.k == Ty.K.INT) { + String tb = fresh(); + into.append("long ").append(t).append(" = 0L;\n"); + indent(into, ind); + into.append("Object ").append(tb).append(" = null;\n"); + indent(into, ind); + storePair(t, tb, v, into, ind); + return dual(t, tb); + } + into.append(jt(v.t)).append(' ').append(t).append(" = ").append(v.c).append(";\n"); + return new E(t, v.t); + } + + private static final Set ORDERS = new HashSet(Arrays.asList("<", "<=", ">", ">=", "==", "!=")); + + /// Whether comparison `c` may be of numbers: no side is None. + private static boolean numeric(Compare c) { + return !(c.left instanceof Constant && ((Constant) c.left).kind == Constant.Kind.NONE) + && !(c.comparators.get(0) instanceof Constant + && ((Constant) c.comparators.get(0)).kind == Constant.Kind.NONE); + } + + private E fastPath(Expr e, List ls, char k) { + Ty t = e instanceof Compare ? Ty.BOOL : k == 'f' ? Ty.FLOAT : Ty.INT; // The generic path goes into a method of its own: with it inline, a // function made only of such arithmetic would call out, and a // ParparVM function that calls out gets a stack guard and is no // longer inlined by the C compiler. - Map nodes = new LinkedHashMap(); - for (Name nm : namesIn(e)) { - nodes.putIfAbsent(nm.id, nm); - } - List names = new ArrayList(nodes.keySet()); - Map sub = new HashMap(); + Map sub = new IdentityHashMap(leafSubst); StringBuilder params = new StringBuilder(); StringBuilder args = new StringBuilder(); - for (int k = 0; k < names.size(); k++) { - E ne = name(nodes.get(names.get(k))); - String pn = "p" + k; - String sep = k == 0 ? "" : ", "; - if (ne.lc != null && longOnly.contains(names.get(k))) { - params.append(sep).append("long ").append(pn); - args.append(sep).append(ne.lc); - sub.put(names.get(k), new E("Long.valueOf(" + pn + ")", ne.t, pn)); - } else { - params.append(sep).append(jt(ne.t)).append(' ').append(pn); - args.append(sep).append(ne.c); - sub.put(names.get(k), new E(pn, ne.t)); - } + for (int i = 0; i < ls.size(); i++) { + E v = leafSubst.get(ls.get(i)); + String pn = "p" + i; + String sep = i == 0 ? "" : ", "; + params.append(sep).append(jt(v.t)).append(' ').append(pn); + args.append(sep).append(v.t.k == Ty.K.INT ? box(v) : v.c); + sub.put(ls.get(i), new E(pn, v.t)); } - Map savedSub = subst; - pre = null; - subst = sub; + String generic; + Map saved = leafSubst; + leafSubst = sub; try { - E g = e instanceof Compare ? compare((Compare) e) : expr(e); - t = g.t; + E g = generic0(e); String m = "S" + (slowCount++); slow.append(" static ").append(jt(t)).append(' ').append(m).append('(').append(params) - .append(") {\n return ").append(g.c).append(";\n }\n"); + .append(") {\n return ").append(t.k == Ty.K.FLOAT ? toD(g) : conv(g, t)).append(";\n }\n"); generic = m + "(" + args + ")"; } finally { - pre = saved; - subst = savedSub; + leafSubst = saved; } - String r = fresh(); String done = fresh(); String fast = fresh(); StringBuilder f = new StringBuilder(); - Map leaves = new HashMap(); + Map guards = new HashMap(); String v; if (e instanceof Compare) { Compare c = (Compare) e; - String a = longOf(c.left, f, leaves, fast); - String b2 = longOf(c.comparators.get(0), f, leaves, fast); - v = a + " " + c.ops.get(0) + " " + b2; - } else if (t.k == Ty.K.FLOAT) { - v = doubleOf(e, f, leaves, fast); + String op = c.ops.get(0); + if (k == 'f') { + v = doubleOf(c.left, f, guards, fast) + " " + op + " " + doubleOf(c.comparators.get(0), f, guards, fast); + } else { + v = longOf(c.left, f, guards, fast) + " " + op + " " + longOf(c.comparators.get(0), f, guards, fast); + } + } else if (k == 'f') { + v = doubleOf(e, f, guards, fast); } else { - v = "Long.valueOf(" + longOf(e, f, leaves, fast) + ")"; + v = longOf(e, f, guards, fast); } + String r = fresh(); + String rb = t.k == Ty.K.INT ? fresh() : null; int ind = preInd; + boolean leaves = f.toString().contains("break " + fast + ";"); indent(pre, ind); - pre.append(jt(t)).append(' ').append(r).append(";\n"); - if (!f.toString().contains("break " + fast + ";")) { + pre.append(t.k == Ty.K.INT ? "long" : jt(t)).append(' ').append(r).append(";\n"); + if (!leaves) { // Nothing in the fast path can leave it (no operand that may be // a big int, no operation that may overflow): the generic // fallback would be unreachable, which javac rejects. @@ -1921,7 +2474,11 @@ private E hoistInt(Expr e) { } indent(pre, ind); pre.append(r).append(" = ").append(v).append(";\n"); - return new E(r, t); + return t.k == Ty.K.INT ? new E("Long.valueOf(" + r + ")", t, r) : new E(r, t); + } + if (rb != null) { + indent(pre, ind); + pre.append("Object ").append(rb).append(" = null;\n"); } indent(pre, ind); pre.append(done).append(": {\n"); @@ -1938,26 +2495,34 @@ private E hoistInt(Expr e) { indent(pre, ind + 1); pre.append("}\n"); indent(pre, ind + 1); - pre.append(r).append(" = ").append(generic).append(";\n"); + if (rb == null) { + pre.append(r).append(" = ").append(generic).append(";\n"); + } else { + String g = fresh(); + pre.append("Object ").append(g).append(" = ").append(generic).append(";\n"); + indent(pre, ind + 1); + pre.append("if (").append(g).append(" instanceof Long) {\n"); + indent(pre, ind + 2); + pre.append(r).append(" = ((Long) ").append(g).append(").longValue();\n"); + indent(pre, ind + 1); + pre.append("} else {\n"); + indent(pre, ind + 2); + pre.append(r).append(" = 0L;\n"); + indent(pre, ind + 2); + pre.append(rb).append(" = ").append(g).append(";\n"); + indent(pre, ind + 1); + pre.append("}\n"); + } indent(pre, ind); pre.append("}\n"); - return new E(r, t); + return rb == null ? new E(r, t) : dual(r, rb); } - private static List namesIn(Object o) { - List out = new ArrayList(); - if (o instanceof Name) { - out.add((Name) o); - } - Infer.children(o, x -> out.addAll(namesIn(x))); - return out; - } - - /// Statements into `f` computing `e` (`pureFloat`, or `pureInt` to be - /// converted) as a double; the expression holding it. - private String doubleOf(Expr e, StringBuilder f, Map leaves, String fast) { - if (pureInt(e)) { - String x = longOf(e, f, leaves, fast); + /// Statements into `f` computing `e` as a double; the expression + /// holding it. + private String doubleOf(Expr e, StringBuilder f, Map guards, String fast) { + if (kindOf(e) == 'i') { + String x = longOf(e, f, guards, fast); String d = fresh(); // Exact below 2^53; Python rounds a wider int the same way, but // the generic path is the one that knows. @@ -1966,18 +2531,19 @@ private String doubleOf(Expr e, StringBuilder f, Map leaves, Str .append(" = (double) ").append(x).append(";\n"); return d; } - if (e instanceof Constant || e instanceof Name) { - return expr(e).c; + E lv = leafSubst.get(e); + if (lv != null) { + return lv.c; } String d = fresh(); if (e instanceof UnaryOp) { - f.append("double ").append(d).append(" = -").append(doubleOf(((UnaryOp) e).operand, f, leaves, fast)) + f.append("double ").append(d).append(" = -").append(doubleOf(((UnaryOp) e).operand, f, guards, fast)) .append(";\n"); return d; } BinOp b = (BinOp) e; - String x = doubleOf(b.left, f, leaves, fast); - String y = doubleOf(b.right, f, leaves, fast); + String x = doubleOf(b.left, f, guards, fast); + String y = doubleOf(b.right, f, guards, fast); if ("/".equals(b.op)) { f.append("if (").append(y).append(" == 0) {\n break ").append(fast).append(";\n}\n"); } @@ -1986,36 +2552,41 @@ private String doubleOf(Expr e, StringBuilder f, Map leaves, Str return d; } - /// Statements into `f` computing `e` as a long; the name holding it. - private String longOf(Expr e, StringBuilder f, Map leaves, String fast) { - if (e instanceof Constant) { - return intConstant(e) + "L"; - } - if (e instanceof Name) { - E ne = name((Name) e); - if (ne.lc != null) { - return ne.lc; + /// Statements into `f` computing int `e` as a long; the name holding it. + private String longOf(Expr e, StringBuilder f, Map guards, String fast) { + E lv = leafSubst.get(e); + if (lv != null) { + if (lv.lc != null) { + return lv.lc; + } + if (lv.t.k == Ty.K.BOOL) { + return "(" + lv.c + " ? 1L : 0L)"; } - String c = ne.c; - String d = leaves.get(c); + if (lv.ll != null) { + if (guards.put(lv.bg, lv.ll) == null) { + f.append("if (").append(lv.bg).append(" != null) {\n break ").append(fast).append(";\n}\n"); + } + return lv.ll; + } + String d = guards.get(lv.c); if (d == null) { d = fresh(); - leaves.put(c, d); - f.append("if (!(").append(c).append(" instanceof Long)) {\n break ").append(fast) - .append(";\n}\nlong ").append(d).append(" = ((Long) ").append(c).append(").longValue();\n"); + guards.put(lv.c, d); + f.append("if (!(").append(lv.c).append(" instanceof Long)) {\n break ").append(fast) + .append(";\n}\nlong ").append(d).append(" = ((Long) ").append(lv.c).append(").longValue();\n"); } return d; } String d = fresh(); if (e instanceof UnaryOp) { - String x = longOf(((UnaryOp) e).operand, f, leaves, fast); + String x = longOf(((UnaryOp) e).operand, f, guards, fast); f.append("if (").append(x).append(" == Long.MIN_VALUE) {\n break ").append(fast).append(";\n}\n") .append("long ").append(d).append(" = -").append(x).append(";\n"); return d; } BinOp b = (BinOp) e; - String x = longOf(b.left, f, leaves, fast); - String y = longOf(b.right, f, leaves, fast); + String x = longOf(b.left, f, guards, fast); + String y = longOf(b.right, f, guards, fast); switch (b.op) { case "+": f.append("long ").append(d).append(" = ").append(x).append(" + ").append(y).append(";\n") @@ -2028,10 +2599,25 @@ private String longOf(Expr e, StringBuilder f, Map leaves, Strin .append(d).append(")) < 0) {\n break ").append(fast).append(";\n}\n"); break; case "*": - // Exact when both fit 31 bits; anything wider takes the runtime. - f.append("if (((").append(x).append(" + 0x80000000L) | (").append(y) - .append(" + 0x80000000L)) >>> 32 != 0) {\n break ").append(fast).append(";\n}\n") - .append("long ").append(d).append(" = ").append(x).append(" * ").append(y).append(";\n"); + f.append("long ").append(d).append(" = ").append(x).append(" * ").append(y).append(";\n"); + // Exact when the high half of the 128-bit product is the + // sign of the low half (Math.multiplyHigh, spelled out). + f.append("if (").append(mulOverflow(x, y, d)).append(") {\n break ").append(fast).append(";\n}\n"); + break; + case "%": + // Python's remainder takes the divisor's sign. + f.append("if (").append(y).append(" == 0L) {\n break ").append(fast).append(";\n}\n") + .append("long ").append(d).append(" = ").append(x).append(" % ").append(y).append(";\n") + .append("if (").append(d).append(" != 0L && (").append(d).append(" ^ ").append(y) + .append(") < 0L) {\n ").append(d).append(" += ").append(y).append(";\n}\n"); + break; + case "//": + // Floor division; MIN // -1 is the one quotient a long cannot hold. + f.append("if (").append(y).append(" == 0L || ").append(x).append(" == Long.MIN_VALUE && ").append(y) + .append(" == -1L) {\n break ").append(fast).append(";\n}\n") + .append("long ").append(d).append(" = ").append(x).append(" / ").append(y).append(";\n") + .append("if (").append(x).append(" % ").append(y).append(" != 0L && (").append(x).append(" ^ ") + .append(y).append(") < 0L) {\n ").append(d).append("--;\n}\n"); break; default: f.append("long ").append(d).append(" = ").append(x).append(' ').append(b.op).append(' ').append(y) @@ -2041,6 +2627,13 @@ private String longOf(Expr e, StringBuilder f, Map leaves, Strin return d; } + /// Java for "`x * y` (whose low 64 bits are in `d`) overflowed": the + /// cheap test for operands of 31 bits, then the exact one. + private static String mulOverflow(String x, String y, String d) { + return "((" + x + " + 0x80000000L) | (" + y + " + 0x80000000L)) >>> 32 != 0 && (" + y + " != 0 && (" + d + + " / " + y + " != " + x + " || " + x + " == Long.MIN_VALUE && " + y + " == -1L))"; + } + /// Helpers emitted into this module: an int operation with the `Long` /// fast path inline, so the common case costs no call into the runtime. private final Set intHelpers = new TreeSet(); @@ -2324,11 +2917,81 @@ private String argList(Infer.Fn f, int from, List args) { } a = expr(defs.get(d)); } - sb.append(i == 0 ? "" : ", ").append(conv(a, f.params[from + i])); + sb.append(i == 0 ? "" : ", ").append(arg(a, f.params[from + i])); } return sb.toString(); } + /// `a` passed for a parameter of type `t`: an int as its two halves. + private String arg(E a, Ty t) { + if (t.k != Ty.K.INT) { + return conv(a, t); + } + String[] p = parts(a); + return p[0] + ", " + p[1]; + } + + /// `struct.unpack(f, struct.pack(g, x))[0]` where `g` packs one double + /// and `f` reads the same eight bytes back as a signed long, or the + /// other way round: a reinterpretation of the bits, which is what the + /// idiom is for. Null for anything else. + private E structCast(Subscript s) { + Long ix = intConstant(s.index); + if (ix == null || ix.longValue() != 0 || !(s.value instanceof Call)) { + return null; + } + Call u = (Call) s.value; + if (!isStructCall(u, "unpack") || !(u.args.get(1) instanceof Call) || !isStructCall((Call) u.args.get(1), "pack") + || ((Call) u.args.get(1)).args.size() != 2) { + return null; + } + Call p = (Call) u.args.get(1); + String uf = strConstant(u.args.get(0)); + String pf = strConstant(p.args.get(0)); + if (uf == null || pf == null || uf.length() != pf.length() || uf.length() > 2 + || uf.length() == 2 && uf.charAt(0) != pf.charAt(0) || uf.length() == 2 && "<>!=".indexOf(uf.charAt(0)) < 0) { + return null; + } + char ut = uf.charAt(uf.length() - 1); + char pt = pf.charAt(pf.length() - 1); + E x = expr(p.args.get(0)); + if (pt == 'd' && ut == 'q' && (x.t.k == Ty.K.FLOAT || x.t.k == Ty.K.INT) && !x.t.opt) { + String v = "Double.doubleToRawLongBits(" + toD(x) + ")"; + if (pre != null) { + String t = fresh(); + indent(pre, preInd); + pre.append("long ").append(t).append(" = ").append(v).append(";\n"); + v = t; + } + return new E("Long.valueOf(" + v + ")", Ty.INT, v); + } + if (pt == 'q' && ut == 'd' && x.lc != null) { + return new E("Double.longBitsToDouble(" + x.lc + ")", Ty.FLOAT); + } + return null; + } + + private boolean isStructCall(Call c, String fn) { + return c.func instanceof Attribute && ((Attribute) c.func).attr.equals(fn) && c.keywords.isEmpty() + && c.args.size() == 2 && in.isStruct(((Attribute) c.func).value, sc); + } + + private static String strConstant(Expr e) { + return e instanceof Constant && ((Constant) e).kind == Constant.Kind.STR ? ((Constant) e).text : null; + } + + /// Int index `i` as {its long, a condition (ending in `&&`, or empty) + /// that it holds}; null when only the object is at hand. + private static String[] index(E i) { + if (i.lc != null) { + return new String[] {i.lc, ""}; + } + if (i.ll != null) { + return new String[] {i.ll, i.bg + " == null && "}; + } + return null; + } + /// `0 <= lc < size` of list `l` (and `l` not None). private static String inRange(E l, String lc) { String r = lc + " >= 0 && " + lc + " < " + l.c + ".size"; @@ -2344,32 +3007,20 @@ private static boolean hasCall(Object o) { return f[0]; } - /// A call of `f`'s `TL_` variant when every argument for one of its - /// long parameters has a long form; else null. - private String longCall(Infer.Fn f, List args) { - if (args.size() != f.params.length || !hasLongParam(f)) { - return null; - } - Set stored = assigned(f.def.body); - StringBuilder sb = new StringBuilder(); - for (int i = 0; i < args.size(); i++) { - E a = args.get(i); - String v; - if (longParam(f, i, stored)) { - if (a.lc == null) { - return null; - } - v = a.lc; - } else { - v = conv(a, f.params[i]); + private E result(Infer.Fn f, String call) { + if (f.ret.k == Ty.K.INT) { + if (pre != null && stmtPure && condDepth == 0) { + // Into a pair before the statement (see the subscript read). + String r = fresh(); + String rb = fresh(); + indent(pre, preInd); + pre.append("long ").append(r).append(" = ").append(call).append(";\n"); + indent(pre, preInd); + pre.append("Object ").append(rb).append(" = ").append(r).append(" != Long.MIN_VALUE ? null : Typed.take();\n"); + return dual(r, rb); } - sb.append(i == 0 ? "" : ", ").append(v); + return new E("Typed.unret(" + call + ")", Ty.INT); } - wantLong.add(f); - return "TL" + typed.get(f).substring(2) + "(" + sb + ")"; - } - - private E result(Infer.Fn f, String call) { return new E(call, f.ret.k == Ty.K.BOT ? Ty.DYN : f.ret); } @@ -2389,6 +3040,17 @@ private E call(Call c) { } private E call0(Call c) { + if (c.args.size() == 1 && c.args.get(0) instanceof Name && codePoints.containsKey(((Name) c.args.get(0)).id) + && ordOf(c, ((Name) c.args.get(0)).id)) { + String l = "((long) " + codePoints.get(((Name) c.args.get(0)).id) + ")"; + return new E("Long.valueOf(" + l + ")", Ty.INT, l); + } + if (c.func instanceof Attribute && ((Attribute) c.func).value instanceof Name + && scalars.containsKey(((Name) ((Attribute) c.func).value).id)) { + String n = ((Name) ((Attribute) c.func).value).id; + Layouts.Cls o = Layouts.resolver(scalars.get(n).k, ((Attribute) c.func).attr); + return scalarRead(n, getterField(in.fns.get(o.typable.get(((Attribute) c.func).attr)))); + } if (c.func instanceof Name) { String n = ((Name) c.func).id; Scope.Binding b = sc.resolve(n); @@ -2397,8 +3059,7 @@ private E call0(Call c) { if (f != null) { List a = args(c); calls.add(f); - String lcall = longCall(f, a); - E r = lcall != null ? result(f, lcall) : result(f, typed.get(f) + "(" + argList(f, 0, a) + ")"); + E r = result(f, typed.get(f) + "(" + argList(f, 0, a) + ")"); Ty site = in.siteRet.get(c); return site == null || site.k == r.t.k ? r : relabel(r, site); } @@ -2462,12 +3123,22 @@ private E call0(Call c) { return result(f, typed.get(f) + "(" + argList(f, 0, args(c)) + ")"); } E r = expr(a.value); + if (r.t.k == Ty.K.LIST && !r.t.opt && "extend".equals(a.attr) && c.keywords.isEmpty() + && c.args.size() == 1 && repeatCount(c.args.get(0)) != null) { + // `l.extend([v] * n)`: filled in place, no list of n made. + BinOp m = (BinOp) c.args.get(0); + boolean left = m.left instanceof ListExpr; + E first = expr(left ? ((ListExpr) m.left).elts.get(0) : m.left); + E second = expr(left ? m.right : ((ListExpr) m.right).elts.get(0)); + return new E("Typed.extendRepeat(" + r.c + ", " + box(left ? first : second) + ", " + + box(left ? second : first) + ")", Ty.NONE); + } List args = args(c); if (r.t.k == Ty.K.LIST && !r.t.opt) { return listMethod(r, a.attr, args); } if (r.t.k != Ty.K.INST) { - throw new Unsupported("method of " + r.t); + return siteCall(r, a.attr, args); } List cand = in.candidates(r.t.cls, a.attr); if (cand == null) { @@ -2477,6 +3148,12 @@ private E call0(Call c) { Layouts.Cls res = Layouts.resolver(r.t.cls, a.attr); if (cand.size() == 1 && res != null && res.typable.get(a.attr) == cand.get(0)) { //NOPMD CompareObjectsWithEquals - identity is meant Infer.Fn f = in.fns.get(cand.get(0)); + String gf = getterField(f); + if (gf != null && args.isEmpty()) { + // `return self.f`: the field, read in place of the call. + E rr = r.t.opt ? new E(recv, r.t.nonNone()) : r; + return fieldRead(rr, gf); + } calls.add(f); String al = argList(f, 1, args); return result(f, typed.get(f) + "(" + recv + (al.isEmpty() ? "" : ", " + al) + ")"); @@ -2499,6 +3176,50 @@ private E call0(Call c) { throw new Unsupported("call"); } + /// `r.attr(args)` as the generic code calls a method: through an inline + /// cache, with an untyped result. + private E siteCall(E r, String attr, List args) { + String site = em.attrSite(attr); + StringBuilder a = new StringBuilder(box(r)); + for (E x : args) { + a.append(", ").append(box(x)); + } + if (args.size() <= 3) { + return new E(site + ".call" + args.size() + "(" + a + ")", Ty.DYN); + } + StringBuilder arr = new StringBuilder(); + for (E x : args) { + arr.append(arr.length() == 0 ? "" : ", ").append(box(x)); + } + return new E(site + ".callN(" + box(r) + ", new Object[] {" + arr + "})", Ty.DYN); + } + + /// The field `f` returns when its body is only `return self.` + /// (a getter), or null. + private String getterField(Infer.Fn f) { + List body = f.def.body; + if (body.size() != 1 || !(body.get(0) instanceof Return) || f.names.size() != 1) { + return null; + } + Expr v = ((Return) body.get(0)).value; + if (!(v instanceof Attribute) || !(((Attribute) v).value instanceof Name) + || !((Name) ((Attribute) v).value).id.equals(f.names.get(0))) { + return null; + } + String attr = ((Attribute) v).attr; + Layouts.Cls o = Infer.fieldOwner(f.cls, attr); + return o != null && in.definitelySet(o, attr) ? attr : null; + } + + /// Field `attr` of `r`. + private E fieldRead(E r, String attr) { + Ty ft = fieldTy(r, attr); + if (ft.k == Ty.K.INT && typedField(owner(r, attr), attr) != null) { + return intField(r, attr); + } + return new E(fieldRef(r, attr), ft); + } + /// A call's result known, at this call site, to be `t` (`Infer.siteRet`) /// though the function returns something wider. private E relabel(E r, Ty t) { @@ -2522,8 +3243,14 @@ private E wrapped(Expr e) { if (!r.fitsLong()) { return null; } + int mark = pre == null ? 0 : pre.length(); String c = mod64(e); if (c == null) { + // Nothing evaluated on the way may run: the caller lowers `e` + // another way, from the start. + if (pre != null) { + pre.setLength(mark); + } return null; } if (pre != null && !plain(c)) { @@ -2544,6 +3271,10 @@ private boolean fits(Expr e) { /// `+ - * & | ^ ~ <<` commute with that reduction; anything else /// (`>>`, `//`, `%`) needs the exact value of its operands. private String mod64(Expr e) { + E sub = leafSubst.get(e); + if (sub != null) { + return sub.t.k == Ty.K.INT || sub.t.k == Ty.K.BOOL ? fits(e) ? fitting(sub) : low64(sub) : null; + } if (e instanceof Constant) { if (((Constant) e).kind != Constant.Kind.INT) { return null; @@ -2590,9 +3321,10 @@ private String mod64(Expr e) { } /// An int expression the long arithmetic cannot follow (a subscript, a - /// call) whose value is known to fit: computed as before and unboxed. + /// call): computed as before, and its exact value when it is known to + /// fit, else its value modulo 2^64. private String generic(Expr e) { - if (!inlined.isEmpty() || !fits(e)) { + if (!inlined.isEmpty()) { return null; } Expr saved = plainExpr; @@ -2602,12 +3334,35 @@ private String generic(Expr e) { if (v.t.k != Ty.K.INT) { return null; } - return v.lc != null ? v.lc : "Typed.toLong(" + box(v) + ")"; + return fits(e) ? fitting(v) : low64(v); } finally { plainExpr = saved; } } + /// Int `v`, known to fit, as a Java long. + private String fitting(E v) { + if (v.lc != null) { + return v.lc; + } + if (v.ll != null) { + // A pair's object is then a bool. + return "(" + v.bg + " == null ? " + v.ll + " : Typed.toLong(" + v.bg + "))"; + } + return "Typed.toLong(" + box(v) + ")"; + } + + /// Int `v` modulo 2^64. + private String low64(E v) { + if (v.lc != null) { + return v.lc; + } + if (v.ll != null) { + return "(" + v.bg + " == null ? " + v.ll + " : Typed.low64(" + v.bg + "))"; + } + return "Typed.low64(" + box(v) + ")"; + } + private static String lit(java.math.BigInteger v) { long l = v.longValue(); return l < 0 ? "(" + l + "L)" : l + "L"; @@ -2623,11 +3378,6 @@ private String nameLong(Name nm) { java.math.BigInteger c = in.intConsts.get(n); return c == null ? null : lit(c); } - if (subst != null && subst.containsKey(n)) { - // Inside an out-of-line slow path, where locals are parameters. - E v = subst.get(n); - return v.t.k != Ty.K.INT ? null : v.lc != null ? v.lc : "Typed.toLong(" + box(v) + ")"; - } switch (sc.resolve(n)) { case LOCAL: if (longLocals.contains(n)) { @@ -2760,6 +3510,297 @@ private static String mathConstant(String n) { } } + // ---- scalar replacement ---------------------------------------------- + + /// A local whose objects never escape the function: every value it + /// holds comes from a constructor that only copies its arguments into + /// fields, and every use reads a field, directly or through a getter. + /// The object is then never made; each field is a local of its own. + private static final class Scalar { + Layouts.Cls k; + /// The field each constructor argument goes to, by position. + String[] attrs; + final Set reads = new HashSet(); + final Set getters = new HashSet(); + } + + private Map scalars = new HashMap(); + + /// The local holding field `attr` of scalar local `n`; the dot keeps it + /// apart from every Python name. + private static String scalarField(String n, String attr) { + return n + "." + attr; + } + + private Map findScalars(Infer.Fn f) { + Map cand = new HashMap(); + for (Map.Entry e : f.locals.entrySet()) { + String n = e.getKey(); + if (e.getValue().k == Ty.K.INST && !sc.params.contains(n) && !sc.deleted.contains(n) + && sc.resolve(n) == Scope.Binding.LOCAL) { + cand.put(n, new Scalar()); + } + } + if (cand.isEmpty()) { + return cand; + } + Set bad = new HashSet(); + scanScalars(f.def.body, cand, bad); + Map out = new HashMap(); + for (Map.Entry e : cand.entrySet()) { + Scalar x = e.getValue(); + if (bad.contains(e.getKey()) || x.k == null || !scalarUses(x)) { + continue; + } + out.put(e.getKey(), x); + } + return out; + } + + /// Whether every use of `x` reads a field its constructor set. + private boolean scalarUses(Scalar x) { + Set fields = new HashSet(Arrays.asList(x.attrs)); + for (String g : x.getters) { + Layouts.Cls o = Layouts.resolver(x.k, g); + FunctionDef d = o == null ? null : o.typable.get(g); + Infer.Fn gf = d == null ? null : in.fns.get(d); + String attr = gf == null ? null : getterField(gf); + if (attr == null) { + return false; + } + x.reads.add(attr); + } + for (String r : x.reads) { + if (!fields.contains(r) || Layouts.resolver(x.k, r) != null) { + return false; + } + Ty t = scalarTy(x, r); + if (t == null || t.k == Ty.K.BOT) { + return false; + } + } + return true; + } + + private Ty scalarTy(Scalar x, String attr) { + Layouts.Cls o = Infer.fieldOwner(x.k, attr); + if (o == null) { + return null; + } + Ty t = typedField(o, attr); + return t != null ? t : in.fieldType(o, attr); + } + + private void scanScalars(Object o, Map cand, Set bad) { + if (o instanceof Assign) { + Assign a = (Assign) o; + for (Expr t : a.targets) { + storeTargets(t, cand, bad); + } + if (a.targets.size() == 1 && a.targets.get(0) instanceof Name + && cand.containsKey(((Name) a.targets.get(0)).id)) { + String n = ((Name) a.targets.get(0)).id; + if (!(a.value instanceof Call) || !scalarCtor(cand.get(n), (Call) a.value)) { + bad.add(n); + scanScalars(a.value, cand, bad); + } else { + for (Expr x : ((Call) a.value).args) { + scanScalars(x, cand, bad); + } + } + return; + } + } else if (o instanceof AugAssign) { + storeTargets(((AugAssign) o).target, cand, bad); + } else if (o instanceof AnnAssign) { + storeTargets(((AnnAssign) o).target, cand, bad); + } else if (o instanceof Delete) { + for (Expr t : ((Delete) o).targets) { + storeTargets(t, cand, bad); + } + } else if (o instanceof For) { + storeTargets(((For) o).target, cand, bad); + } else if (o instanceof Comprehension) { + storeTargets(((Comprehension) o).target, cand, bad); + } else if (o instanceof WithItem && ((WithItem) o).vars != null) { + storeTargets(((WithItem) o).vars, cand, bad); + } else if (o instanceof Call && ((Call) o).func instanceof Attribute) { + Attribute f = (Attribute) ((Call) o).func; + if (f.value instanceof Name && cand.containsKey(((Name) f.value).id)) { + Call c = (Call) o; + String n = ((Name) f.value).id; + if (c.args.isEmpty() && c.keywords.isEmpty() && da.safe.contains((Name) f.value)) { + cand.get(n).getters.add(f.attr); + } else { + bad.add(n); + } + for (Expr x : c.args) { + scanScalars(x, cand, bad); + } + for (Keyword k : c.keywords) { + scanScalars(k.value, cand, bad); + } + return; + } + } else if (o instanceof Attribute) { + Attribute a = (Attribute) o; + if (a.value instanceof Name && cand.containsKey(((Name) a.value).id)) { + if (da.safe.contains((Name) a.value)) { + cand.get(((Name) a.value).id).reads.add(a.attr); + } else { + bad.add(((Name) a.value).id); + } + return; + } + } else if (o instanceof Name) { + bad.add(((Name) o).id); + return; + } + Infer.children(o, x -> scanScalars(x, cand, bad)); + } + + /// Marks every candidate whose field a store target writes. + private static void storeTargets(Expr t, Map cand, Set bad) { + if (t instanceof Attribute && ((Attribute) t).value instanceof Name) { + bad.add(((Name) ((Attribute) t).value).id); + } else if (t instanceof TupleExpr) { + for (Expr x : ((TupleExpr) t).elts) { + storeTargets(x, cand, bad); + } + } else if (t instanceof ListExpr) { + for (Expr x : ((ListExpr) t).elts) { + storeTargets(x, cand, bad); + } + } else if (t instanceof Starred) { + storeTargets(((Starred) t).value, cand, bad); + } + } + + /// Whether `c` constructs, with positional arguments only, a class whose + /// `__init__` only copies each argument into a field of its own; fills + /// `x` on the first one, and requires the same class every time. + private boolean scalarCtor(Scalar x, Call c) { + if (!(c.func instanceof Name) || !c.keywords.isEmpty() + || sc.resolve(((Name) c.func).id) != Scope.Binding.GLOBAL) { + return false; + } + Layouts.Cls k = in.globalClasses.get(((Name) c.func).id); + if (k == null || x.k != null && x.k != k) { //NOPMD CompareObjectsWithEquals - identity is meant + return false; + } + for (String h : new String[] {"__new__", "__setattr__", "__getattribute__", "__getattr__", "__del__"}) { + if (Layouts.resolver(k, h) != null) { + return false; + } + } + Layouts.Cls o = Layouts.resolver(k, "__init__"); + FunctionDef init = o == null ? null : o.typable.get("__init__"); + Infer.Fn f = init == null ? null : in.fns.get(init); + if (f == null || f.names.size() != c.args.size() + 1 || !init.decorators.isEmpty()) { + return false; + } + for (Expr a : c.args) { + if (a instanceof Starred) { + return false; + } + } + String[] attrs = new String[c.args.size()]; + String self = f.names.get(0); + List ts = new ArrayList(); + List vs = new ArrayList(); + for (Stmt s : init.body) { + if (!(s instanceof Assign) || ((Assign) s).targets.size() != 1) { + return false; + } + Expr t = ((Assign) s).targets.get(0); + Expr v = ((Assign) s).value; + if (t instanceof TupleExpr && v instanceof TupleExpr + && ((TupleExpr) t).elts.size() == ((TupleExpr) v).elts.size()) { + ts.addAll(((TupleExpr) t).elts); + vs.addAll(((TupleExpr) v).elts); + } else { + ts.add(t); + vs.add(v); + } + } + Set seen = new HashSet(); + for (int i = 0; i < ts.size(); i++) { + Expr t = ts.get(i); + Expr v = vs.get(i); + if (!(t instanceof Attribute) || !(((Attribute) t).value instanceof Name) + || !((Name) ((Attribute) t).value).id.equals(self) || !(v instanceof Name)) { + return false; + } + int p = f.names.indexOf(((Name) v).id); + String attr = ((Attribute) t).attr; + if (p < 1 || attrs[p - 1] != null || !seen.add(attr)) { + return false; + } + attrs[p - 1] = attr; + } + for (String a : attrs) { + if (a == null) { + return false; + } + } + if (x.attrs != null && !Arrays.equals(x.attrs, attrs)) { + return false; + } + x.k = k; + x.attrs = attrs; + return true; + } + + /// Field `attr` of scalar local `n`. + private E scalarRead(String n, String attr) { + String l = scalarField(n, attr); + Ty t = scalarTy(scalars.get(n), attr); + if (t.k == Ty.K.INT) { + return dual(var(l), big(l)); + } + return new E(var(l), t); + } + + /// `n = C(args)` for scalar local `n`: the arguments, in order, into + /// the field locals. + private void scalarStore(String n, Call c, StringBuilder b, int ind) { + Scalar x = scalars.get(n); + List av = new ArrayList(); + for (Expr a : c.args) { + av.add(spillTo(expr(a), b, ind)); + } + for (int i = 0; i < av.size(); i++) { + String l = scalarField(n, x.attrs[i]); + if (!x.reads.contains(x.attrs[i])) { + continue; + } + Ty t = scalarTy(x, x.attrs[i]); + indent(b, ind); + if (t.k == Ty.K.INT) { + storePair(var(l), big(l), av.get(i), b, ind); + } else { + b.append(var(l)).append(" = ").append(conv(av.get(i), t)).append(";\n"); + } + } + } + + /// Declarations of the field locals of every scalar local. + private void scalarDecls(StringBuilder b) { + for (Map.Entry e : scalars.entrySet()) { + for (String attr : e.getValue().reads) { + String l = scalarField(e.getKey(), attr); + Ty t = scalarTy(e.getValue(), attr); + if (t.k == Ty.K.INT) { + b.append(" long ").append(var(l)).append(" = 0L;\n Object ").append(big(l)) + .append(" = null;\n"); + } else { + b.append(" ").append(jt(t)).append(' ').append(var(l)).append(" = ").append(zero(t)) + .append(";\n"); + } + } + } + } + private E construct(Layouts.Cls k, List args) { Layouts.Cls o = Layouts.resolver(k, "__init__"); if (o == null) { @@ -2785,6 +3826,10 @@ private E builtin(String n, List a) { if (a.size() == 1 && a.get(0).t.k == Ty.K.LIST) { return new E("h_len(" + a.get(0).c + ")", Ty.INT); } + if (a.size() == 1 && a.get(0).t.k == Ty.K.STR && !a.get(0).t.opt) { + String l = "Typed.slen(" + a.get(0).c + ")"; + return new E("Long.valueOf(" + l + ")", Ty.INT, l); + } break; case "float": if (a.size() == 1 && !a.get(0).t.opt @@ -2801,6 +3846,14 @@ private E builtin(String n, List a) { case BOOL: return new E("((" + x.c + ") ? \"True\" : \"False\")", Ty.STR); case INT: + if (x.lc != null) { + return new E("Long.toString(" + x.lc + ")", Ty.STR); + } + if (x.ll != null) { + return new E("(" + x.bg + " == null ? Long.toString(" + x.ll + ") : Py.toStr(" + x.bg + + "))", Ty.STR); + } + return new E("Py.toStr(" + box(x) + ")", Ty.STR); case FLOAT: return new E("Py.toStr(" + box(x) + ")", Ty.STR); default: @@ -2857,6 +3910,21 @@ private E genericCall(String f, Call c) { return new E("Py.callKw(" + f + ", new Object[] {" + a + "}, new String[] {" + names + "})", Ty.DYN); } + /// The count of `[v] * n` or `n * [v]` when it is an int, or null. + private Expr repeatCount(Expr e) { + if (!(e instanceof BinOp) || !"*".equals(((BinOp) e).op)) { + return null; + } + BinOp b = (BinOp) e; + Expr l = b.left instanceof ListExpr ? b.left : b.right instanceof ListExpr ? b.right : null; + if (l == null || ((ListExpr) l).elts.size() != 1 || ((ListExpr) l).elts.get(0) instanceof Starred) { + return null; + } + Expr n = l == b.left ? b.right : b.left; //NOPMD CompareObjectsWithEquals - identity is meant + Ty t = in.type(n, cur, sc); + return t.k == Ty.K.INT && !t.opt ? n : null; + } + private E listMethod(E l, String m, List a) { switch (m) { case "append": @@ -2925,7 +3993,7 @@ String entry(Infer.Fn f, List locals) { case INT: // A guessed parameter (Infer.guess) is checked like any other. check = "(" + v + " instanceof Long || " + v + " instanceof PyInt)"; - arg = v; + arg = "0L, " + v; break; case NONE: check = v + " == Py.NONE"; @@ -2940,7 +4008,8 @@ String entry(Infer.Fn f, List locals) { } args.append(i == 0 ? "" : ", ").append(arg); } - String call = box(new E(typed.get(f) + "(" + args + ")", f.ret)); + String call = f.ret.k == Ty.K.INT ? "Typed.unret(" + typed.get(f) + "(" + args + "))" + : box(new E(typed.get(f) + "(" + args + ")", f.ret)); if (test.length() == 0) { // Always taken; javac must still see the generic body as reachable. test.append("Typed.yes"); @@ -2951,16 +4020,28 @@ String entry(Infer.Fn f, List locals) { /// The Java declaration of a typed field, or null for an object one. String fieldDecl(Layouts.Cls k, String f) { Ty t = typedField(k, f); + if (t != null && t.k == Ty.K.INT) { + return "long t_" + Emitter.mangle(f) + ";\n Object b_" + Emitter.mangle(f) + ";"; + } return t == null ? null : jt(t) + " t_" + Emitter.mangle(f) + ";"; } String fieldGetCase(Layouts.Cls k, String f) { Ty t = typedField(k, f); + if (t.k == Ty.K.INT) { + return dual("t_" + Emitter.mangle(f), "b_" + Emitter.mangle(f)).c; + } return box(new E("t_" + Emitter.mangle(f), t)); } String fieldSetCase(Layouts.Cls k, String f, String v) { Ty t = typedField(k, f); + if (t.k == Ty.K.INT) { + String m = Emitter.mangle(f); + return "if (" + v + " instanceof Long) {\n t_" + m + " = ((Long) " + v + + ").longValue();\n b_" + m + " = null;\n } else {\n" + + " b_" + m + " = " + v + ";\n }"; + } return "t_" + Emitter.mangle(f) + " = " + unbox(v, t) + ";"; } @@ -2976,8 +4057,8 @@ String layoutMembers(Layouts.Cls k) { StringBuilder sig = new StringBuilder(); StringBuilder pass = new StringBuilder(); for (int i = 1; i <= ar; i++) { - sig.append(i == 1 ? "" : ", ").append(jt(any.params[i])).append(" a").append(i); - pass.append(", a").append(i); + sig.append(i == 1 ? "" : ", ").append(param(any.params[i], "a" + i, "b" + i)); + pass.append(", a").append(i).append(any.params[i].k == Ty.K.INT ? ", b" + i : ""); } String head = " " + retJ(any) + " tv_" + Emitter.mangle(name) + "_" + ar + "(" + sig + ") {\n"; if (k == null) { @@ -3006,31 +4087,14 @@ String members() { for (String b : bodies.values()) { o.append(b); } - // A TL_ body may want further variants; emit until none is new. - while (!longBodies.keySet().containsAll(wantLong)) { - for (Infer.Fn f : new ArrayList(wantLong)) { - if (!bodies.containsKey(f)) { - // Its caller turned out untyped too; nothing calls it. - longBodies.put(f, ""); - } else if (!longBodies.containsKey(f)) { - longParams = true; - try { - longBodies.put(f, function(f)); - } finally { - longParams = false; - } - o.append(longBodies.get(f)); - } - } - } for (Layouts.Cls k : constructors) { Layouts.Cls ow = Layouts.resolver(k, "__init__"); Infer.Fn f = in.fns.get(ow.typable.get("__init__")); StringBuilder sig = new StringBuilder(); StringBuilder pass = new StringBuilder(); for (int i = 1; i < f.params.length; i++) { - sig.append(i == 1 ? "" : ", ").append(jt(f.params[i])).append(" a").append(i); - pass.append(", a").append(i); + sig.append(i == 1 ? "" : ", ").append(param(f.params[i], "a" + i, "b" + i)); + pass.append(", a").append(i).append(f.params[i].k == Ty.K.INT ? ", b" + i : ""); } o.append(" static ").append(k.javaName).append(" n_").append(k.javaName).append('(').append(sig) .append(") {\n ").append(k.javaName).append(" o = new ").append(k.javaName).append('(') @@ -3044,6 +4108,12 @@ String members() { for (String h : intHelpers) { o.append(intHelper(h)); } + for (String fb : fieldBoxers) { + String[] p = fb.split(" "); + o.append(" static Object fb_").append(p[0]).append('_').append(p[1]).append('(').append(p[0]) + .append(" o) {\n return o.b_").append(p[1]).append(" != null ? o.b_").append(p[1]) + .append(" : Long.valueOf(o.t_").append(p[1]).append(");\n }\n"); + } for (Layouts.Cls k : unboxers) { o.append(" static ").append(k.javaName).append(" u_").append(k.javaName) .append("(Object o) {\n if (o instanceof ").append(k.javaName).append(") {\n return (") diff --git a/maven/python-transpiler/src/test/resources/conformance/int_divmod.out b/maven/python-transpiler/src/test/resources/conformance/int_divmod.out new file mode 100644 index 00000000000..eda01c7411f --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/int_divmod.out @@ -0,0 +1,50 @@ +7 3 (1, 2, 2, 4) +7 -3 (-2, -3, -1, -6) +7 1 (0, 7, 1, 14) +7 -1 (0, -7, 1, -14) +7 0 division by zero +7 18446744073709551616 (7, 0, 8, 0) +7 -9223372036854775808 (-9223372036854775801, -1, -9223372036854775800, -2) +-7 3 (2, -3, 1, -6) +-7 -3 (-1, 2, 1, 4) +-7 1 (0, -7, 1, -14) +-7 -1 (0, 7, 1, 14) +-7 0 division by zero +-7 18446744073709551616 (18446744073709551609, -1, 250, -2) +-7 -9223372036854775808 (-7, 0, -9223372036854775558, 0) +0 3 (0, 0, 1, 0) +0 -3 (0, 0, 1, 0) +0 1 (0, 0, 1, 0) +0 -1 (0, 0, 1, 0) +0 0 division by zero +0 18446744073709551616 (0, 0, 1, 0) +0 -9223372036854775808 (0, 0, 1, 0) +-9223372036854775808 3 (1, -3074457345618258603, 1, -6148914691236517206) +-9223372036854775808 -3 (-2, 3074457345618258602, 1, 6148914691236517204) +-9223372036854775808 1 (0, -9223372036854775808, 1, -18446744073709551616) +-9223372036854775808 -1 (0, 9223372036854775808, 1, 18446744073709551616) +-9223372036854775808 0 division by zero +-9223372036854775808 18446744073709551616 (9223372036854775808, -1, 1, -2) +-9223372036854775808 -9223372036854775808 (0, 1, 1, 2) +9223372036854775807 3 (1, 3074457345618258602, 1, 6148914691236517204) +9223372036854775807 -3 (-2, -3074457345618258603, 1, -6148914691236517206) +9223372036854775807 1 (0, 9223372036854775807, 1, 18446744073709551614) +9223372036854775807 -1 (0, -9223372036854775807, 1, -18446744073709551614) +9223372036854775807 0 division by zero +9223372036854775807 18446744073709551616 (9223372036854775807, 0, 256, 0) +9223372036854775807 -9223372036854775808 (-1, -1, -9223372036854775552, -2) +1180591620717411303424 3 (1, 393530540239137101141, 1, 787061080478274202282) +1180591620717411303424 -3 (-2, -393530540239137101142, 1, -787061080478274202284) +1180591620717411303424 1 (0, 1180591620717411303424, 1, 2361183241434822606848) +1180591620717411303424 -1 (0, -1180591620717411303424, 1, -2361183241434822606848) +1180591620717411303424 0 division by zero +1180591620717411303424 18446744073709551616 (0, 64, 1, 128) +1180591620717411303424 -9223372036854775808 (0, -128, 1, -256) +-1180591620717411303424 3 (2, -393530540239137101142, 1, -787061080478274202284) +-1180591620717411303424 -3 (-1, 393530540239137101141, 1, 787061080478274202282) +-1180591620717411303424 1 (0, -1180591620717411303424, 1, -2361183241434822606848) +-1180591620717411303424 -1 (0, 1180591620717411303424, 1, 2361183241434822606848) +-1180591620717411303424 0 division by zero +-1180591620717411303424 18446744073709551616 (0, -64, 1, -128) +-1180591620717411303424 -9223372036854775808 (0, 128, 1, 256) +7323 diff --git a/maven/python-transpiler/src/test/resources/conformance/int_divmod.py b/maven/python-transpiler/src/test/resources/conformance/int_divmod.py new file mode 100644 index 00000000000..bb2b901d7bc --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/int_divmod.py @@ -0,0 +1,23 @@ +def dm(a: int, b: int): + try: + return (a % b, a // b, (a & 0xFF) % b + 1, a // b * 2) + except ZeroDivisionError as e: + return str(e) + + +M = -9223372036854775808 +for a in (7, -7, 0, M, 2 ** 63 - 1, 2 ** 70, -(2 ** 70)): + for b in (3, -3, 1, -1, 0, 2 ** 64, M): + print(a, b, dm(a, b)) + + +def scan(n: int) -> int: + t: int = 0 + idx: int = 12345 + for i in range(n): + t += (idx & 0x7FFFFFFF) % 97 - i // 7 + idx = idx * 31 + 7 + return t + + +print(scan(200)) diff --git a/maven/python-transpiler/src/test/resources/conformance/int_pairs.out b/maven/python-transpiler/src/test/resources/conformance/int_pairs.out new file mode 100644 index 00000000000..5785ebda550 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/int_pairs.out @@ -0,0 +1,15 @@ +-9223372036854775808 9223372036854775807 -9223372036854775809 9223372036854775808 +18446744073709551614 -18446744073709551616 6 +15511210043330985984000000 2432902008176640000 51090942171709440000 +832040 +9223372036854775808 -9223372036854775807 0 +-9223372036854775808 9223372036854775812 +True 2 1 +9223372036854775810 5 3 +True 2 +-9223372036854775809 -9223372036854775808 -18446744073709551616 +9223372036854775807 +2417851639230357861040128 +1180591620717411303424 1152921504606846976 1180591620717411303423 +9223372036854775807 True +3 diff --git a/maven/python-transpiler/src/test/resources/conformance/int_pairs.py b/maven/python-transpiler/src/test/resources/conformance/int_pairs.py new file mode 100644 index 00000000000..494cdb3e295 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/int_pairs.py @@ -0,0 +1,102 @@ +# Ints held as a Java long with an overflow object: every place one +# crosses (parameter, return, field, list, method) at the edges of 64 bits. +MIN = -(2 ** 63) +MAX = 2 ** 63 - 1 + + +def ident(x: int) -> int: + return x + + +def twice(x: int) -> int: + return x + x + + +def fact(n: int) -> int: + if n < 2: + return 1 + return n * fact(n - 1) + + +def fib(n: int) -> int: + if n < 2: + return n + return fib(n - 1) + fib(n - 2) + + +def neg(x: int) -> int: + return -x + + +def pick(c: bool, a: int, b: int) -> int: + return a if c else b + + +class Acc: + __slots__ = ("total", "count") + + def __init__(self, start: int): + self.total = start + self.count = 0 + + def add(self, v: int) -> int: + self.total += v + self.count += 1 + return self.total + + def get(self) -> int: + return self.total + + +class Shape: + def area(self, k: int) -> int: + return k + + +class Sq(Shape): + def area(self, k: int) -> int: + return k * k + + +def main() -> None: + print(ident(MIN), ident(MAX), ident(MIN - 1), ident(MAX + 1)) + print(twice(MAX), twice(MIN), twice(3)) + print(fact(25), fact(20), fact(21)) + print(fib(30)) + print(neg(MIN), neg(MAX), neg(0)) + print(pick(True, MIN, 1), pick(False, 1, MAX + 5)) + print(ident(True), twice(True), ident(False) + 1) + a = Acc(MAX - 2) + for i in range(5): + a.add(1) + print(a.get(), a.count, a.total - MAX) + a.total = True + print(a.total, a.get() + 1) + b = Acc(MIN) + b.add(-1) + print(b.total, b.add(1), b.add(MIN)) + xs = [MIN, MAX, 0, MAX + 1] + s: int = 0 + for x in xs: + s += x + print(s) + shapes = [Shape(), Sq(), Sq()] + t: int = 0 + for sh in shapes: + t += sh.area(2 ** 40) + print(t) + k: int = 1 + for i in range(70): + k = k * 2 + print(k, k >> 10, k - 1) + m: int = MAX + m += 1 + m -= 1 + print(m, m == MAX) + c: int = 0 + while c < 3: + c = c + True + print(c) + + +main() diff --git a/maven/python-transpiler/src/test/resources/conformance/scalar_objects.out b/maven/python-transpiler/src/test/resources/conformance/scalar_objects.out new file mode 100644 index 00000000000..e785372486b --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/scalar_objects.out @@ -0,0 +1,8 @@ +1498500 +1180591620717411303423 +arg 1.5 +arg 2 +3.0 +7 +56 12 +93 13 13 diff --git a/maven/python-transpiler/src/test/resources/conformance/scalar_objects.py b/maven/python-transpiler/src/test/resources/conformance/scalar_objects.py new file mode 100644 index 00000000000..1bdfb52955a --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/scalar_objects.py @@ -0,0 +1,70 @@ +class Vec: + __slots__ = ("x", "y") + + def __init__(self, x: int, y: int): + self.x = x + self.y = y + + def get_x(self) -> int: + return self.x + + +class P: + def __init__(self, a, b): + self.a, self.b = b, a + + +def trace(v): + print("arg", v) + return v + + +def sums(n: int) -> int: + total: int = 0 + for i in range(n): + v = Vec(i, i * 2) + total = (total + v.get_x() + v.y) & 0x3FFFFFFF + return total + + +def big() -> int: + v = Vec(2 ** 70, -1) + return v.x + v.y + + +def order() -> float: + p = P(trace(1.5), trace(2)) + return p.a * p.b + + +def escapes() -> int: + v = Vec(3, 4) + keep = [v] + return keep[0].x + v.y + + +def rebound(f: bool) -> int: + v = Vec(1, 2) + if f: + v = Vec(5, 6) + return v.x * 10 + v.y + + +print(sums(1000)) +print(big()) +print(order()) +print(escapes()) +print(rebound(True), rebound(False)) + + +def fill(n: int) -> int: + a = [] + a.extend([7] * n) + a.extend(2 * [1]) + a.extend(a) + b = [1.5] + b.extend([2] * 2) + return len(a) + sum(a) + int(sum(b)) + + +print(fill(5), fill(0), fill(-3)) diff --git a/maven/python-transpiler/src/test/resources/conformance/str_loops.out b/maven/python-transpiler/src/test/resources/conformance/str_loops.out new file mode 100644 index 00000000000..16f3e2f7544 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/str_loops.out @@ -0,0 +1,8 @@ +0 0 [] none 0 +96354 3 ['a', 'b', 'c'] c 1294 +103094734 5 ['h', 'é', 'l', 'l', 'o'] o 664 +123632265 3 ['😀', 'x', '😁'] 😁 257145 +1714298 2 ['\ud800', 'z'] z 55418 +item-5-21809:evenTrue1180591620717411303424 item--12--21722:oddFalse1180591620717411303424 +4 [55296, 122, 128512, 56320] 55296 z 56320 1 True +[98, 99, 100, 'aa', 'cc'] diff --git a/maven/python-transpiler/src/test/resources/conformance/str_loops.py b/maven/python-transpiler/src/test/resources/conformance/str_loops.py new file mode 100644 index 00000000000..0bafc159465 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/str_loops.py @@ -0,0 +1,46 @@ +def jh(s: str) -> int: + h: int = 0 + for c in s: + h = (h * 31 + ord(c)) & 0xFFFFFFFF + return h - 0x100000000 if h > 0x7FFFFFFF else h + + +def chars(s: str) -> list: + out = [] + for c in s: + out.append(c) + return out + + +def last(s: str) -> str: + c = "none" + for c in s: + pass + return c + + +def mixed(s: str) -> int: + t: int = 0 + for c in s: + t += ord(c) + if c == "b": + t += 1000 + return t + + +def label(i: int, flag: bool) -> str: + return "item-" + str(i) + "-" + str(i * 31 ^ 0x55AA) + ":" + ("even" if flag else "odd") + str(flag) + str(2 ** 70) + + +for s in ("", "abc", "héllo", "\U0001F600x\U0001F601", "\ud800z"): + print(jh(s), len(s), chars(s), last(s), mixed(s)) +print(label(5, True), label(-12, False)) +x = "\ud800z\U0001F600\udc00" +print(len(x), [ord(c) for c in x], ord(x[0]), x[1], ord(x[3]), x.find("z"), x[1:3] == "z\U0001F600") + + +def comp(s: str) -> list: + return [ord(c) + 1 for c in s] + [c * 2 for c in s if c != "b"] + + +print(comp("abc")) From d0611f5b9bd3ae39594db4d1a0d977ec663c9210 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Tue, 27 Oct 2026 21:46:58 +0200 Subject: [PATCH 025/112] Python: branch/loop-test int range refinement, threshold widening, typed builtins over BOT, dict clear/len intrinsics Co-Authored-By: Claude Opus 5.5 --- .../codename1/python/runtime/AttrSite.java | 14 + .../com/codename1/python/runtime/PyDict.java | 26 +- .../com/codename1/python/runtime/Typed.java | 13 + .../python/transpiler/codegen/IntRanges.java | 325 +++++++++++++++++- .../python/transpiler/codegen/TypedEmit.java | 6 + .../test/resources/conformance/int_refine.out | 8 + .../test/resources/conformance/int_refine.py | 74 ++++ 7 files changed, 451 insertions(+), 15 deletions(-) create mode 100644 maven/python-transpiler/src/test/resources/conformance/int_refine.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/int_refine.py diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/AttrSite.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/AttrSite.java index 783c6ff9e84..07a34e0763f 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/AttrSite.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/AttrSite.java @@ -63,6 +63,7 @@ public final class AttrSite { private static final int I_LOWER = 4; private static final int I_SPLIT = 5; private static final int I_JOIN = 6; + private static final int I_CLEAR = 7; // Reads and calls. An entry with a null shape is a non-instance receiver // type (BUILTIN or NONE). All entries are valid only while getVersion is @@ -147,6 +148,8 @@ private static int intrinsicOf(String name) { return I_SPLIT; case "join": return I_JOIN; + case "clear": + return I_CLEAR; default: return I_NONE; } @@ -369,6 +372,17 @@ private void fillSet(PyInstance i, Object v) { // ---- calls --------------------------------------------------------------- public Object call0(Object o) { + if (intrinsic == I_CLEAR && o != null) { + // Exact classes: a subclass may override clear. + if (o.getClass() == PyDict.class) { + ((PyDict) o).clear(); + return Py.NONE; + } + if (o instanceof PyList) { + ((PyList) o).clear(); + return Py.NONE; + } + } if (intrinsic != I_NONE && o instanceof String) { String v = ((String) o); if (intrinsic == I_UPPER) { diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyDict.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyDict.java index 532c1984eb9..26fd16d99c0 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyDict.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyDict.java @@ -293,14 +293,32 @@ public long hash() { } public void clear() { - keys = new Object[8]; - values = new Object[8]; - hashes = new long[8]; - index = newIndex(16); + if (keys.length <= KEEP_ON_CLEAR) { + // A dict cleared and refilled, a cache or a window, keeps its + // capacity as Java's HashMap does: refilling it would otherwise + // regrow it from eight entries, a dozen arrays of garbage each + // time. + for (int i = 0; i < used; i++) { + keys[i] = null; + values[i] = null; + } + int[] ix = index; + for (int i = 0; i < ix.length; i++) { + ix[i] = EMPTY; + } + } else { + keys = new Object[8]; + values = new Object[8]; + hashes = new long[8]; + index = newIndex(16); + } used = 0; size = 0; } + /// The largest entry capacity `clear` keeps. + private static final int KEEP_ON_CLEAR = 1 << 17; + /// `dict.update(other)` for a mapping or an iterable of pairs. public void update(Object other) { if (other instanceof PyDict) { diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Typed.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Typed.java index 5c41158563f..3a6f06e856c 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Typed.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Typed.java @@ -412,6 +412,19 @@ public static Object extend(PyList l, PyList other) { return Py.NONE; } + /// `len(o)` where `f` is what the name `len` holds, for typed code, + /// which takes the result to be an int as the builtin's is. + public static long len(Object f, Object o) { + if (f == Builtins.LEN) { //NOPMD CompareObjectsWithEquals - identity is meant + return Py.len(o); + } + Object r = Py.call(f, o); + if (r instanceof Long) { + return ((Long) r).longValue(); + } + throw Py.typeError("len was rebound to a function that does not return an int"); + } + /// `l.extend([v] * n)`. public static Object extendRepeat(PyList l, Object v, Object n) { nzl(l).extendRepeat(v, Py.index(n)); diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/IntRanges.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/IntRanges.java index c14c5e54694..94b5ec97c0e 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/IntRanges.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/IntRanges.java @@ -112,20 +112,38 @@ private static boolean eq(BigInteger a, BigInteger b) { private final List> bound = new ArrayList>(); IntRanges(Infer in, Infer.Fn fn) { + this(in, fn, null, null); + } + + /// The analysis of `fn` with the int parameters in `params` known to + /// hold only values in their intervals (see `TypedEmit.paramRanges`); + /// when `sites` is not null, it receives the interval of every argument + /// of every direct call of a module function the analysis follows. + IntRanges(Infer in, Infer.Fn fn, Map params, Map sites) { this.in = in; this.fn = fn; this.sc = fn.def.scope; + this.sites = sites; for (Map.Entry e : fn.locals.entrySet()) { if (e.getValue().k == Ty.K.INT && !e.getValue().opt && !sc.params.contains(e.getKey())) { ints.add(e.getKey()); } } - if (!ints.isEmpty()) { - Map env = new HashMap(); + Map env = new HashMap(); + if (params != null) { + for (Map.Entry e : params.entrySet()) { + ints.add(e.getKey()); + store(e.getKey(), e.getValue(), env); + } + } + if (!ints.isEmpty() || sites != null) { run(fn.def.body, env, new Loop()); } } + /// Where call arguments are recorded, or null. + private final Map sites; + /// The locals held in a Java `long`. Set longLocals() { Set r = new HashSet(); @@ -221,6 +239,17 @@ private Map run(List body, Map env, Loop loop) { private Map stmt(Stmt s, Map env, Loop loop) { walrus(s, env); + record(s, env); + Map r = stmt0(s, env, loop); + if (r != null && (s instanceof Assign || s instanceof AugAssign || s instanceof ExprStmt)) { + // The statement completed, so every list it indexed by a local + // took that index: the local is within a list's bounds. + indexed(s, r); + } + return r; + } + + private Map stmt0(Stmt s, Map env, Loop loop) { if (s instanceof Assign) { Assign a = (Assign) s; Iv v = eval(a.value, env); @@ -243,15 +272,18 @@ private Map stmt(Stmt s, Map env, Loop loop) { } if (s instanceof If) { If i = (If) s; - return join(run(i.body, new HashMap(env), loop), - run(i.orelse, new HashMap(env), loop)); + Map at = new HashMap(env); + indexed(i.test, at); + Map t = refine(i.test, at, true); + Map f = refine(i.test, at, false); + return join(t == null ? null : run(i.body, t, loop), f == null ? null : run(i.orelse, f, loop)); } if (s instanceof For) { return forLoop((For) s, env); } if (s instanceof While) { While w = (While) s; - Map out = loop(w.body, env, null, null, null); + Map out = loop(w.body, env, null, null, null, forever(w.test) ? null : w.test); return run(w.orelse, out, loop); } if (s instanceof Return || s instanceof Raise) { @@ -309,7 +341,7 @@ private Map forLoop(For f, Map env) { } else { top(TypedEmit.assigned(Collections.singletonList(f)), env); } - Map out = loop(f.body, env, target, tv, trips); + Map out = loop(f.body, env, target, tv, trips, null); return run(f.orelse, out, new Loop()); } @@ -322,9 +354,10 @@ private Map forLoop(For f, Map env) { /// value after `trips - 1` passes, and one more pass must confirm that /// it grows by that same interval and nothing else moves. Whatever fails /// that is unbounded. - private Map loop(List body, Map entry, String target, Iv tv, BigInteger trips) { + private Map loop(List body, Map entry, String target, Iv tv, BigInteger trips, + Expr test) { Map head = new HashMap(entry); - Map exit = new HashMap(entry); + Map exit = test == null ? new HashMap(entry) : leave(test, entry); Map prevDelta = new HashMap(); Map widened = new HashMap(); for (int pass = 0;; pass++) { @@ -332,9 +365,16 @@ private Map loop(List body, Map entry, String targ if (target != null) { store(target, tv, in0); } + if (test != null) { + // The test runs before every pass, and the body runs only + // where it held. + record(test, in0); + indexed(test, in0); + in0 = refine(test, in0, true); + } Loop l = new Loop(); - Map out = join(run(body, in0, l), l.cont); - exit = join(join(exit, out), l.brk); + Map out = in0 == null ? null : join(run(body, in0, l), l.cont); + exit = join(join(exit, test == null ? out : out == null ? null : leave(test, out)), l.brk); Map delta = new HashMap(); Set growing = new HashSet(); if (out != null) { @@ -371,7 +411,10 @@ private Map loop(List body, Map entry, String targ e0.hi.add(d.hi.max(BigInteger.ZERO).multiply(k)))); widened.put(n, d); } else if (pass >= 2) { - next.put(n, Iv.TOP); + // Each moving bound jumps to the next width a value is + // held in, then to unbounded: a test or an index that + // bounds it keeps it there, and the passes end. + next.put(n, widen(head.get(n), next.get(n))); widened.remove(n); } } @@ -380,6 +423,266 @@ private Map loop(List body, Map entry, String targ } } + private static final BigInteger[] LOWS = {BigInteger.valueOf(Integer.MIN_VALUE), LONG_MIN}; + private static final BigInteger[] HIGHS = {BigInteger.valueOf(Integer.MAX_VALUE), LONG_MAX}; + + /// `now`, past `before`, with each bound that moved out to a threshold. + static Iv widen(Iv before, Iv now) { + if (before == null || now == null) { + return Iv.TOP; + } + BigInteger lo = now.lo; + if (lo != null && (before.lo == null || lo.compareTo(before.lo) < 0)) { + BigInteger t = null; + for (BigInteger x : LOWS) { + if (x.compareTo(lo) <= 0) { + t = x; + break; + } + } + lo = t; + } + BigInteger hi = now.hi; + if (hi != null && (before.hi == null || hi.compareTo(before.hi) > 0)) { + BigInteger t = null; + for (BigInteger x : HIGHS) { + if (x.compareTo(hi) >= 0) { + t = x; + break; + } + } + hi = t; + } + return new Iv(lo, hi); + } + + // ---- refinement ------------------------------------------------------ + + /// What a list index can be: a Java array holds at most 2^31 - 1 items. + private static final Iv INDEX = new Iv(BigInteger.valueOf(-Integer.MAX_VALUE), + BigInteger.valueOf(Integer.MAX_VALUE - 1)); + + private static boolean forever(Expr test) { + return test instanceof Constant && ((Constant) test).kind == Constant.Kind.TRUE; + } + + /// The state leaving a while loop through its test. + private Map leave(Expr test, Map env) { + Map at = new HashMap(env); + indexed(test, at); + return refine(test, at, false); + } + + /// `env` where `test` is `truth`; null when it cannot be. + private Map refine(Expr test, Map env, boolean truth) { + if (env == null) { + return null; + } + if (test instanceof UnaryOp && "not".equals(((UnaryOp) test).op)) { + return refine(((UnaryOp) test).operand, env, !truth); + } + if (test instanceof BoolOp && ((BoolOp) test).isAnd == truth) { + // Every operand of a true `and` held; of a false `or`, none did. + Map r = env; + for (Expr v : ((BoolOp) test).values) { + r = refine(v, r, truth); + if (r == null) { + return null; + } + } + return r; + } + if (!(test instanceof Compare) || ((Compare) test).ops.size() != 1) { + return env; + } + Compare c = (Compare) test; + String op = c.ops.get(0); + if (!truth) { + op = negate(op); + } + if (op == null) { + return env; + } + Map r = new HashMap(env); + Expr l = c.left; + Expr rt = c.comparators.get(0); + Iv lv = eval(l, env); + Iv rv = eval(rt, env); + if (!narrow(l, op, rv, r) || !narrow(rt, flip(op), lv, r)) { + return null; + } + return r; + } + + private static String negate(String op) { + switch (op) { + case "<": + return ">="; + case "<=": + return ">"; + case ">": + return "<="; + case ">=": + return "<"; + case "==": + return "!="; + case "!=": + return "=="; + default: + return null; + } + } + + private static String flip(String op) { + switch (op) { + case "<": + return ">"; + case "<=": + return ">="; + case ">": + return "<"; + case ">=": + return "<="; + default: + return op; + } + } + + /// Narrows local `e` to the values that make `e op other` true; false + /// when none can. + private boolean narrow(Expr e, String op, Iv other, Map env) { + if (!(e instanceof Name) || !ints.contains(((Name) e).id) || sc.resolve(((Name) e).id) != Scope.Binding.LOCAL) { + return true; + } + String n = ((Name) e).id; + Iv v = env.get(n); + if (v == null) { + v = Iv.TOP; + } + BigInteger lo = v.lo; + BigInteger hi = v.hi; + switch (op) { + case "<": + hi = min(hi, other.hi == null ? null : other.hi.subtract(BigInteger.ONE)); + break; + case "<=": + hi = min(hi, other.hi); + break; + case ">": + lo = max(lo, other.lo == null ? null : other.lo.add(BigInteger.ONE)); + break; + case ">=": + lo = max(lo, other.lo); + break; + case "==": + lo = max(lo, other.lo); + hi = min(hi, other.hi); + break; + default: + return true; + } + if (lo != null && hi != null && lo.compareTo(hi) > 0) { + return false; + } + env.put(n, new Iv(lo, hi)); + return true; + } + + private static BigInteger min(BigInteger a, BigInteger b) { + return a == null ? b : b == null ? a : a.min(b); + } + + private static BigInteger max(BigInteger a, BigInteger b) { + return a == null ? b : b == null ? a : a.max(b); + } + + /// Narrows every int local that `o` certainly used as a list or str + /// index (not under a branch of `and`, `or` or a conditional) to the + /// indexes such a sequence has. + private void indexed(Object o, Map env) { + if (env == null || o instanceof Lambda || o instanceof Comp) { + return; + } + if (o instanceof BoolOp) { + indexed(((BoolOp) o).values.get(0), env); + return; + } + if (o instanceof IfExp) { + indexed(((IfExp) o).test, env); + return; + } + if (o instanceof Subscript) { + Subscript x = (Subscript) o; + if (x.value instanceof Name && x.index instanceof Name && sequence(((Name) x.value).id)) { + narrowTo(((Name) x.index).id, INDEX, env); + } + } + Infer.children(o, c -> indexed(c, env)); + } + + /// Whether local `n` is always a list or a str. + private boolean sequence(String n) { + if (sc.resolve(n) != Scope.Binding.LOCAL) { + return false; + } + Ty t = fn.locals.get(n); + return t != null && !t.opt && (t.k == Ty.K.LIST || t.k == Ty.K.STR); + } + + private void narrowTo(String n, Iv by, Map env) { + if (!ints.contains(n) || sc.resolve(n) != Scope.Binding.LOCAL) { + return; + } + Iv v = env.get(n); + if (v == null) { + v = Iv.TOP; + } + BigInteger lo = max(v.lo, by.lo); + BigInteger hi = min(v.hi, by.hi); + if (lo != null && hi != null && lo.compareTo(hi) > 0) { + return; + } + env.put(n, new Iv(lo, hi)); + } + + // ---- call sites ------------------------------------------------------ + + /// Records the arguments of the calls `o` makes, as evaluated in `env`. + private void record(Object o, Map env) { + if (sites == null || env == null || !bound.isEmpty() || o instanceof Lambda || o instanceof FunctionDef + || o instanceof ClassDef || o instanceof Comp) { + return; + } + if (o instanceof Stmt && !(o instanceof Assign || o instanceof AugAssign || o instanceof ExprStmt + || o instanceof Return)) { + // Compound statements: their parts are recorded as they run. + if (o instanceof If) { + record(((If) o).test, env); + } else if (o instanceof For) { + record(((For) o).iter, env); + } + return; + } + if (o instanceof Call) { + Call c = (Call) o; + if (c.func instanceof Name && c.keywords.isEmpty() && sc.resolve(((Name) c.func).id) == Scope.Binding.GLOBAL + && in.globalFns.containsKey(((Name) c.func).id)) { + Iv[] a = new Iv[c.args.size()]; + for (int i = 0; i < a.length; i++) { + a[i] = c.args.get(i) instanceof Starred ? Iv.TOP : eval(c.args.get(i), env); + } + Iv[] old = sites.get(c); + if (old != null && old.length == a.length) { + for (int i = 0; i < a.length; i++) { + a[i] = a[i].join(old[i]); + } + } + sites.put(c, a); + } + } + Infer.children(o, x -> record(x, env)); + } + /// How much `b` moved to become `a`, per bound; null when either is /// unbounded there. private static Iv diff(Iv a, Iv b) { diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/TypedEmit.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/TypedEmit.java index 2b60fbc0d1a..cc93888f589 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/TypedEmit.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/TypedEmit.java @@ -3830,6 +3830,12 @@ private E builtin(String n, List a) { String l = "Typed.slen(" + a.get(0).c + ")"; return new E("Long.valueOf(" + l + ")", Ty.INT, l); } + if (a.size() == 1 && a.get(0).t.k == Ty.K.DYN) { + // The builtin's work without a call, unless the name + // was rebound at run time. + String l = "Typed.len(" + em.builtinRef("len") + ", " + a.get(0).c + ")"; + return new E("Long.valueOf(" + l + ")", Ty.INT, l); + } break; case "float": if (a.size() == 1 && !a.get(0).t.opt diff --git a/maven/python-transpiler/src/test/resources/conformance/int_refine.out b/maven/python-transpiler/src/test/resources/conformance/int_refine.out new file mode 100644 index 00000000000..71ff44fe415 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/int_refine.out @@ -0,0 +1,8 @@ +-4995 -100000 -4920 -4913 +13835058055282163712 13835058055282163712 13835058055282163716 +9 -2 +(6, -4) +-9223372036854775807 -1330 3541774862152233910273 -98079714615416886934934209737619787751599303819750539263 1 +500 0 399 -18446744073709551616 +6765 -3 True 1 +5050 0 1 diff --git a/maven/python-transpiler/src/test/resources/conformance/int_refine.py b/maven/python-transpiler/src/test/resources/conformance/int_refine.py new file mode 100644 index 00000000000..a7ec6a2f0cf --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/int_refine.py @@ -0,0 +1,74 @@ +def down(n: int) -> int: + j: int = n + c: int = 0 + while j > -5: + j -= 3 if j < 100 else j // 2 + c += 1 + return j * 1000 + c + + +def up(x: int) -> int: + i: int = x + while i < 2 ** 63 + 5: + i += 2 ** 62 + return i + + +def walk(a: list, j: int) -> int: + s: int = 0 + while j >= 0: + s += a[j] + j -= 1 + return s + j + + +def back(a: list) -> tuple: + j: int = -1 + t: int = 0 + while True: + t += a[j] + j -= 1 + if j < -len(a): + break + return t, j + + +def clamp(v: int) -> int: + if v > 2 ** 63: + return v - 2 ** 64 + if v < -10: + v = v * v * v + elif not (v != 3): + v = v << 70 + return v + 1 + + +def mixed(n: int) -> int: + i: int = 0 + k: int = n + while i < n and k > 0: + i += 1 + k -= 2 + return i * 100 + k + + +def fib(n: int) -> int: + if n < 2: + return n + return fib(n - 1) + fib(n - 2) + + +def tri(n: int) -> int: + if n <= 0: + return 0 + return n + tri(n - 1) + + +print(down(10), down(-100), down(2 ** 63 - 1), down(2 ** 70)) +print(up(0), up(-(2 ** 64)), up(2 ** 63 + 4)) +print(walk([1, 2, 3, 4], 3), walk([5], -2)) +print(back([1, 2, 3])) +print(clamp(2 ** 63 + 1), clamp(-11), clamp(3), clamp(-2 ** 62), clamp(0)) +print(mixed(10), mixed(0), mixed(7), mixed(-2 ** 64)) +print(fib(20), fib(-3), fib(True), fib(2 ** 65 // 2 ** 64)) +print(tri(100), tri(-5), tri(True)) From 537976b8ae1fee2a4c64fec0e1db91b548930a36 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 04:06:51 +0200 Subject: [PATCH 026/112] Python: long-parameter function versions from call-site ranges; typed int results held bare, taken in the slow path Co-Authored-By: Claude Opus 5.5 --- .../com/codename1/python/runtime/Typed.java | 60 ++- .../python/transpiler/codegen/Emitter.java | 12 + .../python/transpiler/codegen/IntRanges.java | 107 +++++- .../python/transpiler/codegen/TypedEmit.java | 363 +++++++++++++++++- .../resources/conformance/pushed_returns.out | 9 + .../resources/conformance/pushed_returns.py | 60 +++ 6 files changed, 593 insertions(+), 18 deletions(-) create mode 100644 maven/python-transpiler/src/test/resources/conformance/pushed_returns.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/pushed_returns.py diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Typed.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Typed.java index 3a6f06e856c..bbbe79bcf6c 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Typed.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Typed.java @@ -134,14 +134,40 @@ public static long low64(Object o) { return toLong(o); } - /// The int a typed function returned in place of its `long` result: - /// a big int, a bool, or a `Long.MIN_VALUE` (the marker itself). - private static final ThreadLocal RETURNED = new ThreadLocal(); + /// The ints typed functions returned in place of their `long` results + /// (a big int, a bool, or a `Long.MIN_VALUE`, the marker itself), last + /// returned on top. A stack rather than one slot so a caller may hold a + /// result as its bare marker while it makes further calls, and take + /// the object only on the slow path, in a method of its own: a ParparVM + /// method with an object local pays for a stack frame on every call, + /// the fast path included. Each marker is taken exactly once, the last + /// first. + private static final class Returned { + Object[] items = new Object[8]; + int size; + } + + private static final ThreadLocal RETURNED = new ThreadLocal(); + + private static Returned returned() { + Returned r = RETURNED.get(); + if (r == null) { + r = new Returned(); + RETURNED.set(r); + } + return r; + } /// Returns int `o` from a typed function whose result is a `long`: the /// marker, with `o` left for the caller's `take`. public static long retBig(Object o) { - RETURNED.set(o); + Returned r = returned(); + if (r.size == r.items.length) { + Object[] n = new Object[r.size * 2]; + System.arraycopy(r.items, 0, n, 0, r.size); + r.items = n; + } + r.items[r.size++] = o; return Long.MIN_VALUE; } @@ -156,13 +182,33 @@ public static long retObj(Object o) { return retBig(o); } - /// The int behind a `Long.MIN_VALUE` a typed function returned. + /// The int behind the last `Long.MIN_VALUE` a typed function returned. public static Object take() { - Object o = RETURNED.get(); - RETURNED.set(null); + Returned r = returned(); + Object o = r.items[--r.size]; + r.items[r.size] = null; return o; } + /// How many results are waiting to be taken: kept at the start of a + /// `try`, for `untake`. + public static int taken() { + Returned r = RETURNED.get(); + return r == null ? 0 : r.size; + } + + /// Drops the results an exception left untaken, back to `mark`: one + /// raised between a call returning a marker and its caller taking it. + public static void untake(int mark) { + Returned r = RETURNED.get(); + if (r == null) { + return; + } + while (r.size > mark) { + r.items[--r.size] = null; + } + } + /// A typed function's `long` result as an object. public static Object unret(long v) { return v != Long.MIN_VALUE ? Long.valueOf(v) : take(); diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Emitter.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Emitter.java index 024f7a13485..d1b58e24301 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Emitter.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Emitter.java @@ -1358,6 +1358,11 @@ private void tryStmt(Try t, StringBuilder out, int ind) { indent(out, ind); out.append("boolean ").append(ok).append(" = false;\n"); } + // A typed call's result an exception left untaken (see + // Typed.take) is dropped where the exception is caught. + String tk = fresh("tk"); + indent(out, ind); + out.append("final int ").append(tk).append(" = Typed.taken();\n"); indent(out, ind); out.append("try {\n"); block(t.body, out, ind + 1); @@ -1370,6 +1375,8 @@ private void tryStmt(Try t, StringBuilder out, int ind) { indent(out, ind); out.append("} catch (Throwable ").append(th).append(") {\n"); indent(out, ind + 1); + out.append("Typed.untake(").append(tk).append(");\n"); + indent(out, ind + 1); out.append("final PyException ").append(pe).append(" = Py.convert(").append(th).append(");\n"); handlers.add(pe); boolean catchAll = false; @@ -1464,12 +1471,17 @@ private void withItems(With w, int i, StringBuilder out, int ind) { } indent(out, ind); out.append("boolean ").append(ok).append(" = true;\n"); + String tk = fresh("tk"); + indent(out, ind); + out.append("final int ").append(tk).append(" = Typed.taken();\n"); indent(out, ind); out.append("try {\n"); withItems(w, i + 1, out, ind + 1); indent(out, ind); out.append("} catch (Throwable ").append(th).append(") {\n"); indent(out, ind + 1); + out.append("Typed.untake(").append(tk).append(");\n"); + indent(out, ind + 1); out.append(ok).append(" = false;\n"); indent(out, ind + 1); out.append("final PyException ").append(pe).append(" = Py.convert(").append(th).append(");\n"); diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/IntRanges.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/IntRanges.java index 94b5ec97c0e..93c374c38bd 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/IntRanges.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/IntRanges.java @@ -91,6 +91,16 @@ Iv join(Iv o) { hi == null || o.hi == null ? null : hi.max(o.hi)); } + @Override + public String toString() { + return "[" + lo + ".." + hi + "]"; + } + + boolean within(Iv o) { + return (o.lo == null || lo != null && lo.compareTo(o.lo) >= 0) + && (o.hi == null || hi != null && hi.compareTo(o.hi) <= 0); + } + boolean same(Iv o) { return o != null && eq(lo, o.lo) && eq(hi, o.hi); } @@ -120,7 +130,18 @@ private static boolean eq(BigInteger a, BigInteger b) { /// when `sites` is not null, it receives the interval of every argument /// of every direct call of a module function the analysis follows. IntRanges(Infer in, Infer.Fn fn, Map params, Map sites) { + this(in, fn, params, sites, null, null); + } + + /// As above, with the long-parameter versions of module functions + /// known: a call whose arguments are within `longParams` of its callee + /// calls that version, and returns within `longRets` of it when that + /// has an entry. + IntRanges(Infer in, Infer.Fn fn, Map params, Map sites, + Map longParams, Map longRets) { this.in = in; + this.longParams = longParams; + this.longRets = longRets; this.fn = fn; this.sc = fn.def.scope; this.sites = sites; @@ -143,6 +164,65 @@ private static boolean eq(BigInteger a, BigInteger b) { /// Where call arguments are recorded, or null. private final Map sites; + private final Map longParams; + private final Map longRets; + /// Every int the function returns; null when it never returns one. + private Iv returned; + private boolean returnsOther; + + /// What the function returns, when it returns only ints the analysis + /// bounds; else null. + Iv returns() { + return returnsOther ? null : returned; + } + + /// The arguments of a call of `g` as the intervals its long version + /// takes (null for a parameter that is not an int), or null when an + /// int argument may not fit a long or `g` takes no int. + static Iv[] longArgs(Infer.Fn g, Iv[] args) { + if (args == null || args.length != g.names.size()) { + return null; + } + Iv[] r = new Iv[args.length]; + boolean any = false; + for (int i = 0; i < args.length; i++) { + if (g.params[i].k == Ty.K.INT && !g.params[i].opt) { + if (!args[i].fitsLong()) { + return null; + } + r[i] = args[i]; + any = true; + } + } + return any ? r : null; + } + + /// Whether `a` has an interval, and every one fits a long. + static boolean fitAll(Iv[] a) { + boolean any = false; + for (Iv v : a) { + if (v != null) { + if (!v.fitsLong()) { + return false; + } + any = true; + } + } + return any; + } + + /// Whether every interval of `a` is within the same one of `s`. + static boolean within(Iv[] a, Iv[] s) { + if (a == null || s == null || a.length != s.length) { + return false; + } + for (int i = 0; i < a.length; i++) { + if (a[i] == null ? s[i] != null : s[i] == null || !a[i].within(s[i])) { + return false; + } + } + return true; + } /// The locals held in a Java `long`. Set longLocals() { @@ -286,7 +366,17 @@ private Map stmt0(Stmt s, Map env, Loop loop) { Map out = loop(w.body, env, null, null, null, forever(w.test) ? null : w.test); return run(w.orelse, out, loop); } - if (s instanceof Return || s instanceof Raise) { + if (s instanceof Return) { + Expr v = ((Return) s).value; + if (v == null) { + returnsOther = true; + } else { + Iv r = eval(v, env); + returned = returned == null ? r : returned.join(r); + } + return null; + } + if (s instanceof Raise) { return null; } if (s instanceof Break) { @@ -804,6 +894,21 @@ private Iv call(Call c, Map env) { } } Infer.Fn f = inlinable(c); + if (f == null && longParams != null && bound.isEmpty() && c.func instanceof Name && c.keywords.isEmpty() + && sc.resolve(((Name) c.func).id) == Scope.Binding.GLOBAL) { + Infer.Fn g = in.globalFns.get(((Name) c.func).id); + Iv[] s = g == null ? null : longParams.get(g); + Iv r = s == null ? null : longRets.get(g); + if (r != null) { + Iv[] a = new Iv[c.args.size()]; + for (int i = 0; i < a.length; i++) { + a[i] = c.args.get(i) instanceof Starred ? Iv.TOP : eval(c.args.get(i), env); + } + if (within(longArgs(g, a), s)) { + return r; + } + } + } if (f != null) { Map b = new HashMap(); for (int i = 0; i < c.args.size(); i++) { diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/TypedEmit.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/TypedEmit.java index cc93888f589..4f3e0d4e581 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/TypedEmit.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/TypedEmit.java @@ -75,6 +75,11 @@ static final class E { /// bool). Both plain Java names; or both null. final String ll; final String bg; + /// An int held as the bare `long` a typed function returned: a + /// plain Java name whose `Long.MIN_VALUE` means the object is + /// waiting in `Typed.take`, which `c` takes. Only the consumers that + /// know to take it exactly once see one (see `pushed`); or null. + String pk; E(String c, Ty t) { this(c, t, null); @@ -93,6 +98,14 @@ static final class E { } } + /// The int a typed function returned as `long` `r`, left as it is + /// (see `E.pk`). + static E pushed(String r) { + E e = new E("Typed.unret(" + r + ")", Ty.INT); + e.pk = r; + return e; + } + /// An int pair `l`/`b` as an expression: the boxed value, built only /// when something needs the object. static E dual(String l, String b) { @@ -129,6 +142,19 @@ static E dual(String l, String b) { /// put in a Java `long` (see `IntRanges`). private IntRanges ranges; private Set longLocals = new HashSet(); + /// The module functions that also have a version taking their int + /// parameters as Java longs (`TL_`), and the interval each such + /// parameter is known to hold there; what that version returns, when + /// it is bounded. See `planLongParams`. + private Map longParams = new HashMap(); + private Map longRets = new HashMap(); + /// The expression a `return` of an int returns, which may lower to a + /// pushed result (see `E.pk`); and the call a hoisted leaf is, likewise. + private Expr pushTop; + private Expr pushLeaf; + private final Map longBodies = new LinkedHashMap(); + /// The argument intervals of the calls the function being emitted makes. + private Map sites = new HashMap(); /// The Java code of each parameter of the expression functions being /// inlined, innermost last, and whether it is the exact value (true) or /// only the value modulo 2^64. @@ -413,6 +439,7 @@ void plan() { } } } + planLongParams(ok); for (Set g : gdeps.values()) { if (g != null) { usedGroups.addAll(g); @@ -442,6 +469,149 @@ private Set liveGroups(Set ok) { return r; } + /// Which typed module functions get a version taking their int + /// parameters as longs, and the interval of each: the union of the + /// arguments of every call whose int arguments all fit a long, found by + /// running the range analysis over every typed body to a fixpoint (a + /// recursive call's arguments depend on its own parameters). A call + /// whose arguments are within those intervals calls that version + /// directly, and the generic version starts with a test that sends + /// arguments within them there; a call outside them is unchanged. + /// `fib(n - 1)` under `if n < 2: return n` then runs on longs from the + /// first call down. + /// + /// What the long version returns is bounded the same way, when the + /// analysis bounds it, and that is only trusted once a round reproduces + /// both the intervals and the returns it assumed. + private void planLongParams(Set ok) { + Map seeds = new HashMap(); + Map rets = new HashMap(); + boolean settled = false; + for (int round = 0; round < 12 && !settled; round++) { + Map ns = new HashMap(); + Map nr = new HashMap(); + for (Infer.Fn f : ok) { + // The generic and the long version's calls apart: one + // site's arguments differ between the two bodies. + Map st = new HashMap(); + new IntRanges(in, f, null, st, seeds, rets); + List> all = new ArrayList>(st.entrySet()); + if (seeds.containsKey(f)) { + Map sl = new HashMap(); + IntRanges r = new IntRanges(in, f, seeds(f, seeds.get(f)), sl, seeds, rets); + all.addAll(sl.entrySet()); + IntRanges.Iv v = f.ret.k == Ty.K.INT && !f.ret.opt ? r.returns() : null; + if (v != null && v.fitsLong()) { + nr.put(f, v); + } + } + for (Map.Entry e : all) { + Infer.Fn g = in.globalFns.get(((Name) e.getKey().func).id); + IntRanges.Iv[] a = g == null || !ok.contains(g) || g.cls != null ? null + : IntRanges.longArgs(g, e.getValue()); + if (a == null) { + continue; + } + IntRanges.Iv[] o = ns.get(g); + if (o != null) { + for (int i = 0; i < a.length; i++) { + a[i] = a[i] == null ? null : a[i].join(o[i]); + } + } + ns.put(g, a); + } + } + if (round >= 3) { + // Out to the next threshold, so the rounds end. + for (Map.Entry e : ns.entrySet()) { + IntRanges.Iv[] o = seeds.get(e.getKey()); + IntRanges.Iv[] a = e.getValue(); + for (int i = 0; o != null && i < a.length; i++) { + a[i] = a[i] == null ? null : IntRanges.widen(o[i], a[i]); + } + } + ns.values().removeIf(a -> !IntRanges.fitAll(a)); + } + settled = sameSeeds(ns, seeds) && sameRets(nr, rets); + seeds = ns; + rets = nr; + } + if (!settled) { + // The intervals are sound whatever they are (every call is + // tested against them); the returns are not. + rets = new HashMap(); + } + while (!seeds.isEmpty()) { + longParams = seeds; + longRets = rets; + Infer.Fn failed; + try { + failed = emitLong(ok); + } catch (Unsupported u) { + // A typed body that only failed beside the long versions: + // without them, it is what it was. + seeds.clear(); + break; + } + if (failed == null) { + return; + } + seeds.remove(failed); + rets = new HashMap(); + } + longParams = seeds; + longRets = rets; + emitLong(ok); + } + + /// Emits every typed body again against `longParams`, and the long + /// versions; the function whose long version could not be emitted, or + /// null. + private Infer.Fn emitLong(Set ok) { + slow.setLength(0); + slowCount = 0; + longBodies.clear(); + for (Infer.Fn f : longParams.keySet()) { + try { + longBodies.put(f, function(f, true)); + } catch (Unsupported u) { + return f; + } + } + for (Infer.Fn f : ok) { + bodies.put(f, function(f)); + } + return null; + } + + private static boolean sameSeeds(Map a, Map b) { + if (!a.keySet().equals(b.keySet())) { + return false; + } + for (Map.Entry e : a.entrySet()) { + IntRanges.Iv[] x = e.getValue(); + IntRanges.Iv[] y = b.get(e.getKey()); + for (int i = 0; i < x.length; i++) { + if (x[i] == null ? y[i] != null : !x[i].same(y[i])) { + return false; + } + } + } + return true; + } + + private static boolean sameRets(Map a, Map b) { + if (!a.keySet().equals(b.keySet())) { + return false; + } + for (Map.Entry e : a.entrySet()) { + if (!e.getValue().same(b.get(e.getKey()))) { + return false; + } + } + return true; + } + private static boolean representable(Ty t) { switch (t.k) { case INT: @@ -456,6 +626,10 @@ private static boolean representable(Ty t) { } } + private static String longName(Infer.Fn f) { + return "TL_" + Emitter.mangle(f.def.name); + } + private String typedName(Infer.Fn f) { if (f.cls != null) { return "TM_" + f.cls.javaName + "_" + Emitter.mangle(f.def.name); @@ -670,6 +844,22 @@ private static void countReads(Object o, String n, For skip, int[] c) { } private String function(Infer.Fn f) { + return function(f, false); + } + + /// The parameters of `f`'s long version, by name, at their intervals. + private static Map seeds(Infer.Fn f, IntRanges.Iv[] iv) { + Map m = new HashMap(); + for (int i = 0; i < iv.length; i++) { + if (iv[i] != null) { + m.put(f.names.get(i), iv[i]); + } + } + return m; + } + + /// `f`'s typed body; with `asLong`, its long version (see `longParams`). + private String function(Infer.Fn f, boolean asLong) { cur = f; sc = f.def.scope; calls = new LinkedHashSet(); @@ -678,10 +868,18 @@ private String function(Infer.Fn f) { nonNull = new HashSet(); shadow = new HashMap(); codePoints.clear(); - ranges = new IntRanges(in, f); + sites = new HashMap(); + IntRanges.Iv[] lp = asLong ? longParams.get(f) : null; + ranges = new IntRanges(in, f, lp == null ? null : seeds(f, lp), sites, longParams, longRets); longLocals = ranges.longLocals(); - // A parameter arrives as a pair, whatever its range. - longLocals.removeAll(f.names); + for (int i = 0; i < f.names.size(); i++) { + if (lp == null || lp[i] == null) { + // A parameter arrives as a pair, whatever its range. + longLocals.remove(f.names.get(i)); + } else if (!longLocals.contains(f.names.get(i))) { + throw new Unsupported("long parameter " + f.names.get(i) + " outgrows a long"); + } + } if (!f.def.scope.cells.isEmpty() || f.def.scope.hasYield || f.def.isAsync || !f.def.decorators.isEmpty() || f.def.args.vararg != null || f.def.args.kwarg != null || !f.def.args.kwonly.isEmpty()) { throw new Unsupported("signature"); @@ -696,11 +894,15 @@ private String function(Infer.Fn f) { recycled = ListRecycling.find(f.def.body, f.names, sc); scalars = findScalars(f); StringBuilder b = new StringBuilder(); - b.append(" static ").append(retJ(f)).append(' ').append(typed.get(f)).append('('); + b.append(" static ").append(retJ(f)).append(' ').append(lp != null ? longName(f) : typed.get(f)).append('('); Set declared = new HashSet(f.names); StringBuilder norm = new StringBuilder(); for (int i = 0; i < f.names.size(); i++) { String n = f.names.get(i); + if (lp != null && lp[i] != null) { + b.append(i == 0 ? "" : ", ").append("long ").append(var(n)); + continue; + } b.append(i == 0 ? "" : ", ").append(param(f.params[i], var(n), big(n))); if (f.params[i].k == Ty.K.INT) { // A caller holding only the object passes it whole. @@ -710,6 +912,31 @@ private String function(Infer.Fn f) { } } b.append(") {\n").append(norm); + IntRanges.Iv[] dl = asLong ? null : longParams.get(f); + if (dl != null) { + // Arguments the long version takes go there. + StringBuilder test = new StringBuilder(); + StringBuilder pass = new StringBuilder(); + for (int i = 0; i < dl.length; i++) { + String n = f.names.get(i); + pass.append(i == 0 ? "" : ", ").append(var(n)); + if (dl[i] == null) { + if (f.params[i].k == Ty.K.INT) { + pass.append(", ").append(big(n)); + } + continue; + } + test.append(test.length() == 0 ? "" : " && ").append(big(n)).append(" == null"); + if (dl[i].lo.compareTo(IntRanges.LONG_MIN) > 0) { + test.append(" && ").append(var(n)).append(" >= ").append(lit(dl[i].lo)); + } + if (dl[i].hi.compareTo(IntRanges.LONG_MAX) < 0) { + test.append(" && ").append(var(n)).append(" <= ").append(lit(dl[i].hi)); + } + } + b.append(" if (").append(test).append(") {\n return ").append(longName(f)).append('(') + .append(pass).append(");\n }\n"); + } for (Map.Entry e : f.locals.entrySet()) { if (longLocals.contains(e.getKey())) { if (declared.contains(e.getKey())) { @@ -919,7 +1146,20 @@ && localType(((Name) ((Assign) s).targets.get(0)).id).k == Ty.K.LIST throw new Unsupported("return None as " + cur.ret); } } else if (cur.ret.k == Ty.K.INT) { - returnInt(expr(v), b, ind); + Expr saved = pushTop; + pushTop = v; + E e; + try { + e = expr(v); + } finally { + pushTop = saved; + } + if (e.lc != null && ranges.eval(v, null).fitsLong() && ranges.eval(v, null).lo.compareTo(IntRanges.LONG_MIN) > 0) { + // Never the marker. + b.append("return ").append(e.lc).append(";\n"); + } else { + returnInt(e, b, ind); + } } else { b.append("return ").append(conv(expr(v), cur.ret)).append(";\n"); } @@ -1031,6 +1271,11 @@ && localType(((Name) ((Assign) s).targets.get(0)).id).k == Ty.K.LIST /// `return e` from a function returning an int (see `retJ`); the /// statement is already indented. private void returnInt(E e, StringBuilder b, int ind) { + if (e.pk != null) { + // The marker and what it stands for, passed on as they are. + b.append("return ").append(e.pk).append(";\n"); + return; + } if (e.lc != null && literal(e.lc) && !e.lc.contains("-9223372036854775808")) { b.append("return ").append(e.lc).append(";\n"); return; @@ -2339,8 +2584,22 @@ private E hoistInt(Expr e) { Map savedLeaves = leafSubst; leafSubst = new IdentityHashMap(savedLeaves); try { + List pushedLeaves = new ArrayList(); for (Expr x : ls) { - leafSubst.put(x, spill(expr(x))); + Expr savedLeaf = pushLeaf; + pushLeaf = x; + E v; + try { + v = expr(x); + } finally { + pushLeaf = savedLeaf; + } + if (v.pk != null) { + pushedLeaves.add(x); + leafSubst.put(x, v); + } else { + leafSubst.put(x, spill(v)); + } } char k; if (e instanceof Compare) { @@ -2352,6 +2611,16 @@ private E hoistInt(Expr e) { k = kindOf(e); } if (k == 0) { + // Taken here, the last first: the generic path reads each + // leaf as an object. + for (int i = pushedLeaves.size() - 1; i >= 0; i--) { + String r = leafSubst.get(pushedLeaves.get(i)).pk; + String rb = fresh(); + indent(pre, preInd); + pre.append("Object ").append(rb).append(" = ").append(r) + .append(" != Long.MIN_VALUE ? null : Typed.take();\n"); + leafSubst.put(pushedLeaves.get(i), dual(r, rb)); + } return generic0(e); } return fastPath(e, ls, k); @@ -2418,22 +2687,36 @@ private E fastPath(Expr e, List ls, char k) { Map sub = new IdentityHashMap(leafSubst); StringBuilder params = new StringBuilder(); StringBuilder args = new StringBuilder(); + StringBuilder takes = new StringBuilder(); for (int i = 0; i < ls.size(); i++) { E v = leafSubst.get(ls.get(i)); String pn = "p" + i; String sep = i == 0 ? "" : ", "; + if (v.pk != null) { + // Passed bare and taken in the slow path, the last first, + // so the caller holds no object (see Typed.take). + params.append(sep).append("long ").append(pn); + args.append(sep).append(v.pk); + takes.insert(0, " Object o" + i + " = " + pn + " != Long.MIN_VALUE ? Long.valueOf(" + pn + + ") : Typed.take();\n"); + sub.put(ls.get(i), new E("o" + i, Ty.INT)); + continue; + } params.append(sep).append(jt(v.t)).append(' ').append(pn); args.append(sep).append(v.t.k == Ty.K.INT ? box(v) : v.c); sub.put(ls.get(i), new E(pn, v.t)); } + boolean push = t.k == Ty.K.INT && e == pushTop; //NOPMD CompareObjectsWithEquals - identity is meant String generic; Map saved = leafSubst; leafSubst = sub; try { E g = generic0(e); String m = "S" + (slowCount++); - slow.append(" static ").append(jt(t)).append(' ').append(m).append('(').append(params) - .append(") {\n return ").append(t.k == Ty.K.FLOAT ? toD(g) : conv(g, t)).append(";\n }\n"); + String body = t.k == Ty.K.FLOAT ? toD(g) : conv(g, t); + slow.append(" static ").append(push ? "long" : jt(t)).append(' ').append(m).append('(').append(params) + .append(") {\n").append(takes).append(" return ") + .append(push ? "Typed.retObj(" + box(new E(body, t)) + ")" : body).append(";\n }\n"); generic = m + "(" + args + ")"; } finally { leafSubst = saved; @@ -2457,8 +2740,16 @@ private E fastPath(Expr e, List ls, char k) { v = longOf(e, f, guards, fast); } String r = fresh(); - String rb = t.k == Ty.K.INT ? fresh() : null; + String rb = t.k == Ty.K.INT && !push ? fresh() : null; int ind = preInd; + if (push) { + // The marker stands for a pushed result only: a true + // Long.MIN_VALUE goes the slow way, and is pushed. + String x = fresh(); + f.append("long ").append(x).append(" = ").append(v).append(";\nif (").append(x) + .append(" == Long.MIN_VALUE) {\n break ").append(fast).append(";\n}\n"); + v = x; + } boolean leaves = f.toString().contains("break " + fast + ";"); indent(pre, ind); pre.append(t.k == Ty.K.INT ? "long" : jt(t)).append(' ').append(r).append(";\n"); @@ -2476,6 +2767,27 @@ private E fastPath(Expr e, List ls, char k) { pre.append(r).append(" = ").append(v).append(";\n"); return t.k == Ty.K.INT ? new E("Long.valueOf(" + r + ")", t, r) : new E(r, t); } + if (push) { + indent(pre, ind); + pre.append(done).append(": {\n"); + indent(pre, ind + 1); + pre.append(fast).append(": {\n"); + for (String line : f.toString().split("\n")) { + indent(pre, ind + 2); + pre.append(line).append('\n'); + } + indent(pre, ind + 2); + pre.append(r).append(" = ").append(v).append(";\n"); + indent(pre, ind + 2); + pre.append("break ").append(done).append(";\n"); + indent(pre, ind + 1); + pre.append("}\n"); + indent(pre, ind + 1); + pre.append(r).append(" = ").append(generic).append(";\n"); + indent(pre, ind); + pre.append("}\n"); + return pushed(r); + } if (rb != null) { indent(pre, ind); pre.append("Object ").append(rb).append(" = null;\n"); @@ -2562,6 +2874,13 @@ private String longOf(Expr e, StringBuilder f, Map guards, Strin if (lv.t.k == Ty.K.BOOL) { return "(" + lv.c + " ? 1L : 0L)"; } + if (lv.pk != null) { + if (guards.put(lv.pk, lv.pk) == null) { + f.append("if (").append(lv.pk).append(" == Long.MIN_VALUE) {\n break ").append(fast) + .append(";\n}\n"); + } + return lv.pk; + } if (lv.ll != null) { if (guards.put(lv.bg, lv.ll) == null) { f.append("if (").append(lv.bg).append(" != null) {\n break ").append(fast).append(";\n}\n"); @@ -2922,6 +3241,16 @@ private String argList(Infer.Fn f, int from, List args) { return sb.toString(); } + /// The arguments of a call of `f`'s long version: its int parameters + /// as longs (the call's ranges proved each fits). + private String longArgList(Infer.Fn f, IntRanges.Iv[] lp, List args) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < args.size(); i++) { + sb.append(i == 0 ? "" : ", ").append(lp[i] != null ? fitting(args.get(i)) : arg(args.get(i), f.params[i])); + } + return sb.toString(); + } + /// `a` passed for a parameter of type `t`: an int as its two halves. private String arg(E a, Ty t) { if (t.k != Ty.K.INT) { @@ -3059,8 +3388,19 @@ && ordOf(c, ((Name) c.args.get(0)).id)) { if (f != null) { List a = args(c); calls.add(f); - E r = result(f, typed.get(f) + "(" + argList(f, 0, a) + ")"); + IntRanges.Iv[] lp = longParams.get(f); + String call = lp != null && IntRanges.within(IntRanges.longArgs(f, sites.get(c)), lp) + ? longName(f) + "(" + longArgList(f, lp, a) + ")" + : typed.get(f) + "(" + argList(f, 0, a) + ")"; Ty site = in.siteRet.get(c); + if (f.ret.k == Ty.K.INT && pre != null && stmtPure && condDepth == 0 && (site == null || site.k == Ty.K.INT) + && (c == pushTop || c == pushLeaf)) { //NOPMD CompareObjectsWithEquals - identity is meant + String r = fresh(); + indent(pre, preInd); + pre.append("long ").append(r).append(" = ").append(call).append(";\n"); + return pushed(r); + } + E r = result(f, call); return site == null || site.k == r.t.k ? r : relabel(r, site); } Layouts.Cls k = in.globalClasses.get(n); @@ -4093,6 +4433,9 @@ String members() { for (String b : bodies.values()) { o.append(b); } + for (String b : longBodies.values()) { + o.append(b); + } for (Layouts.Cls k : constructors) { Layouts.Cls ow = Layouts.resolver(k, "__init__"); Infer.Fn f = in.fns.get(ow.typable.get("__init__")); diff --git a/maven/python-transpiler/src/test/resources/conformance/pushed_returns.out b/maven/python-transpiler/src/test/resources/conformance/pushed_returns.out new file mode 100644 index 00000000000..cfe1f11560a --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/pushed_returns.out @@ -0,0 +1,9 @@ +6765 1 0 +4722366482869645213697 1147535055337323786928850 +-9223372036854775808 -9223372036854775807 -9223372036854775808 +0 4611686018427387904 +1 9223372036854775808 +2 13835058055282163712 +3 18446744073709551616 +1000 2417851639229258349412352 +True 28334198897217871282193 diff --git a/maven/python-transpiler/src/test/resources/conformance/pushed_returns.py b/maven/python-transpiler/src/test/resources/conformance/pushed_returns.py new file mode 100644 index 00000000000..df428d66b57 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/pushed_returns.py @@ -0,0 +1,60 @@ +# Typed int results that are big ints or Long.MIN_VALUE, consumed bare. + + +def fib(n: int) -> int: + if n < 2: + return n + return fib(n - 1) + fib(n - 2) + + +def big(n: int) -> int: + if n == 0: + return 1 << 70 + return big(n - 1) * 3 + n + + +def two(n: int) -> int: + return big(n) + big(n + 1) + + +def edge(x: int) -> int: + return x - 9223372036854775807 - 1 + + +def climb(n: int) -> int: + if n == 0: + return 4611686018427387904 + return climb(n - 1) + 4611686018427387904 + + +def boom(n: int) -> int: + if n > 3: + raise ValueError("boom " + str(n)) + return 1 << 80 + + +def mixed(n: int) -> int: + return boom(1) + boom(n) + + +def cmp(n: int) -> bool: + return big(n) < big(n + 1) - fib(n) + + +def main() -> None: + print(fib(20), fib(1), fib(0)) + print(two(0), two(5)) + print(edge(0), edge(1), edge(-1) + 1) + for i in range(4): + print(i, climb(i)) + caught = 0 + for i in range(1000): + try: + mixed(5) + except ValueError: + caught += 1 + print(caught, mixed(2)) + print(cmp(3), two(2) - two(1)) + + +main() From 2aa2d0428cdf43655941fdcabed83ea9d87580c1 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 04:28:36 +0200 Subject: [PATCH 027/112] Python: one-new construction for plain __init__, no default shape/slot stores per instance, try-free dict key hashing Co-Authored-By: Claude Opus 5.5 --- .../codename1/python/runtime/AttrSite.java | 6 +- .../java/com/codename1/python/runtime/Py.java | 6 ++ .../codename1/python/runtime/PyInstance.java | 43 +++++---- .../python/transpiler/codegen/Emitter.java | 4 + .../python/transpiler/codegen/TypedEmit.java | 90 ++++++++++++++++++- 5 files changed, 130 insertions(+), 19 deletions(-) diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/AttrSite.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/AttrSite.java index 07a34e0763f..a5d30ea4a44 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/AttrSite.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/AttrSite.java @@ -256,7 +256,7 @@ private Object fillGet(PyInstance i) { // A compiled field is read by the generated code itself; this site // only sees it unset, and must not remember that. if (!(clsAttr instanceof PyProperty) && !cls.hasCustomGetattribute() && i.fieldIndex(name) < 0) { - slot = shape.indexOf(name); + slot = shape == null ? -1 : shape.indexOf(name); if (slot >= 0) { kind = SLOT; } else if (clsAttr != null) { @@ -359,7 +359,7 @@ private void fillSet(PyInstance i, Object v) { } Shape after = i.shape; if (before == after) { //NOPMD CompareObjectsWithEquals - identity is meant - addSet(cls, before, after.indexOf(name), null); + addSet(cls, before, after == null ? -1 : after.indexOf(name), null); } else { // Adding the attribute moved the instance to a new shape. Remember // both the transition (the next new instance) and the slot on the @@ -473,7 +473,7 @@ private Object method(Object o) { addGet(i.cls, i.shape, NONE, 0, null); return null; } - int slot = i.shape.indexOf(name); + int slot = i.shape().indexOf(name); if (slot >= 0) { addGet(i.cls, i.shape, SLOT, slot, null); return null; diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Py.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Py.java index cdba7c7ceeb..bd15d1fbac9 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Py.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Py.java @@ -305,6 +305,12 @@ static long keyHash(Object o, String role) { if (o instanceof String) { return o.hashCode(); } + return keyHashSlow(o, role); + } + + /// `keyHash` for anything else, apart: a ParparVM `try` costs a + /// `setjmp` on every entry, the fast cases above included. + private static long keyHashSlow(Object o, String role) { try { return obj(o).hash(); } catch (PyException e) { diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyInstance.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyInstance.java index 5b5d1a27a36..dbf3b2c409c 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyInstance.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyInstance.java @@ -43,9 +43,19 @@ public class PyInstance extends PyObject { /// array shared by every instance of a class instead of a hash table /// per instance, which is both the lookup cost and most of the memory /// a small object would otherwise take. - Shape shape = Shape.EMPTY; - Object[] slots = NO_SLOTS; - private static final Object[] NO_SLOTS = new Object[0]; + /// + /// Null for the empty shape, and `slots` null with it: neither is + /// stored by the constructor, which every object of every class runs. + /// A ParparVM reference store goes through the collector's barrier, a + /// call, so storing the two defaults cost more than the rest of a small + /// object's construction. + Shape shape; + Object[] slots; + + /// The shape, the empty one for null. + final Shape shape() { + return shape == null ? Shape.EMPTY : shape; + } /// For an instance of a subclass of a builtin type, the builtin value it /// extends (see `BuiltinSubclass`); null otherwise. public Object base; @@ -94,7 +104,7 @@ Object own(String name) { return v; } } - int i = shape.indexOf(name); + int i = shape == null ? -1 : shape.indexOf(name); return i < 0 ? null : slots[i]; } @@ -108,7 +118,7 @@ void store(String name, Object value) { setField(f, value); return; } - int i = shape.indexOf(name); + int i = shape == null ? -1 : shape.indexOf(name); if (i >= 0) { slots[i] = value; } else { @@ -117,15 +127,17 @@ void store(String name, Object value) { throw Py.attributeError("'" + cls.name + "' object has no attribute '" + name + "' and no __dict__ for setting new attributes"); } - append(shape.with(name), value); + append(shape().with(name), value); } } /// Moves to `next`, which is this instance's shape plus one name, and /// stores that name's value. void append(Shape next, Object value) { - int n = shape.size(); - if (n == slots.length) { + int n = shape().size(); + if (slots == null) { + slots = new Object[4]; + } else if (n == slots.length) { Object[] grown = new Object[n < 4 ? 4 : n * 2]; System.arraycopy(slots, 0, grown, 0, n); slots = grown; @@ -140,7 +152,7 @@ void append(Shape next, Object value) { String[] attributeNames() { String[] f = fieldNames(); if (f.length == 0) { - return shape.names; + return shape().names; } List out = new ArrayList(); for (int i = 0; i < f.length; i++) { @@ -148,8 +160,9 @@ String[] attributeNames() { out.add(f[i]); } } - for (int i = 0; i < shape.names.length; i++) { - out.add(shape.names[i]); + String[] names = shape().names; + for (int i = 0; i < names.length; i++) { + out.add(names[i]); } return out.toArray(new String[out.size()]); } @@ -258,7 +271,7 @@ void delAttrDirect(String name) { return; } } - int at = shape.indexOf(name); + int at = shape().indexOf(name); if (at < 0) { throw Py.attributeError("'" + cls.name + "' object has no attribute '" + name + "'"); } @@ -266,11 +279,11 @@ void delAttrDirect(String name) { // instance that holds the same names in the same order. String[] names = shape.names; Object[] old = slots; - shape = Shape.EMPTY; - slots = NO_SLOTS; + shape = null; + slots = null; for (int i = 0; i < names.length; i++) { if (i != at) { - append(shape.with(names[i]), old[i]); + append(shape().with(names[i]), old[i]); } } } diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Emitter.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Emitter.java index d1b58e24301..3525e446e31 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Emitter.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Emitter.java @@ -2745,6 +2745,10 @@ private String layoutCode() { } o.append("};\n"); o.append(" ").append(k.javaName).append("(PyType t) {\n super(t);\n }\n"); + String pc = typedEmit != null ? typedEmit.plainCtor(k) : null; + if (pc != null) { + o.append(pc); + } if (!k.fields.isEmpty()) { o.append(" @Override\n protected int fieldIndex(String n) {\n switch (n) {\n"); for (int i = 0; i < k.fields.size(); i++) { diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/TypedEmit.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/TypedEmit.java index 4f3e0d4e581..1c130e7955a 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/TypedEmit.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/TypedEmit.java @@ -4155,8 +4155,11 @@ private E construct(Layouts.Cls k, List args) { } Infer.Fn f = in.fns.get(init); calls.add(f); - constructors.add(k); String al = argList(f, 1, args); + if (plainCtor(k) != null) { + return new E("new " + k.javaName + "(" + k.typeVar + ", " + al + ")", Ty.inst(k, false)); + } + constructors.add(k); return new E("n_" + k.javaName + "(" + al + ")", Ty.inst(k, false)); } @@ -4364,6 +4367,91 @@ String entry(Infer.Fn f, List locals) { } /// The Java declaration of a typed field, or null for an object one. + private final Map plainCtors = new HashMap(); + + /// A Java constructor of `k`'s class doing its `__init__` in place, + /// when that only stores parameters in fields (`self.x = x`); or null. + /// Typed code then constructs with one `new`: ParparVM inlines no call, + /// and the allocation, the generic constructor and a separate + /// `__init__` were three of them on the path every object takes. + String plainCtor(Layouts.Cls k) { + if (!plainCtors.containsKey(k)) { + plainCtors.put(k, plainCtor0(k)); + } + return plainCtors.get(k); + } + + private String plainCtor0(Layouts.Cls k) { + for (String h : new String[] {"__new__", "__setattr__", "__getattribute__", "__getattr__", "__del__"}) { + if (Layouts.resolver(k, h) != null) { + return null; + } + } + Layouts.Cls o = Layouts.resolver(k, "__init__"); + FunctionDef init = o == null ? null : o.typable.get("__init__"); + Infer.Fn f = init == null ? null : in.fns.get(init); + if (f == null || !typed.containsKey(f) || f.names.size() < 2 || !init.decorators.isEmpty() + || init.args.vararg != null || init.args.kwarg != null || !init.args.kwonly.isEmpty()) { + return null; + } + String self = f.names.get(0); + StringBuilder sig = new StringBuilder(); + StringBuilder norm = new StringBuilder(); + for (int i = 1; i < f.names.size(); i++) { + sig.append(", ").append(param(f.params[i], "a" + i, "b" + i)); + } + StringBuilder body = new StringBuilder(); + Set seen = new HashSet(); + Set normed = new HashSet(); + for (Stmt st : init.body) { + if (!(st instanceof Assign) || ((Assign) st).targets.size() != 1 + || !(((Assign) st).targets.get(0) instanceof Attribute) || !(((Assign) st).value instanceof Name)) { + return null; + } + Attribute t = (Attribute) ((Assign) st).targets.get(0); + int p = f.names.indexOf(((Name) ((Assign) st).value).id); + if (!(t.value instanceof Name) || !((Name) t.value).id.equals(self) || p < 1 || !seen.add(t.attr)) { + return null; + } + Layouts.Cls ow = Infer.fieldOwner(k, t.attr); + if (ow == null) { + return null; + } + Ty ft = typedField(ow, t.attr); + Ty pt = f.params[p]; + String m = Emitter.mangle(t.attr); + if (ft != null && ft.k == Ty.K.INT) { + if (pt.k != Ty.K.INT) { + return null; + } + if (normed.add(p)) { + // As the typed __init__ does: a caller holding only the + // object passes it whole. + norm.append(" if (b").append(p).append(" instanceof Long) {\n a").append(p) + .append(" = ((Long) b").append(p).append(").longValue();\n b").append(p) + .append(" = null;\n }\n"); + } + // The object half is almost always null, the field's + // default: not stored, a barrier saved (see PyInstance). + body.append(" t_").append(m).append(" = a").append(p).append(";\n if (b").append(p) + .append(" != null) {\n b_").append(m).append(" = b").append(p) + .append(";\n }\n"); + } else if (ft != null) { + if (pt.k == Ty.K.INT || !jt(ft).equals(jt(pt))) { + return null; + } + body.append(" t_").append(m).append(" = a").append(p).append(";\n"); + } else { + if (!in.definitelySet(ow, t.attr)) { + return null; + } + String v = pt.k == Ty.K.INT ? dual("a" + p, "b" + p).c : box(new E("a" + p, pt)); + body.append(" f_").append(m).append(" = ").append(v).append(";\n"); + } + } + return " " + k.javaName + "(PyType t" + sig + ") {\n super(t);\n" + norm + body + " }\n"; + } + String fieldDecl(Layouts.Cls k, String f) { Ty t = typedField(k, f); if (t != null && t.k == Ty.K.INT) { From 89021bbe6ca4e9c2ac545a4a4db60e41a7530c5c Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 04:40:39 +0200 Subject: [PATCH 028/112] Python: str + chains into one presized StringBuilder; hoist statement-top arithmetic over impure calls Co-Authored-By: Claude Opus 5.5 --- .../python/transpiler/codegen/TypedEmit.java | 98 +++++++++++++++++-- 1 file changed, 91 insertions(+), 7 deletions(-) diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/TypedEmit.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/TypedEmit.java index 1c130e7955a..cead97e54a1 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/TypedEmit.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/TypedEmit.java @@ -152,6 +152,11 @@ static E dual(String l, String b) { /// pushed result (see `E.pk`); and the call a hoisted leaf is, likewise. private Expr pushTop; private Expr pushLeaf; + /// See `top`; and whether the leaves being collected or evaluated are + /// that expression's. + private Expr hoistTop; + private boolean topLeaves; + private boolean leafImpureOk; private final Map longBodies = new LinkedHashMap(); /// The argument intervals of the calls the function being emitted makes. private Map sites = new HashMap(); @@ -1150,7 +1155,7 @@ && localType(((Name) ((Assign) s).targets.get(0)).id).k == Ty.K.LIST pushTop = v; E e; try { - e = expr(v); + e = top(v); } finally { pushTop = saved; } @@ -1702,15 +1707,30 @@ private void storePair(String l, String o, E v, StringBuilder b, int ind) { b.append("}\n"); } + /// `e`, the whole value of a statement that does nothing else with an + /// effect: an augmented assignment to a local, or a `return`. Its + /// arithmetic may then be hoisted even over calls that are not pure + /// (see `collectLeaves`): its leaves run in order, as Python runs them, + /// and nothing else in the statement runs between them. + private E top(Expr e) { + Expr saved = hoistTop; + hoistTop = e; + try { + return expr(e); + } finally { + hoistTop = saved; + } + } + private void augAssign(AugAssign a, StringBuilder b, int ind) { Expr t = a.target; if (t instanceof Name && longLocals.contains(((Name) t).id)) { - store(t, expr(new BinOp(t, a.op, a.value)), b, ind); + store(t, top(new BinOp(t, a.op, a.value)), b, ind); return; } if (t instanceof Name && (localType(((Name) t).id).k == Ty.K.INT || localType(((Name) t).id).k == Ty.K.FLOAT) && sc.resolve(((Name) t).id) == Scope.Binding.LOCAL) { - store(t, expr(new BinOp(t, a.op, a.value)), b, ind); + store(t, top(new BinOp(t, a.op, a.value)), b, ind); return; } if (t instanceof Attribute && ((Attribute) t).value instanceof Name @@ -2132,6 +2152,12 @@ E expr(Expr e) { } } BinOp b = (BinOp) e; + if ("+".equals(b.op) && b.left instanceof BinOp && "+".equals(((BinOp) b.left).op)) { + E c = strChain(b); + if (c != null) { + return c; + } + } return binop(b.op, expr(b.left), expr(b.right), b.right); } if (e instanceof UnaryOp) { @@ -2512,7 +2538,12 @@ private static boolean arith(Expr e) { /// conditionally. private List leaves(Expr e) { List out = new ArrayList(); - return collectLeaves(e, out) ? out : null; + topLeaves = e == hoistTop; //NOPMD CompareObjectsWithEquals - identity is meant + try { + return collectLeaves(e, out) ? out : null; + } finally { + topLeaves = false; + } } private boolean collectLeaves(Expr e, List out) { @@ -2527,7 +2558,7 @@ private boolean collectLeaves(Expr e, List out) { return collectLeaves(((BinOp) e).left, out) && collectLeaves(((BinOp) e).right, out); } boolean quiet = e instanceof Constant || e instanceof Name && sc.resolve(((Name) e).id) == Scope.Binding.LOCAL; - if (!quiet && !(stmtPure && condDepth == 0)) { + if (!quiet && !((stmtPure || topLeaves) && condDepth == 0)) { return false; } out.add(e); @@ -2581,18 +2612,22 @@ private E hoistInt(Expr e) { if (ls == null) { return null; } + boolean impureOk = e == hoistTop; //NOPMD CompareObjectsWithEquals - identity is meant Map savedLeaves = leafSubst; leafSubst = new IdentityHashMap(savedLeaves); try { List pushedLeaves = new ArrayList(); for (Expr x : ls) { Expr savedLeaf = pushLeaf; + boolean savedOk = leafImpureOk; pushLeaf = x; + leafImpureOk = impureOk; E v; try { v = expr(x); } finally { pushLeaf = savedLeaf; + leafImpureOk = savedOk; } if (v.pk != null) { pushedLeaves.add(x); @@ -3031,6 +3066,55 @@ private static String intHelper(String h) { return o.append(";\n }\n").toString(); } + /// `a + b + c ...` of strs, three or more: one `StringBuilder` sized + /// for the result, where nested concatenation made a string per `+`; + /// a `str(i)` of an int is appended as the long itself. Null when the + /// chain is not of strs. + private E strChain(BinOp b) { + List ops = new ArrayList(); + Expr x = b; + while (x instanceof BinOp && "+".equals(((BinOp) x).op)) { + ops.add(0, ((BinOp) x).right); + x = ((BinOp) x).left; + } + ops.add(0, x); + for (Expr o : ops) { + Ty t = in.type(o, cur, sc); + if (t.k != Ty.K.STR || t.opt) { + return null; + } + } + StringBuilder sb = new StringBuilder(); + int cap = 0; + for (Expr o : ops) { + if (o instanceof Call && ((Call) o).func instanceof Name && "str".equals(((Name) ((Call) o).func).id) + && ((Call) o).args.size() == 1 && ((Call) o).keywords.isEmpty() + && !(((Call) o).args.get(0) instanceof Starred) + && sc.resolve("str") == Scope.Binding.BUILTIN) { + E v = expr(((Call) o).args.get(0)); + if (v.t.k == Ty.K.INT && v.lc != null && !v.t.opt) { + sb.append(".append(").append(v.lc).append(')'); + cap += 20; + continue; + } + E sv = builtin("str", java.util.Collections.singletonList(v)); + if (sv == null || sv.t.k != Ty.K.STR || sv.t.opt) { + throw new Unsupported("str() of " + v.t); + } + sb.append(".append(").append(sv.c).append(')'); + cap += 16; + continue; + } + E v = expr(o); + if (v.t.k != Ty.K.STR || v.t.opt) { + throw new Unsupported("str chain over " + v.t); + } + cap += o instanceof Constant ? v.c.length() - 2 : 16; + sb.append(".append(").append(v.c).append(')'); + } + return new E("new StringBuilder(" + cap + ")" + sb + ".toString()", Ty.STR); + } + E binop(String op, E l, E r, Expr right) { Ty t = Infer.binop(op, l.t, r.t, right); if (l.t.k == Ty.K.LIST && "*".equals(op) && (r.t.k == Ty.K.INT || r.t.k == Ty.K.BOOL)) { @@ -3393,8 +3477,8 @@ && ordOf(c, ((Name) c.args.get(0)).id)) { ? longName(f) + "(" + longArgList(f, lp, a) + ")" : typed.get(f) + "(" + argList(f, 0, a) + ")"; Ty site = in.siteRet.get(c); - if (f.ret.k == Ty.K.INT && pre != null && stmtPure && condDepth == 0 && (site == null || site.k == Ty.K.INT) - && (c == pushTop || c == pushLeaf)) { //NOPMD CompareObjectsWithEquals - identity is meant + if (f.ret.k == Ty.K.INT && pre != null && condDepth == 0 && (site == null || site.k == Ty.K.INT) + && (c == pushTop || c == pushLeaf && (stmtPure || leafImpureOk))) { //NOPMD CompareObjectsWithEquals - identity is meant String r = fresh(); indent(pre, preInd); pre.append("long ").append(r).append(" = ").append(call).append(";\n"); From 292091fa2a377fee3da5c59614ecd1b28dae14b4 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 05:54:02 +0200 Subject: [PATCH 029/112] Python: int-keyed dict get/set without boxing the lookup key; bulk dict clear; direct dict len Co-Authored-By: Claude Opus 5.5 --- .../com/codename1/python/runtime/Hashing.java | 3 + .../com/codename1/python/runtime/PyDict.java | 72 +++++++++++++++++-- .../codename1/python/runtime/PyInstance.java | 5 ++ .../com/codename1/python/runtime/Typed.java | 22 ++++++ .../python/transpiler/codegen/TypedEmit.java | 16 +++++ 5 files changed, 114 insertions(+), 4 deletions(-) diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Hashing.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Hashing.java index e688807f5c6..59a7f94774d 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Hashing.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Hashing.java @@ -48,6 +48,9 @@ private static long fix(long h) { } static long ofLong(long v) { + if (v >= 0 && v < MODULUS) { + return v; + } if (v >= 0) { return fix(v % MODULUS); } diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyDict.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyDict.java index 26fd16d99c0..7cef358dcb9 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyDict.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyDict.java @@ -158,6 +158,62 @@ Object count(Object key, Object dflt, Object inc) { return r; } + /// The slot of int key `k` (hash `h`), as `find` answers; the key is + /// compared without being boxed. + private int findLong(long k, long h) { + int mask = index.length - 1; + int i = (int) (h & mask); + long perturb = h & Long.MAX_VALUE; + int freeSlot = -1; + while (true) { + int e = index[i]; + if (e == EMPTY) { + return -1 - (freeSlot >= 0 ? freeSlot : i); + } + if (e == DUMMY) { + if (freeSlot < 0) { + freeSlot = i; + } + } else if (hashes[e] == h) { + Object x = keys[e]; + if (x instanceof Long ? ((Long) x).longValue() == k : keyEq(x, h, Long.valueOf(k), h)) { + return i; + } + } + perturb >>>= 5; + i = (int) ((i * 5L + perturb + 1) & mask); + } + } + + /// `self.get(k)` for an int key, or `null` when absent. + Object getLong(long k) { + int s = findLong(k, Hashing.ofLong(k)); + return s >= 0 ? values[index[s]] : null; + } + + /// `self[k] = value` for an int key; the key is boxed only when it is new. + void setLong(long k, Object value) { + long h = Hashing.ofLong(k); + int s = findLong(k, h); + if (s >= 0) { + values[index[s]] = value; + return; + } + if (used == keys.length) { + grow(); + s = findLong(k, h); + } + index[-1 - s] = used; + keys[used] = Long.valueOf(k); + values[used] = value; + hashes[used] = h; + used++; + size++; + if (used * 3 >= index.length * 2) { + rebuild(index.length * 2); + } + } + /// `self[key]`, or `null` when absent. public Object getOrNull(Object key) { int s = find(key, Py.keyHash(key, "a dict key")); @@ -298,10 +354,8 @@ public void clear() { // capacity as Java's HashMap does: refilling it would otherwise // regrow it from eight entries, a dozen arrays of garbage each // time. - for (int i = 0; i < used; i++) { - keys[i] = null; - values[i] = null; - } + nullOut(keys, used); + nullOut(values, used); int[] ix = index; for (int i = 0; i < ix.length; i++) { ix[i] = EMPTY; @@ -316,6 +370,16 @@ public void clear() { size = 0; } + private static final Object[] NULLS = new Object[256]; + + /// Nulls the first `n` slots of `a` in bulk copies: one store per slot + /// would pass each through the collector's write barrier. + private static void nullOut(Object[] a, int n) { + for (int i = 0; i < n; i += NULLS.length) { + System.arraycopy(NULLS, 0, a, i, Math.min(NULLS.length, n - i)); + } + } + /// The largest entry capacity `clear` keeps. private static final int KEEP_ON_CLEAR = 1 << 17; diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyInstance.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyInstance.java index dbf3b2c409c..445d51e4272 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyInstance.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyInstance.java @@ -629,6 +629,11 @@ public void setItem(Object key, Object value) { self.store(Py.toStr(key), value); } + @Override + void setLong(long k, Object value) { + setItem(Long.valueOf(k), value); + } + @Override public void delItem(Object key) { super.delItem(key); diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Typed.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Typed.java index bbbe79bcf6c..ad0fcbe7c0c 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Typed.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Typed.java @@ -460,8 +460,30 @@ public static Object extend(PyList l, PyList other) { /// `len(o)` where `f` is what the name `len` holds, for typed code, /// which takes the result to be an int as the builtin's is. + /// `m.get(k)` for an int key: a dict answers without the call or a + /// boxed key; anything else goes through `site`. + public static Object dget(AttrSite site, Object m, long k) { + if (m instanceof PyDict) { + Object v = ((PyDict) m).getLong(k); + return v != null ? v : Py.NONE; + } + return site.call1(m, Long.valueOf(k)); + } + + /// `m[k] = v` for an int key. + public static void dset(Object m, long k, Object v) { + if (m instanceof PyDict) { + ((PyDict) m).setLong(k, v); + return; + } + Py.setitem(m, Long.valueOf(k), v); + } + public static long len(Object f, Object o) { if (f == Builtins.LEN) { //NOPMD CompareObjectsWithEquals - identity is meant + if (o instanceof PyDict) { + return ((PyDict) o).size; + } return Py.len(o); } Object r = Py.call(f, o); diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/TypedEmit.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/TypedEmit.java index cead97e54a1..08c6bc8f4ea 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/TypedEmit.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/TypedEmit.java @@ -1613,6 +1613,11 @@ private void store(Expr t, E v, StringBuilder b, int ind) { v = spillTo(v, b, ind); } indent(b, ind); + String k = l.t.k == Ty.K.LIST ? null : intKey(i); + if (k != null) { + b.append("Typed.dset(").append(box(l)).append(", ").append(k).append(", ").append(box(v)).append(");\n"); + return; + } b.append("Py.setitem(").append(box(l)).append(", ").append(box(i)).append(", ").append(box(v)) .append(");\n"); return; @@ -3562,6 +3567,11 @@ && ordOf(c, ((Name) c.args.get(0)).id)) { return listMethod(r, a.attr, args); } if (r.t.k != Ty.K.INST) { + String k = "get".equals(a.attr) && args.size() == 1 && c.keywords.isEmpty() ? intKey(args.get(0)) : null; + if (k != null) { + // A plain dict looks an int key up without boxing it. + return new E("Typed.dget(" + em.attrSite(a.attr) + ", " + box(r) + ", " + k + ")", Ty.DYN); + } return siteCall(r, a.attr, args); } List cand = in.candidates(r.t.cls, a.attr); @@ -3600,6 +3610,12 @@ && ordOf(c, ((Name) c.args.get(0)).id)) { throw new Unsupported("call"); } + /// `x` as a Java `long` when it is an int at hand as one, cheap and free + /// of effects (see `E.lc`); or null. + private static String intKey(E x) { + return x.t.k == Ty.K.INT && !x.t.opt && x.pk == null && x.lc != null && (plain(x.lc) || literal(x.lc)) ? x.lc : null; + } + /// `r.attr(args)` as the generic code calls a method: through an inline /// cache, with an untyped result. private E siteCall(E r, String attr, List args) { From 8b2f406ecd3fc1161114a93149a73ce28d830601 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 05:58:57 +0200 Subject: [PATCH 030/112] Python: int lists hold 32-bit items, as Java's int[] does; wider ints box the list Co-Authored-By: Claude Opus 5.5 --- .../java/com/codename1/python/runtime/Py.java | 8 +- .../com/codename1/python/runtime/PyList.java | 74 ++++++++++--------- .../com/codename1/python/runtime/Typed.java | 8 +- .../python/transpiler/codegen/TypedEmit.java | 6 +- .../resources/conformance/wide_int_lists.out | 7 ++ .../resources/conformance/wide_int_lists.py | 41 ++++++++++ 6 files changed, 99 insertions(+), 45 deletions(-) create mode 100644 maven/python-transpiler/src/test/resources/conformance/wide_int_lists.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/wide_int_lists.py diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Py.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Py.java index bd15d1fbac9..584b3520543 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Py.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Py.java @@ -1283,7 +1283,7 @@ public static Object getitem(Object o, Object key) { if (key instanceof Long) { long i = ((Long) key).longValue(); if (i >= 0 && i < l.size) { - long[] li = l.ints; + int[] li = l.ints; if (li != null) { return Long.valueOf(li[(int) i]); } @@ -1609,10 +1609,10 @@ public static void setitem(Object o, Object key, Object v) { if (key instanceof Long) { long i = ((Long) key).longValue(); if (i >= 0 && i < l.size) { - long[] li = l.ints; + int[] li = l.ints; if (li != null) { - if (v instanceof Long) { - li[(int) i] = ((Long) v).longValue(); + if (v instanceof Long && PyList.fits(((Long) v).longValue())) { + li[(int) i] = (int) ((Long) v).longValue(); return; } } else if (l.floats == null) { diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyList.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyList.java index f82064a836f..21479f1eb4e 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyList.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyList.java @@ -42,11 +42,12 @@ public final class PyList extends PyObject { /// objects first ([#objects()]), once. public double[] floats; - /// The items of a list that has only ever held ints, unboxed, as - /// `floats` is for floats; never both. A Long that fits 61 bits is an - /// immediate under ParparVM, so reading one back allocates nothing, and - /// storing one needs no GC write barrier. - public long[] ints; + /// The items of a list that has only ever held ints that fit 32 bits, + /// unboxed, as `floats` is for floats; never both. Four bytes an item, as + /// a Java `int[]` takes; a wider int turns the list into objects, where + /// a Long that fits 61 bits is an immediate under ParparVM, so reading + /// one back allocates nothing either. + public int[] ints; /// Set on a list Py.recycleSlice or Py.recycleList made for a local the /// transpiler proved never shares it; only such a list is reused. A list @@ -80,7 +81,7 @@ static PyList of(Object[] items) { /// The items as objects, boxing an unboxed list for good. Object[] objects() { - long[] l = ints; + int[] l = ints; if (l != null) { Object[] o = new Object[Math.max(4, l.length)]; for (int i = 0; i < size; i++) { @@ -140,7 +141,7 @@ public int size() { } public Object get(int i) { - long[] l = ints; + int[] l = ints; if (l != null) { return Long.valueOf(l[i]); } @@ -153,8 +154,8 @@ public void append(Object v) { appendFloat(((Double) v).doubleValue()); return; } - if (v instanceof Long && floats == null) { - appendInt(((Long) v).longValue()); + if (v instanceof Long && floats == null && fits(((Long) v).longValue())) { + appendInt((int) ((Long) v).longValue()); return; } objects(); @@ -162,13 +163,18 @@ public void append(Object v) { appendObject(v); } - private void appendInt(long v) { - long[] l = ints; + /// Whether `v` may go into `ints`. + static boolean fits(long v) { + return (int) v == v; + } + + private void appendInt(int v) { + int[] l = ints; if (l == null) { - l = new long[4]; + l = new int[4]; ints = l; } else if (size == l.length) { - long[] n = new long[size * 2 + 4]; + int[] n = new int[size * 2 + 4]; System.arraycopy(l, 0, n, 0, size); ints = n; l = n; @@ -206,7 +212,7 @@ public void extend(Object iterable) { if (o.ints != null && intsOrEmpty()) { // One copy, into an array of the exact size when this list // was empty: no element by element growth. - long[] l = intRoom(n); + int[] l = intRoom(n); System.arraycopy(o.ints, 0, l, size, n); size += n; return; @@ -228,10 +234,10 @@ private boolean intsOrEmpty() { } /// `ints`, with room for `n` more after `size`. - private long[] intRoom(int n) { - long[] l = ints; + private int[] intRoom(int n) { + int[] l = ints; if (l == null || l.length - size < n) { - long[] g = new long[l == null ? n : Math.max(size + n, size * 2 + 4)]; + int[] g = new int[l == null ? n : Math.max(size + n, size * 2 + 4)]; if (l != null) { System.arraycopy(l, 0, g, 0, size); } @@ -246,9 +252,9 @@ public void extendRepeat(Object v, long n) { if (n <= 0) { return; } - if (v instanceof Long && n <= Integer.MAX_VALUE - size && intsOrEmpty()) { - long x = ((Long) v).longValue(); - long[] l = intRoom((int) n); + if (v instanceof Long && n <= Integer.MAX_VALUE - size && intsOrEmpty() && fits(((Long) v).longValue())) { + int x = (int) ((Long) v).longValue(); + int[] l = intRoom((int) n); int end = size + (int) n; for (int i = size; i < end; i++) { l[i] = x; @@ -412,13 +418,13 @@ public Object mul(Object o) { if (ints == null && floats == null && size > 0) { boolean all = true; for (int i = 0; i < size && all; i++) { - all = items[i] instanceof Long; + all = items[i] instanceof Long && fits(((Long) items[i]).longValue()); } if (all) { // `[0] * n` is a list of ints: unboxed. - long[] one = new long[size]; + int[] one = new int[size]; for (int i = 0; i < size; i++) { - one[i] = ((Long) items[i]).longValue(); + one[i] = (int) ((Long) items[i]).longValue(); } return repeatInts(one, n); } @@ -444,9 +450,9 @@ public Object mul(Object o) { return new PyList(r); } - private PyList repeatInts(long[] one, long n) { + private PyList repeatInts(int[] one, long n) { int k = size; - long[] l = new long[Math.max(4, (int) (k * n))]; + int[] l = new int[Math.max(4, (int) (k * n))]; for (int i = 0; i < n; i++) { System.arraycopy(one, 0, l, i * k, k); } @@ -505,7 +511,7 @@ public PyList getSlice(Object start, Object stop, Object step) { int lo = PySlice.lower(start, stop, st, size); int hi = PySlice.upper(stop, st, size); int n = PySlice.count(lo, hi, st); - long[] out = new long[Math.max(4, n)]; + int[] out = new int[Math.max(4, n)]; if (st == 1) { System.arraycopy(ints, lo, out, 0, n); } else { @@ -546,9 +552,9 @@ PyList sliceInto(PyList dst, Object start, Object stop, Object step) { int lo = PySlice.lower(start, stop, st, size); int hi = PySlice.upper(stop, st, size); int n = PySlice.count(lo, hi, st); - long[] out = dst.ints; + int[] out = dst.ints; if (out == null || out.length < n) { - out = new long[Math.max(4, n)]; + out = new int[Math.max(4, n)]; } if (st == 1) { System.arraycopy(ints, lo, out, 0, n); @@ -673,9 +679,9 @@ void setSliceFrom(Object start, Object stop, Object step, PyList src, Object sSt s0 = size - 1; } if (lo == 0 && hr >= 0 && hi == s0 + 1 && li) { - long[] it = ints; + int[] it = ints; for (int i = 0, j = (int) s0; i < j; i++, j--) { - long t = it[i]; + int t = it[i]; it[i] = it[j]; it[j] = t; } @@ -747,8 +753,8 @@ private static int bound(Object b, int dflt, int n) { @Override public void setItem(Object key, Object value) { - if (ints != null && value instanceof Long && !(key instanceof PySlice)) { - ints[checkIndex(key)] = ((Long) value).longValue(); + if (ints != null && value instanceof Long && !(key instanceof PySlice) && fits(((Long) value).longValue())) { + ints[checkIndex(key)] = (int) ((Long) value).longValue(); return; } if (floats != null && value instanceof Double && !(key instanceof PySlice)) { @@ -911,9 +917,9 @@ private static boolean before(Object x, Object y, boolean rev) { public void reverse() { if (ints != null) { - long[] l = ints; + int[] l = ints; for (int i = 0, j = size - 1; i < j; i++, j--) { - long t = l[i]; + int t = l[i]; l[i] = l[j]; l[j] = t; } diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Typed.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Typed.java index ad0fcbe7c0c..af50b65b982 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Typed.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Typed.java @@ -323,7 +323,7 @@ public static Object get(PyList l, Object i) { if (i instanceof Long) { long k = ((Long) i).longValue(); if (k >= 0 && k < l.size) { - long[] n = l.ints; + int[] n = l.ints; if (n != null) { return Long.valueOf(n[(int) k]); } @@ -354,10 +354,10 @@ public static void set(PyList l, Object i, Object v) { if (i instanceof Long) { long k = ((Long) i).longValue(); if (k >= 0 && k < l.size) { - long[] n = l.ints; + int[] n = l.ints; if (n != null) { - if (v instanceof Long) { - n[(int) k] = ((Long) v).longValue(); + if (v instanceof Long && PyList.fits(((Long) v).longValue())) { + n[(int) k] = (int) ((Long) v).longValue(); return; } } else if (l.floats == null) { diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/TypedEmit.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/TypedEmit.java index 08c6bc8f4ea..38e2f237265 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/TypedEmit.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/TypedEmit.java @@ -1635,9 +1635,9 @@ private void store(Expr t, E v, StringBuilder b, int ind) { String[] vp = parts(v); b.append("if (").append(ix[1]).append(inRange(l, ix[0])).append(" && ").append(l.c) .append(".ints != null").append("null".equals(vp[1]) ? "" : " && " + vp[1] + " == null") - .append(") {\n"); + .append(" && (int) ").append(vp[0]).append(" == ").append(vp[0]).append(") {\n"); indent(b, ind + 1); - b.append(l.c).append(".ints[(int) ").append(ix[0]).append("] = ").append(vp[0]).append(";\n"); + b.append(l.c).append(".ints[(int) ").append(ix[0]).append("] = (int) ").append(vp[0]).append(";\n"); indent(b, ind); b.append("} else {\n"); indent(b, ind + 1); @@ -2450,7 +2450,7 @@ private static String pyOp(String op) { " if (l != null && i instanceof Long) {", " long k = ((Long) i).longValue();", " if (k >= 0 && k < l.size) {", - " long[] n = l.ints;", + " int[] n = l.ints;", " if (n != null) {", " return Long.valueOf(n[(int) k]);", " }", diff --git a/maven/python-transpiler/src/test/resources/conformance/wide_int_lists.out b/maven/python-transpiler/src/test/resources/conformance/wide_int_lists.out new file mode 100644 index 00000000000..24f381949dd --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/wide_int_lists.out @@ -0,0 +1,7 @@ +[0, 3, 6, 2147483648, -2147483649, 1180591620717411303424, 18, 21] +[1, 2, 3, 1099511627776, 7, 7, 7] 1099511627803 +[2147483647, -2147483648, 5, 5] -1 +[8589934592, 8589934592, 8589934592] [8589934592, 8589934592] +[0, 3000000000, 6000000000, 9000000000, 12000000000, 15000000000] +9 4 None 10 +t diff --git a/maven/python-transpiler/src/test/resources/conformance/wide_int_lists.py b/maven/python-transpiler/src/test/resources/conformance/wide_int_lists.py new file mode 100644 index 00000000000..3599f22be98 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/wide_int_lists.py @@ -0,0 +1,41 @@ +# Lists of ints are held four bytes an item while every item fits 32 bits; +# a wider one must turn the list into objects without losing a value. +def fill(n: int) -> list: + a = [0] * n + for i in range(n): + a[i] = i * 3 + return a + + +def run() -> None: + a = fill(8) + a[3] = 2 ** 31 + a[4] = -2 ** 31 - 1 + a[5] = 2 ** 70 + print(a) + b = [1, 2, 3] + b.append(1 << 40) + b.extend([7] * 3) + print(b, sum(b)) + c = [5] * 4 + c[0] = 2147483647 + c[1] = -2147483648 + print(c, c[0] + c[1]) + d = [2 ** 33] * 3 + d.reverse() + print(d, d[1:]) + e = fill(6) + for i in range(6): + e[i] = e[i] * 1000000000 + print(e) + m = {} + for k in range(5): + m[k] = k * k + m[k + 2 ** 62] = k + print(m.get(3), m.get(2 ** 62 + 4), m.get(99), len(m)) + m.clear() + m[True] = "t" + print(m.get(1)) + + +run() From 1c4dc1de0d9ac31e9dcf9e82cab6d97abb448e45 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 06:01:58 +0200 Subject: [PATCH 031/112] Python: math.sqrt/sin/cos/fmod domain checks in line, Math called directly Co-Authored-By: Claude Opus 5.5 --- .../python/transpiler/codegen/TypedEmit.java | 39 +++++++++++++++++-- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/TypedEmit.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/TypedEmit.java index 38e2f237265..cb7920587e0 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/TypedEmit.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/TypedEmit.java @@ -3527,15 +3527,20 @@ && ordOf(c, ((Name) c.args.get(0)).id)) { throw new Unsupported("call of " + n); } if (in.isMathCall(c, sc)) { - StringBuilder sb = new StringBuilder("PyMath.").append(((Attribute) c.func).attr).append('('); + String fn = ((Attribute) c.func).attr; + StringBuilder sb = new StringBuilder("PyMath.").append(fn).append('('); + List ds = new ArrayList(); for (int i = 0; i < c.args.size(); i++) { E x = expr(c.args.get(i)); if (x.t.opt || x.t.k != Ty.K.INT && x.t.k != Ty.K.FLOAT) { throw new Unsupported("math of " + x.t); } - sb.append(i == 0 ? "" : ", ").append(toD(x)); + ds.add(toD(x)); + sb.append(i == 0 ? "" : ", ").append(ds.get(i)); } - return new E(sb.append(')').toString(), Ty.FLOAT); + sb.append(')'); + String in0 = inlineMath(fn, ds); + return new E(in0 != null ? "(" + in0 + " : " + sb + ")" : sb.toString(), Ty.FLOAT); } if (c.func instanceof Attribute) { Attribute a = (Attribute) c.func; @@ -3610,6 +3615,34 @@ && ordOf(c, ((Name) c.args.get(0)).id)) { throw new Unsupported("call"); } + /// The guard and the `Math` call of `math.fn(ds)` when its domain can be + /// tested in line (`guard ? Math.fn(..)`, the `PyMath` call raising the + /// error after the colon); or null. A `PyMath` call is two calls ParparVM + /// does not inline, in what is usually an inner loop. `x - x == 0` holds + /// for every finite `x`, not for an infinity or NaN. + private static String inlineMath(String fn, List ds) { + for (String d : ds) { + if (!plain(d) && !literal(d)) { + return null; + } + } + if (ds.size() == 1) { + String x = ds.get(0); + if ("sqrt".equals(fn)) { + return x + " >= 0.0 ? Math.sqrt(" + x + ")"; + } + if ("sin".equals(fn) || "cos".equals(fn)) { + return x + " - " + x + " == 0.0 ? Math." + fn + "(" + x + ")"; + } + } else if (ds.size() == 2 && "fmod".equals(fn)) { + String x = ds.get(0); + String y = ds.get(1); + // Finite `x` and a nonzero `y`: C's fmod, which `%` is. + return x + " - " + x + " == 0.0 && " + y + " != 0.0 ? " + x + " % " + y; + } + return null; + } + /// `x` as a Java `long` when it is an int at hand as one, cheap and free /// of effects (see `E.lc`); or null. private static String intKey(E x) { From b9c559760e5548e73865c448986fa78536238fdd Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 06:33:38 +0200 Subject: [PATCH 032/112] Python: compare against Codename One Java; startup, kernel and Flask charts with GraalPy and RSS; rerun on CPython 3.14, PyPy 3.11, GraalPy 25.4 Co-Authored-By: Claude Opus 5.5 --- docs/developer-guide/Python-Interop.asciidoc | 144 +++++----- .../img/python-flask-memory.svg | 45 +++ docs/developer-guide/img/python-flask-rps.svg | 27 ++ .../img/python-kernels-rss.svg | 69 +++++ .../img/python-kernels-time.svg | 67 +++++ docs/developer-guide/img/python-perf-rss.svg | 188 ++++++------- docs/developer-guide/img/python-perf-time.svg | 224 +++++++-------- docs/developer-guide/img/python-startup.svg | 36 +++ .../codename1/python/runtime/PyInstance.java | 5 +- maven/python-transpiler/spotbugs-exclude.xml | 16 ++ .../python/transpiler/codegen/TypedEmit.java | 18 +- scripts/python-compare/README.md | 9 +- scripts/python-compare/RESULTS.md | 68 ++--- scripts/python-compare/common-results.json | 264 +++++++++--------- scripts/python-compare/common_compare.py | 11 +- scripts/python-compare/compare-results.json | 154 ++++++++++ scripts/python-compare/compare.py | 26 +- scripts/python-compare/render_charts.py | 54 +++- 18 files changed, 949 insertions(+), 476 deletions(-) create mode 100644 docs/developer-guide/img/python-flask-memory.svg create mode 100644 docs/developer-guide/img/python-flask-rps.svg create mode 100644 docs/developer-guide/img/python-kernels-rss.svg create mode 100644 docs/developer-guide/img/python-kernels-time.svg create mode 100644 docs/developer-guide/img/python-startup.svg create mode 100644 scripts/python-compare/compare-results.json diff --git a/docs/developer-guide/Python-Interop.asciidoc b/docs/developer-guide/Python-Interop.asciidoc index 21694c51d9e..6259b2ac3cf 100644 --- a/docs/developer-guide/Python-Interop.asciidoc +++ b/docs/developer-guide/Python-Interop.asciidoc @@ -180,49 +180,62 @@ FastAPI WebSockets and middleware. === Compared with CPython, PyPy and GraalPy -The same programs, run on one Apple silicon Mac, with -`scripts/python-compare/compare.py`. Codename One rows are native binaries; -GraalPy is the Community edition, 25.4, on its own JVM. +The same programs, run on one Apple silicon Mac with +`scripts/python-compare/compare.py` and +`scripts/python-compare/common_compare.py`. Codename One rows are native +executables. The interpreters are CPython 3.14, PyPy 3.11 (7.3.20) and GraalPy +3.13 (Community edition 25.4, on its own JVM). Every program runs in a process +of its own, so each memory figure is that program's peak resident memory. -Startup, a program that prints one line: +==== Startup + +A program that prints one line, best of ten runs: + +image::img/python-startup.svg["Startup wall time and peak memory on CPython, PyPy, GraalPy and Codename One"] [cols="1,1,1",options="header"] |=== | | Wall time (ms) | Peak memory (MB) -| CPython 3.14 | 20.3 | 14.3 -| PyPy 3.11 | 18.2 | 29.2 -| GraalPy 3.13 | 34.9 | 116.6 -| Codename One | 5.9 | 8.7 +| CPython 3.14 | 15.1 | 14.4 +| PyPy 3.11 | 13.9 | 29.1 +| GraalPy 3.13 | 28.2 | 118.6 +| Codename One | 4.7 | 8.6 |=== -CPU kernels, fastest round in milliseconds, then the process's peak memory in -MB: +==== Python kernels + +Classic benchmark programs written for Python (`scripts/python-bench/kernels`), +fastest round in milliseconds, then peak memory in MB: + +image::img/python-kernels-time.svg["Run time of the Python kernels on CPython, PyPy, GraalPy and Codename One"] + +image::img/python-kernels-rss.svg["Peak memory of the Python kernels on CPython, PyPy, GraalPy and Codename One"] [cols="2,1,1,1,1,1,1,1,1",options="header"] |=== | Kernel 2+| CPython 3.14 2+| PyPy 3.11 2+| GraalPy 3.13 2+| Codename One -| nbody | 776.0 | 15 | 62.7 | 32 | 192.5 | 234 | 89.2 | 10 -| richards | 634.8 | 16 | 39.9 | 34 | 146.5 | 260 | 44.7 | 11 -| fannkuch | 2652.3 | 15 | 604.7 | 33 | 541.2 | 244 | 723.4 | 10 -| spectral_norm | 729.4 | 15 | 12.0 | 32 | 7.4 | 248 | 11.1 | 10 -| dicts_strings | 404.2 | 63 | 141.5 | 87 | 274.4 | 383 | 208.6 | 127 +| nbody | 764 | 15 | 62 | 32 | 180 | 236 | 88 | 10 +| richards | 632 | 15 | 39 | 34 | 104 | 272 | 36 | 11 +| fannkuch | 2606 | 15 | 582 | 33 | 519 | 248 | 568 | 10 +| spectral_norm | 711 | 15 | 12 | 32 | 7.2 | 253 | 32 | 10 +| dicts_strings | 387 | 63 | 134 | 80 | 213 | 434 | 189 | 125 |=== -The two JITs and Codename One's ahead of time compilation land within a small -factor of each other, from three to sixty times as fast as CPython. PyPy is -the fastest on most kernels and GraalPy on two of them; Codename One gets -close without a warm-up and in a third of PyPy's memory and a twentieth of -GraalPy's on the numeric kernels. +==== A Flask server + +The Flask sample above, measured with `ab -n 20000 -c 8` after a warm-up: -The Flask sample above, measured with `ab -n 20000 -c 8`: +image::img/python-flask-rps.svg["Requests per second of the Flask sample on CPython, PyPy, GraalPy and Codename One"] + +image::img/python-flask-memory.svg["Memory and deployed size of the Flask sample on CPython, PyPy, GraalPy and Codename One"] [cols="1,1,1,1,1,1",options="header"] |=== | | First response (ms) | Requests per second | Idle memory (MB) | Memory after load (MB) | Deployed size (MB) -| CPython 3.14 | 85 | 685 | 39.0 | 24.5 | 82.0 -| PyPy 3.11 | 196 | 688 | 65.3 | 88.7 | 137.7 -| GraalPy 3.13 | 609 | 561 | 325.7 | 931.0 | 343.1 -| Codename One | 58 | 13,564 | 26.8 | 423.4 | 22.7 +| CPython 3.14 | 81 | 611 | 39.1 | 39.9 | 82.0 +| PyPy 3.11 | 184 | 610 | 64.8 | 102.3 | 110.5 +| GraalPy 3.13 | 438 | 468 | 322.8 | 1270.4 | 305.8 +| Codename One | 33 | 16,749 | 26.6 | 375.2 | 22.8 |=== The deployed size is the interpreter's installation plus Flask and its @@ -231,69 +244,66 @@ Werkzeug's threaded server, which is what `app.run()` starts; a production CPython deployment behind a WSGI server with several worker processes serves more, using a full interpreter's memory per worker. The Codename One server's heap grows under load and stays at that size, which is the memory after load -column; it isn't a leak, and it doesn't grow further on longer runs. +column; it doesn't grow further on longer runs. Where an application needs CPython instead: packages with C extensions that have no Codename One implementation (for example `pandas`, `scipy` or `torch`), and code that builds source text and runs it with `exec`. -=== Performance and Java hot spots +=== Python against Java on Codename One -The comparisons above are Python programs written for Python. The workloads -below are the ones every Codename One port is measured with, +The workloads below are the ones every Codename One port is measured with, `com.bench.CommonWorkloads`, ported line by line to Python in `scripts/python-bench/common`. Each one returns a checksum, and every runtime has to return the checksum Java returns, so all five columns compute the same thing. The Python port reproduces Java's 32 and 64-bit wrap-around with masks, since a Python `int` never overflows. -Five ways to run them, each workload in a process of its own so that the peak -memory column belongs to that workload alone -(`scripts/python-compare/common_compare.py`): - -* CPython 3.14, PyPy 3.11 and GraalPy 3.13 (Community, 25.4) running the - Python source. -* Codename One, the same Python compiled to a native executable. -* Codename One with the hot spot in Java: the same executable, where the - Python calls the Java method for the workload instead of its own loop. +The *Codename One Java* column is the Java original compiled by Codename One +into the same executable. It's the floor: the speed and memory Codename One +reaches for this work when written in Java, and the target for the same +program written in Python. -image::img/python-perf-time.svg["Run time of each CommonWorkloads benchmark on CPython, PyPy, GraalPy and Codename One, with and without a Java hot spot"] +image::img/python-perf-time.svg["Run time of each CommonWorkloads benchmark on CPython, PyPy, GraalPy, Codename One Python and Codename One Java"] -image::img/python-perf-rss.svg["Peak memory of each CommonWorkloads benchmark on CPython, PyPy, GraalPy and Codename One, with and without a Java hot spot"] +image::img/python-perf-rss.svg["Peak memory of each CommonWorkloads benchmark on CPython, PyPy, GraalPy, Codename One Python and Codename One Java"] [cols="2,1,1,1,1,1,1,1,1,1,1",options="header"] |=== -| Workload 2+| CPython (ms, MB) 2+| PyPy 2+| GraalPy 2+| Codename One 2+| Hot spot in Java -| `intArithmetic` | 15125 | 15 | 142 | 32 | 114 | 353 | 1754 | 86 | 65 | 11 -| `longArithmetic` | 11839 | 15 | 5410 | 42 | 11762 | 259 | 26379 | 135 | 41 | 11 -| `mathTranscendental` | 1024 | 15 | 192 | 32 | 292 | 343 | 2016 | 93 | 181 | 11 -| `arraySequential` | 1778 | 382 | 34 | 154 | 26 | 331 | 282 | 273 | 10 | 42 -| `arrayRandom` | 4662 | 199 | 308 | 93 | 252 | 248 | 550 | 105 | 240 | 27 -| `objectAllocation` | 528 | 15 | 19 | 63 | 62 | 344 | 68 | 133 | 17 | 192 -| `valueEscape` | 805 | 15 | 6.3 | 32 | 59 | 331 | 114 | 91 | 5.0 | 11 -| `hashMapChurn` | 161 | 26 | 44 | 78 | 98 | 243 | 127 | 127 | 21 | 30 -| `stringBuilding` | 670 | 15 | 30 | 40 | 71 | 257 | 695 | 86 | 14 | 86 -| `recursion` | 2228 | 15 | 182 | 57 | 193 | 280 | 355 | 11 | 88 | 11 -| `quicksort` | 1780 | 84 | 171 | 58 | 160 | 268 | 410 | 114 | 89 | 17 +| Workload 2+| CPython (ms, MB) 2+| PyPy 2+| GraalPy 2+| Codename One Python 2+| Codename One Java +| `intArithmetic` | 14452 | 15 | 132 | 32 | 102 | 365 | 66 | 11 | 58 | 11 +| `longArithmetic` | 11709 | 15 | 5301 | 42 | 11134 | 263 | 39 | 11 | 38 | 11 +| `mathTranscendental` | 1010 | 15 | 186 | 32 | 289 | 354 | 180 | 11 | 167 | 12 +| `arraySequential` | 1745 | 382 | 33 | 154 | 26 | 338 | 32 | 42 | 9.3 | 42 +| `arrayRandom` | 4542 | 199 | 275 | 93 | 234 | 248 | 244 | 26 | 220 | 27 +| `objectAllocation` | 522 | 15 | 18 | 63 | 58 | 352 | 31 | 164 | 14 | 179 +| `valueEscape` | 801 | 15 | 6.1 | 32 | 56 | 331 | 3.5 | 11 | 4.5 | 11 +| `hashMapChurn` | 158 | 23 | 41 | 71 | 87 | 251 | 44 | 27 | 20 | 30 +| `stringBuilding` | 630 | 15 | 32 | 40 | 67 | 263 | 29 | 84 | 14 | 86 +| `recursion` | 2152 | 15 | 177 | 57 | 189 | 283 | 86 | 11 | 82 | 11 +| `quicksort` | 1519 | 84 | 168 | 58 | 111 | 247 | 118 | 17 | 80 | 17 |=== What the numbers say: -* **Python as written.** Compiled ahead of time, Codename One runs seven of - these eleven workloads 4 to 9 times faster than CPython, without a warm-up - and in less memory than GraalPy. PyPy and GraalPy are faster still on most of - them: a JIT watches the program run and specializes the loop for the values - it sees, which a compiler that runs before the program can't do. -* **Where Codename One is slowest.** Arithmetic that leaves 64 bits - (`longArithmetic` masks the result of a multiplication that needs 128) runs - on arbitrary precision integers, and code the compiler can't type, such as - the string loop in `stringBuilding` and the `struct` call in - `mathTranscendental`, runs on the generic object path. Those are the long - blue bars, and the workloads where CPython keeps up or wins. -* **Hot spot in Java.** Moving the measured hot spot into a Java method made - each of these workloads the fastest of the five runs, and usually the - smallest in memory as well because the loop then runs on unboxed `int`, - `long` and `double` values and arrays. +* **Python reaches Java's memory.** On every workload the compiled Python uses + the memory the Java program uses, within a few percent and sometimes less: + ints, floats and int lists are held unboxed, a list of ints that fit 32 bits + takes four bytes an item as a Java `int[]` does, and objects that don't + escape a function are never allocated. +* **And mostly its speed.** Six of the eleven run within 15% of Java, among + them the arithmetic, recursion and the transcendental math; the rest take + 1.5 to 3.4 times as long. What remains is the cost of a Python `int` being + unbounded: where the compiler can't prove a value stays within 64 bits, the + generated code carries the check and a slow path Java doesn't need. +* **Against the interpreters.** Codename One Python is the fastest of the four + Python runtimes on six workloads and within 10% of the fastest JIT on three + more, without a warm-up; GraalPy is faster at sequential array access and + PyPy at allocating objects. It uses a + third of PyPy's memory and a twentieth of GraalPy's on most of them. `longArithmetic` + multiplies 64-bit values whose product needs 128 bits and masks it back: the + interpreters compute it with arbitrary precision integers, and the compiler + recognizes the wrap-around and computes it in a Java `long`. ==== Tuning a hot spot in Java diff --git a/docs/developer-guide/img/python-flask-memory.svg b/docs/developer-guide/img/python-flask-memory.svg new file mode 100644 index 00000000000..0bd1558f5ed --- /dev/null +++ b/docs/developer-guide/img/python-flask-memory.svg @@ -0,0 +1,45 @@ + + +Flask server memory and deployed size, MB (lower is better) + +CPython + +PyPy + +GraalPy + +Codename One Python + +0 MB + +1000 MB + +2000 MB +idle + +39 + +65 + +323 + +27 +after load + +40 + +102 + +1270 + +375 +deployed size + +82 + +111 + +306 + +23 + diff --git a/docs/developer-guide/img/python-flask-rps.svg b/docs/developer-guide/img/python-flask-rps.svg new file mode 100644 index 00000000000..8d5df5fe914 --- /dev/null +++ b/docs/developer-guide/img/python-flask-rps.svg @@ -0,0 +1,27 @@ + + +Flask JSON API throughput (higher is better) + +CPython + +PyPy + +GraalPy + +Codename One Python + +0 + +10k + +20k +requests/s + +611 + +610 + +468 + +16749 + diff --git a/docs/developer-guide/img/python-kernels-rss.svg b/docs/developer-guide/img/python-kernels-rss.svg new file mode 100644 index 00000000000..f66f5d9ceef --- /dev/null +++ b/docs/developer-guide/img/python-kernels-rss.svg @@ -0,0 +1,69 @@ + + +Kernel peak memory (lower is better) + +CPython + +PyPy + +GraalPy + +Codename One Python + +0 MB + +100 MB + +200 MB + +300 MB + +400 MB + +500 MB +nbody + +15 + +32 + +236 + +10 +richards + +15 + +34 + +272 + +11 +fannkuch + +15 + +33 + +248 + +10 +spectral_norm + +15 + +32 + +253 + +10 +dicts_strings + +63 + +80 + +434 + +125 + diff --git a/docs/developer-guide/img/python-kernels-time.svg b/docs/developer-guide/img/python-kernels-time.svg new file mode 100644 index 00000000000..d026f5ae301 --- /dev/null +++ b/docs/developer-guide/img/python-kernels-time.svg @@ -0,0 +1,67 @@ + + +Kernel run time (lower is better, log scale) + +CPython + +PyPy + +GraalPy + +Codename One Python + +1 ms + +10 ms + +100 ms + +1000 ms + +10k ms +nbody + +764 + +62 + +180 + +88 +richards + +632 + +39 + +104 + +36 +fannkuch + +2606 + +582 + +519 + +568 +spectral_norm + +711 + +12 + +7.2 + +32 +dicts_strings + +386 + +134 + +213 + +189 + diff --git a/docs/developer-guide/img/python-perf-rss.svg b/docs/developer-guide/img/python-perf-rss.svg index 98df6a70331..7a0f01ea87d 100644 --- a/docs/developer-guide/img/python-perf-rss.svg +++ b/docs/developer-guide/img/python-perf-rss.svg @@ -8,9 +8,9 @@ GraalPy -Codename One +Codename One Python -Codename One, hot spot in Java +Codename One Java 0 MB @@ -22,91 +22,91 @@ 400 MB intArithmetic - -15 - -32 - -353 - -86 - -11 + +15 + +32 + +365 + +11 + +11 longArithmetic - -15 - -42 - -259 - -135 + +15 + +42 + +263 + +11 11 mathTranscendental - -15 - -32 - -343 - -93 - -11 + +15 + +32 + +354 + +11 + +12 arraySequential - -382 - -154 - -331 - -273 + +382 + +154 + +338 + +42 42 arrayRandom - -199 - -93 - -248 - -105 + +199 + +93 + +248 + +26 27 objectAllocation - -15 - -63 - -344 - -133 - -192 + +15 + +63 + +352 + +164 + +179 valueEscape - -15 + +15 32 - -331 - -91 - -11 + +331 + +11 + +11 hashMapChurn - -26 - -78 - -243 - -127 + +23 + +71 + +251 + +27 30 stringBuilding @@ -114,32 +114,32 @@ 15 40 - -257 - -86 - -86 + +263 + +84 + +86 recursion - -15 - -57 - -280 + +15 + +57 + +283 11 11 quicksort - -84 - -58 - -268 - -114 - -17 + +84 + +58 + +247 + +17 + +17 diff --git a/docs/developer-guide/img/python-perf-time.svg b/docs/developer-guide/img/python-perf-time.svg index 4bfabebcc0e..8f0f6f69d19 100644 --- a/docs/developer-guide/img/python-perf-time.svg +++ b/docs/developer-guide/img/python-perf-time.svg @@ -8,9 +8,9 @@ GraalPy -Codename One +Codename One Python -Codename One, hot spot in Java +Codename One Java 1 ms @@ -24,124 +24,124 @@ 100k ms intArithmetic - -15125 - -142 - -114 - -1754 - -65 + +14452 + +132 + +102 + +66 + +58 longArithmetic - -11839 - -5410 - -11762 - -26379 - -41 + +11709 + +5301 + +11134 + +39 + +38 mathTranscendental - -1024 - -192 - -292 - -2016 - -181 + +1010 + +186 + +288 + +180 + +167 arraySequential - -1778 - -34 - -26 - -282 - -10 + +1745 + +33 + +26 + +32 + +9.3 arrayRandom - -4662 - -308 - -252 - -550 - -240 + +4542 + +275 + +234 + +244 + +220 objectAllocation - -528 - -19 - -62 - -68 - -17 + +522 + +18 + +58 + +31 + +14 valueEscape - -805 - -6.3 - -59 - -114 - -5.0 + +801 + +6.1 + +56 + +3.5 + +4.5 hashMapChurn - -161 - -44 - -98 - -127 - -21 + +158 + +41 + +87 + +44 + +20 stringBuilding - -670 - -30 - -71 - -695 - -14 + +630 + +32 + +67 + +29 + +14 recursion - -2228 - -182 - -193 - -355 - -88 + +2152 + +177 + +189 + +86 + +82 quicksort - -1780 - -171 - -160 - -410 - -89 + +1519 + +168 + +111 + +118 + +80 diff --git a/docs/developer-guide/img/python-startup.svg b/docs/developer-guide/img/python-startup.svg new file mode 100644 index 00000000000..a7bd792373a --- /dev/null +++ b/docs/developer-guide/img/python-startup.svg @@ -0,0 +1,36 @@ + + +Startup, a program that prints one line (lower is better) + +CPython + +PyPy + +GraalPy + +Codename One Python + +0 + +100 + +200 +wall time (ms) + +15 + +14 + +28 + +4.7 +peak memory (MB) + +14 + +29 + +119 + +8.6 + diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyInstance.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyInstance.java index 445d51e4272..6eb1b5455d2 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyInstance.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyInstance.java @@ -160,9 +160,8 @@ String[] attributeNames() { out.add(f[i]); } } - String[] names = shape().names; - for (int i = 0; i < names.length; i++) { - out.add(names[i]); + for (String n : shape().names) { + out.add(n); } return out.toArray(new String[out.size()]); } diff --git a/maven/python-transpiler/spotbugs-exclude.xml b/maven/python-transpiler/spotbugs-exclude.xml index 8fef78dc488..14ca316a137 100644 --- a/maven/python-transpiler/spotbugs-exclude.xml +++ b/maven/python-transpiler/spotbugs-exclude.xml @@ -34,4 +34,20 @@ + + + + + + + + + + + + + + diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/TypedEmit.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/TypedEmit.java index cb7920587e0..8ace95cd341 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/TypedEmit.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/TypedEmit.java @@ -98,6 +98,14 @@ static final class E { } } + /// Appends a copy of each entry of `m` to `out`: an entry set may hand + /// out one reused entry object. + private static void copyEntries(Map m, List> out) { + for (Map.Entry e : m.entrySet()) { + out.add(new java.util.AbstractMap.SimpleImmutableEntry(e.getKey(), e.getValue())); + } + } + /// The int a typed function returned as `long` `r`, left as it is /// (see `E.pk`). static E pushed(String r) { @@ -500,11 +508,12 @@ private void planLongParams(Set ok) { // site's arguments differ between the two bodies. Map st = new HashMap(); new IntRanges(in, f, null, st, seeds, rets); - List> all = new ArrayList>(st.entrySet()); + List> all = new ArrayList>(); + copyEntries(st, all); if (seeds.containsKey(f)) { Map sl = new HashMap(); IntRanges r = new IntRanges(in, f, seeds(f, seeds.get(f)), sl, seeds, rets); - all.addAll(sl.entrySet()); + copyEntries(sl, all); IntRanges.Iv v = f.ret.k == Ty.K.INT && !f.ret.opt ? r.returns() : null; if (v != null && v.fitsLong()) { nr.put(f, v); @@ -806,11 +815,6 @@ static String param(Ty t, String name, String bigName) { return t.k == Ty.K.INT ? "long " + name + ", Object " + bigName : jt(t) + " " + name; } - /// Whether `n` is an int local held as a pair. - private boolean pairLocal(String n) { - return sc.resolve(n) == Scope.Binding.LOCAL && localType(n).k == Ty.K.INT && !longLocals.contains(n); - } - /// The long and object halves of int (or bool) `e`, for a pair: the /// object may be a `Long` here when nothing better is at hand, which /// the receiving end normalizes. diff --git a/scripts/python-compare/README.md b/scripts/python-compare/README.md index 9b2ad54d0da..01c6b83dfc4 100644 --- a/scripts/python-compare/README.md +++ b/scripts/python-compare/README.md @@ -22,7 +22,8 @@ scripts/python-backend/build.sh scripts/python-backend/samples/flask-todo /tmp/f python3 scripts/python-compare/compare.py \ --cpython --pypy [--graalpy ] \ --cn1-kernels scripts/python-bench/target --cn1-hello scripts/python-bench/target/hello/hello \ - --cn1-server /tmp/flask-todo --out scripts/python-compare/RESULTS.md + --cn1-server /tmp/flask-todo --out scripts/python-compare/RESULTS.md \ + --json scripts/python-compare/compare-results.json ``` GraalPy is an optional arm (`--graalpy`); `RESULTS.md` includes it. Install @@ -34,7 +35,8 @@ Flask into it with `graalpy -m ensurepip` and `graalpy -m pip install flask`. Codename One port runs, ported to Python in `scripts/python-bench/common`. Each workload runs in its own process, so the peak RSS is that workload's, and every checksum must equal Java's. Codename One runs twice: the Python as -written, and the same executable calling the Java method for the hot spot. +written, and Codename One Java, the same executable running the Java original +-- the floor the compiled Python is held against. ```bash C=scripts/python-bench/common @@ -44,5 +46,6 @@ PY_SOURCES=$C PY_EXTRA_FILES="$C/common_workloads.py $C/java_hotspots.py" \ python3 scripts/python-compare/common_compare.py \ --cn1 scripts/python-bench/target/common_bench/common_bench \ --cpython --pypy --graalpy --json common.json -python3 scripts/python-compare/render_charts.py common.json --out docs/developer-guide/img +python3 scripts/python-compare/render_charts.py common.json \ + --compare scripts/python-compare/compare-results.json --out docs/developer-guide/img ``` diff --git a/scripts/python-compare/RESULTS.md b/scripts/python-compare/RESULTS.md index 04e9f40b2ae..322cf23a4c2 100644 --- a/scripts/python-compare/RESULTS.md +++ b/scripts/python-compare/RESULTS.md @@ -1,29 +1,29 @@ # Codename One Python vs CPython, PyPy, GraalPy -Measured on Darwin arm64, 2026-10-27. Codename One rows are native binaries (Python -> Java -> ParparVM C -> clang -O3). +Measured on Darwin arm64, 2026-10-28. Codename One rows are native binaries (Python -> Java -> ParparVM C -> clang -O3). -- CPython: `python3.14 (venv with Flask)` (3.14.5 cpython) -- PyPy: `~/.pyenv/versions/pypy3.11-7.3.19/bin/pypy3` (3.11.11 pypy) -- GraalPy: `graalpy-community3.13-25.4.4.1.1-macos-aarch64/bin/graalpy` (3.13.14 graalpy) +- CPython: `python3.14` in a venv with Flask (3.14.5 cpython) +- PyPy: `pypy3` in a venv with Flask (3.11.13 pypy) +- GraalPy: `graalpy` in a venv with Flask (3.13.14 graalpy) ## Startup: `print('hello')` | | wall ms (best of 10) | peak RSS MB | |---|---:|---:| -| CPython | 20.3 | 14.3 | -| PyPy | 18.2 | 29.2 | -| GraalPy | 34.9 | 116.6 | -| Codename One | 5.9 | 8.7 | +| CPython | 15.1 | 14.4 | +| PyPy | 13.9 | 29.1 | +| GraalPy | 28.2 | 118.6 | +| Codename One Python | 4.7 | 8.6 | ## CPU kernels (fastest of the kernel's rounds, ms; peak RSS MB) -| kernel | CPython ms | CPython MB | PyPy ms | PyPy MB | GraalPy ms | GraalPy MB | Codename One ms | Codename One MB | +| kernel | CPython ms | CPython MB | PyPy ms | PyPy MB | GraalPy ms | GraalPy MB | Codename One Python ms | Codename One Python MB | |---|---:|---:|---:|---:|---:|---:|---:|---:| -| nbody | 776.0 | 14.9 | 62.7 | 32.3 | 192.5 | 233.6 | 89.2 | 10.2 | -| richards | 634.8 | 15.5 | 39.9 | 33.8 | 146.5 | 260.1 | 44.7 | 10.6 | -| fannkuch | 2652.3 | 14.5 | 604.7 | 32.8 | 541.2 | 244.0 | 723.4 | 10.4 | -| spectral_norm | 729.4 | 14.6 | 12.0 | 32.2 | 7.4 | 248.3 | 11.1 | 10.2 | -| dicts_strings | 404.2 | 62.6 | 141.5 | 87.2 | 274.4 | 382.8 | 208.6 | 126.5 | +| nbody | 764.4 | 14.9 | 61.9 | 32.4 | 180.4 | 236.2 | 88.0 | 10.3 | +| richards | 631.7 | 15.2 | 38.5 | 33.8 | 104.2 | 271.6 | 36.0 | 10.6 | +| fannkuch | 2606.4 | 14.6 | 581.9 | 32.9 | 519.0 | 247.8 | 567.9 | 10.0 | +| spectral_norm | 711.3 | 14.6 | 11.7 | 32.2 | 7.2 | 252.9 | 32.4 | 10.2 | +| dicts_strings | 386.5 | 62.6 | 133.8 | 80.4 | 213.3 | 433.9 | 188.9 | 125.1 | ## Flask JSON API (`scripts/python-backend/samples/flask-todo`) @@ -31,33 +31,33 @@ Measured on Darwin arm64, 2026-10-27. Codename One rows are native binaries (Pyt | | first response ms | idle RSS MB | RSS after load MB | req/s | p99 ms | |---|---:|---:|---:|---:|---:| -| CPython | 85 | 39.0 | 24.5 | 685 | 18 | -| PyPy | 196 | 65.3 | 88.7 | 688 | 17 | -| GraalPy | 609 | 325.7 | 931.0 | 561 | 44 | -| Codename One | 58 | 26.8 | 423.4 | 13564 | 2 | +| CPython | 81 | 39.1 | 39.9 | 611 | 16 | +| PyPy | 184 | 64.8 | 102.3 | 610 | 24 | +| GraalPy | 438 | 322.8 | 1270.4 | 468 | 59 | +| Codename One Python | 33 | 26.6 | 375.2 | 16749 | 1 | ## What you deploy | | size MB | what | |---|---:|---| | CPython | 82.0 | interpreter install (`/opt/homebrew/opt/python@3.14/Frameworks/Python.framework/Versions/3.14`) + Flask and its dependencies (5.2) | -| PyPy | 137.7 | interpreter install (`~/.pyenv/versions/pypy3.11-7.3.19`) + Flask and its dependencies (4.9) | -| GraalPy | 343.1 | interpreter install (`graalpy-community3.13-25.4.4.1.1-macos-aarch64`) + Flask and its dependencies (11.2) | -| Codename One | 22.7 | one self-contained executable, Flask included | +| PyPy | 110.5 | interpreter install (`pypy3.11-v7.3.20-macos_arm64`) + Flask and its dependencies (5.1) | +| GraalPy | 305.8 | interpreter install (`graalpy-community3.13-25.4.4.1.1-macos-aarch64`) + Flask and its dependencies (11.2) | +| Codename One Python | 22.8 | one self-contained executable, Flask included | -## CommonWorkloads (common_compare.py; ms, peak RSS MB) +## CommonWorkloads (`common_compare.py`; ms, peak RSS MB) -| workload | CPython ms | MB | PyPy ms | MB | GraalPy ms | MB | Codename One ms | MB | Codename One, hot spot in Java ms | MB | +| workload | CPython ms | MB | PyPy ms | MB | GraalPy ms | MB | Codename One Python ms | MB | Codename One Java ms | MB | |---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| -| intArithmetic | 15124.8 | 15.0 | 141.8 | 32.1 | 114.2 | 353.0 | 1753.8 | 85.6 | 65.2 | 11.4 | -| longArithmetic | 11839.2 | 14.8 | 5410.0 | 41.6 | 11761.8 | 259.0 | 26378.6 | 135.3 | 40.6 | 11.3 | -| mathTranscendental | 1024.5 | 15.0 | 191.6 | 31.8 | 292.2 | 342.9 | 2015.7 | 93.0 | 180.8 | 11.4 | -| arraySequential | 1778.4 | 382.2 | 33.7 | 154.0 | 26.2 | 331.3 | 282.4 | 273.1 | 10.5 | 41.8 | -| arrayRandom | 4662.5 | 198.6 | 308.2 | 93.1 | 252.1 | 248.1 | 549.5 | 104.6 | 239.8 | 26.6 | -| objectAllocation | 527.8 | 15.1 | 19.1 | 63.3 | 62.0 | 344.0 | 68.2 | 133.1 | 16.6 | 191.7 | -| valueEscape | 805.3 | 14.8 | 6.3 | 32.0 | 59.1 | 330.7 | 114.2 | 91.0 | 5.0 | 11.5 | -| hashMapChurn | 160.9 | 25.5 | 43.6 | 77.6 | 97.8 | 243.5 | 127.3 | 126.6 | 20.8 | 30.3 | -| stringBuilding | 670.0 | 14.9 | 30.5 | 40.1 | 70.7 | 256.9 | 695.2 | 86.5 | 14.4 | 86.4 | -| recursion | 2227.7 | 14.8 | 181.9 | 57.2 | 193.3 | 280.4 | 355.1 | 11.0 | 87.6 | 11.3 | -| quicksort | 1779.8 | 83.6 | 171.4 | 58.0 | 160.0 | 267.6 | 410.4 | 113.5 | 89.3 | 17.2 | +| intArithmetic | 14452.1 | 14.8 | 131.6 | 32.1 | 101.6 | 365.0 | 65.7 | 10.9 | 57.5 | 11.3 | +| longArithmetic | 11708.8 | 14.9 | 5300.7 | 41.6 | 11133.7 | 263.1 | 38.6 | 10.9 | 38.4 | 11.3 | +| mathTranscendental | 1009.9 | 14.8 | 186.4 | 31.8 | 288.5 | 354.1 | 179.8 | 11.3 | 167.2 | 11.6 | +| arraySequential | 1745.4 | 382.0 | 32.6 | 154.2 | 25.7 | 337.6 | 31.5 | 41.8 | 9.3 | 41.8 | +| arrayRandom | 4541.6 | 198.6 | 275.4 | 93.2 | 233.5 | 248.0 | 244.4 | 26.3 | 219.7 | 26.6 | +| objectAllocation | 521.6 | 14.9 | 18.1 | 63.1 | 58.1 | 351.9 | 30.8 | 163.8 | 14.2 | 179.1 | +| valueEscape | 801.2 | 14.9 | 6.1 | 32.0 | 55.5 | 330.6 | 3.5 | 10.9 | 4.5 | 11.2 | +| hashMapChurn | 158.4 | 23.1 | 41.1 | 70.8 | 86.7 | 250.6 | 43.7 | 26.9 | 20.4 | 30.3 | +| stringBuilding | 629.5 | 14.9 | 31.5 | 40.1 | 66.7 | 263.0 | 28.9 | 83.6 | 13.9 | 86.0 | +| recursion | 2151.7 | 14.9 | 177.3 | 57.0 | 188.6 | 282.6 | 85.7 | 10.9 | 81.5 | 11.3 | +| quicksort | 1519.2 | 83.8 | 168.0 | 57.9 | 110.9 | 247.0 | 117.7 | 16.7 | 79.8 | 17.0 | diff --git a/scripts/python-compare/common-results.json b/scripts/python-compare/common-results.json index 237a5d31a41..2e19377774e 100644 --- a/scripts/python-compare/common-results.json +++ b/scripts/python-compare/common-results.json @@ -3,250 +3,250 @@ "CPython", "PyPy", "GraalPy", - "Codename One", - "Codename One, hot spot in Java" + "Codename One Python", + "Codename One Java" ], "results": { "intArithmetic": { "CPython": { - "ms": 15124.8, - "rss_mb": 14.96875 + "ms": 14452.1, + "rss_mb": 14.84375 }, "PyPy": { - "ms": 141.8, - "rss_mb": 32.125 + "ms": 131.6, + "rss_mb": 32.0625 }, "GraalPy": { - "ms": 114.2, - "rss_mb": 352.953125 + "ms": 101.6, + "rss_mb": 365.046875 }, - "Codename One": { - "ms": 1753.8, - "rss_mb": 85.5625 + "Codename One Python": { + "ms": 65.7, + "rss_mb": 10.90625 }, - "Codename One, hot spot in Java": { - "ms": 65.2, - "rss_mb": 11.375 + "Codename One Java": { + "ms": 57.5, + "rss_mb": 11.28125 } }, "longArithmetic": { "CPython": { - "ms": 11839.2, - "rss_mb": 14.796875 + "ms": 11708.8, + "rss_mb": 14.9375 }, "PyPy": { - "ms": 5410.0, - "rss_mb": 41.578125 + "ms": 5300.7, + "rss_mb": 41.625 }, "GraalPy": { - "ms": 11761.8, - "rss_mb": 259.015625 + "ms": 11133.7, + "rss_mb": 263.0625 }, - "Codename One": { - "ms": 26378.6, - "rss_mb": 135.328125 + "Codename One Python": { + "ms": 38.6, + "rss_mb": 10.921875 }, - "Codename One, hot spot in Java": { - "ms": 40.6, - "rss_mb": 11.3125 + "Codename One Java": { + "ms": 38.4, + "rss_mb": 11.296875 } }, "mathTranscendental": { "CPython": { - "ms": 1024.5, - "rss_mb": 14.953125 + "ms": 1009.9, + "rss_mb": 14.8125 }, "PyPy": { - "ms": 191.6, - "rss_mb": 31.765625 + "ms": 186.4, + "rss_mb": 31.84375 }, "GraalPy": { - "ms": 292.2, - "rss_mb": 342.9375 + "ms": 288.5, + "rss_mb": 354.125 }, - "Codename One": { - "ms": 2015.7, - "rss_mb": 93.0 + "Codename One Python": { + "ms": 179.8, + "rss_mb": 11.34375 }, - "Codename One, hot spot in Java": { - "ms": 180.8, - "rss_mb": 11.40625 + "Codename One Java": { + "ms": 167.2, + "rss_mb": 11.578125 } }, "arraySequential": { "CPython": { - "ms": 1778.4, - "rss_mb": 382.203125 + "ms": 1745.4, + "rss_mb": 382.046875 }, "PyPy": { - "ms": 33.7, - "rss_mb": 154.046875 + "ms": 32.6, + "rss_mb": 154.171875 }, "GraalPy": { - "ms": 26.2, - "rss_mb": 331.28125 + "ms": 25.7, + "rss_mb": 337.640625 }, - "Codename One": { - "ms": 282.4, - "rss_mb": 273.09375 + "Codename One Python": { + "ms": 31.5, + "rss_mb": 41.75 }, - "Codename One, hot spot in Java": { - "ms": 10.5, - "rss_mb": 41.796875 + "Codename One Java": { + "ms": 9.3, + "rss_mb": 41.8125 } }, "arrayRandom": { "CPython": { - "ms": 4662.5, - "rss_mb": 198.609375 + "ms": 4541.6, + "rss_mb": 198.5625 }, "PyPy": { - "ms": 308.2, - "rss_mb": 93.078125 + "ms": 275.4, + "rss_mb": 93.1875 }, "GraalPy": { - "ms": 252.1, - "rss_mb": 248.125 + "ms": 233.5, + "rss_mb": 247.96875 }, - "Codename One": { - "ms": 549.5, - "rss_mb": 104.59375 + "Codename One Python": { + "ms": 244.4, + "rss_mb": 26.265625 }, - "Codename One, hot spot in Java": { - "ms": 239.8, - "rss_mb": 26.625 + "Codename One Java": { + "ms": 219.7, + "rss_mb": 26.609375 } }, "objectAllocation": { "CPython": { - "ms": 527.8, - "rss_mb": 15.078125 + "ms": 521.6, + "rss_mb": 14.875 }, "PyPy": { - "ms": 19.1, - "rss_mb": 63.28125 + "ms": 18.1, + "rss_mb": 63.140625 }, "GraalPy": { - "ms": 62.0, - "rss_mb": 344.03125 + "ms": 58.1, + "rss_mb": 351.859375 }, - "Codename One": { - "ms": 68.2, - "rss_mb": 133.0625 + "Codename One Python": { + "ms": 30.8, + "rss_mb": 163.84375 }, - "Codename One, hot spot in Java": { - "ms": 16.6, - "rss_mb": 191.6875 + "Codename One Java": { + "ms": 14.2, + "rss_mb": 179.09375 } }, "valueEscape": { "CPython": { - "ms": 805.3, - "rss_mb": 14.78125 + "ms": 801.2, + "rss_mb": 14.921875 }, "PyPy": { - "ms": 6.3, - "rss_mb": 32.03125 + "ms": 6.1, + "rss_mb": 32.015625 }, "GraalPy": { - "ms": 59.1, - "rss_mb": 330.71875 + "ms": 55.5, + "rss_mb": 330.59375 }, - "Codename One": { - "ms": 114.2, - "rss_mb": 90.984375 + "Codename One Python": { + "ms": 3.5, + "rss_mb": 10.921875 }, - "Codename One, hot spot in Java": { - "ms": 5.0, - "rss_mb": 11.453125 + "Codename One Java": { + "ms": 4.5, + "rss_mb": 11.25 } }, "hashMapChurn": { "CPython": { - "ms": 160.9, - "rss_mb": 25.546875 + "ms": 158.4, + "rss_mb": 23.125 }, "PyPy": { - "ms": 43.6, - "rss_mb": 77.625 + "ms": 41.1, + "rss_mb": 70.84375 }, "GraalPy": { - "ms": 97.8, - "rss_mb": 243.46875 + "ms": 86.7, + "rss_mb": 250.609375 }, - "Codename One": { - "ms": 127.3, - "rss_mb": 126.5625 + "Codename One Python": { + "ms": 43.7, + "rss_mb": 26.890625 }, - "Codename One, hot spot in Java": { - "ms": 20.8, + "Codename One Java": { + "ms": 20.4, "rss_mb": 30.265625 } }, "stringBuilding": { "CPython": { - "ms": 670.0, - "rss_mb": 14.921875 + "ms": 629.5, + "rss_mb": 14.9375 }, "PyPy": { - "ms": 30.5, + "ms": 31.5, "rss_mb": 40.078125 }, "GraalPy": { - "ms": 70.7, - "rss_mb": 256.90625 + "ms": 66.7, + "rss_mb": 262.984375 }, - "Codename One": { - "ms": 695.2, - "rss_mb": 86.5 + "Codename One Python": { + "ms": 28.9, + "rss_mb": 83.609375 }, - "Codename One, hot spot in Java": { - "ms": 14.4, - "rss_mb": 86.421875 + "Codename One Java": { + "ms": 13.9, + "rss_mb": 86.0 } }, "recursion": { "CPython": { - "ms": 2227.7, - "rss_mb": 14.796875 + "ms": 2151.7, + "rss_mb": 14.921875 }, "PyPy": { - "ms": 181.9, - "rss_mb": 57.15625 + "ms": 177.3, + "rss_mb": 57.015625 }, "GraalPy": { - "ms": 193.3, - "rss_mb": 280.40625 + "ms": 188.6, + "rss_mb": 282.609375 }, - "Codename One": { - "ms": 355.1, - "rss_mb": 10.96875 + "Codename One Python": { + "ms": 85.7, + "rss_mb": 10.9375 }, - "Codename One, hot spot in Java": { - "ms": 87.6, - "rss_mb": 11.34375 + "Codename One Java": { + "ms": 81.5, + "rss_mb": 11.328125 } }, "quicksort": { "CPython": { - "ms": 1779.8, - "rss_mb": 83.609375 + "ms": 1519.2, + "rss_mb": 83.796875 }, "PyPy": { - "ms": 171.4, - "rss_mb": 57.96875 + "ms": 168.0, + "rss_mb": 57.859375 }, "GraalPy": { - "ms": 160.0, - "rss_mb": 267.625 + "ms": 110.9, + "rss_mb": 247.046875 }, - "Codename One": { - "ms": 410.4, - "rss_mb": 113.53125 + "Codename One Python": { + "ms": 117.7, + "rss_mb": 16.703125 }, - "Codename One, hot spot in Java": { - "ms": 89.3, - "rss_mb": 17.171875 + "Codename One Java": { + "ms": 79.8, + "rss_mb": 17.03125 } } } diff --git a/scripts/python-compare/common_compare.py b/scripts/python-compare/common_compare.py index 359f1c97099..4c8f9ac28fb 100644 --- a/scripts/python-compare/common_compare.py +++ b/scripts/python-compare/common_compare.py @@ -1,8 +1,9 @@ #!/usr/bin/env python3 # Runs vm/benchmarks' CommonWorkloads, ported to Python # (scripts/python-bench/common), on every Python runtime given, plus the -# Codename One build twice: the Python as written, and the same program with -# the hot spot in Java. Each workload runs in a process of its own, so every +# Codename One build twice: the Python as written, and Codename One Java, the +# same executable running vm/benchmarks' own Java method for the workload -- +# the floor the compiled Python is measured against. Each workload runs in a process of its own, so every # row has its own peak RSS. Checksums are checked against Java's. # # common_compare.py --cn1 --cpython P [--pypy P] @@ -55,14 +56,16 @@ def main(): ap.add_argument("--md") ap.add_argument("--only", help="comma separated workloads") a = ap.parse_args() + # The workloads run from the Python sources' directory. + a.cn1 = os.path.abspath(a.cn1) script = os.path.join(COMMON, "common_bench.py") arms = [("CPython", lambda w: [a.cpython, script, w])] if a.pypy: arms.append(("PyPy", lambda w: [a.pypy, script, w])) if a.graalpy: arms.append(("GraalPy", lambda w: [a.graalpy, script, w])) - arms.append(("Codename One", lambda w: [a.cn1, w, "python"])) - arms.append(("Codename One, hot spot in Java", lambda w: [a.cn1, w, "java"])) + arms.append(("Codename One Python", lambda w: [a.cn1, w, "python"])) + arms.append(("Codename One Java", lambda w: [a.cn1, w, "java"])) reps = [str(a.warmup), str(a.measured)] names = [w for w in EXPECTED if not a.only or w in a.only.split(",")] results = {} diff --git a/scripts/python-compare/compare-results.json b/scripts/python-compare/compare-results.json new file mode 100644 index 00000000000..00260d15a2b --- /dev/null +++ b/scripts/python-compare/compare-results.json @@ -0,0 +1,154 @@ +{ + "arms": [ + "CPython", + "PyPy", + "GraalPy", + "Codename One Python" + ], + "startup": { + "CPython": { + "ms": 15.0733329992363, + "rss_mb": 14.359375 + }, + "PyPy": { + "ms": 13.8918749998993, + "rss_mb": 29.140625 + }, + "GraalPy": { + "ms": 28.15879199988558, + "rss_mb": 118.578125 + }, + "Codename One Python": { + "ms": 4.749292000269634, + "rss_mb": 8.609375 + } + }, + "kernels": { + "nbody": { + "CPython": { + "ms": 764.37, + "rss_mb": 14.875 + }, + "PyPy": { + "ms": 61.914, + "rss_mb": 32.359375 + }, + "GraalPy": { + "ms": 180.422, + "rss_mb": 236.234375 + }, + "Codename One Python": { + "ms": 87.998, + "rss_mb": 10.265625 + } + }, + "richards": { + "CPython": { + "ms": 631.718, + "rss_mb": 15.234375 + }, + "PyPy": { + "ms": 38.523, + "rss_mb": 33.796875 + }, + "GraalPy": { + "ms": 104.166, + "rss_mb": 271.578125 + }, + "Codename One Python": { + "ms": 36.045, + "rss_mb": 10.625 + } + }, + "fannkuch": { + "CPython": { + "ms": 2606.358, + "rss_mb": 14.5625 + }, + "PyPy": { + "ms": 581.941, + "rss_mb": 32.859375 + }, + "GraalPy": { + "ms": 519.041, + "rss_mb": 247.828125 + }, + "Codename One Python": { + "ms": 567.906, + "rss_mb": 10.046875 + } + }, + "spectral_norm": { + "CPython": { + "ms": 711.285, + "rss_mb": 14.640625 + }, + "PyPy": { + "ms": 11.683, + "rss_mb": 32.171875 + }, + "GraalPy": { + "ms": 7.217, + "rss_mb": 252.875 + }, + "Codename One Python": { + "ms": 32.403, + "rss_mb": 10.21875 + } + }, + "dicts_strings": { + "CPython": { + "ms": 386.494, + "rss_mb": 62.59375 + }, + "PyPy": { + "ms": 133.808, + "rss_mb": 80.40625 + }, + "GraalPy": { + "ms": 213.251, + "rss_mb": 433.921875 + }, + "Codename One Python": { + "ms": 188.941, + "rss_mb": 125.140625 + } + } + }, + "flask": { + "CPython": { + "first_ms": 80.58633299879148, + "idle_mb": 39.0625, + "loaded_mb": 39.875, + "rps": 611.47, + "p99_ms": 16 + }, + "PyPy": { + "first_ms": 183.99095800123177, + "idle_mb": 64.84375, + "loaded_mb": 102.28125, + "rps": 610.14, + "p99_ms": 24 + }, + "GraalPy": { + "first_ms": 438.2767499992042, + "idle_mb": 322.828125, + "loaded_mb": 1270.375, + "rps": 468.22, + "p99_ms": 59 + }, + "Codename One Python": { + "first_ms": 32.780792000266956, + "idle_mb": 26.59375, + "loaded_mb": 375.1875, + "rps": 16748.68, + "p99_ms": 1 + } + }, + "deploy": { + "CPython": 82.028018, + "PyPy": 110.51205999999999, + "GraalPy": 305.841158, + "Codename One Python": 22.783288 + } +} \ No newline at end of file diff --git a/scripts/python-compare/compare.py b/scripts/python-compare/compare.py index 3d579546939..1232d8c5b69 100644 --- a/scripts/python-compare/compare.py +++ b/scripts/python-compare/compare.py @@ -25,6 +25,7 @@ SERVER_APP = os.path.join(REPO, "scripts", "python-backend", "samples", "flask-todo", "src", "main.py") KERNEL_NAMES = ["nbody", "richards", "fannkuch", "spectral_norm", "dicts_strings"] PORT = 8080 +CN1 = "Codename One Python" def timed(cmd, cwd=None): @@ -76,7 +77,7 @@ def kernel_rows(arms, cn1_dir): out, wall, rss = timed([exe, os.path.join(KERNELS, k + ".py")], cwd=KERNELS) row[name] = (best_ms(out), rss) out, wall, rss = timed([os.path.join(cn1_dir, k, k)]) - row["Codename One"] = (best_ms(out), rss) + row[CN1] = (best_ms(out), rss) rows.append(row) return rows @@ -136,13 +137,14 @@ def main(): ap.add_argument("--cn1-hello", help="native binary of hello.py, for the startup row") ap.add_argument("--cn1-server", required=True) ap.add_argument("--out") + ap.add_argument("--json", help="the measurements, for render_charts.py --compare") a = ap.parse_args() arms = [("CPython", a.cpython)] if a.pypy: arms.append(("PyPy", a.pypy)) if a.graalpy: arms.append(("GraalPy", a.graalpy)) - names = [n for n, _ in arms] + ["Codename One"] + names = [n for n, _ in arms] + [CN1] lines = [] w = lines.append w("# Codename One Python vs %s\n" % ", ".join(n for n, _ in arms)) @@ -162,16 +164,20 @@ def main(): w("|---|---:|---:|") startup = [(n, [exe, hello]) for n, exe in arms] if a.cn1_hello: - startup.append(("Codename One", [a.cn1_hello])) + startup.append((CN1, [a.cn1_hello])) + data = {"arms": names, "startup": {}, "kernels": {}, "flask": {}, "deploy": {}} for n, cmd in startup: runs = [timed(cmd) for _ in range(10)] - w("| %s | %.1f | %.1f |" % (n, min(r[1] for r in runs) * 1000, min(r[2] for r in runs))) + ms, mb = min(r[1] for r in runs) * 1000, min(r[2] for r in runs) + data["startup"][n] = {"ms": ms, "rss_mb": mb} + w("| %s | %.1f | %.1f |" % (n, ms, mb)) w("") w("## CPU kernels (fastest of the kernel's rounds, ms; peak RSS MB)\n") w("| kernel | " + " | ".join("%s ms | %s MB" % (n, n) for n in names) + " |") w("|---|" + "---:|---:|" * len(names)) for r in kernel_rows(arms, a.cn1_kernels): + data["kernels"][r["kernel"]] = {n: {"ms": r[n][0], "rss_mb": r[n][1]} for n in names} w("| %s | " % r["kernel"] + " | ".join("%.1f | %.1f" % r[n] for n in names) + " |") w("") @@ -186,9 +192,11 @@ def main(): w("| %s | (Flask not installed) | | | | |" % n) continue r = server_row([exe, SERVER_APP]) + data["flask"][n] = dict(zip(("first_ms", "idle_mb", "loaded_mb", "rps", "p99_ms"), r)) w("| %s | %.0f | %.1f | %.1f | %.0f | %s |" % ((n,) + r)) r = server_row([a.cn1_server]) - w("| Codename One | %.0f | %.1f | %.1f | %.0f | %s |" % (r[0], r[1], r[2], r[3], r[4])) + data["flask"][CN1] = dict(zip(("first_ms", "idle_mb", "loaded_mb", "rps", "p99_ms"), r)) + w("| " + CN1 + " | %.0f | %.1f | %.1f | %.0f | %s |" % (r[0], r[1], r[2], r[3], r[4])) w("") w("## What you deploy\n") @@ -197,10 +205,16 @@ def main(): for n, exe in arms: home = interpreter_home(exe) extra = flask_size(exe) if has_flask(exe) else 0.0 + data["deploy"][n] = du_mb(home) + extra w("| %s | %.1f | interpreter install (`%s`) + Flask and its dependencies (%.1f) |" % (n, du_mb(home) + extra, home, extra)) - w("| Codename One | %.1f | one self-contained executable, Flask included |" % du_mb(a.cn1_server)) + w("| " + CN1 + " | %.1f | one self-contained executable, Flask included |" % du_mb(a.cn1_server)) w("") + data["deploy"][CN1] = du_mb(a.cn1_server) + if a.json: + import json + with open(a.json, "w") as f: + json.dump(data, f, indent=1) text = "\n".join(lines) if a.out: with open(a.out, "w") as f: diff --git a/scripts/python-compare/render_charts.py b/scripts/python-compare/render_charts.py index 35232028841..efae4643015 100644 --- a/scripts/python-compare/render_charts.py +++ b/scripts/python-compare/render_charts.py @@ -22,12 +22,14 @@ # need additional information or have any questions. """Renders the developer guide's Python performance charts. - render_charts.py common.json --out docs/developer-guide/img + render_charts.py [common.json] [--compare compare.json] --out docs/developer-guide/img -The input is what common_compare.py --json writes. Two SVGs come out: -python-perf-time.svg (milliseconds, log scale, because the runtimes differ by -two orders of magnitude) and python-perf-rss.svg (peak RSS, linear). The SVGs -are self-contained so the website and the PDF build render them alike. +common.json is what common_compare.py --json writes: python-perf-time.svg +(milliseconds, log scale, because the runtimes differ by two orders of +magnitude) and python-perf-rss.svg (peak RSS, linear). compare.json is what +compare.py --json writes: python-kernels-time.svg, python-kernels-rss.svg, +python-startup.svg, python-flask-rps.svg and python-flask-memory.svg. The +SVGs are self-contained so the website and the PDF build render them alike. """ import argparse @@ -39,8 +41,8 @@ "CPython": "#9aa0a6", "PyPy": "#f9ab00", "GraalPy": "#a142f4", - "Codename One": "#1a73e8", - "Codename One, hot spot in Java": "#0b8043", + "Codename One Python": "#1a73e8", + "Codename One Java": "#0b8043", } TEXT = "#202124" MUTED = "#5f6368" @@ -59,6 +61,7 @@ def chart(data, key, title, unit, log): arms = data["arms"] rows = list(data["results"].items()) plot = WIDTH - LEFT - RIGHT + arms = [a for a in arms if all(a in r for _, r in rows)] values = [r[a][key] for _, r in rows for a in arms] hi = max(values) lo = min(v for v in values if v > 0) @@ -116,15 +119,38 @@ def x(v): def main(): ap = argparse.ArgumentParser() - ap.add_argument("json") + ap.add_argument("json", nargs="?") + ap.add_argument("--compare") ap.add_argument("--out", required=True) a = ap.parse_args() - with open(a.json) as f: - data = json.load(f) - with open(os.path.join(a.out, "python-perf-time.svg"), "w") as f: - f.write(chart(data, "ms", "Run time per workload (lower is better, log scale)", "ms", True)) - with open(os.path.join(a.out, "python-perf-rss.svg"), "w") as f: - f.write(chart(data, "rss_mb", "Peak memory per workload (lower is better)", "MB", False)) + out = [] + if a.json: + with open(a.json) as f: + data = json.load(f) + out.append(("python-perf-time.svg", chart(data, "ms", "Run time per workload (lower is better, log scale)", "ms", True))) + out.append(("python-perf-rss.svg", chart(data, "rss_mb", "Peak memory per workload (lower is better)", "MB", False))) + if a.compare: + with open(a.compare) as f: + c = json.load(f) + kernels = {"arms": c["arms"], "results": c["kernels"]} + out.append(("python-kernels-time.svg", chart(kernels, "ms", "Kernel run time (lower is better, log scale)", "ms", True))) + out.append(("python-kernels-rss.svg", chart(kernels, "rss_mb", "Kernel peak memory (lower is better)", "MB", False))) + startup = {"arms": c["arms"], "results": { + "wall time (ms)": {n: {"v": r["ms"]} for n, r in c["startup"].items()}, + "peak memory (MB)": {n: {"v": r["rss_mb"]} for n, r in c["startup"].items()}}} + out.append(("python-startup.svg", chart(startup, "v", "Startup, a program that prints one line (lower is better)", "", False))) + fl = c["flask"] + out.append(("python-flask-rps.svg", chart( + {"arms": c["arms"], "results": {"requests/s": {n: {"v": r["rps"]} for n, r in fl.items()}}}, + "v", "Flask JSON API throughput (higher is better)", "", False))) + mem = {"idle": "idle_mb", "after load": "loaded_mb"} + rows = {k: {n: {"v": r[f]} for n, r in fl.items()} for k, f in mem.items()} + rows["deployed size"] = {n: {"v": v} for n, v in c["deploy"].items()} + out.append(("python-flask-memory.svg", chart( + {"arms": c["arms"], "results": rows}, "v", "Flask server memory and deployed size, MB (lower is better)", "MB", False))) + for name, svg in out: + with open(os.path.join(a.out, name), "w") as f: + f.write(svg) if __name__ == "__main__": From 78f3c4aa7e780b487d9619e6a46c782660b4c90b Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 08:18:27 +0200 Subject: [PATCH 033/112] Python: classes report the Python file and lines to stack traces and debuggers The transpiler writes a /* line N */ comment above each statement and, on the class declaration's own line, a table pairing Java lines with Python lines. PythonLineMapper (remap-compat, cn1RemapCompat, the backend build script) rewrites the line tables and source file of each module's classes from it, adds a JSR-45 source map with the file's path and drops the table. Co-Authored-By: Claude Opus 5.5 --- docs/developer-guide/Python-Interop.asciidoc | 30 +++ .../com/codename1/maven/PythonLineMapper.java | 189 ++++++++++++++++++ .../codename1/maven/PythonLineMapperTest.java | 143 +++++++++++++ .../gradle/tasks/RemapCompatAction.java | 7 + .../com/codename1/maven/RemapCompatMojo.java | 8 + .../transpiler/api/PythonTranspiler.java | 61 ++++++ .../python/transpiler/codegen/Emitter.java | 16 ++ .../python/transpiler/codegen/TypedEmit.java | 1 + scripts/python-backend/build.sh | 6 +- 9 files changed, 460 insertions(+), 1 deletion(-) create mode 100644 maven/build-engine/src/main/java/com/codename1/maven/PythonLineMapper.java create mode 100644 maven/build-engine/src/test/java/com/codename1/maven/PythonLineMapperTest.java diff --git a/docs/developer-guide/Python-Interop.asciidoc b/docs/developer-guide/Python-Interop.asciidoc index 6259b2ac3cf..fd2b69eea73 100644 --- a/docs/developer-guide/Python-Interop.asciidoc +++ b/docs/developer-guide/Python-Interop.asciidoc @@ -48,6 +48,36 @@ The goal reads these properties: | `cn1.python.skip` | `true` skips the step. |=== +=== Debugging + +The translated classes carry the Python source's file and line numbers in +place of the generated Java's. A stack trace names the Python file and line: + +---- +com.codename1.python.runtime.PyException: ValueError: bad 6 + at com.codename1.generated.python.py_app$2eutil.M_L1_Thing_boom(util.py:7) + at com.codename1.generated.python.py_main.lambda$exec$1(main.py:6) + at com.codename1.generated.python.py_main.exec(main.py:10) +---- + +The same lines reach a Java debugger attached to the simulator or to a server +binary run on the JVM. Each class also carries a JSR-45 source map whose +default stratum, `Python`, gives the file's path within the source directory, +for example `app/util.py`. Breakpoints are set by Python line number, and a +debugger that resolves sources through the source map opens the `.py` file. +With `jdb`: + +[source,bash] +---- +jdb -sourcepath common/src/main/python -classpath ... MyAppStub +stop at com.codename1.generated.python.py_app$2eutil:6 +---- + +The mapping is applied to the compiled classes by the build's `remap-compat` +step (`cn1RemapCompat` in Gradle), so the generated Java in +`target/generated-sources/python` still shows which Python line each +statement came from, as a `/* line N */` comment above it. + === Calling Codename One and Java from Python Any class on the project's class path is importable under the `java.` prefix: diff --git a/maven/build-engine/src/main/java/com/codename1/maven/PythonLineMapper.java b/maven/build-engine/src/main/java/com/codename1/maven/PythonLineMapper.java new file mode 100644 index 00000000000..eaaa5a16fb5 --- /dev/null +++ b/maven/build-engine/src/main/java/com/codename1/maven/PythonLineMapper.java @@ -0,0 +1,189 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.maven; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.HashMap; +import java.util.Map; +import java.util.TreeMap; + +import org.objectweb.asm.ClassReader; +import org.objectweb.asm.ClassVisitor; +import org.objectweb.asm.ClassWriter; +import org.objectweb.asm.FieldVisitor; +import org.objectweb.asm.Label; +import org.objectweb.asm.MethodVisitor; +import org.objectweb.asm.Opcodes; + +/// Points the classes translated from Python at the Python source. +/// +/// The transpiler writes, into each module's class, a constant naming the +/// `.py` file and pairing Java lines with the Python lines they came from. +/// This rewrites every line number of the module's classes (nested ones +/// included) to the Python line, sets the source file to the `.py` file, +/// adds a JSR-45 source map carrying its path, and removes the constant. +/// A Java debugger then steps through, and sets breakpoints in, the Python +/// source, and a stack trace names `main.py:12` -- on every target, since the +/// device translators read the same line tables. +/// +/// Runs over the compiled classes, after javac; a class without the +/// constant, or already rewritten, is left alone. +public final class PythonLineMapper { + /// The package the transpiler writes into. + static final String PACKAGE_DIR = "com/codename1/generated/python"; + /// Must match `PythonTranspiler.LINE_MAP_FIELD`. + static final String FIELD = "$cn1PyLines"; + + private PythonLineMapper() { + } + + /// `PythonLineMapper `, for a build that compiles the + /// generated Java itself. + public static void main(String[] args) throws IOException { + System.out.println("Mapped " + apply(new File(args[0])) + " Python classes to their sources"); + } + + /// Rewrites the classes under `classesDir`; returns how many. + public static int apply(File classesDir) throws IOException { + File dir = new File(classesDir, PACKAGE_DIR); + File[] files = dir.listFiles(); + if (files == null) { + return 0; + } + Map tables = new HashMap(); + for (File f : files) { + String n = f.getName(); + // Module class names may hold a '$' themselves (`py_app$2eutil`), + // so every class is read for the constant, not only top-level + // looking ones. + if (n.endsWith(".class")) { + Table t = read(Files.readAllBytes(f.toPath())); + if (t != null) { + tables.put(n.substring(0, n.length() - ".class".length()), t); + } + } + } + int count = 0; + for (File f : files) { + String n = f.getName(); + if (!n.endsWith(".class")) { + continue; + } + String stem = n.substring(0, n.length() - ".class".length()); + Table t = null; + String outer = stem; + while (t == null) { + t = tables.get(outer); + int cut = outer.lastIndexOf('$'); + if (t != null || cut < 0) { + break; + } + outer = outer.substring(0, cut); + } + if (t != null) { + Files.write(f.toPath(), rewrite(Files.readAllBytes(f.toPath()), t)); + count++; + } + } + return count; + } + + /// A module's line table. + static final class Table { + final String path; + final TreeMap lines = new TreeMap(); + int max; + + Table(String value) { + int nl = value.indexOf('\n'); + path = value.substring(0, nl); + for (String pair : value.substring(nl + 1).split(",")) { + int colon = pair.indexOf(':'); + int py = Integer.parseInt(pair.substring(colon + 1)); + lines.put(Integer.valueOf(pair.substring(0, colon)), py); + max = Math.max(max, py); + } + } + + int map(int javaLine) { + Map.Entry e = lines.floorEntry(javaLine); + return e != null ? e.getValue() : lines.firstEntry().getValue(); + } + + String fileName() { + return path.substring(path.lastIndexOf('/') + 1); + } + + /// One file, Python lines mapped to themselves: the line table + /// already holds Python lines, and the map adds the file's path, + /// which the source file attribute cannot carry. + String smap(String javaFile) { + return "SMAP\n" + javaFile + "\nPython\n*S Python\n*F\n+ 1 " + fileName() + "\n" + path + + "\n*L\n1#1," + max + ":1\n*E\n"; + } + } + + static Table read(byte[] bytes) { + final String[] value = new String[1]; + new ClassReader(bytes).accept(new ClassVisitor(Opcodes.ASM9) { + @Override + public FieldVisitor visitField(int access, String name, String desc, String signature, Object v) { + if (FIELD.equals(name) && v instanceof String) { + value[0] = (String) v; + } + return null; + } + }, ClassReader.SKIP_CODE); + return value[0] == null ? null : new Table(value[0]); + } + + static byte[] rewrite(byte[] bytes, final Table t) { + ClassReader r = new ClassReader(bytes); + ClassWriter w = new ClassWriter(r, 0); + r.accept(new ClassVisitor(Opcodes.ASM9, w) { + @Override + public void visitSource(String source, String debug) { + super.visitSource(t.fileName(), t.smap(source == null ? "" : source)); + } + + @Override + public FieldVisitor visitField(int access, String name, String desc, String signature, Object v) { + return FIELD.equals(name) ? null : super.visitField(access, name, desc, signature, v); + } + + @Override + public MethodVisitor visitMethod(int access, String name, String desc, String signature, + String[] exceptions) { + return new MethodVisitor(Opcodes.ASM9, super.visitMethod(access, name, desc, signature, exceptions)) { + @Override + public void visitLineNumber(int line, Label start) { + super.visitLineNumber(t.map(line), start); + } + }; + } + }, 0); + return w.toByteArray(); + } +} diff --git a/maven/build-engine/src/test/java/com/codename1/maven/PythonLineMapperTest.java b/maven/build-engine/src/test/java/com/codename1/maven/PythonLineMapperTest.java new file mode 100644 index 00000000000..942dce5dbb0 --- /dev/null +++ b/maven/build-engine/src/test/java/com/codename1/maven/PythonLineMapperTest.java @@ -0,0 +1,143 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.maven; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.List; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.objectweb.asm.ClassReader; +import org.objectweb.asm.ClassVisitor; +import org.objectweb.asm.FieldVisitor; +import org.objectweb.asm.Label; +import org.objectweb.asm.MethodVisitor; +import org.objectweb.asm.Opcodes; + +/// A class translated from Python reports the Python file and lines. +public class PythonLineMapperTest { + /// What the transpiler writes: a line comment before each statement and + /// the table, computed from them, on the class declaration's line. + private static final String SOURCE = "package com.codename1.generated.python;\n" + + "public final class py_app$2eutil { private static final String $cn1PyLines = " + + "\"app/util.py\\n5:3,7:4,9:9\";\n" + + " static int f(int a) {\n" + + " /* line 3 */\n" + + " int b = a + 1;\n" + + " /* line 4 */\n" + + " int c = b * 2;\n" + + " /* line 9 */\n" + + " return new Inner().g(c);\n" + + " }\n" + + " static final class Inner {\n" + + " int g(int x) {\n" + + " return x - 1;\n" + + " }\n" + + " }\n" + + "}\n"; + + @Rule + public TemporaryFolder tmp = new TemporaryFolder(); + + @Test + public void rewritesLinesSourceAndDropsTheTable() throws Exception { + File classes = tmp.newFolder("classes"); + CompatFixtures.compile(tmp.newFolder(), classes, "com/codename1/generated/python/py_app$2eutil.java", SOURCE); + assertEquals(2, PythonLineMapper.apply(classes)); + + File dir = new File(classes, PythonLineMapper.PACKAGE_DIR); + Seen outer = scan(new File(dir, "py_app$2eutil.class")); + assertEquals("util.py", outer.source); + assertTrue(outer.debug, outer.debug.contains("\napp/util.py\n")); + assertTrue(outer.debug, outer.debug.startsWith("SMAP\n")); + assertFalse("the table is removed", outer.hasTable); + assertTrue(outer.lines.toString(), outer.lines.contains(3)); + assertTrue(outer.lines.toString(), outer.lines.contains(4)); + assertTrue(outer.lines.toString(), outer.lines.contains(9)); + for (int line : outer.lines) { + assertTrue("no Java line survives: " + outer.lines, line == 3 || line == 4 || line == 9); + } + // The nested class carries no table of its own and takes its + // module's, its lines being past the last statement's. + Seen inner = scan(new File(dir, "py_app$2eutil$Inner.class")); + assertEquals("util.py", inner.source); + assertEquals("[9]", new java.util.TreeSet(inner.lines).toString()); + + // Running it again changes nothing: the table is gone. + assertEquals(0, PythonLineMapper.apply(classes)); + } + + @Test + public void otherClassesAreLeftAlone() throws Exception { + File classes = tmp.newFolder("classes"); + CompatFixtures.compile(tmp.newFolder(), classes, "com/codename1/generated/python/PythonModules.java", + "package com.codename1.generated.python;\npublic final class PythonModules { }\n"); + File f = new File(classes, PythonLineMapper.PACKAGE_DIR + "/PythonModules.class"); + byte[] before = Files.readAllBytes(f.toPath()); + assertEquals(0, PythonLineMapper.apply(classes)); + assertTrue(java.util.Arrays.equals(before, Files.readAllBytes(f.toPath()))); + assertEquals(0, PythonLineMapper.apply(tmp.newFolder("empty"))); + } + + private static final class Seen { + String source; + String debug; + boolean hasTable; + final List lines = new ArrayList(); + } + + private static Seen scan(File f) throws Exception { + final Seen s = new Seen(); + new ClassReader(Files.readAllBytes(f.toPath())).accept(new ClassVisitor(Opcodes.ASM9) { + @Override + public void visitSource(String source, String debug) { + s.source = source; + s.debug = debug; + } + + @Override + public FieldVisitor visitField(int access, String name, String desc, String sig, Object v) { + s.hasTable |= PythonLineMapper.FIELD.equals(name); + return null; + } + + @Override + public MethodVisitor visitMethod(int access, String name, String desc, String sig, String[] ex) { + return new MethodVisitor(Opcodes.ASM9) { + @Override + public void visitLineNumber(int line, Label start) { + s.lines.add(line); + } + }; + } + }, 0); + return s; + } +} diff --git a/maven/codenameone-gradle-plugin/src/main/java/com/codename1/gradle/tasks/RemapCompatAction.java b/maven/codenameone-gradle-plugin/src/main/java/com/codename1/gradle/tasks/RemapCompatAction.java index 7dac1a03921..8a9fb7a9754 100644 --- a/maven/codenameone-gradle-plugin/src/main/java/com/codename1/gradle/tasks/RemapCompatAction.java +++ b/maven/codenameone-gradle-plugin/src/main/java/com/codename1/gradle/tasks/RemapCompatAction.java @@ -97,6 +97,13 @@ public RemapCompatAction shippingWhenEmpty(FileCollection javaSources) { @Override public void execute(Task task) { + // Classes translated from Python get the Python file and lines, so + // a debugger and a stack trace show the Python source. + try { + com.codename1.maven.PythonLineMapper.apply(classesDir); + } catch (java.io.IOException ex) { + throw new GradleException("Could not map Python line numbers in " + classesDir, ex); + } Set classpath = compileClasspath.getFiles(); // By the jars alone, as the first test: with none there is nothing to // read the classes for. diff --git a/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/RemapCompatMojo.java b/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/RemapCompatMojo.java index 65a0c409e96..d785005ff08 100644 --- a/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/RemapCompatMojo.java +++ b/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/RemapCompatMojo.java @@ -67,6 +67,14 @@ protected void executeImpl() throws MojoExecutionException, MojoFailureException classpath.add(artifact.getFile()); } } + // Classes translated from Python get the Python file and lines, so + // a debugger and a stack trace show the Python source. + File compiled = new File(project.getBuild().getOutputDirectory()); + try { + PythonLineMapper.apply(compiled); + } catch (java.io.IOException ex) { + throw new MojoExecutionException("Could not map Python line numbers in " + compiled, ex); + } // By the jars alone: whether a desktop layer is also used can only be // read from the classes, and Kotlin's are not all in place yet. if (CompatLayers.active(classpath).isEmpty()) { diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java index 4bc21e10a31..41d7971fd65 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java @@ -147,6 +147,7 @@ public static Map transpile(Map sources, String // whole-program types, and is left out of the closed-world check. String java = new Emitter(path, module, isPackage, PACKAGE, cls, scope) .closed(closed && !stdlib.contains(path)).emit(tree); + java = withLineMap(java, path.replace('\\', '/')); out.put(PACKAGE.replace('.', '/') + "/" + cls + ".java", java); classes.put(module, cls); } @@ -428,6 +429,66 @@ private static String backendServer() { + "}\n"; } + /// The name of the constant that carries a module's line table: the + /// Python file, a newline, then `javaLine:pythonLine` pairs at every + /// line where the Python line changes. The build's class post-processing + /// (`PythonLineMapper` in build-engine) rewrites the class's line + /// numbers and source file from it and removes the constant. + public static final String LINE_MAP_FIELD = "$cn1PyLines"; + + /// Adds the line table, computed from the line comments the emitter + /// wrote before each statement, to the class declaration's own line so + /// that no line moves. + static String withLineMap(String java, String pyFile) { + StringBuilder pairs = new StringBuilder(); + int line = 0; + int py = 0; + int last = -1; + int start = 0; + while (start < java.length()) { + int end = java.indexOf('\n', start); + if (end < 0) { + end = java.length(); + } + line++; + int i = start; + while (i < end && java.charAt(i) == ' ') { + i++; + } + if (java.startsWith(Emitter.LINE_MARK, i)) { + int stop = java.indexOf(' ', i + Emitter.LINE_MARK.length()); + py = Integer.parseInt(java.substring(i + Emitter.LINE_MARK.length(), stop)); + } else if (py > 0 && py != last && i < end) { + pairs.append(pairs.length() == 0 ? "" : ",").append(line).append(':').append(py); + last = py; + } + start = end + 1; + } + String decl = "public final class "; + int at = java.indexOf(decl); + int brace = at < 0 ? -1 : java.indexOf("{\n", at); + if (brace < 0 || pairs.length() == 0) { + return java; + } + String value = pyFile + "\n" + pairs; + StringBuilder lit = new StringBuilder("\""); + for (int k = 0; k < value.length(); k++) { + char c = value.charAt(k); + if (c == '\n') { + lit.append("\\n"); + } else if (c == '"' || c == '\\') { + lit.append('\\').append(c); + } else if (c < 0x20 || c > 0x7e) { + lit.append(String.format("\\u%04x", (int) c)); + } else { + lit.append(c); + } + } + lit.append('"'); + return java.substring(0, brace + 1) + " private static final String " + LINE_MAP_FIELD + " = " + lit + ";" + + java.substring(brace + 1); + } + private static String registry(Map classes, String mainModule, boolean interop, boolean server) { StringBuilder sb = new StringBuilder(); sb.append("// Generated by the Codename One Python transpiler.\n"); diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Emitter.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Emitter.java index 3525e446e31..b00b134cbc2 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Emitter.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Emitter.java @@ -455,6 +455,21 @@ String nameLvalue(String n) { // ---- statements ------------------------------------------------------ + /// Prefix of the comment that names the Python line the code after it + /// comes from. [com.codename1.python.transpiler.api.PythonTranspiler] + /// turns these into the line table the build later writes into the + /// class file, so a debugger and a stack trace show the Python source. + public static final String LINE_MARK = "/* line "; + + static void lineMark(int line, StringBuilder out, int ind) { + if (line > 0) { + indent(out, ind); + // A block comment, which stays a comment wherever the statement + // is spliced, even inside an expression. + out.append(LINE_MARK).append(line).append(" */\n"); + } + } + private static void indent(StringBuilder out, int ind) { for (int i = 0; i < ind; i++) { out.append(" "); @@ -474,6 +489,7 @@ private void block(List body, StringBuilder out, int ind) { } void stmt(Stmt s, StringBuilder out, int ind) { + lineMark(s.line, out, ind); if (s instanceof ExprStmt) { Expr v = ((ExprStmt) s).value; if (v instanceof Constant) { diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/TypedEmit.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/TypedEmit.java index 8ace95cd341..d0714a497c8 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/TypedEmit.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/TypedEmit.java @@ -1022,6 +1022,7 @@ private boolean stmt(Stmt s, StringBuilder b, int ind) { StringBuilder sb = new StringBuilder(); try { boolean r = stmt0(s, sb, ind); + Emitter.lineMark(s.line, b, ind); if (pre != null) { b.append(pre); } diff --git a/scripts/python-backend/build.sh b/scripts/python-backend/build.sh index ca067db8562..f38970af2ee 100755 --- a/scripts/python-backend/build.sh +++ b/scripts/python-backend/build.sh @@ -31,8 +31,10 @@ CORE=$(art com/codenameone codenameone-core 8.0-SNAPSHOT) RUNTIME=$(art com/codenameone codenameone-python-runtime 8.0-SNAPSHOT) TRANSPILER=$(art com/codenameone codenameone-python-transpiler 8.0-SNAPSHOT) BACKEND=$(art com/codenameone codenameone-backend 8.0-SNAPSHOT) +ENGINE=$(art com/codenameone codenameone-build-engine 8.0-SNAPSHOT) +ASM=$(art org/ow2/asm asm 9.8) JAVAAPI=$(art com/codename1/parparvm JavaAPI 1.0-SNAPSHOT) -for j in "$CORE" "$RUNTIME" "$TRANSPILER" "$BACKEND" "$JAVAAPI"; do +for j in "$CORE" "$RUNTIME" "$TRANSPILER" "$BACKEND" "$JAVAAPI" "$ENGINE" "$ASM"; do [ -f "$j" ] || { echo "missing $j -- install it (see the header of this script)" >&2; exit 1; } done @@ -62,6 +64,8 @@ JAVA (cd "$CLASSES" && unzip -qo "$RUNTIME" 'com/*' && unzip -qo "$CORE" 'com/codename1/util/*') "$J17/bin/javac" -nowarn --release 17 -d "$CLASSES" $(find "$REPO/scripts/python-bench/hostfs" -name '*.java') "$J17/bin/javac" -nowarn --release 17 -cp "$CLASSES:$BACKEND" -d "$CLASSES" $(find "$GEN" -name '*.java') +# Line numbers and source file of the Python, for stack traces and debuggers. +"$J17/bin/java" -cp "$ENGINE:$ASM" com.codename1.maven.PythonLineMapper "$CLASSES" # The backend build compiles a demo directory with JDK 8; the application is # already compiled, so it is an empty one and the classes come in through From a4d02084ce15a837016551ff507c7bf76f396d07 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 08:29:51 +0200 Subject: [PATCH 034/112] Python: sqlite3 over Codename One's Database; rows report SQLite storage classes; tracebacks list Python frames Database.getColumnType(Row, int) and the TypedRow interface report how SQLite stored a value (JavaSE through the driver's value class, Android through Cursor.getType, iOS through sqlite3_column_type); the other ports answer COLUMN_UNKNOWN and sqlite3 infers the type from the text there. sqlite3 is the DB-API over a PySql engine: Cn1Sql in an application, a JDBC one in the conformance runs. float() accepts bytes, as CPython's does. Co-Authored-By: Claude Opus 5.5 --- .../src/com/codename1/db/Database.java | 39 + .../com/codename1/db/ThreadSafeDatabase.java | 12 +- .../src/com/codename1/db/TypedRow.java | 50 + .../com/codename1/impl/AbstractDBCursor.java | 19 +- .../codename1/impl/android/AndroidCursor.java | 27 +- .../com/codename1/impl/javase/SECursor.java | 27 + Ports/iOSPort/nativeSources/IOSNative.m | 13 + .../com/codename1/impl/ios/DatabaseImpl.java | 5 + .../src/com/codename1/impl/ios/IOSNative.java | 1 + .../codename1/python/runtime/Builtins.java | 13 + .../com/codename1/python/runtime/Cn1Sql.java | 194 ++++ .../com/codename1/python/runtime/PySql.java | 122 +++ .../codename1/python/runtime/StdModules.java | 171 ++++ maven/python-transpiler/pom.xml | 7 + .../transpiler/api/PythonTranspiler.java | 3 + .../python/stdlib/sqlite3/__init__.py | 902 ++++++++++++++++++ .../codename1/python/stdlib/sqlite3/dbapi2.py | 6 + .../python/transpiler/ConformanceTest.java | 7 +- .../codename1/python/transpiler/JvmSql.java | 147 +++ .../test/resources/conformance/sqlite3_.out | 51 + .../test/resources/conformance/sqlite3_.py | 132 +++ 21 files changed, 1944 insertions(+), 4 deletions(-) create mode 100644 CodenameOne/src/com/codename1/db/TypedRow.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/Cn1Sql.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/PySql.java create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/sqlite3/__init__.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/sqlite3/dbapi2.py create mode 100644 maven/python-transpiler/src/test/java/com/codename1/python/transpiler/JvmSql.java create mode 100644 maven/python-transpiler/src/test/resources/conformance/sqlite3_.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/sqlite3_.py diff --git a/CodenameOne/src/com/codename1/db/Database.java b/CodenameOne/src/com/codename1/db/Database.java index 5cb6c1c4852..d994a04cb05 100644 --- a/CodenameOne/src/com/codename1/db/Database.java +++ b/CodenameOne/src/com/codename1/db/Database.java @@ -819,6 +819,45 @@ public void changeKey(DatabaseConfig config) throws IOException { "Changing the database key is not supported on this platform"); } + /// `#getColumnType(Row, int)`: the value is SQL NULL. + public static final int COLUMN_NULL = 0; + /// `#getColumnType(Row, int)`: the value is stored as a signed integer. + public static final int COLUMN_INTEGER = 1; + /// `#getColumnType(Row, int)`: the value is stored as a floating point number. + public static final int COLUMN_FLOAT = 2; + /// `#getColumnType(Row, int)`: the value is stored as text. + public static final int COLUMN_TEXT = 3; + /// `#getColumnType(Row, int)`: the value is stored as a blob. + public static final int COLUMN_BLOB = 4; + /// `#getColumnType(Row, int)`: the port cannot tell how the value is stored. + public static final int COLUMN_UNKNOWN = -1; + + /// Returns how the value in a column of the row's current position is stored. + /// + /// SQLite types values rather than columns, so this is per row. The typed getters convert + /// whatever is stored; use this to read a value back in the type it was written with. + /// + /// #### Parameters + /// + /// - `row`: the row, as returned by `Cursor#getRow()` + /// + /// - `index`: the zero-based column index + /// + /// #### Returns + /// + /// one of `#COLUMN_NULL`, `#COLUMN_INTEGER`, `#COLUMN_FLOAT`, `#COLUMN_TEXT`, + /// `#COLUMN_BLOB`, or `#COLUMN_UNKNOWN` when the port cannot tell + /// + /// #### Throws + /// + /// - `IOException`: if the cursor is closed, not on a row, or the index is out of range + public static int getColumnType(Row row, int index) throws IOException { + if (row instanceof TypedRow) { + return ((TypedRow) row).getColumnType(index); + } + return COLUMN_UNKNOWN; + } + /// Checks if the last value accessed from a given row was null. Not all platforms /// support wasNull(). If the platform does not support it, this will just return false. /// diff --git a/CodenameOne/src/com/codename1/db/ThreadSafeDatabase.java b/CodenameOne/src/com/codename1/db/ThreadSafeDatabase.java index 4aa467dc114..2d643c6f4f2 100644 --- a/CodenameOne/src/com/codename1/db/ThreadSafeDatabase.java +++ b/CodenameOne/src/com/codename1/db/ThreadSafeDatabase.java @@ -526,7 +526,7 @@ interface RunnableWithResponseOrIOException { Object run() throws IOException; } - private class RowWrapper implements RowExt { + private class RowWrapper implements RowExt, TypedRow { private final Row underlyingRow; public RowWrapper(Row underlyingRow) { @@ -613,6 +613,16 @@ public Object run() throws IOException { } }); } + + @Override + public int getColumnType(final int index) throws IOException { + return (Integer) invokeWithException(new RunnableWithResponseOrIOException() { + @Override + public Object run() throws IOException { + return Database.getColumnType(underlyingRow, index); + } + }); + } } /// Implements `CursorExt` as well as `Cursor`, because every cursor a Codename One port diff --git a/CodenameOne/src/com/codename1/db/TypedRow.java b/CodenameOne/src/com/codename1/db/TypedRow.java new file mode 100644 index 00000000000..2392a2a8a6f --- /dev/null +++ b/CodenameOne/src/com/codename1/db/TypedRow.java @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.db; + +import java.io.IOException; + +/// A row that reports the storage class SQLite holds a value in. +/// +/// SQLite types values, not columns: one column can hold an integer in one +/// row and text in the next, and the typed getters of `Row` convert whatever +/// is stored. A caller that has to hand the value back as it was stored -- +/// a scripting layer, a generic exporter -- asks for its type first. Reach +/// this through `Database#getColumnType(Row, int)`, which answers +/// `Database#COLUMN_UNKNOWN` on a row that does not implement it. +public interface TypedRow extends Row { + /// Returns the storage class of the given column of the current row: + /// one of `Database#COLUMN_NULL`, `Database#COLUMN_INTEGER`, + /// `Database#COLUMN_FLOAT`, `Database#COLUMN_TEXT`, + /// `Database#COLUMN_BLOB`, or `Database#COLUMN_UNKNOWN` where the + /// engine cannot tell. + /// + /// #### Parameters + /// + /// - `index`: the zero-based column index + /// + /// #### Throws + /// + /// - `IOException`: if the cursor is closed, not on a row, or the index is out of range + int getColumnType(int index) throws IOException; +} diff --git a/CodenameOne/src/com/codename1/impl/AbstractDBCursor.java b/CodenameOne/src/com/codename1/impl/AbstractDBCursor.java index adba2842438..da639cbe588 100644 --- a/CodenameOne/src/com/codename1/impl/AbstractDBCursor.java +++ b/CodenameOne/src/com/codename1/impl/AbstractDBCursor.java @@ -55,7 +55,7 @@ /// /// Ports whose underlying cursor is genuinely random access, such as Android's, do not need this /// class. -public abstract class AbstractDBCursor implements Cursor, CursorExt, Row, RowExt { +public abstract class AbstractDBCursor implements Cursor, CursorExt, Row, RowExt, com.codename1.db.TypedRow { /// Position of the cursor, -1 before the first row, and the row count once exhausted. private int position = -1; @@ -444,6 +444,23 @@ protected boolean legacyWasNullBeforeAnyRead() { return false; } + /// Returns the storage class of a value as one of the `Database.COLUMN_*` constants, or + /// `Database.COLUMN_UNKNOWN` (the default) where the engine cannot say. Only called on an + /// open cursor positioned on a row, with a valid index. + protected int columnType(int index) throws IOException { + return com.codename1.db.Database.COLUMN_UNKNOWN; + } + + @Override + public int getColumnType(int index) throws IOException { + checkOpen(); + if (!onRow) { + throw new IOException("The cursor is not on a row"); + } + checkColumn(index); + return columnType(index); + } + /// Guards a value read and updates the null latch. private void beginRead(int index) throws IOException { checkOpen(); diff --git a/Ports/Android/src/com/codename1/impl/android/AndroidCursor.java b/Ports/Android/src/com/codename1/impl/android/AndroidCursor.java index 452e0952b4e..cf0e953e122 100644 --- a/Ports/Android/src/com/codename1/impl/android/AndroidCursor.java +++ b/Ports/Android/src/com/codename1/impl/android/AndroidCursor.java @@ -37,7 +37,7 @@ * slower rather than faster. The externally visible behaviour is the same either way, and the * portable conformance suite checks that. */ -public class AndroidCursor implements Cursor, CursorExt, RowExt { +public class AndroidCursor implements Cursor, CursorExt, RowExt, com.codename1.db.TypedRow { private android.database.Cursor c; private int last_read_column_index = -1; @@ -632,6 +632,31 @@ private void checkColumn(int index) throws IOException { } } + @Override + public int getColumnType(int index) throws IOException { + checkOpen(); + checkOnARow(); + checkColumn(index); + try { + switch (c.getType(index)) { + case android.database.Cursor.FIELD_TYPE_NULL: + return Database.COLUMN_NULL; + case android.database.Cursor.FIELD_TYPE_INTEGER: + return Database.COLUMN_INTEGER; + case android.database.Cursor.FIELD_TYPE_FLOAT: + return Database.COLUMN_FLOAT; + case android.database.Cursor.FIELD_TYPE_STRING: + return Database.COLUMN_TEXT; + case android.database.Cursor.FIELD_TYPE_BLOB: + return Database.COLUMN_BLOB; + default: + return Database.COLUMN_UNKNOWN; + } + } catch (RuntimeException failed) { + throw readFailed(index, failed); + } + } + @Override public boolean wasNull() throws IOException { // Checked first: without it a Row retained past the close of its cursor reached straight diff --git a/Ports/JavaSE/src/com/codename1/impl/javase/SECursor.java b/Ports/JavaSE/src/com/codename1/impl/javase/SECursor.java index 7ee6fcc3b7a..9cda35da6b8 100644 --- a/Ports/JavaSE/src/com/codename1/impl/javase/SECursor.java +++ b/Ports/JavaSE/src/com/codename1/impl/javase/SECursor.java @@ -158,6 +158,33 @@ protected String columnLabel(int columnIndex) throws IOException { } } + @Override + protected int columnType(int index) throws IOException { + Object v; + try { + v = resultSet.getObject(index + 1); + } catch (SQLException ex) { + throw new IOException(ex.getMessage(), ex); + } + // The SQLite driver hands each value back in the class of its storage. + if (v == null) { + return com.codename1.db.Database.COLUMN_NULL; + } + if (v instanceof Integer || v instanceof Long || v instanceof Short || v instanceof Byte) { + return com.codename1.db.Database.COLUMN_INTEGER; + } + if (v instanceof Double || v instanceof Float) { + return com.codename1.db.Database.COLUMN_FLOAT; + } + if (v instanceof String) { + return com.codename1.db.Database.COLUMN_TEXT; + } + if (v instanceof byte[]) { + return com.codename1.db.Database.COLUMN_BLOB; + } + return com.codename1.db.Database.COLUMN_UNKNOWN; + } + @Override protected boolean isNullAt(int index) throws IOException { try { diff --git a/Ports/iOSPort/nativeSources/IOSNative.m b/Ports/iOSPort/nativeSources/IOSNative.m index db360e2242e..7a14c573de4 100644 --- a/Ports/iOSPort/nativeSources/IOSNative.m +++ b/Ports/iOSPort/nativeSources/IOSNative.m @@ -13782,6 +13782,19 @@ JAVA_BOOLEAN com_codename1_impl_ios_IOSNative_sqlCursorNullValueAtColumn___long_ return sqlite3_column_type((sqlite3_stmt*)statement, col) == SQLITE_NULL ? JAVA_TRUE : JAVA_FALSE; } +// The storage class as Database.COLUMN_*. Asked before any getter reads the value, since a +// getter's conversion leaves sqlite3_column_type undefined. +JAVA_INT com_codename1_impl_ios_IOSNative_sqlCursorColumnType___long_int_R_int(CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject, JAVA_LONG statement, JAVA_INT col) { + switch (sqlite3_column_type((sqlite3_stmt*)statement, col)) { + case SQLITE_NULL: return 0; + case SQLITE_INTEGER: return 1; + case SQLITE_FLOAT: return 2; + case SQLITE_TEXT: return 3; + case SQLITE_BLOB: return 4; + default: return -1; + } +} + JAVA_INT com_codename1_impl_ios_IOSNative_sqlCursorGetColumnCount___long(CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject, JAVA_LONG statement) { return sqlite3_column_count((sqlite3_stmt*)statement); } diff --git a/Ports/iOSPort/src/com/codename1/impl/ios/DatabaseImpl.java b/Ports/iOSPort/src/com/codename1/impl/ios/DatabaseImpl.java index bd8fac9ac43..10b7d9f7b02 100644 --- a/Ports/iOSPort/src/com/codename1/impl/ios/DatabaseImpl.java +++ b/Ports/iOSPort/src/com/codename1/impl/ios/DatabaseImpl.java @@ -607,6 +607,11 @@ protected boolean isNullAt(int index) throws IOException { return IOSImplementation.nativeInstance.sqlCursorNullValueAtColumn(stmt, index); } + @Override + protected int columnType(int index) throws IOException { + return IOSImplementation.nativeInstance.sqlCursorColumnType(stmt, index); + } + @Override protected boolean legacyWasNullBeforeAnyRead() { // iOS answered wasNull() by asking the statement about the last column index it was diff --git a/Ports/iOSPort/src/com/codename1/impl/ios/IOSNative.java b/Ports/iOSPort/src/com/codename1/impl/ios/IOSNative.java index 05c191d083e..03294d8897d 100644 --- a/Ports/iOSPort/src/com/codename1/impl/ios/IOSNative.java +++ b/Ports/iOSPort/src/com/codename1/impl/ios/IOSNative.java @@ -1000,6 +1000,7 @@ native void gl3dDrawArrays(long contextPeer, long pipelinePeer, long vboPeer, in native short sqlCursorValueAtColumnShort(long statement, int col); native byte[] sqlCursorValueAtColumnText(long statement, int col); native boolean sqlCursorNullValueAtColumn(long statement, int col); //Warning. This function only works if no automatic type conversions have occurred for the value in question. So it must be called before any of the sqlCursorValueAtColumn* methods. After a type conversion, the result of calling this method is undefined, though harmless + native int sqlCursorColumnType(long statement, int col); native int sqlCursorGetColumnCount(long statementPeer); diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Builtins.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Builtins.java index 5b3bbbded2e..fa427075b67 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Builtins.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Builtins.java @@ -823,6 +823,19 @@ static Object toFloat(Object x) { } throw Py.valueError("could not convert string to float: " + Py.toRepr(x)); } + byte[] b = x instanceof PyStr ? null : PyBytes.like(x); + if (b != null) { + // float(b"1.5") parses the bytes as ASCII text. + char[] cs = new char[b.length]; + for (int i = 0; i < b.length; i++) { + cs[i] = (char) (b[i] & 0xff); + } + try { + return toFloat(new String(cs)); + } catch (PyException e) { + throw Py.valueError("could not convert string to float: " + Py.toRepr(x)); + } + } return Py.ofFloat(Py.asDouble(x)); } diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Cn1Sql.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Cn1Sql.java new file mode 100644 index 00000000000..46fca22e8c1 --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Cn1Sql.java @@ -0,0 +1,194 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +import com.codename1.db.Cursor; +import com.codename1.db.Database; +import com.codename1.db.Row; +import java.io.IOException; + +/// SQLite through Codename One's `Database`: the platform's own engine on +/// every target. Values come back in their storage class where the port +/// reports it ([Database#getColumnType(Row, int)]), and are inferred from +/// their text where it does not. +public final class Cn1Sql extends PySql { + private static int memoryCount; + + /// Makes this the engine unless a host installed its own first. + public static void installDefault() { + if (!installed()) { + install(new Cn1Sql()); + } + } + + @Override + public Connection open(String name) throws IOException { + boolean memory = ":memory:".equals(name) || name.length() == 0; + String dbName = memory ? "cn1py-memory-" + System.currentTimeMillis() + "-" + (memoryCount++) + ".db" + : localName(name); + if (memory && Database.exists(dbName)) { + Database.delete(dbName); + } + return new Conn(Database.openOrCreate(dbName), memory ? dbName : null); + } + + /// A path where the platform takes one; the file name alone elsewhere, + /// which puts the database in the application's database directory. + private static String localName(String name) { + if (name.indexOf('/') >= 0 && !Database.isCustomPathSupported()) { + return name.substring(name.lastIndexOf('/') + 1); + } + return name; + } + + private static final class Conn extends Connection { + private final Database db; + private final String deleteOnClose; + + Conn(Database db, String deleteOnClose) { + this.db = db; + this.deleteOnClose = deleteOnClose; + } + + @Override + public Result run(String sql, Object[] params, boolean query) throws IOException { + Result r = new Result(); + if (query) { + Cursor c = params.length == 0 ? db.executeQuery(sql) : db.executeQuery(sql, params); + try { + int n = c.getColumnCount(); + r.columns = new String[n]; + for (int i = 0; i < n; i++) { + r.columns[i] = c.getColumnName(i); + } + while (c.next()) { + Row row = c.getRow(); + Object[] values = new Object[n]; + for (int i = 0; i < n; i++) { + values[i] = value(row, i); + } + r.rows.add(values); + } + } finally { + c.close(); + } + return r; + } + if (params.length == 0) { + db.execute(sql); + } else { + db.execute(sql, params); + } + if (isChange(sql)) { + Cursor c = db.executeQuery("SELECT changes(), last_insert_rowid()"); + try { + if (c.next()) { + Row row = c.getRow(); + r.changes = row.getLong(0); + r.lastRowId = row.getLong(1); + } + } finally { + c.close(); + } + } + return r; + } + + private static Object value(Row row, int i) throws IOException { + switch (Database.getColumnType(row, i)) { + case Database.COLUMN_NULL: + return null; + case Database.COLUMN_INTEGER: + return Long.valueOf(row.getLong(i)); + case Database.COLUMN_FLOAT: + return Double.valueOf(row.getDouble(i)); + case Database.COLUMN_TEXT: + return row.getString(i); + case Database.COLUMN_BLOB: + return row.getBlob(i); + default: + return inferred(row.getString(i)); + } + } + + @Override + public void begin() throws IOException { + db.beginTransaction(); + } + + @Override + public void commit() throws IOException { + db.commitTransaction(); + } + + @Override + public void rollback() throws IOException { + db.rollbackTransaction(); + } + + @Override + public void close() throws IOException { + db.close(); + if (deleteOnClose != null) { + Database.delete(deleteOnClose); + } + } + } + + /// Whether a statement is one whose changes and row id are reported. + static boolean isChange(String sql) { + String w = firstWord(sql); + return "INSERT".equalsIgnoreCase(w) || "UPDATE".equalsIgnoreCase(w) || "DELETE".equalsIgnoreCase(w) + || "REPLACE".equalsIgnoreCase(w); + } + + static String firstWord(String sql) { + int i = 0; + int n = sql.length(); + while (i < n) { + char c = sql.charAt(i); + if (c == '-' && i + 1 < n && sql.charAt(i + 1) == '-') { + while (i < n && sql.charAt(i) != '\n') { + i++; + } + } else if (c == '/' && i + 1 < n && sql.charAt(i + 1) == '*') { + int e = sql.indexOf("*/", i + 2); + i = e < 0 ? n : e + 2; + } else if (c <= ' ' || c == '(') { + i++; + } else { + break; + } + } + int s = i; + while (i < n && isAsciiLetter(sql.charAt(i))) { + i++; + } + return sql.substring(s, i); + } + + private static boolean isAsciiLetter(char c) { + return c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z'; + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PySql.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PySql.java new file mode 100644 index 00000000000..1f2fe47e013 --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PySql.java @@ -0,0 +1,122 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/// The SQLite engine under the `sqlite3` module. +/// +/// A Codename One application uses [Cn1Sql], over `com.codename1.db.Database` +/// and so over each platform's own SQLite. A host that is not a Codename One +/// application (a test, a server) installs its own. +public abstract class PySql { + private static PySql current; + + /// The engine in use. + public static PySql get() { + if (current == null) { + throw Py.error(Py.OS_ERROR, "no SQLite engine is installed"); + } + return current; + } + + static boolean installed() { + return current != null; + } + + /// Replaces the engine. + public static void install(PySql sql) { + current = sql; + } + + /// Opens, creating it if needed, the database `name`: a file name, a + /// path, or `:memory:` for a private database that is gone once closed. + public abstract Connection open(String name) throws IOException; + + /// One open database. + public abstract static class Connection { + /// Runs one statement with its parameters bound in order. Each + /// parameter is null, a `Long`, a `Double`, a `String` or a `byte[]`. + /// `query` says whether the statement returns rows; when it does not, + /// the result has no columns, and the changes and the last row id are + /// filled for an `INSERT`, `UPDATE`, `DELETE` or `REPLACE`. + public abstract Result run(String sql, Object[] params, boolean query) throws IOException; + + public abstract void begin() throws IOException; + + public abstract void commit() throws IOException; + + public abstract void rollback() throws IOException; + + public abstract void close() throws IOException; + } + + /// What a statement produced. Values are null, `Long`, `Double`, + /// `String` or `byte[]`, as SQLite stored them. + public static final class Result { + public String[] columns = new String[0]; + public final List rows = new ArrayList(); + public long changes = -1; + public long lastRowId = -1; + } + + /// The value a SQLite engine that cannot report storage classes most + /// likely stored, from its text: an integer or a real when the text is + /// exactly the canonical form of one, text otherwise. + public static Object inferred(String text) { + if (text == null) { + return null; + } + int n = text.length(); + if (n == 0 || n > 24) { + return text; + } + int i = text.charAt(0) == '-' ? 1 : 0; + boolean digits = i < n; + boolean dot = false; + for (int k = i; k < n; k++) { + char c = text.charAt(k); + if (c == '.' && !dot) { + dot = true; + } else if (c < '0' || c > '9') { + digits = false; + break; + } + } + if (!digits) { + return text; + } + try { + if (!dot) { + long v = Long.parseLong(text); + return Long.toString(v).equals(text) ? (Object) Long.valueOf(v) : text; + } + return Double.valueOf(Double.parseDouble(text)); + } catch (NumberFormatException e) { + return text; + } + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java index 6523e579a18..d9b823a0c6e 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java @@ -74,6 +74,10 @@ static boolean provide(String name) { PyModule.registerModule(m); return true; } + if ("_cn1sql".equals(name)) { + PyModule.registerModule(sql()); + return true; + } if ("_cn1http".equals(name)) { PyModule.registerModule(http()); return true; @@ -115,10 +119,45 @@ public static int exitStatus(PyException e) { return 1; } System.err.println("Traceback (most recent call last):"); + System.err.print(traceback(e)); System.err.println(e.getMessage()); return 1; } + /// The Python frames of an exception's stack, outermost first, as + /// CPython prints them. The build points the translated classes at + /// their `.py` files and lines (`PythonLineMapper`), so the frames are + /// found by their file name; without that mapping there are none, and + /// the traceback is the message alone. + static String traceback(Throwable e) { + StackTraceElement[] st = e.getStackTrace(); + StringBuilder b = new StringBuilder(); + StackTraceElement previous = null; + java.util.List lines = new java.util.ArrayList(); + for (StackTraceElement f : st) { + StackTraceElement callee = previous; + previous = f; + String file = f.getFileName(); + if (file == null || !file.endsWith(".py") || f.getLineNumber() <= 0) { + continue; + } + // A Python function may be a lambda that only calls the method + // holding its body; that lambda's frame, at the `def` line, is + // not a frame of the Python program. + String m = f.getMethodName(); + if (m != null && m.startsWith("lambda$") && callee != null + && f.getClassName().equals(callee.getClassName()) + && !String.valueOf(callee.getMethodName()).startsWith("lambda$")) { + continue; + } + lines.add(" File \"" + file + "\", line " + f.getLineNumber() + "\n"); + } + for (int i = lines.size() - 1; i >= 0; i--) { + b.append(lines.get(i)); + } + return b.toString(); + } + public static void runExitHandlers() { while (!EXIT_CALLS.isEmpty()) { int last = EXIT_CALLS.size() - 1; @@ -272,6 +311,138 @@ private static PyModule fileSystem() { return m; } + /// The open `_cn1sql` connections, by the handle given to Python. + private static final java.util.Map SQL = new java.util.HashMap(); + private static long sqlHandles; + + /// `_cn1sql`: `open(name)` answering a handle; `run(handle, sql, params, + /// query)` answering `(columns, rows, changes, lastrowid)` with each row + /// a tuple; `begin`, `commit`, `rollback` and `close` of a handle. An + /// engine failure raises `OSError` with the engine's message, which the + /// `sqlite3` module turns into its own exceptions. + private static PyModule sql() { + PyModule m = new PyModule("_cn1sql", new String[0]); + m.setAttr("open", PyBuiltin.positional("open", a -> { + try { + PySql.Connection c = PySql.get().open(Py.toStr(a[0])); + long h = ++sqlHandles; + SQL.put(h, c); + return PyInt.of(h); + } catch (java.io.IOException e) { + throw Py.error(Py.OS_ERROR, String.valueOf(e.getMessage())); + } + })); + m.setAttr("run", PyBuiltin.positional("run", a -> { + PySql.Connection c = sqlConnection(a[0]); + Object[] ps = Py.toArray(a[2]); + Object[] params = new Object[ps.length]; + for (int i = 0; i < ps.length; i++) { + params[i] = sqlParam(ps[i], i); + } + PySql.Result r; + try { + r = c.run(Py.toStr(a[1]), params, Py.isTrue(a[3])); + } catch (java.io.IOException e) { + throw Py.error(Py.OS_ERROR, String.valueOf(e.getMessage())); + } + Object[] cols = new Object[r.columns.length]; + for (int i = 0; i < cols.length; i++) { + cols[i] = Py.str(r.columns[i]); + } + Object[] rows = new Object[r.rows.size()]; + for (int i = 0; i < rows.length; i++) { + Object[] values = r.rows.get(i); + Object[] t = new Object[values.length]; + for (int k = 0; k < values.length; k++) { + t[k] = sqlValue(values[k]); + } + rows[i] = new PyTuple(t); + } + return new PyTuple(new Object[] {new PyTuple(cols), new PyList(rows), PyInt.of(r.changes), + PyInt.of(r.lastRowId)}); + })); + sqlControl(m, "begin", 0); + sqlControl(m, "commit", 1); + sqlControl(m, "rollback", 2); + sqlControl(m, "close", 3); + return m; + } + + private static void sqlControl(PyModule m, String name, int op) { + m.setAttr(name, PyBuiltin.positional(name, a -> { + PySql.Connection c = sqlConnection(a[0]); + try { + if (op == 0) { + c.begin(); + } else if (op == 1) { + c.commit(); + } else if (op == 2) { + c.rollback(); + } else { + SQL.remove(Py.asLong(a[0])); + c.close(); + } + } catch (java.io.IOException e) { + throw Py.error(Py.OS_ERROR, String.valueOf(e.getMessage())); + } + return Py.NONE; + })); + } + + private static PySql.Connection sqlConnection(Object handle) { + PySql.Connection c = SQL.get(Py.asLong(handle)); + if (c == null) { + throw Py.error(Py.OS_ERROR, "Cannot operate on a closed database."); + } + return c; + } + + /// A parameter as the engine binds it. The `sqlite3` module has applied + /// the adapters already, so anything else is an error. + private static Object sqlParam(Object v, int index) { + if (v == Py.NONE) { //NOPMD CompareObjectsWithEquals - identity is meant + return null; + } + if (Py.isInt(v)) { + if (v instanceof Long) { + return v; + } + PyInt big = (PyInt) v; + if (big.isBig()) { + throw Py.error(Py.OVERFLOW_ERROR, "Python int too large to convert to SQLite INTEGER"); + } + return Long.valueOf(big.asLong()); + } + if (Py.isFloat(v)) { + return Double.valueOf(Py.asDouble(v)); + } + if (v instanceof String) { + return v; + } + byte[] b = PyBytes.like(v); + if (b != null) { + return b; + } + if (v instanceof PyStr) { + return Py.toStr(v); + } + throw Py.error(Py.TYPE_ERROR, "Error binding parameter " + (index + 1) + ": type '" + + Py.typeOf(v).name + "' is not supported"); + } + + private static Object sqlValue(Object v) { + if (v == null) { + return Py.NONE; + } + if (v instanceof byte[]) { + return new PyBytes((byte[]) v); + } + if (v instanceof String) { + return Py.str((String) v); + } + return v; + } + /// `_cn1http`: `request(method, url, headers, body, timeout, redirects)` /// over [PyHttp], answering `(status, reason, url, headers, body)` with /// the headers a list of pairs. No response at all raises `OSError`. diff --git a/maven/python-transpiler/pom.xml b/maven/python-transpiler/pom.xml index 38ae8f97f1c..d8da8a3ce9a 100644 --- a/maven/python-transpiler/pom.xml +++ b/maven/python-transpiler/pom.xml @@ -130,6 +130,13 @@ junit-jupiter test + + + org.xerial + sqlite-jdbc + 3.46.1.0 + test + diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java index 41d7971fd65..b908d644d2a 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java @@ -510,6 +510,9 @@ private static String registry(Map classes, String mainModule, b if (classes.containsKey("_cn1_http")) { sb.append(" com.codename1.python.runtime.Cn1Http.installDefault();\n"); } + if (classes.containsKey("sqlite3")) { + sb.append(" com.codename1.python.runtime.Cn1Sql.installDefault();\n"); + } for (String cls : classes.values()) { sb.append(" ").append(cls).append(".register();\n"); } diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/sqlite3/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/sqlite3/__init__.py new file mode 100644 index 00000000000..53327183d0d --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/sqlite3/__init__.py @@ -0,0 +1,902 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""The DB-API 2.0 interface to SQLite, over the platform's own engine. + +In an application the database is Codename One's `Database`: a bare file +name lands in the application's database directory, `:memory:` is a +private database removed when it is closed. Connections, cursors, `Row`, +qmark, numbered and named parameters, adapters and converters +(`PARSE_COLNAMES`, and `PARSE_DECLTYPES` for columns read straight from +one table), the legacy and the `autocommit` transaction control, +`executescript`, `iterdump` and the exception hierarchy behave as in +CPython. + +Not provided, raising `NotSupportedError`: user-defined SQL functions, +aggregates and collations, `backup`, `blobopen`, loading extensions. +""" + +import datetime as _datetime + +import _cn1sql + +apilevel = "2.0" +threadsafety = 1 +paramstyle = "qmark" +version = "2.6.0" +version_info = (2, 6, 0) +sqlite_version = "3.45.0" +sqlite_version_info = (3, 45, 0) + +PARSE_DECLTYPES = 1 +PARSE_COLNAMES = 2 +LEGACY_TRANSACTION_CONTROL = -1 + +SQLITE_OK = 0 +SQLITE_DENY = 1 +SQLITE_IGNORE = 2 + + +class Warning(Exception): + pass + + +class Error(Exception): + sqlite_errorcode = None + sqlite_errorname = None + + +class InterfaceError(Error): + pass + + +class DatabaseError(Error): + pass + + +class DataError(DatabaseError): + pass + + +class OperationalError(DatabaseError): + pass + + +class IntegrityError(DatabaseError): + pass + + +class InternalError(DatabaseError): + pass + + +class ProgrammingError(DatabaseError): + pass + + +class NotSupportedError(DatabaseError): + pass + + +class PrepareProtocol: + pass + + +Binary = memoryview + +adapters = {} +converters = {} + + +def register_adapter(type_, adapter): + adapters[(type_, PrepareProtocol)] = adapter + + +def register_converter(name, converter): + converters[name.upper()] = converter + + +def adapt(obj, proto=PrepareProtocol, alt=None): + a = adapters.get((type(obj), proto)) + if a is not None: + return a(obj) + conform = getattr(obj, "__conform__", None) + if conform is not None: + r = conform(proto) + if r is not None: + return r + if alt is not None: + return alt + raise ProgrammingError("can't adapt") + + +register_adapter(_datetime.date, lambda d: d.isoformat()) +register_adapter(_datetime.datetime, lambda d: d.isoformat(" ")) + + +def _convert_date(b): + return _datetime.date(*map(int, b.split(b"-"))) + + +def _convert_timestamp(b): + datepart, timepart = b.split(b" ") + year, month, day = map(int, datepart.split(b"-")) + timepart_full = timepart.split(b".") + hours, minutes, seconds = map(int, timepart_full[0].split(b":")) + microseconds = 0 + if len(timepart_full) == 2: + microseconds = int("{:0<6.6}".format(timepart_full[1].decode())) + return _datetime.datetime(year, month, day, hours, minutes, seconds, microseconds) + + +register_converter("date", _convert_date) +register_converter("timestamp", _convert_timestamp) + + +def _engine_error(e): + """The DB-API exception for an engine failure's message.""" + msg = str(e) + # Drivers wrap SQLite's own message: "[SQLITE_ERROR] SQL error or + # missing database (no such table: t)". The part in the last + # parentheses is what SQLite said. + if msg.startswith("[SQLITE_") and msg.endswith(")"): + depth = 0 + i = len(msg) - 1 + while i >= 0: + c = msg[i] + if c == ")": + depth += 1 + elif c == "(": + depth -= 1 + if depth == 0: + break + i -= 1 + if i > 0: + msg = msg[i + 1:-1] + low = msg.lower() + if "constraint failed" in low or "constraint violation" in low or "not unique" in low: + return IntegrityError(msg) + if "datatype mismatch" in low: + return IntegrityError(msg) + return OperationalError(msg) + + +def _skip_space(sql, i): + n = len(sql) + while i < n: + c = sql[i] + if c == "-" and sql.startswith("--", i): + j = sql.find("\n", i) + i = n if j < 0 else j + 1 + elif c == "/" and sql.startswith("/*", i): + j = sql.find("*/", i + 2) + i = n if j < 0 else j + 2 + elif c in " \t\r\n\f": + i += 1 + else: + break + return i + + +def _first_word(sql): + i = _skip_space(sql, 0) + j = i + n = len(sql) + while j < n and (sql[j].isalpha() or sql[j] == "_"): + j += 1 + return sql[i:j].upper() + + +_QUERY_WORDS = ("SELECT", "PRAGMA", "WITH", "VALUES", "EXPLAIN") +_DML_WORDS = ("INSERT", "UPDATE", "DELETE", "REPLACE") + + +def _scan(sql): + """Splits off the first statement and finds its parameters. + + Answers (statement with every placeholder written as ``?``, the + placeholders in order -- None for a bare ``?``, an int for ``?NNN``, a + name for ``:name``, ``@name`` and ``$name`` -- the rest after the + statement's ``;``, and whether the statement creates a trigger). + """ + out = [] + params = [] + n = len(sql) + i = 0 + start = 0 + trigger = _first_word(sql) == "CREATE" and _is_trigger(sql) + depth_begin = 0 + word = "" + while i < n: + c = sql[i] + if c == "'" or c == '"' or c == "`" or c == "[": + close = "]" if c == "[" else c + j = i + 1 + while j < n: + if sql[j] == close: + if close != "]" and j + 1 < n and sql[j + 1] == close: + j += 2 + continue + break + j += 1 + i = j + 1 + continue + if c == "-" and sql.startswith("--", i): + j = sql.find("\n", i) + i = n if j < 0 else j + 1 + continue + if c == "/" and sql.startswith("/*", i): + j = sql.find("*/", i + 2) + i = n if j < 0 else j + 2 + continue + if c.isalpha() or c == "_": + j = i + while j < n and (sql[j].isalnum() or sql[j] == "_"): + j += 1 + word = sql[i:j].upper() + if trigger: + if word == "BEGIN" or word == "CASE": + depth_begin += 1 + elif word == "END": + depth_begin -= 1 + i = j + continue + if c == "?": + out.append(sql[start:i]) + j = i + 1 + while j < n and sql[j].isdigit(): + j += 1 + params.append(int(sql[i + 1:j]) if j > i + 1 else None) + out.append("?") + start = j + i = j + continue + if (c == ":" or c == "@" or c == "$") and i + 1 < n and (sql[i + 1].isalpha() or sql[i + 1] == "_"): + out.append(sql[start:i]) + j = i + 1 + while j < n and (sql[j].isalnum() or sql[j] == "_"): + j += 1 + params.append(sql[i + 1:j]) + out.append("?") + start = j + i = j + continue + if c == ";" and depth_begin <= 0: + out.append(sql[start:i]) + return "".join(out), params, sql[i + 1:], trigger + i += 1 + out.append(sql[start:]) + return "".join(out), params, "", trigger + + +def _is_trigger(sql): + words = [] + i = 0 + n = len(sql) + while i < n and len(words) < 4: + i = _skip_space(sql, i) + j = i + while j < n and (sql[j].isalnum() or sql[j] == "_"): + j += 1 + if j == i: + break + words.append(sql[i:j].upper()) + i = j + return "TRIGGER" in words + + +def _is_blank(sql): + return _skip_space(sql, 0) >= len(sql) + + +def complete_statement(statement): + rest = statement + while not _is_blank(rest): + stmt, params, after, trig = _scan(rest) + if after == "" and not rest.rstrip().endswith(";"): + return False + rest = after + return statement.rstrip().endswith(";") or _is_blank(statement) + + +class Row: + """A result row indexable by position and, case-insensitively, by name.""" + + def __init__(self, cursor, values): + self._keys = tuple(d[0] for d in cursor.description) + self._values = tuple(values) + + def keys(self): + return list(self._keys) + + def __getitem__(self, key): + if isinstance(key, str): + low = key.lower() + for i, k in enumerate(self._keys): + if k.lower() == low: + return self._values[i] + raise IndexError("No item with that key") + return self._values[key] + + def __len__(self): + return len(self._values) + + def __iter__(self): + return iter(self._values) + + def __eq__(self, other): + if not isinstance(other, Row): + return NotImplemented + return self._keys == other._keys and self._values == other._values + + def __ne__(self, other): + r = self.__eq__(other) + return r if r is NotImplemented else not r + + def __hash__(self): + return hash(self._keys) ^ hash(self._values) + + def __repr__(self): + return "" + + +class Cursor: + def __init__(self, connection): + if not isinstance(connection, Connection): + raise TypeError("argument 1 must be sqlite3.Connection") + self.connection = connection + self.arraysize = 1 + self.row_factory = None + self.description = None + self.rowcount = -1 + self.lastrowid = None + self._rows = [] + self._pos = 0 + self._closed = False + self._inited = True + + def _check(self): + if self._closed: + raise ProgrammingError("Cannot operate on a closed cursor.") + self.connection._check() + + def close(self): + self._closed = True + self._rows = [] + + def setinputsizes(self, sizes): + pass + + def setoutputsize(self, size, column=None): + pass + + def execute(self, sql, parameters=()): + self._check() + if not isinstance(sql, str): + raise TypeError("execute() argument 1 must be str, not " + type(sql).__name__) + stmt, names, rest, trig = _scan(sql) + if not _is_blank(rest): + raise ProgrammingError("You can only execute one statement at a time.") + self._run(stmt, names, parameters, False) + return self + + def executemany(self, sql, seq_of_parameters): + self._check() + stmt, names, rest, trig = _scan(sql) + if not _is_blank(rest): + raise ProgrammingError("You can only execute one statement at a time.") + word = _first_word(stmt) + if word in _QUERY_WORDS and word != "WITH": + raise ProgrammingError("executemany() can only execute DML statements.") + total = 0 + any_run = False + for params in seq_of_parameters: + self._run(stmt, names, params, True) + any_run = True + if self.rowcount > 0: + total += self.rowcount + self.rowcount = total if any_run else 0 + self.description = None + self._rows = [] + self._pos = 0 + return self + + def executescript(self, sql_script): + self._check() + if not isinstance(sql_script, str): + raise ValueError("script argument must be unicode.") + conn = self.connection + if conn._autocommit == LEGACY_TRANSACTION_CONTROL and conn.in_transaction: + conn.commit() + rest = sql_script + while not _is_blank(rest): + stmt, names, rest, trig = _scan(rest) + if _is_blank(stmt): + continue + self._run(stmt, [], (), False, script=True) + self.description = None + self._rows = [] + return self + + def _bind(self, names, parameters): + if not names: + if parameters is None: + return [] + if isinstance(parameters, dict) or (hasattr(parameters, "keys") and hasattr(parameters, "__getitem__") + and not isinstance(parameters, (list, tuple, Row))): + return [] + ps = list(parameters) + if ps: + raise ProgrammingError("Incorrect number of bindings supplied. The current statement uses 0, " + "and there are %d supplied." % len(ps)) + return [] + named = [n for n in names if isinstance(n, str)] + is_map = isinstance(parameters, dict) or (hasattr(parameters, "keys") and hasattr(parameters, "__getitem__") + and not isinstance(parameters, (list, tuple, Row))) + values = [] + if named and is_map: + for n in names: + if not isinstance(n, str): + raise ProgrammingError("Binding %d has no name, but you supplied a dictionary (which has " + "only names)." % (len(values) + 1)) + try: + values.append(parameters[n]) + except KeyError: + raise ProgrammingError("You did not supply a value for binding parameter :%s." % n) from None + return values + if is_map: + raise ProgrammingError("Binding 1 has no name, but you supplied a dictionary (which has only names).") + try: + ps = list(parameters) + except TypeError: + raise ProgrammingError("parameters are of unsupported type") from None + numbered = [n for n in names if isinstance(n, int)] + if numbered: + count = max(numbered) + if len(ps) != count: + raise ProgrammingError("Incorrect number of bindings supplied. The current statement uses %d, " + "and there are %d supplied." % (count, len(ps))) + nxt = 1 + for n in names: + k = n if isinstance(n, int) else nxt + values.append(ps[k - 1]) + nxt = k + 1 + return values + if len(ps) != len(names): + raise ProgrammingError("Incorrect number of bindings supplied. The current statement uses %d, " + "and there are %d supplied." % (len(names), len(ps))) + return ps + + def _adapt(self, v): + if v is None or type(v) in (int, float, str, bytes): + a = adapters.get((type(v), PrepareProtocol)) + return v if a is None else a(v) + a = adapters.get((type(v), PrepareProtocol)) + if a is not None: + return a(v) + conform = getattr(v, "__conform__", None) + if conform is not None: + r = conform(PrepareProtocol) + if r is not None: + return r + if isinstance(v, (bytearray, memoryview)): + return bytes(v) + if isinstance(v, bool): + return int(v) + if isinstance(v, int): + return int(v) + if isinstance(v, float): + return float(v) + if isinstance(v, str): + return str(v) + if isinstance(v, bytes): + return bytes(v) + raise ProgrammingError("Error binding parameter %d: type '%s' is not supported" + % (0, type(v).__name__)) + + def _run(self, stmt, names, parameters, many, script=False): + conn = self.connection + values = self._bind(names, parameters) + bound = [] + for i, v in enumerate(values): + try: + bound.append(self._adapt(v)) + except ProgrammingError: + raise ProgrammingError("Error binding parameter %d: type '%s' is not supported" + % (i + 1, type(v).__name__)) from None + word = _first_word(stmt) + self._rows = [] + self._pos = 0 + if word == "BEGIN": + conn._begin() + self.description = None + self.rowcount = -1 + return + if word == "COMMIT" or word == "END": + conn._commit_raw() + self.description = None + self.rowcount = -1 + return + if word == "ROLLBACK" and "TO" not in stmt.upper().split(): + conn._rollback_raw() + self.description = None + self.rowcount = -1 + return + if word in _DML_WORDS and conn._autocommit == LEGACY_TRANSACTION_CONTROL \ + and conn._isolation_level is not None and not conn.in_transaction: + conn._begin() + query = word in _QUERY_WORDS or " RETURNING " in (" " + stmt.upper().replace("\n", " ") + " ") + try: + cols, rows, changes, rowid = _cn1sql.run(conn._handle, stmt, tuple(bound), query) + except OSError as e: + raise _engine_error(e) from None + if word in ("CREATE", "DROP", "ALTER"): + conn._decltypes = {} + if word in _DML_WORDS: + conn.total_changes += max(changes, 0) + if cols: + self.description = tuple((self._colname(c), None, None, None, None, None, None) for c in cols) + self._rows = self._convert(stmt, cols, rows) + self.rowcount = -1 if word in _QUERY_WORDS else len(rows) + else: + self.description = None + self.rowcount = changes if word in _DML_WORDS else -1 + if word in ("INSERT", "REPLACE") and not many: + self.lastrowid = rowid + elif word in ("INSERT", "REPLACE") and many: + self.lastrowid = rowid + + def _colname(self, name): + if self.connection.detect_types & PARSE_COLNAMES: + k = name.find(" [") + if k >= 0 and name.endswith("]"): + return name[:k] + return name + + def _convert(self, stmt, cols, rows): + conn = self.connection + convs = [None] * len(cols) + if conn.detect_types: + decl = conn._declared(stmt) if conn.detect_types & PARSE_DECLTYPES else {} + for i, c in enumerate(cols): + kind = None + if conn.detect_types & PARSE_COLNAMES: + k = c.find(" [") + if k >= 0 and c.endswith("]"): + kind = c[k + 2:-1] + if kind is None and decl: + kind = decl.get(c.lower()) + if kind is not None: + kind = kind.split("(")[0].strip().split(" ")[0] + convs[i] = converters.get(kind.upper()) + tf = conn.text_factory + out = [] + for row in rows: + vals = list(row) + for i, v in enumerate(vals): + f = convs[i] + if f is not None and v is not None: + if isinstance(v, str): + v = v.encode("utf-8") + elif not isinstance(v, bytes): + v = str(v).encode("ascii") + vals[i] = f(v) + elif isinstance(v, str) and tf is not str: + b = v.encode("utf-8") + vals[i] = b if tf is bytes else tf(b) + out.append(tuple(vals)) + return out + + def _make(self, values): + rf = self.row_factory + if rf is None: + rf = self.connection.row_factory + if rf is None: + return values + return rf(self, values) + + def fetchone(self): + self._check() + if self._pos >= len(self._rows): + return None + r = self._rows[self._pos] + self._pos += 1 + return self._make(r) + + def fetchmany(self, size=None): + if size is None: + size = self.arraysize + out = [] + while len(out) < size: + r = self.fetchone() + if r is None: + break + out.append(r) + return out + + def fetchall(self): + out = [] + while True: + r = self.fetchone() + if r is None: + return out + out.append(r) + + def __iter__(self): + return self + + def __next__(self): + r = self.fetchone() + if r is None: + raise StopIteration + return r + + +class Connection: + def __init__(self, database, timeout=5.0, detect_types=0, isolation_level="", check_same_thread=True, + factory=None, cached_statements=128, uri=False, *, autocommit=LEGACY_TRANSACTION_CONTROL): + name = database + if hasattr(name, "__fspath__"): + name = name.__fspath__() + if isinstance(name, bytes): + name = name.decode("utf-8") + if uri: + name = _from_uri(name) + if autocommit not in (True, False, LEGACY_TRANSACTION_CONTROL): + raise ValueError("autocommit must be True, False, or sqlite3.LEGACY_TRANSACTION_CONTROL") + self.detect_types = detect_types + self.row_factory = None + self.text_factory = str + self.total_changes = 0 + self.in_transaction = False + self._isolation_level = isolation_level + self._autocommit = autocommit + self._decltypes = {} + self._closed = False + try: + self._handle = _cn1sql.open(name) + except OSError as e: + raise OperationalError("unable to open database file") from None + self.Warning = Warning + self.Error = Error + self.InterfaceError = InterfaceError + self.DatabaseError = DatabaseError + self.DataError = DataError + self.OperationalError = OperationalError + self.IntegrityError = IntegrityError + self.InternalError = InternalError + self.ProgrammingError = ProgrammingError + self.NotSupportedError = NotSupportedError + if autocommit is False: + self._begin() + + def _check(self): + if self._closed: + raise ProgrammingError("Cannot operate on a closed database.") + + @property + def isolation_level(self): + return self._isolation_level + + @isolation_level.setter + def isolation_level(self, value): + if value is not None and str(value).upper() not in ("", "DEFERRED", "IMMEDIATE", "EXCLUSIVE"): + raise ValueError("isolation_level string must be '', 'DEFERRED', 'IMMEDIATE', or 'EXCLUSIVE'") + if value is None and self.in_transaction and self._autocommit == LEGACY_TRANSACTION_CONTROL: + self.commit() + self._isolation_level = value + + @property + def autocommit(self): + return self._autocommit + + @autocommit.setter + def autocommit(self, value): + if value not in (True, False, LEGACY_TRANSACTION_CONTROL): + raise ValueError("autocommit must be True, False, or sqlite3.LEGACY_TRANSACTION_CONTROL") + if value is True and self.in_transaction: + self._commit_raw() + self._autocommit = value + if value is False and not self.in_transaction: + self._begin() + + def _begin(self): + if self.in_transaction: + raise OperationalError("cannot start a transaction within a transaction") + try: + _cn1sql.begin(self._handle) + except OSError as e: + raise _engine_error(e) from None + self.in_transaction = True + + def _commit_raw(self): + if not self.in_transaction: + raise OperationalError("cannot commit - no transaction is active") + try: + _cn1sql.commit(self._handle) + except OSError as e: + raise _engine_error(e) from None + self.in_transaction = False + + def _rollback_raw(self): + if not self.in_transaction: + raise OperationalError("cannot rollback - no transaction is active") + try: + _cn1sql.rollback(self._handle) + except OSError as e: + raise _engine_error(e) from None + self.in_transaction = False + + def _declared(self, stmt): + """The declared column types of the one table a query reads from.""" + words = stmt.replace("\n", " ").replace("\t", " ").split() + table = None + for i, w in enumerate(words): + if w.upper() == "FROM" and i + 1 < len(words): + table = words[i + 1].strip("`\"[];,()") + if i + 2 < len(words) and words[i + 2].upper() == "JOIN": + return {} + break + if not table: + return {} + if table in self._decltypes: + return self._decltypes[table] + try: + cols, rows, ch, rid = _cn1sql.run(self._handle, "PRAGMA table_info(\"%s\")" + % table.replace('"', '""'), (), True) + except OSError: + rows = [] + decl = {} + for r in rows: + decl[str(r[1]).lower()] = str(r[2]) if r[2] is not None else "" + decl = {k: v for k, v in decl.items() if v} + self._decltypes[table] = decl + return decl + + def cursor(self, factory=Cursor): + self._check() + c = factory(self) + if self.row_factory is not None and c.row_factory is None: + pass + return c + + def execute(self, sql, parameters=()): + return self.cursor().execute(sql, parameters) + + def executemany(self, sql, parameters): + return self.cursor().executemany(sql, parameters) + + def executescript(self, sql_script): + return self.cursor().executescript(sql_script) + + def commit(self): + self._check() + if self.in_transaction: + self._commit_raw() + if self._autocommit is False: + self._begin() + + def rollback(self): + self._check() + if self.in_transaction: + self._rollback_raw() + if self._autocommit is False: + self._begin() + + def close(self): + if self._closed: + return + if self._autocommit is False and self.in_transaction: + self._rollback_raw() + self._closed = True + _cn1sql.close(self._handle) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + if self._closed: + raise ProgrammingError("Cannot operate on a closed database.") + if exc_type is None: + self.commit() + else: + self.rollback() + return False + + def iterdump(self): + """The SQL that recreates the database, as CPython's iterdump writes it.""" + yield "BEGIN TRANSACTION;" + cols, rows, ch, rid = _cn1sql.run( + self._handle, "SELECT name, type, sql FROM sqlite_master WHERE sql NOT NULL AND type == 'table' " + "ORDER BY name", (), True) + sequence = False + for name, kind, sql in rows: + if name == "sqlite_sequence": + yield "DELETE FROM \"sqlite_sequence\";" + sequence = True + continue + elif name == "sqlite_stat1": + yield "ANALYZE \"sqlite_master\";" + elif name.startswith("sqlite_"): + continue + else: + yield "%s;" % sql + qname = name.replace('"', '""') + c2, info, x, y = _cn1sql.run(self._handle, "PRAGMA table_info(\"%s\")" % qname, (), True) + names = [str(r[1]) for r in info] + q = "SELECT 'INSERT INTO \"%s\" VALUES(' || %s || ')' FROM \"%s\";" % ( + qname, "||','||".join("quote(\"%s\")" % n.replace('"', '""') for n in names), qname) + c3, data, x, y = _cn1sql.run(self._handle, q[:-1], (), True) + for r in data: + yield "%s;" % r[0] + cols, rows, ch, rid = _cn1sql.run( + self._handle, "SELECT name, type, sql FROM sqlite_master WHERE sql NOT NULL AND type IN " + "('index', 'trigger', 'view') ORDER BY name", (), True) + for name, kind, sql in rows: + yield "%s;" % sql + if sequence: + c4, seq, x, y = _cn1sql.run(self._handle, "SELECT name, seq FROM sqlite_sequence", (), True) + for name, seq_value in seq: + yield "INSERT INTO \"sqlite_sequence\" VALUES('%s',%d);" % (name.replace("'", "''"), seq_value) + yield "COMMIT;" + + def create_function(self, name, narg, func, *, deterministic=False): + raise NotSupportedError("user-defined SQL functions are not supported on this platform") + + def create_aggregate(self, name, n_arg, aggregate_class): + raise NotSupportedError("user-defined aggregates are not supported on this platform") + + def create_window_function(self, name, num_params, aggregate_class): + raise NotSupportedError("user-defined window functions are not supported on this platform") + + def create_collation(self, name, callable): + raise NotSupportedError("user-defined collations are not supported on this platform") + + def set_authorizer(self, authorizer_callback): + raise NotSupportedError("authorizers are not supported on this platform") + + def set_progress_handler(self, progress_handler, n): + if progress_handler is not None: + raise NotSupportedError("progress handlers are not supported on this platform") + + def set_trace_callback(self, trace_callback): + if trace_callback is not None: + raise NotSupportedError("trace callbacks are not supported on this platform") + + def enable_load_extension(self, enabled): + raise NotSupportedError("extensions cannot be loaded on this platform") + + def load_extension(self, path, *, entrypoint=None): + raise NotSupportedError("extensions cannot be loaded on this platform") + + def backup(self, target, *, pages=-1, progress=None, name="main", sleep=0.25): + raise NotSupportedError("backup is not supported on this platform") + + def blobopen(self, table, column, row, *, readonly=False, name="main"): + raise NotSupportedError("blob I/O is not supported on this platform") + + def interrupt(self): + pass + + +def _from_uri(uri): + if not uri.startswith("file:"): + raise OperationalError("unable to open database file") + rest = uri[5:] + query = "" + if "?" in rest: + rest, query = rest.split("?", 1) + if rest.startswith("//"): + rest = rest[2:] + k = rest.find("/") + rest = rest[k:] if k >= 0 else "" + for part in query.split("&"): + if part == "mode=memory": + return ":memory:" + return rest or ":memory:" + + +def connect(database, timeout=5.0, detect_types=0, isolation_level="", check_same_thread=True, + factory=Connection, cached_statements=128, uri=False, *, autocommit=LEGACY_TRANSACTION_CONTROL): + return factory(database, timeout, detect_types, isolation_level, check_same_thread, None, + cached_statements, uri, autocommit=autocommit) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/sqlite3/dbapi2.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/sqlite3/dbapi2.py new file mode 100644 index 00000000000..f37a14843ac --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/sqlite3/dbapi2.py @@ -0,0 +1,6 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""`sqlite3.dbapi2`, the same module as `sqlite3`.""" + +from sqlite3 import * diff --git a/maven/python-transpiler/src/test/java/com/codename1/python/transpiler/ConformanceTest.java b/maven/python-transpiler/src/test/java/com/codename1/python/transpiler/ConformanceTest.java index 8f819db5947..6e7c3688255 100644 --- a/maven/python-transpiler/src/test/java/com/codename1/python/transpiler/ConformanceTest.java +++ b/maven/python-transpiler/src/test/java/com/codename1/python/transpiler/ConformanceTest.java @@ -138,7 +138,9 @@ private static void run(String stem, Map sources, boolean http) assertEquals(0, rc, "generated Java did not compile:\n" + diagnostics.toString("UTF-8")); URL[] urls = {cls.toUri().toURL(), new File(runtime).toURI().toURL(), new File(core).toURI().toURL(), - new File(location(JvmFileSystem.class)).toURI().toURL()}; + new File(location(JvmFileSystem.class)).toURI().toURL(), + new File(location(org.sqlite.JDBC.class)).toURI().toURL(), + new File(location(org.slf4j.LoggerFactory.class)).toURI().toURL()}; Path fsRoot = work.resolve("fs"); Files.createDirectories(fsRoot.resolve("home")); ByteArrayOutputStream out = new ByteArrayOutputStream(); @@ -154,6 +156,9 @@ private static void run(String stem, Map sources, boolean http) Object fs = loader.loadClass(JvmFileSystem.class.getName()).getConstructor(String.class) .newInstance(fsRoot.toString()); fsType.getMethod("install", fsType).invoke(null, fs); + Class sqlType = loader.loadClass("com.codename1.python.runtime.PySql"); + sqlType.getMethod("install", sqlType).invoke(null, loader.loadClass(JvmSql.class.getName()) + .getConstructor(String.class).newInstance(fsRoot.resolve("home").toString())); if (http) { Class httpType = loader.loadClass("com.codename1.python.runtime.PyHttp"); httpType.getMethod("install", httpType) diff --git a/maven/python-transpiler/src/test/java/com/codename1/python/transpiler/JvmSql.java b/maven/python-transpiler/src/test/java/com/codename1/python/transpiler/JvmSql.java new file mode 100644 index 00000000000..2bd58e485a8 --- /dev/null +++ b/maven/python-transpiler/src/test/java/com/codename1/python/transpiler/JvmSql.java @@ -0,0 +1,147 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.transpiler; + +import com.codename1.python.runtime.PySql; +import java.io.File; +import java.io.IOException; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.sql.Types; +import java.util.Properties; + +/// The `sqlite3` engine of the conformance runs: SQLite through its JDBC +/// driver, which reports each value in its storage class as the device +/// ports do. Relative names are files under the run's own directory. +public final class JvmSql extends PySql { + private final String root; + + public JvmSql(String root) { + this.root = root; + } + + @Override + public PySql.Connection open(String name) throws IOException { + String url = ":memory:".equals(name) || name.isEmpty() ? "jdbc:sqlite::memory:" + : "jdbc:sqlite:" + (new File(name).isAbsolute() ? name : new File(root, name).getPath()); + try { + return new Conn(new org.sqlite.JDBC().connect(url, new Properties())); + } catch (SQLException e) { + throw new IOException(e.getMessage(), e); + } + } + + private static final class Conn extends PySql.Connection { + private final java.sql.Connection c; + + Conn(java.sql.Connection c) { + this.c = c; + } + + @Override + public Result run(String sql, Object[] params, boolean query) throws IOException { + Result r = new Result(); + try (PreparedStatement st = c.prepareStatement(sql)) { + for (int i = 0; i < params.length; i++) { + Object p = params[i]; + if (p == null) { + st.setNull(i + 1, Types.NULL); + } else if (p instanceof Long) { + st.setLong(i + 1, (Long) p); + } else if (p instanceof Double) { + st.setDouble(i + 1, (Double) p); + } else if (p instanceof byte[]) { + st.setBytes(i + 1, (byte[]) p); + } else { + st.setString(i + 1, p.toString()); + } + } + if (query) { + try (ResultSet rs = st.executeQuery()) { + ResultSetMetaData md = rs.getMetaData(); + int n = md.getColumnCount(); + r.columns = new String[n]; + for (int i = 0; i < n; i++) { + r.columns[i] = md.getColumnLabel(i + 1); + } + while (rs.next()) { + Object[] row = new Object[n]; + for (int i = 0; i < n; i++) { + Object v = rs.getObject(i + 1); + row[i] = v instanceof Integer ? Long.valueOf((Integer) v) : v; + } + r.rows.add(row); + } + } + return r; + } + r.changes = st.executeUpdate(); + } catch (SQLException e) { + throw new IOException(e.getMessage(), e); + } + try (Statement s = c.createStatement(); ResultSet rs = s.executeQuery("SELECT last_insert_rowid()")) { + rs.next(); + r.lastRowId = rs.getLong(1); + } catch (SQLException e) { + throw new IOException(e.getMessage(), e); + } + return r; + } + + private void exec(String sql) throws IOException { + try (Statement s = c.createStatement()) { + s.execute(sql); + } catch (SQLException e) { + throw new IOException(e.getMessage(), e); + } + } + + @Override + public void begin() throws IOException { + exec("BEGIN"); + } + + @Override + public void commit() throws IOException { + exec("COMMIT"); + } + + @Override + public void rollback() throws IOException { + exec("ROLLBACK"); + } + + @Override + public void close() throws IOException { + try { + c.close(); + } catch (SQLException e) { + throw new IOException(e.getMessage(), e); + } + } + } +} diff --git a/maven/python-transpiler/src/test/resources/conformance/sqlite3_.out b/maven/python-transpiler/src/test/resources/conformance/sqlite3_.out new file mode 100644 index 00000000000..4f669c6db77 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/sqlite3_.out @@ -0,0 +1,51 @@ +None -1 +lastrowid 1 rowcount 1 in_tx True +many rowcount 3 +after commit in_tx False total 4 +(2, 'And Now for Something', 1971, 7.5) ['int', 'str', 'int', 'float'] +(1, 'Monty Python', 1975, 8.2) ['int', 'str', 'int', 'float'] +(3, 'Life of Brian', 1979, 8.0) ['int', 'str', 'int', 'float'] +(4, 'Meaning of Life', 1983, 7.6) ['int', 'str', 'int', 'float'] +['id', 'title', 'year', 'score'] +[('Monty Python',), ('Life of Brian',)] +('b', 'a') +(4, 8.2, 'And Now for Something') +None +updated 3 +(8.2,) +(b'\x00\x01\xff', 1975, 8.2) +(None, None, None) +IntegrityError UNIQUE constraint failed: movie.title +OperationalError no such table: no_such_table +ProgrammingError Incorrect number of bindings supplied. The current statement uses 1, and there are 2 supplied. +ProgrammingError You can only execute one statement at a time. +ProgrammingError You did not supply a value for binding parameter :a. +True True +And Now for Something And Now for Something 2 ['id', 'name'] (2, 'And Now for Something') 2 +[('one; two',), ('three',), ('FOUR',)] +(4,) +((1.0, 2.5), datetime.date(2024, 2, 29), datetime.datetime(2024, 1, 2, 3, 4, 5, 600)) +((1.0, 2.5),) +pt +ProgrammingError Cannot operate on a closed database. +(b'abc',) +CREATE TABLE log(id INTEGER PRIMARY KEY AUTOINCREMENT, msg TEXT); +INSERT INTO "log" VALUES(1,'one; two'); +INSERT INTO "log" VALUES(2,'three'); +INSERT INTO "log" VALUES(3,'FOUR'); +INSERT INTO "log" VALUES(4,'CTX'); +CREATE TABLE movie(id INTEGER PRIMARY KEY, title TEXT NOT NULL UNIQUE, year INTEGER, score REAL, poster BLOB); +INSERT INTO "movie" VALUES(1,'Monty Python',1975,8.2,X'0001FF'); +INSERT INTO "movie" VALUES(2,'And Now for Something',1971,7.5,NULL); +INSERT INTO "movie" VALUES(3,'Life of Brian',1979,8.0,NULL); +INSERT INTO "movie" VALUES(4,'Meaning of Life',1983,7.6,NULL); +INSERT INTO "movie" VALUES(5,'Untitled',NULL,NULL,NULL); +CREATE TRIGGER t AFTER INSERT ON log BEGIN UPDATE log SET msg = upper(msg) WHERE id = new.id; END; +INSERT INTO "sqlite_sequence" VALUES('log',4); +True False +autocommit False in_tx True +still True +isolation None in_tx False +explicit True +[(1,)] +qmark 2.0 diff --git a/maven/python-transpiler/src/test/resources/conformance/sqlite3_.py b/maven/python-transpiler/src/test/resources/conformance/sqlite3_.py new file mode 100644 index 00000000000..4b4e23f74bc --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/sqlite3_.py @@ -0,0 +1,132 @@ +import datetime +import sqlite3 + +con = sqlite3.connect(":memory:") +cur = con.cursor() +cur.execute("CREATE TABLE movie(id INTEGER PRIMARY KEY, title TEXT NOT NULL UNIQUE, year INTEGER, score REAL, poster BLOB)") +print(cur.description, cur.rowcount) +cur.execute("INSERT INTO movie(title, year, score) VALUES (?, ?, ?)", ("Monty Python", 1975, 8.2)) +print("lastrowid", cur.lastrowid, "rowcount", cur.rowcount, "in_tx", con.in_transaction) +data = [("And Now for Something", 1971, 7.5), ("Life of Brian", 1979, 8.0), ("Meaning of Life", 1983, 7.6)] +cur.executemany("INSERT INTO movie(title, year, score) VALUES (?, ?, ?)", data) +print("many rowcount", cur.rowcount) +con.commit() +print("after commit in_tx", con.in_transaction, "total", con.total_changes) + +for row in cur.execute("SELECT id, title, year, score FROM movie ORDER BY year"): + print(row, [type(v).__name__ for v in row]) +print([d[0] for d in cur.description]) + +cur.execute("SELECT title FROM movie WHERE year > :y AND score >= :s", {"y": 1974, "s": 7.9}) +print(cur.fetchall()) +cur.execute("SELECT ?2, ?1", ("a", "b")) +print(cur.fetchone()) +cur.execute("SELECT count(*), max(score), min(title) FROM movie") +print(cur.fetchone()) +print(cur.fetchone()) + +cur.execute("UPDATE movie SET score = score + 0.5 WHERE year < ?", (1980,)) +print("updated", cur.rowcount) +con.rollback() +print(con.execute("SELECT score FROM movie WHERE year = 1975").fetchone()) + +con.execute("UPDATE movie SET poster = ? WHERE id = 1", (b"\x00\x01\xff",)) +con.execute("INSERT INTO movie(title, year) VALUES ('Untitled', NULL)") +for r in con.execute("SELECT poster, year, score FROM movie WHERE id IN (1, 5) ORDER BY id"): + print(r) + +try: + con.execute("INSERT INTO movie(title) VALUES (?)", ("Life of Brian",)) +except sqlite3.IntegrityError as e: + print("IntegrityError", e) +try: + con.execute("SELECT * FROM no_such_table") +except sqlite3.OperationalError as e: + print("OperationalError", e) +try: + con.execute("SELECT ?", (1, 2)) +except sqlite3.ProgrammingError as e: + print("ProgrammingError", e) +try: + con.execute("SELECT 1; SELECT 2") +except sqlite3.ProgrammingError as e: + print("ProgrammingError", e) +try: + con.execute("SELECT :a", {"b": 1}) +except sqlite3.ProgrammingError as e: + print("ProgrammingError", e) +print(issubclass(sqlite3.IntegrityError, sqlite3.DatabaseError), issubclass(sqlite3.Error, Exception)) +con.commit() + +con.row_factory = sqlite3.Row +r = con.execute("SELECT id, title AS name FROM movie WHERE id = 2").fetchone() +print(r["name"], r["NAME"], r[0], r.keys(), tuple(r), len(r)) +con.row_factory = None + +con.executescript(""" + CREATE TABLE log(id INTEGER PRIMARY KEY AUTOINCREMENT, msg TEXT); + -- a comment; with a semicolon + INSERT INTO log(msg) VALUES ('one; two'); + INSERT INTO log(msg) VALUES ('three'); + CREATE TRIGGER t AFTER INSERT ON log BEGIN UPDATE log SET msg = upper(msg) WHERE id = new.id; END; + INSERT INTO log(msg) VALUES ('four'); +""") +print(con.execute("SELECT msg FROM log ORDER BY id").fetchall()) + +with con: + con.execute("INSERT INTO log(msg) VALUES ('ctx')") +try: + with con: + con.execute("INSERT INTO log(msg) VALUES ('lost')") + raise ValueError("boom") +except ValueError: + pass +print(con.execute("SELECT count(*) FROM log").fetchone()) + + +class Point: + def __init__(self, x, y): + self.x, self.y = x, y + + +sqlite3.register_adapter(Point, lambda p: "%f;%f" % (p.x, p.y)) +sqlite3.register_converter("point", lambda b: tuple(float(v) for v in b.split(b";"))) +con2 = sqlite3.connect(":memory:", detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES) +con2.execute("CREATE TABLE p(pt point, d date, ts timestamp)") +con2.execute("INSERT INTO p VALUES (?, ?, ?)", (Point(1, 2.5), datetime.date(2024, 2, 29), + datetime.datetime(2024, 1, 2, 3, 4, 5, 600))) +print(con2.execute("SELECT pt, d, ts FROM p").fetchone()) +print(con2.execute('SELECT pt AS "p [point]" FROM p').fetchone()) +print(con2.execute("SELECT pt FROM p").description[0][0]) +con2.close() +try: + con2.execute("SELECT 1") +except sqlite3.ProgrammingError as e: + print("ProgrammingError", e) + +con.text_factory = bytes +print(con.execute("SELECT 'abc'").fetchone()) +con.text_factory = str + +for line in con.iterdump(): + if "movie" in line or "log" in line: + print(line) +print(sqlite3.complete_statement("SELECT 1;"), sqlite3.complete_statement("SELECT 1")) + +con3 = sqlite3.connect(":memory:", autocommit=False) +print("autocommit False in_tx", con3.in_transaction) +con3.execute("CREATE TABLE a(x)") +con3.commit() +print("still", con3.in_transaction) +con3.close() +con4 = sqlite3.connect(":memory:", isolation_level=None) +con4.execute("CREATE TABLE a(x)") +con4.execute("INSERT INTO a VALUES (1)") +print("isolation None in_tx", con4.in_transaction) +con4.execute("BEGIN") +print("explicit", con4.in_transaction) +con4.execute("INSERT INTO a VALUES (2)") +con4.execute("ROLLBACK") +print(con4.execute("SELECT * FROM a").fetchall()) +print(sqlite3.paramstyle, sqlite3.apilevel) +con.close() From 79525b97d31416471b692eb613c1ae2f4bb4b485 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 08:31:44 +0200 Subject: [PATCH 035/112] Python: unittest (TestCase, suites, loader, runner, mock/patch) and a small pytest (raises, approx, marks, fixtures, run_module) Co-Authored-By: Claude Opus 5.5 --- .../codename1/python/site/pytest/__init__.py | 694 +++++++++ .../python/stdlib/unittest/__init__.py | 1340 +++++++++++++++++ .../codename1/python/stdlib/unittest/mock.py | 636 ++++++++ .../test/resources/conformance/pytest_.out | 22 + .../src/test/resources/conformance/pytest_.py | 233 +++ .../test/resources/conformance/unittest_.out | 66 + .../test/resources/conformance/unittest_.py | 261 ++++ .../resources/conformance/unittest_mock_.out | 67 + .../resources/conformance/unittest_mock_.py | 198 +++ 9 files changed, 3517 insertions(+) create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/pytest/__init__.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/unittest/__init__.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/unittest/mock.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/pytest_.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/pytest_.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/unittest_.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/unittest_.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/unittest_mock_.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/unittest_mock_.py diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/pytest/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/pytest/__init__.py new file mode 100644 index 00000000000..e0a18149554 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/pytest/__init__.py @@ -0,0 +1,694 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""A small pytest: the API tests are written against, and a runner. + +Supported: raises (as context manager with match=, excinfo.value / .type / +.typename, and as a call), approx, fail, skip, xfail, importorskip (for modules already imported), +mark.parametrize / skip / skipif / xfail (other marks are accepted and +ignored), param(..., id=, marks=), fixture (function scope, found by parameter name from the test's class, +the module or the fixtures registered so far; yield fixtures run their teardown; +fixtures may use other fixtures; autouse is honoured), the built-in +monkeypatch fixture (capsys, tmp_path and the rest are absent), and run_module(module) / +main([...]) which run every test_* function and Test* class of a module +object and print a summary such as "3 passed, 1 failed". + +Not supported: collecting files from disk, conftest.py, plugins, fixture +scopes above function (accepted, treated as function), assertion rewriting (a failed assert reports its message only), capture of +stdout, and the long traceback report. Plain `assert` is Python's assert. +""" + +import sys +import re +import inspect +import math + +__all__ = ["raises", "approx", "fail", "skip", "xfail", "mark", "fixture", + "main", "run_module", "param", "Failed", "Skipped", "XFailed", + "ExceptionInfo", "warns", "importorskip"] + + +class Failed(BaseException): + pass + + +class Skipped(BaseException): + pass + + +class XFailed(BaseException): + pass + + +def fail(reason="", pytrace=True): + raise Failed(reason) + + +def skip(reason="", allow_module_level=False): + raise Skipped(reason) + + +def xfail(reason=""): + raise XFailed(reason) + + +def importorskip(modname, minversion=None, reason=None): + mod = sys.modules.get(modname) + if mod is None: + raise Skipped(reason or "could not import %r" % modname) + return mod + + +class ExceptionInfo: + def __init__(self): + self.value = None + self.type = None + self.tb = None + + @property + def typename(self): + return self.type.__name__ + + def match(self, regexp): + if not re.search(regexp, str(self.value)): + raise AssertionError("Regex pattern did not match.\n Regex: %r\n Input: %r" % ( + regexp, str(self.value))) + return True + + def errisinstance(self, exc): + return isinstance(self.value, exc) + + def __repr__(self): + if self.type is None: + return "" + return "" % (self.type.__name__, str(self.value)) + + +class _RaisesContext: + def __init__(self, expected, match): + self.expected = expected + self.match_expr = match + self.excinfo = ExceptionInfo() + + def __enter__(self): + return self.excinfo + + def __exit__(self, exc_type, exc, tb): + if exc_type is None: + raise Failed("DID NOT RAISE %s" % _expected_repr(self.expected)) + if not issubclass(exc_type, self.expected): + return False + self.excinfo.value = exc + self.excinfo.type = exc_type + if self.match_expr is not None: + self.excinfo.match(self.match_expr) + return True + + +def _expected_repr(expected): + if isinstance(expected, tuple): + return "any of (" + ", ".join(_expected_repr(e) for e in expected) + ")" + return getattr(expected, "__name__", repr(expected)) + + +def raises(expected_exception, *args, match=None, **kwargs): + ctx = _RaisesContext(expected_exception, match) + if not args: + return ctx + func = args[0] + with ctx: + func(*args[1:], **kwargs) + return ctx.excinfo + + +class _WarnsContext: + def __init__(self, expected): + self.expected = expected + + def __enter__(self): + import warnings + self._cm = warnings.catch_warnings(record=True) + self.list = self._cm.__enter__() + warnings.simplefilter("always") + return self + + def __exit__(self, exc_type, exc, tb): + self._cm.__exit__(exc_type, exc, tb) + if exc_type is not None: + return False + for w in self.list: + if isinstance(w.message, self.expected): + return True + raise Failed("DID NOT WARN. No warnings of type %s were emitted." % (self.expected,)) + + +def warns(expected_warning=Warning, *args, **kwargs): + return _WarnsContext(expected_warning) + + +# -- approx --------------------------------------------------------------- + +class _Approx: + def __init__(self, expected, rel=None, abs=None, nan_ok=False): + self.expected = expected + self.rel = rel + self.abs = abs + self.nan_ok = nan_ok + + def _tolerance(self, exp): + mag = exp if exp >= 0 else -exp + if self.rel is None and self.abs is None: + tol = 1e-6 * mag + return tol if tol > 1e-12 else 1e-12 + if self.abs is None: + return self.rel * mag + if self.rel is None: + return self.abs + tol = self.rel * mag + return tol if tol > self.abs else self.abs + + def _close(self, actual, exp): + if isinstance(exp, bool) or not isinstance(exp, (int, float)): + return actual == exp + if isinstance(actual, bool) or not isinstance(actual, (int, float)): + return False + if actual == exp: + return True + if math.isnan(exp): + return self.nan_ok and math.isnan(actual) + if math.isinf(exp) or math.isinf(actual): + return False + diff = actual - exp + if diff < 0: + diff = -diff + return diff <= self._tolerance(exp) + + def __eq__(self, actual): + exp = self.expected + if isinstance(exp, dict): + if not isinstance(actual, dict) or set(actual) != set(exp): + return False + return all(self._close(actual[k], exp[k]) for k in exp) + if isinstance(exp, (list, tuple)): + if not isinstance(actual, (list, tuple)) or len(actual) != len(exp): + return False + return all(self._close(a, e) for a, e in zip(actual, exp)) + return self._close(actual, exp) + + def __ne__(self, actual): + return not self.__eq__(actual) + + def __hash__(self): + return id(self) + + def __repr__(self): + exp = self.expected + if isinstance(exp, (int, float)) and not isinstance(exp, bool): + tol = self._tolerance(exp) + if math.isnan(exp) or math.isinf(exp): + return "%s" % (exp,) + return "%s ± %.1e" % (exp, tol) + if isinstance(exp, dict): + return "approx({" + ", ".join("%r: %r" % (k, _Approx(v, self.rel, self.abs, self.nan_ok)) + for k, v in exp.items()) + "})" + if isinstance(exp, (list, tuple)): + body = ", ".join(repr(_Approx(v, self.rel, self.abs, self.nan_ok)) for v in exp) + return "approx([" + body + "])" if isinstance(exp, list) else "approx((" + body + ",))" + return "approx(%r)" % (exp,) + + +def approx(expected, rel=None, abs=None, nan_ok=False): + return _Approx(expected, rel, abs, nan_ok) + + +# -- marks ------------------------------------------------------------------ + +class _ParamSet: + def __init__(self, values, marks, id): + self.values = values + self.marks = marks + self.id = id + + +def param(*values, marks=(), id=None): + if not isinstance(marks, (list, tuple)): + marks = (marks,) + return _ParamSet(values, [m.mark if isinstance(m, _MarkDecorator) else m for m in marks], id) + + +class _Mark: + def __init__(self, name, args, kwargs): + self.name = name + self.args = args + self.kwargs = kwargs + + def _apply(self, func): + marks = list(getattr(func, "pytestmark", [])) + marks.append(self) + func.pytestmark = marks + return func + + def __repr__(self): + return "Mark(name=%r, args=%r, kwargs=%r)" % (self.name, self.args, self.kwargs) + + +class _MarkGenerator: + def __getattr__(self, name): + if name.startswith("__"): + raise AttributeError(name) + return _MarkMaker(name) + + +class _MarkMaker: + """mark.name(...) returns a decorator that attaches the mark.""" + + def __init__(self, name): + self.name = name + + def __call__(self, *args, **kwargs): + if len(args) == 1 and not kwargs and callable(args[0]) and self.name not in ("parametrize", "skipif"): + return _Mark(self.name, (), {})._apply(args[0]) + return _MarkDecorator(_Mark(self.name, args, kwargs)) + + +class _MarkDecorator: + def __init__(self, mark): + self.mark = mark + + def __call__(self, func): + return self.mark._apply(func) + + +mark = _MarkGenerator() + + +# -- fixtures ----------------------------------------------------------------- + +_registry = {} + + +class _FixtureDef: + def __init__(self, func, name, autouse): + self.func = func + self.name = name + self.autouse = autouse + + +def fixture(func=None, *, scope="function", params=None, autouse=False, ids=None, name=None): + def register(f): + fname = name or f.__name__ + f.__cn1_fixture__ = _FixtureDef(f, fname, autouse) + _registry[fname] = f.__cn1_fixture__ + return f + if func is not None: + return register(func) + return register + + +class _MonkeyPatch: + def __init__(self): + self._undo = [] + + def setattr(self, target, name, value=None, raising=True): + if value is None and isinstance(target, str): + raise TypeError("monkeypatch.setattr with a string target is not supported") + old = getattr(target, name, _MISSING) + if old is _MISSING and raising: + raise AttributeError("%r has no attribute %r" % (target, name)) + self._undo.append((target, name, old)) + setattr(target, name, value) + + def delattr(self, target, name, raising=True): + old = getattr(target, name, _MISSING) + if old is _MISSING: + if raising: + raise AttributeError(name) + return + self._undo.append((target, name, old)) + delattr(target, name) + + def setitem(self, dic, name, value): + self._undo.append((dic, name, dic.get(name, _MISSING))) + dic[name] = value + + def delitem(self, dic, name, raising=True): + if name not in dic: + if raising: + raise KeyError(name) + return + self._undo.append((dic, name, dic[name])) + del dic[name] + + def undo(self): + while self._undo: + target, name, old = self._undo.pop() + if isinstance(target, dict): + if old is _MISSING: + target.pop(name, None) + else: + target[name] = old + elif old is _MISSING: + delattr(target, name) + else: + setattr(target, name, old) + + +_MISSING = object() + + +# -- running ---------------------------------------------------------------- + +def _param_names(func, skip_self): + names = list(inspect.signature(func).parameters) + if skip_self and names: + names = names[1:] + return names + + +def _split_argnames(argnames): + if isinstance(argnames, str): + return [a.strip() for a in argnames.split(",") if a.strip()] + return list(argnames) + + +def _marks_of(func): + return list(getattr(func, "pytestmark", [])) + + +def _expand_params(func): + """The (id, {name: value}, marks) combinations parametrize marks give.""" + combos = [("", {}, [])] + for m in reversed(_marks_of(func)): + if m.name != "parametrize": + continue + names = _split_argnames(m.args[0]) + values = m.args[1] + ids = m.kwargs.get("ids") + nxt = [] + for base_id, base_vals, base_marks in combos: + for i, v in enumerate(values): + marks = [] + if isinstance(v, _ParamSet): + marks = v.marks + tup = v.values + pid = v.id + elif len(names) == 1: + tup = (v,) + pid = None + else: + tup = tuple(v) + pid = None + if pid is None: + if ids is not None and i < len(ids): + pid = str(ids[i]) + else: + pid = "-".join(_idpart(x, names[j] if j < len(names) else "", i) + for j, x in enumerate(tup)) + vals = dict(base_vals) + for j, n in enumerate(names): + vals[n] = tup[j] + nid = pid if not base_id else base_id + "-" + pid + nxt.append((nid, vals, base_marks + marks)) + combos = nxt + return combos + + +def _idpart(value, argname, index): + if isinstance(value, bool) or value is None: + return str(value) + if isinstance(value, (int, float)): + return str(value) + if isinstance(value, str): + return value + return argname + str(index) + + +class _Result: + def __init__(self): + self.passed = 0 + self.failed = 0 + self.skipped = 0 + self.xfailed = 0 + self.xpassed = 0 + self.errors = 0 + self.outcomes = [] + + def record(self, nodeid, outcome, detail=""): + self.outcomes.append((nodeid, outcome, detail)) + if outcome == "passed": + self.passed += 1 + elif outcome == "failed": + self.failed += 1 + elif outcome == "skipped": + self.skipped += 1 + elif outcome == "xfailed": + self.xfailed += 1 + elif outcome == "xpassed": + self.xpassed += 1 + elif outcome == "error": + self.errors += 1 + + def summary(self): + parts = [] + if self.failed: + parts.append("%d failed" % self.failed) + if self.passed: + parts.append("%d passed" % self.passed) + if self.skipped: + parts.append("%d skipped" % self.skipped) + if self.xfailed: + parts.append("%d xfailed" % self.xfailed) + if self.xpassed: + parts.append("%d xpassed" % self.xpassed) + if self.errors: + parts.append("%d error%s" % (self.errors, "s" if self.errors != 1 else "")) + return ", ".join(parts) if parts else "no tests ran" + + +class _Session: + def __init__(self, module, quiet): + self.module = module + self.quiet = quiet + self.result = _Result() + self.teardowns = [] + + # fixture lookup: class attribute, module attribute, then the registry + def find_fixture(self, name, instance): + if instance is not None: + f = getattr(type(instance), name, None) + if f is not None and hasattr(f, "__cn1_fixture__"): + return f, True + f = getattr(self.module, name, None) + if f is not None and hasattr(f, "__cn1_fixture__"): + return f, False + d = _registry.get(name) + if d is not None: + return d.func, False + return None, False + + def make(self, name, instance, cache, stack): + if name in cache: + return cache[name] + if name == "monkeypatch": + mp = _MonkeyPatch() + self.teardowns.append(mp.undo) + cache[name] = mp + return mp + func, is_method = self.find_fixture(name, instance) + if func is None: + raise Failed("fixture '%s' not found" % name) + if name in stack: + raise Failed("recursive dependency involving fixture '%s'" % name) + stack.append(name) + names = _param_names(func, is_method) + kwargs = {} + for n in names: + kwargs[n] = self.make(n, instance, cache, stack) + stack.pop() + if is_method: + bound = getattr(instance, func.__name__) + value = bound(**kwargs) + else: + value = func(**kwargs) + if _is_generator(value): + gen = value + value = next(gen) + self.teardowns.append(_finish(gen)) + cache[name] = value + return value + + def run_teardowns(self): + errors = [] + while self.teardowns: + t = self.teardowns.pop() + try: + t() + except Exception as e: + errors.append(e) + return errors + + def autouse_names(self, instance): + names = [] + for n in dir(self.module): + f = getattr(self.module, n) + d = getattr(f, "__cn1_fixture__", None) + if d is not None and d.autouse: + names.append(d.name) + if instance is not None: + for n in dir(type(instance)): + f = getattr(type(instance), n) + d = getattr(f, "__cn1_fixture__", None) + if d is not None and d.autouse and d.name not in names: + names.append(d.name) + return names + + def run_one(self, nodeid, func, instance, is_method, extra_marks, values): + marks = _marks_of(func) + extra_marks + if instance is not None: + marks = marks + _marks_of(type(instance)) + xfail_mark = None + for m in marks: + if m.name == "skip": + reason = m.kwargs.get("reason", m.args[0] if m.args else "") + self.result.record(nodeid, "skipped", reason) + return "s" + if m.name == "skipif": + cond = m.args[0] if m.args else True + if cond: + self.result.record(nodeid, "skipped", m.kwargs.get("reason", "")) + return "s" + if m.name == "xfail": + cond = m.args[0] if m.args else True + if cond: + xfail_mark = m + cache = {} + outcome = None + detail = "" + try: + try: + for n in self.autouse_names(instance): + self.make(n, instance, cache, []) + names = _param_names(func, is_method) + kwargs = {} + for n in names: + if n in values: + kwargs[n] = values[n] + else: + kwargs[n] = self.make(n, instance, cache, []) + if is_method: + getattr(instance, func.__name__)(**kwargs) + else: + func(**kwargs) + outcome = "passed" + except Skipped as e: + outcome = "skipped" + detail = str(e) + except XFailed as e: + outcome = "xfailed" + detail = str(e) + except BaseException as e: + if xfail_mark is not None: + want = xfail_mark.kwargs.get("raises") + if want is None or isinstance(e, want): + outcome = "xfailed" + else: + outcome = "failed" + else: + outcome = "failed" + detail = _describe(e) + finally: + errs = self.run_teardowns() + if outcome == "passed" and xfail_mark is not None: + if xfail_mark.kwargs.get("strict"): + outcome = "failed" + detail = "[XPASS(strict)] " + str(xfail_mark.kwargs.get("reason", "")) + else: + outcome = "xpassed" + if errs and outcome in ("passed", "xpassed"): + self.result.record(nodeid, outcome, detail) + self.result.record(nodeid, "error", _describe(errs[0])) + return "E" + self.result.record(nodeid, outcome, detail) + return {"passed": ".", "failed": "F", "skipped": "s", "xfailed": "x", "xpassed": "X"}[outcome] + + def run_function(self, prefix, func, instance, is_method, extra_marks): + out = "" + for pid, values, pmarks in _expand_params(func): + nodeid = prefix + func.__name__ + ("[" + pid + "]" if pid else "") + out += self.run_one(nodeid, func, instance, is_method, extra_marks + pmarks, values) + return out + + def run_class(self, name, cls): + out = "" + for attr in dir(cls): + if not attr.startswith("test"): + continue + member = getattr(cls, attr) + if not callable(member): + continue + for pid, values, pmarks in _expand_params(member): + nodeid = "%s::%s::%s%s" % (self.module_name(), name, attr, "[" + pid + "]" if pid else "") + instance = cls() + if hasattr(instance, "setup_method"): + instance.setup_method(getattr(instance, attr)) + out += self.run_one(nodeid, member, instance, True, pmarks, values) + if hasattr(instance, "teardown_method"): + instance.teardown_method(getattr(instance, attr)) + return out + + def module_name(self): + return getattr(self.module, "__name__", "module") + + +def _is_generator(value): + return type(value).__name__ == "generator" + + +def _finish(gen): + def run(): + try: + next(gen) + except StopIteration: + return + raise Failed("fixture function has more than one 'yield'") + return run + + +def _describe(e): + text = str(e) + name = type(e).__name__ + return "%s: %s" % (name, text) if text else name + + +def run_module(module, quiet=False): + """Run every test_* function and Test* class of a module object. Prints + one line per test when not quiet, then the summary; returns the exit code + pytest would use (0 all passed, 1 failures).""" + session = _Session(module, quiet) + prefix = session.module_name() + "::" + line = "" + for name in dir(module): + obj = getattr(module, name) + if name.startswith("test") and callable(obj) and not isinstance(obj, type) and \ + not hasattr(obj, "__cn1_fixture__"): + line += session.run_function(prefix, obj, None, False, []) + elif isinstance(obj, type) and name.startswith("Test"): + line += session.run_class(name, obj) + result = session.result + if not quiet: + print(line) + for nodeid, outcome, detail in result.outcomes: + if outcome in ("failed", "error"): + print("%s %s - %s" % (outcome.upper(), nodeid, detail)) + print("=== %s ===" % result.summary()) + run_module.last = result + return 0 if result.failed == 0 and result.errors == 0 else 1 + + +def main(args=None, plugins=None, module=None): + """main([...]): the module to run is the keyword `module`, or __main__. + Paths on the command line cannot be loaded from disk, so `args` is read + only for -q, which prints the summary line alone.""" + if module is None: + module = sys.modules["__main__"] + return run_module(module, "-q" in (args or [])) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/unittest/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/unittest/__init__.py new file mode 100644 index 00000000000..457ccc8716d --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/unittest/__init__.py @@ -0,0 +1,1340 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Unit testing framework. + +Supported: TestCase with setUp/tearDown/setUpClass/tearDownClass/addCleanup, +the assert* family (including assertRaises, assertRaisesRegex, assertWarns, +assertLogs-free), subTest, skip/skipIf/skipUnless/expectedFailure, TestSuite, +TestLoader (loadTestsFromTestCase, loadTestsFromModule, loadTestsFromName for +a module object), TestResult, TextTestRunner and main(). + +Not supported: test discovery from the file system, loadTestsFromName with a +dotted string (it names a module by a computed name), assertLogs, and the +difflib-style diff appended to the message of a failed container comparison; +the first line of every message matches CPython's. A failure report has no +traceback lines because the program is compiled ahead of time. main() runs +the module given as its module argument, or __main__. +""" + +import sys +import time +import re +import warnings as _warnings + +__all__ = ["TestCase", "TestSuite", "TestLoader", "TestResult", "TextTestRunner", + "TextTestResult", "SkipTest", "skip", "skipIf", "skipUnless", + "expectedFailure", "main", "defaultTestLoader", "FunctionTestCase", + "installHandler", "IsolatedAsyncioTestCase", "TestProgram"] + +__unittest = True + + +class SkipTest(Exception): + pass + + +class _ShouldStop(Exception): + pass + + +def skip(reason): + def decorator(item): + if isinstance(item, type): + item.__unittest_skip__ = True + item.__unittest_skip_why__ = reason + return item + + def skip_wrapper(*args, **kwargs): + raise SkipTest(reason) + skip_wrapper.__name__ = item.__name__ + skip_wrapper.__unittest_skip__ = True + skip_wrapper.__unittest_skip_why__ = reason + return skip_wrapper + return decorator + + +def skipIf(condition, reason): + if condition: + return skip(reason) + return _id + + +def skipUnless(condition, reason): + if not condition: + return skip(reason) + return _id + + +def _id(obj): + return obj + + +def expectedFailure(test_item): + test_item.__unittest_expecting_failure__ = True + return test_item + + +def installHandler(): + pass + + +class _Outcome: + pass + + +class _SubTest: + def __init__(self, test_case, message, params): + self.test_case = test_case + self.message = message + self.params = params + + def id(self): + return "%s %s" % (self.test_case.id(), self._desc()) + + def _desc(self): + parts = [] + if self.message is not _marker: + parts.append("[%s]" % (self.message,)) + if self.params: + parts.append("(" + ", ".join("%s=%r" % (k, v) for k, v in self.params.items()) + ")") + return " ".join(parts) or "()" + + def __str__(self): + return "%s %s" % (self.test_case, self._desc()) + + def shortDescription(self): + return None + + +_marker = object() + + +class _SubTestContext: + def __init__(self, case, msg, params): + self.case = case + self.msg = msg + self.params = params + + def __enter__(self): + return None + + def __exit__(self, exc_type, exc, tb): + if exc_type is None: + return False + result = self.case._result + if result is None: + return False + sub = _SubTest(self.case, self.msg, self.params) + if issubclass(exc_type, SkipTest): + result.addSkip(sub, str(exc)) + elif issubclass(exc_type, self.case.failureException): + result.addSubTest(self.case, sub, (exc_type, exc, tb)) + self.case._subtest_failed = True + else: + result.addSubTest(self.case, sub, (exc_type, exc, tb)) + self.case._subtest_failed = True + return True + + +class _AssertRaisesContext: + def __init__(self, expected, test_case, expected_regex=None): + self.expected = expected + self.test_case = test_case + self.expected_regex = expected_regex + self.exception = None + self.obj_name = None + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, tb): + if exc_type is None: + name = getattr(self.expected, "__name__", str(self.expected)) + if self.obj_name: + raise self.test_case.failureException("%s not raised by %s" % (name, self.obj_name)) + raise self.test_case.failureException("%s not raised" % name) + if not issubclass(exc_type, self.expected): + return False + self.exception = exc_value + if self.expected_regex is None: + return True + expected_regex = self.expected_regex + if isinstance(expected_regex, str): + expected_regex = re.compile(expected_regex) + if not expected_regex.search(str(exc_value)): + raise self.test_case.failureException( + '"%s" does not match "%s"' % (expected_regex.pattern, str(exc_value))) + return True + + +class _AssertWarnsContext: + def __init__(self, expected, test_case, expected_regex=None): + self.expected = expected + self.test_case = test_case + self.expected_regex = expected_regex + self.warning = None + self.warnings = [] + self.filename = None + self.lineno = None + + def __enter__(self): + self._cm = _warnings.catch_warnings(record=True) + self._log = self._cm.__enter__() + _warnings.simplefilter("always") + return self + + def __exit__(self, exc_type, exc_value, tb): + self._cm.__exit__(exc_type, exc_value, tb) + if exc_type is not None: + return False + first_matching = None + for m in self._log: + w = m.message + if not isinstance(w, self.expected): + continue + if first_matching is None: + first_matching = w + if self.expected_regex is not None and not re.search(self.expected_regex, str(w)): + continue + self.warning = w + self.warnings = self._log + return True + if first_matching is not None: + raise self.test_case.failureException('"%s" does not match "%s"' % ( + self.expected_regex if isinstance(self.expected_regex, str) else self.expected_regex.pattern, + str(first_matching))) + name = getattr(self.expected, "__name__", str(self.expected)) + raise self.test_case.failureException("%s not triggered" % name) + + +def _safe_repr(obj, short=False): + try: + result = repr(obj) + except Exception: + result = object.__repr__(obj) + if not short or len(result) < 80: + return result + return result[:80] + " [truncated]..." + + +def _count_diff_all_purpose(actual, expected): + s, t = list(actual), list(expected) + m, n = len(s), len(t) + NULL = _marker + result = [] + for i, elem in enumerate(s): + if elem is NULL: + continue + cnt_s = cnt_t = 0 + for j in range(i, m): + if s[j] == elem: + cnt_s += 1 + s[j] = NULL + for j, other_elem in enumerate(t): + if other_elem == elem: + cnt_t += 1 + t[j] = NULL + if cnt_s != cnt_t: + result.append((cnt_s, cnt_t, elem)) + for i, elem in enumerate(t): + if elem is NULL: + continue + cnt_t = 0 + for j in range(i, n): + if t[j] == elem: + cnt_t += 1 + t[j] = NULL + result.append((0, cnt_t, elem)) + return result + + +class TestCase: + failureException = AssertionError + longMessage = True + maxDiff = 80 * 8 + + def __init__(self, methodName="runTest"): + self._testMethodName = methodName + self._testMethodDoc = "No test" + self._cleanups = [] + self._result = None + self._subtest_failed = False + self._type_equality_funcs = {} + try: + testMethod = getattr(self, methodName) + except AttributeError: + if methodName != "runTest": + raise ValueError("no such test method in %s: %s" % (self.__class__, methodName)) + else: + self._testMethodDoc = testMethod.__doc__ + + # -- hooks ------------------------------------------------------------ + def setUp(self): + pass + + def tearDown(self): + pass + + @classmethod + def setUpClass(cls): + pass + + @classmethod + def tearDownClass(cls): + pass + + def countTestCases(self): + return 1 + + def defaultTestResult(self): + return TestResult() + + def shortDescription(self): + doc = self._testMethodDoc + return doc.strip().split("\n")[0].strip() if doc else None + + def id(self): + return "%s.%s.%s" % (self.__class__.__module__, self.__class__.__qualname__, self._testMethodName) + + def __eq__(self, other): + if type(self) is not type(other): + return NotImplemented + return self._testMethodName == other._testMethodName + + def __hash__(self): + return hash((type(self), self._testMethodName)) + + def __str__(self): + return "%s (%s.%s)" % (self._testMethodName, self.__class__.__module__, self.__class__.__qualname__ + "." + self._testMethodName) + + def __repr__(self): + return "<%s.%s testMethod=%s>" % (self.__class__.__module__, self.__class__.__qualname__, self._testMethodName) + + def addCleanup(self, function, *args, **kwargs): + self._cleanups.append((function, args, kwargs)) + + def doCleanups(self): + ok = True + while self._cleanups: + function, args, kwargs = self._cleanups.pop() + try: + function(*args, **kwargs) + except BaseException as e: + ok = False + if self._result is not None: + self._result.addError(self, (type(e), e, None)) + return ok + + def addTypeEqualityFunc(self, typeobj, function): + self._type_equality_funcs[typeobj] = function + + def subTest(self, msg=_marker, **params): + return _SubTestContext(self, msg, params) + + # -- running ---------------------------------------------------------- + def run(self, result=None): + if result is None: + result = self.defaultTestResult() + self._result = result + result.startTest(self) + testMethod = getattr(self, self._testMethodName) + try: + if getattr(self.__class__, "__unittest_skip__", False) or \ + getattr(testMethod, "__unittest_skip__", False): + why = getattr(self.__class__, "__unittest_skip_why__", "") or \ + getattr(testMethod, "__unittest_skip_why__", "") + result.addSkip(self, why) + return result + expecting_failure = getattr(testMethod, "__unittest_expecting_failure__", False) or \ + getattr(self.__class__, "__unittest_expecting_failure__", False) + self._subtest_failed = False + outcome = None + ok = False + try: + self.setUp() + except SkipTest as e: + result.addSkip(self, str(e)) + self.doCleanups() + return result + except BaseException as e: + result.addError(self, (type(e), e, None)) + self.doCleanups() + return result + try: + testMethod() + ok = True + except SkipTest as e: + result.addSkip(self, str(e)) + outcome = "skip" + except self.failureException as e: + if expecting_failure: + result.addExpectedFailure(self, (type(e), e, None)) + outcome = "xfail" + else: + result.addFailure(self, (type(e), e, None)) + outcome = "fail" + except BaseException as e: + if expecting_failure: + result.addExpectedFailure(self, (type(e), e, None)) + outcome = "xfail" + else: + result.addError(self, (type(e), e, None)) + outcome = "error" + try: + self.tearDown() + except BaseException as e: + result.addError(self, (type(e), e, None)) + ok = False + outcome = "error" + if not self.doCleanups(): + ok = False + if ok and not self._subtest_failed: + if expecting_failure: + result.addUnexpectedSuccess(self) + else: + result.addSuccess(self) + return result + finally: + result.stopTest(self) + self._result = None + + def __call__(self, *args, **kwds): + return self.run(*args, **kwds) + + def debug(self): + self.setUp() + getattr(self, self._testMethodName)() + self.tearDown() + while self._cleanups: + function, args, kwargs = self._cleanups.pop() + function(*args, **kwargs) + + # -- failing ---------------------------------------------------------- + def skipTest(self, reason): + raise SkipTest(reason) + + def fail(self, msg=None): + raise self.failureException(msg) + + def _formatMessage(self, msg, standardMsg): + if not self.longMessage: + return msg or standardMsg + if msg is None: + return standardMsg + try: + return "%s : %s" % (standardMsg, msg) + except UnicodeDecodeError: + return "%s : %s" % (_safe_repr(standardMsg), _safe_repr(msg)) + + # -- simple asserts --------------------------------------------------- + def assertFalse(self, expr, msg=None): + if expr: + msg = self._formatMessage(msg, "%s is not false" % _safe_repr(expr)) + raise self.failureException(msg) + + def assertTrue(self, expr, msg=None): + if not expr: + msg = self._formatMessage(msg, "%s is not true" % _safe_repr(expr)) + raise self.failureException(msg) + + def _getAssertEqualityFunc(self, first, second): + if type(first) == type(second): + asserter = self._type_equality_funcs.get(type(first)) + if asserter is not None: + return asserter + if isinstance(first, str): + return self.assertMultiLineEqual + if isinstance(first, dict): + return self.assertDictEqual + if isinstance(first, list): + return self.assertListEqual + if isinstance(first, tuple): + return self.assertTupleEqual + if isinstance(first, (set, frozenset)): + return self.assertSetEqual + return self._baseAssertEqual + + def _baseAssertEqual(self, first, second, msg=None): + if not first == second: + standardMsg = "%s != %s" % (_safe_repr(first), _safe_repr(second)) + msg = self._formatMessage(msg, standardMsg) + raise self.failureException(msg) + + def assertEqual(self, first, second, msg=None): + self._getAssertEqualityFunc(first, second)(first, second, msg) + + def assertNotEqual(self, first, second, msg=None): + if not first != second: + msg = self._formatMessage(msg, "%s == %s" % (_safe_repr(first), _safe_repr(second))) + raise self.failureException(msg) + + def assertAlmostEqual(self, first, second, places=None, msg=None, delta=None): + if first == second: + return + if delta is not None and places is not None: + raise TypeError("specify delta or places not both") + diff = abs(first - second) + if delta is not None: + if diff <= delta: + return + standardMsg = "%s != %s within %s delta (%s difference)" % ( + _safe_repr(first), _safe_repr(second), _safe_repr(delta), _safe_repr(diff)) + else: + if places is None: + places = 7 + if round(diff, places) == 0: + return + standardMsg = "%s != %s within %r places (%s difference)" % ( + _safe_repr(first), _safe_repr(second), places, _safe_repr(diff)) + raise self.failureException(self._formatMessage(msg, standardMsg)) + + def assertNotAlmostEqual(self, first, second, places=None, msg=None, delta=None): + if delta is not None and places is not None: + raise TypeError("specify delta or places not both") + diff = abs(first - second) + if delta is not None: + if not (first == second) and diff > delta: + return + standardMsg = "%s == %s within %s delta (%s difference)" % ( + _safe_repr(first), _safe_repr(second), _safe_repr(delta), _safe_repr(diff)) + else: + if places is None: + places = 7 + if not (first == second) and round(diff, places) != 0: + return + standardMsg = "%s == %s within %r places" % (_safe_repr(first), _safe_repr(second), places) + raise self.failureException(self._formatMessage(msg, standardMsg)) + + def assertGreater(self, a, b, msg=None): + if not a > b: + raise self.failureException(self._formatMessage(msg, "%s not greater than %s" % (_safe_repr(a), _safe_repr(b)))) + + def assertGreaterEqual(self, a, b, msg=None): + if not a >= b: + raise self.failureException(self._formatMessage(msg, "%s not greater than or equal to %s" % (_safe_repr(a), _safe_repr(b)))) + + def assertLess(self, a, b, msg=None): + if not a < b: + raise self.failureException(self._formatMessage(msg, "%s not less than %s" % (_safe_repr(a), _safe_repr(b)))) + + def assertLessEqual(self, a, b, msg=None): + if not a <= b: + raise self.failureException(self._formatMessage(msg, "%s not less than or equal to %s" % (_safe_repr(a), _safe_repr(b)))) + + def assertIs(self, expr1, expr2, msg=None): + if expr1 is not expr2: + raise self.failureException(self._formatMessage(msg, "%s is not %s" % (_safe_repr(expr1), _safe_repr(expr2)))) + + def assertIsNot(self, expr1, expr2, msg=None): + if expr1 is expr2: + raise self.failureException(self._formatMessage(msg, "unexpectedly identical: %s" % (_safe_repr(expr1),))) + + def assertIsNone(self, obj, msg=None): + if obj is not None: + raise self.failureException(self._formatMessage(msg, "%s is not None" % (_safe_repr(obj),))) + + def assertIsNotNone(self, obj, msg=None): + if obj is None: + raise self.failureException(self._formatMessage(msg, "unexpectedly None")) + + def assertIn(self, member, container, msg=None): + if member not in container: + raise self.failureException(self._formatMessage(msg, "%s not found in %s" % (_safe_repr(member), _safe_repr(container)))) + + def assertNotIn(self, member, container, msg=None): + if member in container: + raise self.failureException(self._formatMessage(msg, "%s unexpectedly found in %s" % (_safe_repr(member), _safe_repr(container)))) + + def assertIsInstance(self, obj, cls, msg=None): + if not isinstance(obj, cls): + raise self.failureException(self._formatMessage(msg, "%s is not an instance of %r" % (_safe_repr(obj), cls))) + + def assertNotIsInstance(self, obj, cls, msg=None): + if isinstance(obj, cls): + raise self.failureException(self._formatMessage(msg, "%s is an instance of %r" % (_safe_repr(obj), cls))) + + def assertIsSubclass(self, cls, superclass, msg=None): + if not issubclass(cls, superclass): + raise self.failureException(self._formatMessage(msg, "%r is not a subclass of %r" % (cls, superclass))) + + def assertNotIsSubclass(self, cls, superclass, msg=None): + if issubclass(cls, superclass): + raise self.failureException(self._formatMessage(msg, "%r is a subclass of %r" % (cls, superclass))) + + # -- raises / warns --------------------------------------------------- + def assertRaises(self, expected_exception, *args, **kwargs): + context = _AssertRaisesContext(expected_exception, self) + if not args: + msg = kwargs.pop("msg", None) + if kwargs: + raise TypeError("%r is an invalid keyword argument for this function" % (next(iter(kwargs)),)) + return context + callable_obj = args[0] + context.obj_name = getattr(callable_obj, "__name__", str(callable_obj)) + with context: + callable_obj(*args[1:], **kwargs) + return None + + def assertRaisesRegex(self, expected_exception, expected_regex, *args, **kwargs): + context = _AssertRaisesContext(expected_exception, self, expected_regex) + if not args: + kwargs.pop("msg", None) + return context + callable_obj = args[0] + context.obj_name = getattr(callable_obj, "__name__", str(callable_obj)) + with context: + callable_obj(*args[1:], **kwargs) + return None + + def assertWarns(self, expected_warning, *args, **kwargs): + context = _AssertWarnsContext(expected_warning, self) + if not args: + kwargs.pop("msg", None) + return context + with context: + args[0](*args[1:], **kwargs) + return None + + def assertWarnsRegex(self, expected_warning, expected_regex, *args, **kwargs): + context = _AssertWarnsContext(expected_warning, self, expected_regex) + if not args: + kwargs.pop("msg", None) + return context + with context: + args[0](*args[1:], **kwargs) + return None + + # -- regex ------------------------------------------------------------ + def assertRegex(self, text, expected_regex, msg=None): + if isinstance(expected_regex, (str, bytes)): + expected_regex = re.compile(expected_regex) + if not expected_regex.search(text): + standardMsg = "Regex didn't match: %r not found in %r" % (expected_regex.pattern, text) + raise self.failureException(self._formatMessage(msg, standardMsg)) + + def assertNotRegex(self, text, unexpected_regex, msg=None): + if isinstance(unexpected_regex, (str, bytes)): + unexpected_regex = re.compile(unexpected_regex) + match = unexpected_regex.search(text) + if match: + standardMsg = "Regex matched: %r matches %r in %r" % ( + text[match.start():match.end()], unexpected_regex.pattern, text) + raise self.failureException(self._formatMessage(msg, standardMsg)) + + # -- containers ------------------------------------------------------- + def assertCountEqual(self, first, second, msg=None): + differences = [] + for cnt_s, cnt_t, elem in _count_diff_all_purpose(list(first), list(second)): + differences.append("First has %d, Second has %d: %r" % (cnt_s, cnt_t, elem)) + if not differences: + return + standardMsg = "Element counts were not equal:\n" + "\n".join(differences) + raise self.failureException(self._formatMessage(msg, standardMsg)) + + def _assertSequenceFirstDiff(self, seq1, seq2, seq_type, name): + len1 = len(seq1) + len2 = len(seq2) + differing = "%ss differ: %s != %s\n" % (name.capitalize(), _safe_repr(seq1, True), _safe_repr(seq2, True)) + for i in range(min(len1, len2)): + try: + item1 = seq1[i] + item2 = seq2[i] + except (TypeError, IndexError, NotImplementedError): + differing += "\nUnable to index element %d of %s\n" % (i, name) + break + if item1 != item2: + differing += "\nFirst differing element %d:\n%s\n%s\n" % (i, _safe_repr(item1), _safe_repr(item2)) + break + else: + if len1 == len2: + return None + if len1 > len2: + differing += "\nFirst %s contains %d additional elements.\n" % (name, len1 - len2) + try: + differing += "First extra element %d:\n%s\n" % (len2, _safe_repr(seq1[len2])) + except (TypeError, IndexError, NotImplementedError): + pass + elif len1 < len2: + differing += "\nSecond %s contains %d additional elements.\n" % (name, len2 - len1) + try: + differing += "First extra element %d:\n%s\n" % (len1, _safe_repr(seq2[len1])) + except (TypeError, IndexError, NotImplementedError): + pass + return differing + + def assertSequenceEqual(self, seq1, seq2, msg=None, seq_type=None): + if seq_type is not None: + if not isinstance(seq1, seq_type): + raise self.failureException("First sequence is not a %s: %s" % (seq_type.__name__, _safe_repr(seq1))) + if not isinstance(seq2, seq_type): + raise self.failureException("Second sequence is not a %s: %s" % (seq_type.__name__, _safe_repr(seq2))) + name = seq_type.__name__ + else: + name = "sequence" + if seq1 == seq2: + return + differing = self._assertSequenceFirstDiff(seq1, seq2, seq_type, name) + if differing is None: + differing = "%ss differ: %s != %s\n" % (name.capitalize(), _safe_repr(seq1, True), _safe_repr(seq2, True)) + raise self.failureException(self._formatMessage(msg, differing + "\n- %s\n+ %s" % (_safe_repr(seq1), _safe_repr(seq2)))) + + def assertListEqual(self, list1, list2, msg=None): + self.assertSequenceEqual(list1, list2, msg, seq_type=list) + + def assertTupleEqual(self, tuple1, tuple2, msg=None): + self.assertSequenceEqual(tuple1, tuple2, msg, seq_type=tuple) + + def assertSetEqual(self, set1, set2, msg=None): + try: + difference1 = set1.difference(set2) + except TypeError as e: + raise self.failureException("invalid type when attempting set difference: %s" % e) + try: + difference2 = set2.difference(set1) + except TypeError as e: + raise self.failureException("invalid type when attempting set difference: %s" % e) + if not (difference1 or difference2): + return + lines = [] + if difference1: + lines.append("Items in the first set but not the second:") + for item in difference1: + lines.append(repr(item)) + if difference2: + lines.append("Items in the second set but not the first:") + for item in difference2: + lines.append(repr(item)) + raise self.failureException(self._formatMessage(msg, "\n".join(lines))) + + def assertDictEqual(self, d1, d2, msg=None): + self.assertIsInstance(d1, dict, "First argument is not a dictionary") + self.assertIsInstance(d2, dict, "Second argument is not a dictionary") + if d1 != d2: + standardMsg = "%s != %s\n- %s\n+ %s" % (_safe_repr(d1, True), _safe_repr(d2, True), + _safe_repr(d1), _safe_repr(d2)) + raise self.failureException(self._formatMessage(msg, standardMsg)) + + def assertMultiLineEqual(self, first, second, msg=None): + self.assertIsInstance(first, str, "First argument is not a string") + self.assertIsInstance(second, str, "Second argument is not a string") + if first != second: + standardMsg = "%s != %s\n- %s\n+ %s" % (_safe_repr(first, True), _safe_repr(second, True), + first, second) + raise self.failureException(self._formatMessage(msg, standardMsg)) + + def assertLogs(self, logger=None, level=None): + raise NotImplementedError("assertLogs is not supported") + + # -- deprecated aliases ------------------------------------------------- + assertEquals = assertEqual + assertNotEquals = assertNotEqual + assert_ = assertTrue + failUnlessEqual = assertEqual + failUnless = assertTrue + assertRegexpMatches = assertRegex + assertRaisesRegexp = assertRaisesRegex + assertAlmostEquals = assertAlmostEqual + + +class IsolatedAsyncioTestCase(TestCase): + pass + + +class FunctionTestCase(TestCase): + def __init__(self, testFunc, setUp=None, tearDown=None, description=None): + TestCase.__init__(self) + self._testFunc = testFunc + self._setUpFunc = setUp + self._tearDownFunc = tearDown + self._description = description + + def setUp(self): + if self._setUpFunc is not None: + self._setUpFunc() + + def tearDown(self): + if self._tearDownFunc is not None: + self._tearDownFunc() + + def runTest(self): + self._testFunc() + + def id(self): + return self._testFunc.__name__ + + def __str__(self): + return "%s (%s)" % (self.__class__.__name__, self._testFunc.__name__) + + def shortDescription(self): + if self._description is not None: + return self._description + doc = self._testFunc.__doc__ + return doc.split("\n")[0].strip() if doc else None + + +# -- suites --------------------------------------------------------------- + +class TestSuite: + def __init__(self, tests=()): + self._tests = [] + self._removed_tests = 0 + self.addTests(tests) + + def __repr__(self): + return "<%s tests=%s>" % (self.__class__.__name__, list(self._tests)) + + def __iter__(self): + return iter(self._tests) + + def countTestCases(self): + cases = self._removed_tests + for test in self: + if test: + cases += test.countTestCases() + return cases + + def addTest(self, test): + if not callable(test): + raise TypeError("%r is not callable" % (test,)) + if isinstance(test, type) and issubclass(test, (TestCase, TestSuite)): + raise TypeError("TestCases and TestSuites must be instantiated before passing them to addTest()") + self._tests.append(test) + + def addTests(self, tests): + if isinstance(tests, str): + raise TypeError("tests must be an iterable of tests, not a string") + for test in tests: + self.addTest(test) + + def run(self, result, debug=False): + topLevel = False + if getattr(result, "_testRunEntered", False) is False: + result._testRunEntered = topLevel = True + for index, test in enumerate(self): + if result.shouldStop: + break + if isinstance(test, TestSuite): + test.run(result) + else: + self._handleClassSetUp(test, result) + if getattr(result, "_classSetupFailed", False): + self._tearDownPrevious(test, result) + continue + test(result) + if topLevel: + self._tearDownPreviousClass(None, result) + result._testRunEntered = False + return result + + def __call__(self, *args, **kwds): + return self.run(*args, **kwds) + + def debug(self): + for test in self: + test.debug() + + def _handleClassSetUp(self, test, result): + previousClass = getattr(result, "_previousTestClass", None) + currentClass = test.__class__ + if currentClass == previousClass: + return + self._tearDownPreviousClass(test, result) + result._previousTestClass = currentClass + result._classSetupFailed = False + if getattr(currentClass, "__unittest_skip__", False): + return + setUpClass = getattr(currentClass, "setUpClass", None) + if setUpClass is not None: + try: + setUpClass() + except BaseException as e: + result._classSetupFailed = True + if isinstance(e, SkipTest): + return + result.addError(_ErrorHolder("setUpClass (%s.%s)" % ( + currentClass.__module__, currentClass.__qualname__)), (type(e), e, None)) + + def _tearDownPrevious(self, test, result): + pass + + def _tearDownPreviousClass(self, test, result): + previousClass = getattr(result, "_previousTestClass", None) + currentClass = test.__class__ if test is not None else None + if currentClass == previousClass or previousClass is None: + return + if getattr(result, "_classSetupFailed", False): + return + if getattr(previousClass, "__unittest_skip__", False): + return + tearDownClass = getattr(previousClass, "tearDownClass", None) + if tearDownClass is not None: + try: + tearDownClass() + except BaseException as e: + result.addError(_ErrorHolder("tearDownClass (%s.%s)" % ( + previousClass.__module__, previousClass.__qualname__)), (type(e), e, None)) + result._previousTestClass = None + + +class _ErrorHolder: + failureException = None + + def __init__(self, description): + self.description = description + + def id(self): + return self.description + + def shortDescription(self): + return None + + def __repr__(self): + return "" % (self.description,) + + def __str__(self): + return self.id() + + def run(self, result): + pass + + def __call__(self, result): + return self.run(result) + + def countTestCases(self): + return 0 + + +# -- loader --------------------------------------------------------------- + +class TestLoader: + testMethodPrefix = "test" + sortTestMethodsUsing = None + suiteClass = TestSuite + testNamePatterns = None + + def loadTestsFromTestCase(self, testCaseClass): + if issubclass(testCaseClass, TestSuite): + raise TypeError("Test cases should not be derived from TestSuite. Maybe you meant to derive from TestCase?") + names = self.getTestCaseNames(testCaseClass) + if not names and hasattr(testCaseClass, "runTest"): + names = ["runTest"] + return self.suiteClass([testCaseClass(name) for name in names]) + + def loadTestsFromModule(self, module, *args, pattern=None, **kws): + tests = [] + for name in dir(module): + obj = getattr(module, name) + if isinstance(obj, type) and issubclass(obj, TestCase): + tests.append(self.loadTestsFromTestCase(obj)) + load_tests = getattr(module, "load_tests", None) + tests = self.suiteClass(tests) + if load_tests is not None: + return load_tests(self, tests, pattern) + return tests + + def loadTestsFromName(self, name, module=None): + if module is None: + raise ValueError("loadTestsFromName needs a module object: dotted names are resolved at run time") + obj = module + for part in name.split("."): + obj = getattr(obj, part) + if isinstance(obj, type) and issubclass(obj, TestCase): + return self.loadTestsFromTestCase(obj) + if isinstance(obj, TestSuite): + return obj + if callable(obj): + parent = None + parts = name.split(".") + if len(parts) > 1: + holder = module + for part in parts[:-1]: + holder = getattr(holder, part) + if isinstance(holder, type) and issubclass(holder, TestCase): + return self.suiteClass([holder(parts[-1])]) + return self.suiteClass([FunctionTestCase(obj)]) + raise TypeError("don't know how to make test from: %s" % obj) + + def loadTestsFromNames(self, names, module=None): + return self.suiteClass([self.loadTestsFromName(n, module) for n in names]) + + def getTestCaseNames(self, testCaseClass): + names = [] + for attrname in dir(testCaseClass): + if not attrname.startswith(self.testMethodPrefix): + continue + attr = getattr(testCaseClass, attrname) + if not callable(attr): + continue + if self.testNamePatterns is not None: + ok = False + for pat in self.testNamePatterns: + if "*" in pat: + if _fnmatch(attrname, pat): + ok = True + elif pat in attrname: + ok = True + if not ok: + continue + names.append(attrname) + if self.sortTestMethodsUsing is not None: + names.sort(key=_CmpToKey(self.sortTestMethodsUsing)) + return names + + +def _fnmatch(name, pat): + import fnmatch + return fnmatch.fnmatchcase(name, pat) + + +class _CmpToKey: + def __init__(self, cmp): + self.cmp = cmp + + def __call__(self, obj): + cmp = self.cmp + + class K: + def __init__(self, o): + self.o = o + + def __lt__(self, other): + return cmp(self.o, other.o) < 0 + return K(obj) + + +defaultTestLoader = TestLoader() + + +def findTestCases(module, prefix="test", sortUsing=None, suiteClass=TestSuite): + loader = TestLoader() + loader.testMethodPrefix = prefix + loader.suiteClass = suiteClass + return loader.loadTestsFromModule(module) + + +# -- results ---------------------------------------------------------------- + +class TestResult: + def __init__(self, stream=None, descriptions=None, verbosity=None): + self.failfast = False + self.failures = [] + self.errors = [] + self.testsRun = 0 + self.skipped = [] + self.expectedFailures = [] + self.unexpectedSuccesses = [] + self.shouldStop = False + self.buffer = False + self.tb_locals = False + self._testRunEntered = False + self._previousTestClass = None + self._classSetupFailed = False + + def printErrors(self): + pass + + def startTest(self, test): + self.testsRun += 1 + + def startTestRun(self): + pass + + def stopTestRun(self): + pass + + def stopTest(self, test): + pass + + def _exc_info_to_string(self, err, test): + exctype, value, tb = err + name = exctype.__name__ + text = str(value) + if text: + line = "%s: %s" % (name, text) + else: + line = name + if exctype.__module__ not in ("builtins", "__main__") and exctype is not AssertionError: + line = "%s.%s" % (exctype.__module__, line) if "." not in name else line + return "Traceback (most recent call last):\n" + line + "\n" + + def addError(self, test, err): + self.errors.append((test, self._exc_info_to_string(err, test))) + if self.failfast: + self.stop() + + def addFailure(self, test, err): + self.failures.append((test, self._exc_info_to_string(err, test))) + if self.failfast: + self.stop() + + def addSubTest(self, test, subtest, err): + if err is not None: + if issubclass(err[0], test.failureException): + self.failures.append((subtest, self._exc_info_to_string(err, test))) + else: + self.errors.append((subtest, self._exc_info_to_string(err, test))) + if self.failfast: + self.stop() + + def addSuccess(self, test): + pass + + def addSkip(self, test, reason): + self.skipped.append((test, reason)) + + def addExpectedFailure(self, test, err): + self.expectedFailures.append((test, self._exc_info_to_string(err, test))) + + def addUnexpectedSuccess(self, test): + self.unexpectedSuccesses.append(test) + + def wasSuccessful(self): + return len(self.failures) == len(self.errors) == 0 and ( + not hasattr(self, "unexpectedSuccesses") or len(self.unexpectedSuccesses) == 0) + + def stop(self): + self.shouldStop = True + + def __repr__(self): + return "<%s run=%i errors=%i failures=%i>" % ( + self.__class__.__name__, self.testsRun, len(self.errors), len(self.failures)) + + +class _WritelnDecorator: + def __init__(self, stream): + self.stream = stream + + def __getattr__(self, attr): + return getattr(self.stream, attr) + + def writeln(self, arg=None): + if arg: + self.write(arg) + self.write("\n") + + +class TextTestResult(TestResult): + separator1 = "=" * 70 + separator2 = "-" * 70 + + def __init__(self, stream, descriptions, verbosity): + TestResult.__init__(self, stream, descriptions, verbosity) + self.stream = stream + self.showAll = verbosity > 1 + self.dots = verbosity == 1 + self.descriptions = descriptions + + def getDescription(self, test): + doc_first_line = test.shortDescription() + if self.descriptions and doc_first_line: + return "\n".join((str(test), doc_first_line)) + return str(test) + + def startTest(self, test): + TestResult.startTest(self, test) + if self.showAll: + self.stream.write(self.getDescription(test)) + self.stream.write(" ... ") + self.stream.flush() + + def addSuccess(self, test): + TestResult.addSuccess(self, test) + if self.showAll: + self.stream.writeln("ok") + elif self.dots: + self.stream.write(".") + self.stream.flush() + + def addError(self, test, err): + TestResult.addError(self, test, err) + if self.showAll: + self.stream.writeln("ERROR") + elif self.dots: + self.stream.write("E") + self.stream.flush() + + def addFailure(self, test, err): + TestResult.addFailure(self, test, err) + if self.showAll: + self.stream.writeln("FAIL") + elif self.dots: + self.stream.write("F") + self.stream.flush() + + def addSubTest(self, test, subtest, err): + if err is not None: + if self.showAll: + if issubclass(err[0], subtest.test_case.failureException): + self.stream.writeln("FAIL") + else: + self.stream.writeln("ERROR") + elif self.dots: + if issubclass(err[0], subtest.test_case.failureException): + self.stream.write("F") + else: + self.stream.write("E") + self.stream.flush() + TestResult.addSubTest(self, test, subtest, err) + + def addSkip(self, test, reason): + TestResult.addSkip(self, test, reason) + if self.showAll: + self.stream.writeln("skipped {0!r}".format(reason)) + elif self.dots: + self.stream.write("s") + self.stream.flush() + + def addExpectedFailure(self, test, err): + TestResult.addExpectedFailure(self, test, err) + if self.showAll: + self.stream.writeln("expected failure") + elif self.dots: + self.stream.write("x") + self.stream.flush() + + def addUnexpectedSuccess(self, test): + TestResult.addUnexpectedSuccess(self, test) + if self.showAll: + self.stream.writeln("unexpected success") + elif self.dots: + self.stream.write("u") + self.stream.flush() + + def printErrors(self): + if self.dots or self.showAll: + self.stream.writeln() + self.stream.flush() + self.printErrorList("ERROR", self.errors) + self.printErrorList("FAIL", self.failures) + unexpectedSuccesses = self.unexpectedSuccesses + if unexpectedSuccesses: + self.stream.writeln(self.separator1) + for test in unexpectedSuccesses: + self.stream.writeln("UNEXPECTED SUCCESS: %s" % self.getDescription(test)) + self.stream.flush() + + def printErrorList(self, flavour, errors): + for test, err in errors: + self.stream.writeln(self.separator1) + self.stream.writeln("%s: %s" % (flavour, self.getDescription(test))) + self.stream.writeln(self.separator2) + self.stream.writeln("%s" % err) + self.stream.flush() + + +class TextTestRunner: + resultclass = TextTestResult + + def __init__(self, stream=None, descriptions=True, verbosity=1, failfast=False, buffer=False, + resultclass=None, warnings=None, *, tb_locals=False): + if stream is None: + stream = sys.stderr + self.stream = _WritelnDecorator(stream) + self.descriptions = descriptions + self.verbosity = verbosity + self.failfast = failfast + self.buffer = buffer + self.tb_locals = tb_locals + self.warnings = warnings + if resultclass is not None: + self.resultclass = resultclass + + def _makeResult(self): + return self.resultclass(self.stream, self.descriptions, self.verbosity) + + def run(self, test): + result = self._makeResult() + result.failfast = self.failfast + result.buffer = self.buffer + result.tb_locals = self.tb_locals + startTime = time.perf_counter() + result.startTestRun() + try: + test(result) + finally: + result.stopTestRun() + stopTime = time.perf_counter() + timeTaken = stopTime - startTime + result.printErrors() + if hasattr(result, "separator2"): + self.stream.writeln(result.separator2) + run = result.testsRun + self.stream.writeln("Ran %d test%s in %.3fs" % (run, run != 1 and "s" or "", timeTaken)) + self.stream.writeln() + expectedFails = len(result.expectedFailures) + unexpectedSuccesses = len(result.unexpectedSuccesses) + skipped = len(result.skipped) + infos = [] + if not result.wasSuccessful(): + self.stream.write("FAILED") + failed = len(result.failures) + errored = len(result.errors) + if failed: + infos.append("failures=%d" % failed) + if errored: + infos.append("errors=%d" % errored) + else: + self.stream.write("OK") + if skipped: + infos.append("skipped=%d" % skipped) + if expectedFails: + infos.append("expected failures=%d" % expectedFails) + if unexpectedSuccesses: + infos.append("unexpected successes=%d" % unexpectedSuccesses) + if infos: + self.stream.writeln(" (%s)" % (", ".join(infos),)) + else: + self.stream.write("\n") + self.stream.flush() + return result + + +class TestProgram: + def __init__(self, module="__main__", defaultTest=None, argv=None, testRunner=None, + testLoader=defaultTestLoader, exit=True, verbosity=1, failfast=None, + catchbreak=None, buffer=None, warnings=None, *, tb_locals=False): + if module is None or isinstance(module, str): + module = sys.modules[module or "__main__"] + self.module = module + self.exit = exit + self.verbosity = verbosity + self.failfast = failfast + self.testRunner = testRunner + self.testLoader = testLoader + if argv is None: + argv = sys.argv + self.progName = argv[0] if argv else "python -m unittest" + names = [] + for arg in argv[1:]: + if arg == "-v" or arg == "--verbose": + self.verbosity = 2 + elif arg == "-q" or arg == "--quiet": + self.verbosity = 0 + elif arg == "-f" or arg == "--failfast": + self.failfast = True + elif not arg.startswith("-"): + names.append(arg) + if defaultTest is not None and not names: + names = [defaultTest] if isinstance(defaultTest, str) else list(defaultTest) + if names: + self.test = self.testLoader.loadTestsFromNames(names, self.module) + else: + self.test = self.testLoader.loadTestsFromModule(self.module) + self.runTests() + + def runTests(self): + testRunner = self.testRunner + if testRunner is None: + testRunner = TextTestRunner + if isinstance(testRunner, type): + kw = {"verbosity": self.verbosity} + if self.failfast is not None: + kw["failfast"] = self.failfast + testRunner = testRunner(**kw) + self.result = testRunner.run(self.test) + if self.exit: + sys.exit(0 if self.result.wasSuccessful() else 1) + + +main = TestProgram diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/unittest/mock.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/unittest/mock.py new file mode 100644 index 00000000000..52de1275a49 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/unittest/mock.py @@ -0,0 +1,636 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Mock objects and patching. + +Supported: Mock and MagicMock (child mocks by attribute, return_value, +side_effect as exception, callable or iterable, call recording, call_count, +call_args, call_args_list, mock_calls, method_calls, the assert_called* +family, assert_has_calls, assert_any_call, reset_mock, configure_mock, +common magic methods on MagicMock), call, ANY, DEFAULT, sentinel, and patch +as decorator and context manager: patch.object, patch.dict, and patch with a +dotted name that names a module the program has already imported. + +Not supported: mutating a magic method in place (m.__len__.return_value = 3; +assign m.__len__ = Mock(return_value=3) instead), spec and autospec (accepted and ignored), create_autospec, +PropertyMock, AsyncMock, mock_open, and patch of a module that was not +imported, because that would import by a computed name. +""" + +import sys + +__all__ = ["Mock", "MagicMock", "NonCallableMock", "call", "ANY", "DEFAULT", + "sentinel", "patch", "PropertyMock"] + + +class _Sentinel: + def __init__(self, name): + self.name = name + + def __repr__(self): + return "sentinel." + self.name + + +class _SentinelFactory: + def __init__(self): + self._items = {} + + def __getattr__(self, name): + if name.startswith("__"): + raise AttributeError(name) + item = self._items.get(name) + if item is None: + item = _Sentinel(name) + self._items[name] = item + return item + + +sentinel = _SentinelFactory() +DEFAULT = sentinel.DEFAULT + + +class _ANY: + def __eq__(self, other): + return True + + def __ne__(self, other): + return False + + def __repr__(self): + return "" + + +ANY = _ANY() + + +def _format_call(name, args, kwargs): + parts = [repr(a) for a in args] + for k in kwargs: + parts.append("%s=%r" % (k, kwargs[k])) + return "%s(%s)" % (name, ", ".join(parts)) + + +class _Call: + """A recorded call: (name, args, kwargs). `call(...)` builds one to + compare with.""" + + def __init__(self, args=(), kwargs=None, name="", parent=None): + self._args = tuple(args) + self._kwargs = dict(kwargs) if kwargs else {} + self._name = name + self._parent = parent + + @property + def args(self): + return self._args + + @property + def kwargs(self): + return self._kwargs + + def __call__(self, *args, **kwargs): + return _Call(args, kwargs, self._name, self._parent) + + def __getattr__(self, attr): + if attr.startswith("__"): + raise AttributeError(attr) + name = attr if not self._name else self._name + "." + attr + return _Call((), {}, name, self) + + def __eq__(self, other): + if not isinstance(other, _Call): + if isinstance(other, tuple): + if len(other) == 2: + return self._args == other[0] and self._kwargs == other[1] + if len(other) == 3: + return self._name == other[0] and self._args == other[1] and self._kwargs == other[2] + return False + return NotImplemented + return self._args == other._args and self._kwargs == other._kwargs and \ + (not self._name or not other._name or self._name == other._name) + + def __ne__(self, other): + r = self.__eq__(other) + return r if r is NotImplemented else not r + + def __hash__(self): + return id(self) + + def __iter__(self): + return iter((self._args, self._kwargs)) + + def __len__(self): + return 2 + + def __getitem__(self, i): + return (self._args, self._kwargs)[i] + + def __repr__(self): + if not self._name: + return _format_call("call", self._args, self._kwargs) + n = "call" + self._name if self._name.startswith("(") else "call." + self._name + return _format_call(n, self._args, self._kwargs) + + +call = _Call(name="") +call._name = "" + + +class NonCallableMock: + def __init__(self, spec=None, side_effect=None, return_value=DEFAULT, wraps=None, + name=None, spec_set=None, parent=None, _new_name="", **kwargs): + d = self.__dict__ + d["_children"] = {} + d["_mock_name"] = name + d["_mock_new_name"] = _new_name + d["_mock_parent"] = parent + d["_mock_return_value"] = return_value + d["side_effect"] = side_effect + d["_mock_wraps"] = wraps + d["_mock_spec"] = spec + d["called"] = False + d["call_count"] = 0 + d["call_args"] = None + d["call_args_list"] = [] + d["mock_calls"] = [] + d["method_calls"] = [] + for k in kwargs: + setattr(self, k, kwargs[k]) + + # -- naming ------------------------------------------------------------- + def __repr__(self): + kind = type(self).__name__ + if self._mock_parent is not None and self._mock_new_name: + return "<%s name=%r id='%d'>" % (kind, self._full_name(), id(self)) + if self._mock_name is None: + return "<%s id='%d'>" % (kind, id(self)) + return "<%s name=%r id='%d'>" % (kind, self._mock_name, id(self)) + + def _full_name(self): + parts = [] + node = self + while node is not None: + if node._mock_new_name: + parts.append(node._mock_new_name) + elif node._mock_name: + parts.append(node._mock_name) + node = node._mock_parent + if not parts: + return "mock" + parts.reverse() + if parts[0] != "mock" and self._root_unnamed(): + parts.insert(0, "mock") + return ".".join(parts) + + def _root_unnamed(self): + node = self + while node._mock_parent is not None: + node = node._mock_parent + return node._mock_name is None + + # -- attributes --------------------------------------------------------- + def _make_child(self, name): + return self._child_class()(parent=self, _new_name=name) + + def _child_class(self): + return MagicMock if isinstance(self, MagicMock) else Mock + + def __getattr__(self, name): + if name == "return_value": + return self._get_return_value() + if name.startswith("__") and name.endswith("__") or name.startswith("_mock_"): + raise AttributeError(name) + children = self.__dict__["_children"] + child = children.get(name) + if child is None: + if self.__dict__["_mock_spec"] is not None and not hasattr(self.__dict__["_mock_spec"], name): + raise AttributeError("Mock object has no attribute %r" % name) + child = self._make_child(name) + children[name] = child + return child + + def __setattr__(self, name, value): + if name == "return_value": + self.__dict__["_mock_return_value"] = value + return + if name in ("side_effect", "called", "call_count", "call_args", + "call_args_list", "mock_calls", "method_calls"): + self.__dict__[name] = value + return + if isinstance(value, NonCallableMock) and value._mock_parent is None and \ + not value._mock_new_name: + value.__dict__["_mock_parent"] = self + value.__dict__["_mock_new_name"] = name + self.__dict__["_children"][name] = value + + def __delattr__(self, name): + children = self.__dict__["_children"] + if name in children: + del children[name] + else: + raise AttributeError(name) + + def _get_return_value(self): + rv = self._mock_return_value + if rv is DEFAULT: + rv = self._child_class()(parent=self, _new_name="()") + self.__dict__["_mock_return_value"] = rv + return rv + + def configure_mock(self, **kwargs): + for k in kwargs: + target = self + parts = k.split(".") + for p in parts[:-1]: + target = getattr(target, p) + setattr(target, parts[-1], kwargs[k]) + + def reset_mock(self, visited=None, return_value=False, side_effect=False): + d = self.__dict__ + d["called"] = False + d["call_count"] = 0 + d["call_args"] = None + d["call_args_list"] = [] + d["mock_calls"] = [] + d["method_calls"] = [] + if return_value: + d["_mock_return_value"] = DEFAULT + elif isinstance(d["_mock_return_value"], NonCallableMock): + d["_mock_return_value"].reset_mock() + if side_effect: + d["side_effect"] = None + for c in list(d["_children"].values()): + if isinstance(c, NonCallableMock): + c.reset_mock() + + # -- assertions --------------------------------------------------------- + def _name_for_message(self): + return self._mock_name or self._mock_new_name or "mock" + + def _call_str(self, args, kwargs): + return _format_call(self._name_for_message(), args, kwargs) + + def assert_not_called(self): + if self.call_count != 0: + raise AssertionError("Expected '%s' to not have been called. Called %d times.%s" % ( + self._name_for_message(), self.call_count, self._calls_repr())) + + def assert_called(self): + if self.call_count == 0: + raise AssertionError("Expected '%s' to have been called." % (self._name_for_message(),)) + + def assert_called_once(self): + if not self.call_count == 1: + raise AssertionError("Expected '%s' to have been called once. Called %d times.%s" % ( + self._name_for_message(), self.call_count, self._calls_repr())) + + def _calls_repr(self): + if not self.call_args_list: + return "" + return "\nCalls: %s." % (self._calls_text(),) + + def _calls_text(self): + return "[" + ", ".join(self._call_str(c._args, c._kwargs).replace( + self._name_for_message(), "call", 1) for c in self.call_args_list) + "]" + + def assert_called_with(self, *args, **kwargs): + if self.call_args is None: + expected = self._call_str(args, kwargs) + raise AssertionError("expected call not found.\nExpected: %s\n Actual: not called." % expected) + actual = self.call_args + if not (actual._args == args and actual._kwargs == kwargs): + raise AssertionError("expected call not found.\nExpected: %s\n Actual: %s" % ( + self._call_str(args, kwargs), self._call_str(actual._args, actual._kwargs))) + + def assert_called_once_with(self, *args, **kwargs): + if not self.call_count == 1: + raise AssertionError("Expected '%s' to be called once. Called %d times.%s" % ( + self._name_for_message(), self.call_count, self._calls_repr())) + self.assert_called_with(*args, **kwargs) + + def assert_any_call(self, *args, **kwargs): + for c in self.call_args_list: + if c._args == args and c._kwargs == kwargs: + return + raise AssertionError("%s call not found" % self._call_str(args, kwargs)) + + def assert_has_calls(self, calls, any_order=False): + expected = list(calls) + actual = list(self.mock_calls) + if not any_order: + n = len(expected) + for i in range(len(actual) - n + 1): + if all(actual[i + j] == expected[j] for j in range(n)): + return + raise AssertionError("Calls not found.\nExpected: %r\n Actual: %r" % (expected, actual)) + remaining = list(actual) + missing = [] + for e in expected: + for i in range(len(remaining)): + if remaining[i] == e: + del remaining[i] + break + else: + missing.append(e) + if missing: + raise AssertionError("%r not all found in call list" % (missing,)) + + # -- calling -------------------------------------------------------------- + def _execute(self, args, kwargs): + self.__dict__["called"] = True + self.__dict__["call_count"] = self.call_count + 1 + c = _Call(args, kwargs) + self.__dict__["call_args"] = c + self.__dict__["call_args_list"].append(c) + # mock_calls on this mock and every ancestor, with the dotted path + node = self + path = "" + through_call = False + while node is not None: + node.__dict__["mock_calls"].append(_Call(args, kwargs, path)) + parent = node._mock_parent + nn = node._mock_new_name + if parent is not None and nn: + if nn == "()": + path = "()" + ("." + path if path else "") + through_call = True + else: + if path.startswith("("): + path = nn + path + elif path: + path = nn + "." + path + else: + path = nn + if not through_call: + parent.__dict__["method_calls"].append(_Call(args, kwargs, path)) + node = parent + effect = self.side_effect + if effect is not None: + if isinstance(effect, BaseException) or (isinstance(effect, type) and issubclass(effect, BaseException)): + raise effect + if callable(effect): + result = effect(*args, **kwargs) + if result is not DEFAULT: + return result + else: + try: + it = self.__dict__["_effect_iter"] + except KeyError: + it = iter(effect) + self.__dict__["_effect_iter"] = it + value = next(it) + if isinstance(value, BaseException) or (isinstance(value, type) and issubclass(value, BaseException)): + raise value + if value is not DEFAULT: + return value + if self._mock_return_value is not DEFAULT: + return self._mock_return_value + if self._mock_wraps is not None: + return self._mock_wraps(*args, **kwargs) + return self._get_return_value() + + +class Mock(NonCallableMock): + def __call__(self, *args, **kwargs): + return self._execute(args, kwargs) + + +_MAGIC_DEFAULTS = {"__len__": 0, "__bool__": True, "__contains__": False, "__getitem__": DEFAULT, + "__setitem__": None, "__delitem__": None, "__enter__": DEFAULT, + "__exit__": False, "__int__": 1, "__float__": 1.0, "__lt__": NotImplemented, + "__gt__": NotImplemented, "__le__": NotImplemented, "__ge__": NotImplemented} + + +class MagicMock(Mock): + def _magic(self, name, default): + child = self.__dict__["_children"].get(name) + if child is None: + child = Mock(return_value=default, parent=self, _new_name=name) + self.__dict__["_children"][name] = child + return child + + def __len__(self): + return self._magic("__len__", 0)() + + def __bool__(self): + return self._magic("__bool__", True)() + + def __iter__(self): + return iter(self._magic("__iter__", iter([]))()) + + def __contains__(self, item): + return self._magic("__contains__", False)(item) + + def __getitem__(self, key): + return self._magic("__getitem__", DEFAULT)(key) + + def __setitem__(self, key, value): + self._magic("__setitem__", None)(key, value) + + def __delitem__(self, key): + self._magic("__delitem__", None)(key) + + def __enter__(self): + return self._magic("__enter__", DEFAULT)() + + def __exit__(self, exc_type, exc, tb): + return self._magic("__exit__", False)(exc_type, exc, tb) + + def __int__(self): + return self._magic("__int__", 1)() + + def __float__(self): + return self._magic("__float__", 1.0)() + + def __str__(self): + return self._magic("__str__", object.__repr__(self))() + + def __hash__(self): + return id(self) + + def __eq__(self, other): + return self is other + + def __ne__(self, other): + return self is not other + + def __lt__(self, other): + return self._magic("__lt__", NotImplemented)(other) + + def __gt__(self, other): + return self._magic("__gt__", NotImplemented)(other) + + def __le__(self, other): + return self._magic("__le__", NotImplemented)(other) + + def __ge__(self, other): + return self._magic("__ge__", NotImplemented)(other) + + +class PropertyMock(Mock): + pass + + +# -- patch ------------------------------------------------------------------- + +def _resolve(target): + """Find the object a dotted name such as "pkg.mod.attr" starts from: + the longest prefix that is an imported module, then attribute lookups.""" + parts = target.split(".") + for n in range(len(parts), 0, -1): + name = ".".join(parts[:n]) + mod = sys.modules.get(name) + if mod is not None: + obj = mod + for p in parts[n:]: + obj = getattr(obj, p) + return obj + raise ImportError("patch target %r names a module that was not imported" % target) + + +class _Patch: + def __init__(self, getter, attribute, new, kwargs, create=False): + self.getter = getter + self.attribute = attribute + self.new = new + self.kwargs = kwargs + self.create = create + self.target = None + self.original = None + self.had = False + self.is_local = False + + def __enter__(self): + target = self.getter() + self.target = target + d = getattr(target, "__dict__", None) + self.had = hasattr(target, self.attribute) + if not self.had and not self.create: + raise AttributeError("%r does not have the attribute %r" % (target, self.attribute)) + self.original = getattr(target, self.attribute) if self.had else None + self.is_local = d is not None and self.attribute in d + new = self.new + if new is DEFAULT: + new = MagicMock(**self.kwargs) + setattr(target, self.attribute, new) + self.active = new + return new + + def __exit__(self, exc_type, exc, tb): + if self.is_local or not self.had: + if self.had: + setattr(self.target, self.attribute, self.original) + else: + delattr(self.target, self.attribute) + else: + try: + delattr(self.target, self.attribute) + except AttributeError: + setattr(self.target, self.attribute, self.original) + if getattr(self.target, self.attribute, None) is not self.original: + setattr(self.target, self.attribute, self.original) + return False + + start = __enter__ + + def stop(self): + return self.__exit__(None, None, None) + + def __call__(self, func): + if isinstance(func, type): + for name in dir(func): + if name.startswith("test"): + attr = getattr(func, name) + if callable(attr): + setattr(func, name, self.__class__(self.getter, self.attribute, self.new, + self.kwargs, self.create)(attr)) + return func + patcher = self + + def patched(*args, **kw): + p = patcher.__class__(patcher.getter, patcher.attribute, patcher.new, patcher.kwargs, patcher.create) + with p as value: + if patcher.new is DEFAULT: + args = args + (value,) + return func(*args, **kw) + patched.__name__ = getattr(func, "__name__", "patched") + patched.__wrapped__ = func + return patched + + +class _PatchDict: + def __init__(self, in_dict, values, clear, kwargs): + self.in_dict = in_dict + self.values = dict(values) + self.values.update(kwargs) + self.clear = clear + self.saved = None + + def __enter__(self): + d = self.in_dict + if isinstance(d, str): + d = _resolve(d) + self._d = d + self.saved = dict(d) + if self.clear: + d.clear() + d.update(self.values) + return d + + def __exit__(self, exc_type, exc, tb): + self._d.clear() + self._d.update(self.saved) + return False + + def __call__(self, func): + pd = self + + def patched(*args, **kw): + with _PatchDict(pd.in_dict, pd.values, pd.clear, {}): + return func(*args, **kw) + patched.__name__ = getattr(func, "__name__", "patched") + patched.__wrapped__ = func + return patched + + +def _patch(target, new=DEFAULT, spec=None, create=False, spec_set=None, autospec=None, + new_callable=None, **kwargs): + if not isinstance(target, str): + raise TypeError("patch target must be a string such as 'module.name'") + idx = target.rfind(".") + if idx < 0: + raise TypeError("Need a valid target to patch. You supplied: %r" % (target,)) + path = target[:idx] + attribute = target[idx + 1:] + return _Patch(lambda: _resolve(path), attribute, new, kwargs, create) + + +def _patch_object(target, attribute, new=DEFAULT, spec=None, create=False, spec_set=None, + autospec=None, new_callable=None, **kwargs): + return _Patch(lambda: target, attribute, new, kwargs, create) + + +def _patch_multiple(target, spec=None, create=False, spec_set=None, autospec=None, + new_callable=None, **kwargs): + raise NotImplementedError("patch.multiple is not supported") + + +class _PatchFacade: + def __call__(self, target, new=DEFAULT, spec=None, create=False, spec_set=None, + autospec=None, new_callable=None, **kwargs): + return _patch(target, new, spec, create, spec_set, autospec, new_callable, **kwargs) + + def object(self, target, attribute, new=DEFAULT, spec=None, create=False, spec_set=None, + autospec=None, new_callable=None, **kwargs): + return _patch_object(target, attribute, new, spec, create, spec_set, autospec, + new_callable, **kwargs) + + def dict(self, in_dict, values=(), clear=False, **kwargs): + return _PatchDict(in_dict, values, clear, kwargs) + + def multiple(self, target, spec=None, create=False, spec_set=None, autospec=None, + new_callable=None, **kwargs): + return _patch_multiple(target) + + +patch = _PatchFacade() diff --git a/maven/python-transpiler/src/test/resources/conformance/pytest_.out b/maven/python-transpiler/src/test/resources/conformance/pytest_.out new file mode 100644 index 00000000000..dc446541d0b --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/pytest_.out @@ -0,0 +1,22 @@ +type ValueError +value invalid literal for int() with base 10: 'zz' +typename ValueError +call ZeroDivisionError +no raise Failed: DID NOT RAISE ValueError +no match Regex pattern did not match. +passes through TypeError +approx1 True +approx2 False +approx3 True +approx4 True +approx5 True +approx6 True +approx7 False +approx8 True +approx9 True +approx10 True +fail Failed: boom +skip Skipped: later +xfail XFailed: known +=== 5 failed, 21 passed, 4 skipped, 1 xfailed, 1 xpassed === +exit 1 diff --git a/maven/python-transpiler/src/test/resources/conformance/pytest_.py b/maven/python-transpiler/src/test/resources/conformance/pytest_.py new file mode 100644 index 00000000000..07d7679d768 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/pytest_.py @@ -0,0 +1,233 @@ +import io +import sys +import contextlib +import pytest + + +def show(label, value): + print(label, value) + + +def main_checks(): + # raises: context manager, excinfo, match + with pytest.raises(ValueError) as info: + int("zz") + show("type", info.type.__name__) + show("value", str(info.value)) + show("typename", info.typename) + with pytest.raises(KeyError, match="missing"): + raise KeyError("missing key") + r = pytest.raises(ZeroDivisionError, lambda: 1 / 0) + show("call", r.type.__name__) + try: + with pytest.raises(ValueError): + pass + except BaseException as e: + show("no raise", type(e).__name__ + ": " + str(e)) + try: + with pytest.raises(ValueError, match="abc"): + raise ValueError("xyz") + except AssertionError as e: + show("no match", str(e).splitlines()[0]) + try: + with pytest.raises(ValueError): + raise TypeError("other") + except TypeError: + show("passes through", "TypeError") + + # approx + show("approx1", 0.1 + 0.2 == pytest.approx(0.3)) + show("approx2", 1.0 == pytest.approx(1.1)) + show("approx3", 1.0 == pytest.approx(1.1, abs=0.2)) + show("approx4", 100 == pytest.approx(101, rel=0.02)) + show("approx5", [0.1 + 0.2, 1.0] == pytest.approx([0.3, 1.0])) + show("approx6", {"a": 0.1 + 0.2} == pytest.approx({"a": 0.3})) + show("approx7", (1, 2) == pytest.approx((1, 3))) + show("approx8", pytest.approx(3.0) != 3.5) + show("approx9", float("nan") == pytest.approx(float("nan"), nan_ok=True)) + show("approx10", 1 == pytest.approx(1.0)) + + # fail / skip / xfail raise their own exceptions + for label, fn, args in (("fail", pytest.fail, ("boom",)), ("skip", pytest.skip, ("later",)), + ("xfail", pytest.xfail, ("known",))): + try: + fn(*args) + except BaseException as e: + show(label, type(e).__name__ + ": " + str(e)) + + +@pytest.fixture +def numbers(): + return [1, 2, 3] + + +@pytest.fixture +def total(numbers): + return sum(numbers) + + +EVENTS = [] + + +@pytest.fixture +def resource(): + EVENTS.append("open") + yield "res" + EVENTS.append("close") + + +@pytest.fixture(autouse=True) +def always(): + EVENTS.append("auto") + + +def test_plain(): + assert 1 + 1 == 2 + + +def test_fixture_chain(numbers, total): + assert total == 6 + assert numbers == [1, 2, 3] + + +def test_yield_fixture(resource): + assert resource == "res" + assert "open" in EVENTS and "close" not in EVENTS + + +@pytest.mark.parametrize("a, b, expected", [(1, 2, 3), (2, 3, 5), (10, 5, 15)]) +def test_add(a, b, expected): + assert a + b == expected + + +@pytest.mark.parametrize("n", [1, 2]) +@pytest.mark.parametrize("s", ["x", "y"]) +def test_stacked(n, s): + assert n in (1, 2) and s in ("x", "y") + + +@pytest.mark.parametrize("v", [pytest.param(1, id="one"), pytest.param(2, id="two"), + pytest.param(3, marks=pytest.mark.skip(reason="no"))]) +def test_params_ids(v): + assert v < 3 + + +@pytest.mark.parametrize("x", [1, 2, 3]) +def test_some_fail(x): + assert x != 2, "x was two" + + +def test_failing(): + assert 1 == 2 + + +def test_error_in_body(): + raise RuntimeError("oops") + + +@pytest.mark.skip(reason="not now") +def test_skipped(): + assert False + + +@pytest.mark.skipif(True, reason="cond") +def test_skipif_true(): + assert False + + +@pytest.mark.skipif(False, reason="cond") +def test_skipif_false(): + assert True + + +def test_skip_inside(): + pytest.skip("inside") + + +@pytest.mark.xfail(reason="known bug") +def test_xfail_fails(): + assert 1 == 2 + + +@pytest.mark.xfail +def test_xfail_passes(): + assert True + + +def test_fail_call(): + pytest.fail("explicit") + + +def test_raises_and_approx(): + with pytest.raises(ZeroDivisionError): + 1 / 0 + assert 0.1 + 0.2 == pytest.approx(0.3) + + +def test_monkeypatch(monkeypatch): + d = {"a": 1} + monkeypatch.setitem(d, "a", 2) + assert d["a"] == 2 + + +class TestGroup: + def setup_method(self, method): + self.base = 10 + + @pytest.fixture + def extra(self): + return 5 + + def test_method(self): + assert self.base == 10 + + def test_method_fixture(self, extra): + assert self.base + extra == 15 + + @pytest.mark.parametrize("k", [1, 2]) + def test_method_param(self, k): + assert k > 0 + + def test_method_failing(self): + assert self.base == 11 + + +class Counter: + def __init__(self): + self.counts = {} + + def pytest_runtest_logreport(self, report): + if report.when == "call" or (report.when == "setup" and report.outcome != "passed"): + key = report.outcome + if hasattr(report, "wasxfail"): + key = "xpassed" if report.outcome == "passed" else "xfailed" + elif report.when == "setup" and report.outcome == "failed": + key = "error" + self.counts[key] = self.counts.get(key, 0) + 1 + + +def summary(counts): + parts = [] + for key, word in (("failed", "failed"), ("passed", "passed"), ("skipped", "skipped"), + ("xfailed", "xfailed"), ("xpassed", "xpassed"), ("error", "errors")): + n = counts.get(key, 0) + if n: + parts.append("%d %s" % (n, word)) + return "=== " + ", ".join(parts) + " ===" + + +if __name__ == "__main__": + main_checks() + if hasattr(pytest, "run_module"): + # Our runner prints per test results and the summary line. + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + code = pytest.run_module(sys.modules["__main__"], quiet=True) + print(buf.getvalue().strip()) + print("exit", code) + else: + counter = Counter() + with contextlib.redirect_stdout(io.StringIO()): + code = pytest.main([__file__, "-q", "-p", "no:cacheprovider"], plugins=[counter]) + print(summary(counter.counts)) + print("exit", int(code)) diff --git a/maven/python-transpiler/src/test/resources/conformance/unittest_.out b/maven/python-transpiler/src/test/resources/conformance/unittest_.out new file mode 100644 index 00000000000..815f5e6197d --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/unittest_.out @@ -0,0 +1,66 @@ +['test_equal', 'test_raises', 'test_skip_inside', 'test_skipped', 'test_skipped_if', 'test_skipped_unless', 'test_subtests', 'test_xfail'] +8 +..ssss.x +---------------------------------------------------------------------- +Ran 8 tests in X.XXXs + +OK (skipped=4, expected failures=1) + +8 0 0 4 1 0 True +setUpClass tearDownClass 5 5 +['cond', 'cond2', 'inside', 'not now'] +15 13 1 False +FFFFFEFFFFFFFFu +1 +test_a_equal | AssertionError: 1 != 2 +test_b_msg | AssertionError: 'a' != 'b' +test_c_true | AssertionError: 0 is not true +test_d_in | AssertionError: 1 not found in [2] +test_e_list | AssertionError: Lists differ: [1, 2, 3] != [1, 2, 4] + +First differing element 2: +3 +4 + +test_f_error | KeyError: 'boom' +test_g_raises | AssertionError: ValueError not raised +test_h_regex | AssertionError: "abc" does not match "xyz" +test_i_almost | AssertionError: 1.0 != 1.5 within 7 places (0.5 difference) +test_j_less | AssertionError: 2 not less than 1 +test_k_count | AssertionError: Element counts were not equal: +First has 1, Second has 0: 2 +First has 0, Second has 1: 3 +test_l_isnone | AssertionError: 5 is not None +test_n_set | AssertionError: Items in the first set but not the second: +1 +Items in the second set but not the first: +2 +'Lists differ: [1, 2, 3] != [1, 2]\n\nFirst list contains 1 additional elements.\nFirst extra element 2:\n3\n' +"{'a': 1} != {'a': 2}" +'Tuples differ: (1, 2) != (1, 3)\n\nFirst differing element 1:\n2\n3\n' +'1 == 1' +'1 is not 2' +'unexpectedly identical: None' +'unexpectedly None' +'1 unexpectedly found in [1]' +"1 is not an instance of " +"1 is an instance of " +'1 not greater than 2' +'1 not greater than or equal to 2' +'2 not less than or equal to 1' +'1 is not false' +"Regex didn't match: 'x' not found in 'abc'" +"Regex matched: 'b' matches 'b' in 'abc'" +'1.0 == 1.0 within 7 places' +'1.0 != 1.5 within 0.1 delta (0.5 difference)' +'1 != 2 : why' +'explicit' +'ValueError not raised by ' +FAILED (failures=13, errors=1, unexpected successes=1) +['test', 'tearDown', 'cleanup'] True +test_one (__main__.Teardown.test_one) ... ok +['Basics', 'Failing', 'Teardown'] +2 +1 True +1 True +exit 0 diff --git a/maven/python-transpiler/src/test/resources/conformance/unittest_.py b/maven/python-transpiler/src/test/resources/conformance/unittest_.py new file mode 100644 index 00000000000..0ec6992fdb9 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/unittest_.py @@ -0,0 +1,261 @@ +import io +import re +import sys +import unittest + + +class Basics(unittest.TestCase): + log = [] + + @classmethod + def setUpClass(cls): + Basics.log.append("setUpClass") + + @classmethod + def tearDownClass(cls): + Basics.log.append("tearDownClass") + + def setUp(self): + self.value = 10 + Basics.log.append("setUp") + + def tearDown(self): + Basics.log.append("tearDown") + + def test_equal(self): + self.assertEqual(self.value, 10) + self.assertNotEqual(self.value, 11) + self.assertTrue(self.value) + self.assertFalse(0) + self.assertIs(None, None) + self.assertIsNot([], []) + self.assertIsNone(None) + self.assertIsNotNone(0) + self.assertIn(1, [1, 2]) + self.assertNotIn(3, [1, 2]) + self.assertIsInstance(1, int) + self.assertNotIsInstance("a", int) + self.assertGreater(2, 1) + self.assertGreaterEqual(2, 2) + self.assertLess(1, 2) + self.assertLessEqual(2, 2) + self.assertAlmostEqual(1.00000001, 1.0) + self.assertAlmostEqual(1.5, 1.4, delta=0.2) + self.assertNotAlmostEqual(1.5, 1.0) + self.assertCountEqual([1, 2, 2, 3], [3, 2, 1, 2]) + self.assertDictEqual({"a": 1}, {"a": 1}) + self.assertListEqual([1, 2], [1, 2]) + self.assertTupleEqual((1, 2), (1, 2)) + self.assertSetEqual({1, 2}, {2, 1}) + self.assertMultiLineEqual("a\nb", "a\nb") + self.assertRegex("hello world", r"w.r") + self.assertNotRegex("hello", r"xyz") + + def test_raises(self): + with self.assertRaises(ZeroDivisionError) as cm: + 1 / 0 + self.assertIsInstance(cm.exception, ZeroDivisionError) + self.assertRaises(KeyError, lambda: {}["x"]) + self.assertRaises(ValueError, int, "zz") + with self.assertRaisesRegex(ValueError, "invalid literal"): + int("zz") + self.assertRaisesRegex(ValueError, "^invalid", int, "q") + + def test_subtests(self): + for i in range(3): + with self.subTest(i=i): + self.assertLess(i, 5) + + @unittest.skip("not now") + def test_skipped(self): + self.fail("must not run") + + @unittest.skipIf(True, "cond") + def test_skipped_if(self): + self.fail("must not run") + + @unittest.skipUnless(False, "cond2") + def test_skipped_unless(self): + self.fail("must not run") + + def test_skip_inside(self): + self.skipTest("inside") + + @unittest.expectedFailure + def test_xfail(self): + self.assertEqual(1, 2) + + def helper_not_a_test(self): + pass + + +class Failing(unittest.TestCase): + def test_a_equal(self): + self.assertEqual(1, 2) + + def test_b_msg(self): + self.assertEqual("a", "b", "custom") + + def test_c_true(self): + self.assertTrue(0) + + def test_d_in(self): + self.assertIn(1, [2]) + + def test_e_list(self): + self.assertEqual([1, 2, 3], [1, 2, 4]) + + def test_f_error(self): + raise KeyError("boom") + + def test_g_raises(self): + with self.assertRaises(ValueError): + pass + + def test_h_regex(self): + with self.assertRaisesRegex(ValueError, "abc"): + raise ValueError("xyz") + + def test_i_almost(self): + self.assertAlmostEqual(1.0, 1.5) + + def test_j_less(self): + self.assertLess(2, 1) + + def test_k_count(self): + self.assertCountEqual([1, 2], [1, 3]) + + def test_l_isnone(self): + self.assertIsNone(5) + + def test_m_sub(self): + for i in range(3): + with self.subTest(i=i): + self.assertEqual(i % 2, 0) + + def test_n_set(self): + self.assertSetEqual({1}, {2}) + + @unittest.expectedFailure + def test_o_unexpected(self): + pass + + +class Teardown(unittest.TestCase): + order = [] + + def setUp(self): + self.addCleanup(Teardown.order.append, "cleanup") + + def test_one(self): + Teardown.order.append("test") + + def tearDown(self): + Teardown.order.append("tearDown") + + +def first_line(result_text): + return result_text + + +def run(suite, verbosity=1): + stream = io.StringIO() + result = unittest.TextTestRunner(stream=stream, verbosity=verbosity).run(suite) + out = stream.getvalue() + out = re.sub(r"in \d+\.\d+s", "in X.XXXs", out) + return result, out + + +loader = unittest.TestLoader() +print(loader.getTestCaseNames(Basics)) +suite = loader.loadTestsFromTestCase(Basics) +print(suite.countTestCases()) +result, out = run(suite) +print(out) +print(result.testsRun, len(result.failures), len(result.errors), len(result.skipped), + len(result.expectedFailures), len(result.unexpectedSuccesses), result.wasSuccessful()) +print(Basics.log[0], Basics.log[-1], Basics.log.count("setUp"), Basics.log.count("tearDown")) +print(sorted(r for _, r in result.skipped)) + +result, out = run(loader.loadTestsFromTestCase(Failing)) +print(result.testsRun, len(result.failures), len(result.errors), result.wasSuccessful()) +print(out.splitlines()[0]) +print(sorted(type(t).__name__ for t, _ in result.failures + result.errors).count("_SubTest")) + + +def trim(text): + # The difflib-style diff after the first line block is not supported. + return re.split(r"\n[-+] ", text)[0] + + +def message(name, *args): + case = Failing(name) + try: + getattr(case, name)() + except Exception as e: + return type(e).__name__ + ": " + trim(str(e)) + return "no exception" + + +for name in sorted(loader.getTestCaseNames(Failing)): + if name != "test_m_sub" and name != "test_o_unexpected": + print(name, "|", message(name)) +c = Failing("test_a_equal") +for call in [lambda: c.assertEqual([1, 2, 3], [1, 2]), + lambda: c.assertEqual({"a": 1}, {"a": 2}), + lambda: c.assertEqual((1, 2), (1, 3)), + lambda: c.assertNotEqual(1, 1), + lambda: c.assertIs(1, 2), + lambda: c.assertIsNot(None, None), + lambda: c.assertIsNotNone(None), + lambda: c.assertNotIn(1, [1]), + lambda: c.assertIsInstance(1, str), + lambda: c.assertNotIsInstance(1, int), + lambda: c.assertGreater(1, 2), + lambda: c.assertGreaterEqual(1, 2), + lambda: c.assertLessEqual(2, 1), + lambda: c.assertFalse(1), + lambda: c.assertRegex("abc", "x"), + lambda: c.assertNotRegex("abc", "b"), + lambda: c.assertNotAlmostEqual(1.0, 1.0), + lambda: c.assertAlmostEqual(1.0, 1.5, delta=0.1), + lambda: c.assertEqual(1, 2, msg="why"), + lambda: c.fail("explicit"), + lambda: c.assertRaises(ValueError, lambda: None)]: + try: + call() + except AssertionError as e: + print(repr(trim(str(e)))) +print(out.rstrip().splitlines()[-1]) + +result, out = run(loader.loadTestsFromTestCase(Teardown)) +print(Teardown.order, result.wasSuccessful()) + +# verbose output +result, out = run(loader.loadTestsFromTestCase(Teardown), 2) +print(out.splitlines()[0]) + +# module loading, suite composition +mod = sys.modules["__main__"] +names = [type(t).__name__ for s in loader.loadTestsFromModule(mod) for t in s] +print(sorted(set(names))) +combined = unittest.TestSuite([loader.loadTestsFromTestCase(Teardown), unittest.TestSuite()]) +combined.addTest(Teardown("test_one")) +print(combined.countTestCases()) + +def plain(): + assert 1 + 1 == 2 + +result, out = run(unittest.TestSuite([unittest.FunctionTestCase(plain)])) +print(result.testsRun, result.wasSuccessful()) + +# main() with exit=False +stream = io.StringIO() +prog = unittest.main(module=mod, argv=["x", "Teardown"], exit=False, + testRunner=unittest.TextTestRunner(stream=stream)) +print(prog.result.testsRun, prog.result.wasSuccessful()) +try: + unittest.main(module=mod, argv=["x", "Teardown"], + testRunner=unittest.TextTestRunner(stream=io.StringIO())) +except SystemExit as e: + print("exit", e.code) diff --git a/maven/python-transpiler/src/test/resources/conformance/unittest_mock_.out b/maven/python-transpiler/src/test/resources/conformance/unittest_mock_.out new file mode 100644 index 00000000000..78975c0ad57 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/unittest_mock_.out @@ -0,0 +1,67 @@ +False 0 None +True 2 +call('again') +[call(1, 2, a=3), call('again')] +True ('again',) {} +(1, 2) {'a': 3} +AssertionError: Expected 'mock' to have been called once. Called 2 times. +Calls: [call(1, 2, a=3), call('again')]. +AssertionError: expected call not found. +Expected: mock('nope') + Actual: mock('again') +AssertionError: Expected 'mock' to be called once. Called 2 times. +Calls: [call(1, 2, a=3), call('again')]. +AssertionError: Expected 'mock' to not have been called. Called 2 times. +Calls: [call(1, 2, a=3), call('again')]. +AssertionError: Expected 'mock' to have been called. +AssertionError: expected call not found. +Expected: mock(1) + Actual: not called. +AssertionError: mock(9) call not found +False 0 [] +42 42 2 +7 +True +raised bad +raised KeyError +1 2 3 +exhausted +8 +1 +second raised two +2 call(2) +[call.child.method(1), call.child.method(2), call.other(3)] +[call.child.method(1), call.child.method(2), call.other(3)] +True +5 +9 +1 two +call(1, k=2) call.foo(3) call.foo.bar() +True False True True +3 True [] False +item item +True +[1, 2] call('q') +5 +patched call('x') +hello x +lambda x +hello x +fake 1 +real k +inst +real z +D +x +dec! hello x +new a real a +['a', 'b'] +['a'] +['c'] +['a'] +True +hello q +missing attribute +True +False +2 True diff --git a/maven/python-transpiler/src/test/resources/conformance/unittest_mock_.py b/maven/python-transpiler/src/test/resources/conformance/unittest_mock_.py new file mode 100644 index 00000000000..c3698c4667e --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/unittest_mock_.py @@ -0,0 +1,198 @@ +import unittest +from unittest import mock +from unittest.mock import Mock, MagicMock, patch, call, ANY +import textwrap + + +def greet(name): + return "hello " + name + + +class Service: + def fetch(self, key): + return "real " + key + + +def use_greet(): + return greet("x") + + +m = Mock() +print(m.called, m.call_count, m.call_args) +m(1, 2, a=3) +m("again") +print(m.called, m.call_count) +print(m.call_args) +print(m.call_args_list) +print(m.call_args == call("again"), m.call_args.args, m.call_args.kwargs) +args, kwargs = m.call_args_list[0] +print(args, kwargs) +m.assert_called() +m.assert_called_with("again") +m.assert_any_call(1, 2, a=3) +for check in [lambda: m.assert_called_once(), lambda: m.assert_called_with("nope"), + lambda: m.assert_called_once_with("again"), lambda: m.assert_not_called(), + lambda: Mock().assert_called(), lambda: Mock().assert_called_with(1), + lambda: m.assert_any_call(9)]: + try: + check() + except AssertionError as e: + print("AssertionError:", str(e)) + +one = Mock() +one(5) +one.assert_called_once_with(5) +one.assert_called_once() +one.reset_mock() +print(one.called, one.call_count, one.call_args_list) + +r = Mock(return_value=42) +print(r(), r(1), r.call_count) +r.return_value = 7 +print(r()) +print(Mock().return_value is not None) + +e = Mock(side_effect=ValueError("bad")) +try: + e() +except ValueError as ex: + print("raised", ex) +e2 = Mock(side_effect=KeyError) +try: + e2() +except KeyError: + print("raised KeyError") +it = Mock(side_effect=[1, 2, 3]) +print(it(), it(), it()) +try: + it() +except StopIteration: + print("exhausted") +f = Mock(side_effect=lambda x: x * 2) +print(f(4)) +mixed = Mock(side_effect=[1, ValueError("two")]) +print(mixed()) +try: + mixed() +except ValueError as ex: + print("second raised", ex) + +parent = Mock() +parent.child.method(1) +parent.child.method(2) +print(parent.child.method.call_count, parent.child.method.call_args) +parent.other(3) +print(parent.method_calls) +print(parent.mock_calls) +parent.child.method.assert_called_with(2) +parent.assert_has_calls([call.child.method(1), call.child.method(2)]) +parent.assert_has_calls([call.other(3), call.child.method(2)], any_order=True) +print(parent.child is parent.child) +parent.attr = 5 +print(parent.attr) +parent.configure_mock(**{"x.return_value": 9}) +print(parent.x()) +cfg = Mock(a=1, b="two") +print(cfg.a, cfg.b) +print(call(1, k=2), call.foo(3), call.foo.bar()) +print(call(1, 2) == call(1, 2), call(1) == call(2), m.call_args == ANY, ANY == 3) + +mm = MagicMock() +mm.__len__ = Mock(return_value=3) +print(len(mm), bool(mm), list(MagicMock()), 1 in MagicMock()) +getitem = Mock(return_value="item") +mm.__getitem__ = getitem +print(mm["k"], mm[0]) +getitem.assert_called_with(0) +with MagicMock() as ctx: + print(ctx is not None) +mm2 = MagicMock() +mm2.method.return_value = [1, 2] +print(mm2.method("q"), mm2.method.call_args) +print(MagicMock(return_value=5)()) + + +# patch +with patch("__main__.greet") as g: + g.return_value = "patched" + print(use_greet(), g.call_args) +print(use_greet()) + +with patch("__main__.greet", lambda n: "lambda " + n): + print(use_greet()) +print(use_greet()) + +with patch.object(Service, "fetch", return_value="fake") as fetch: + print(Service().fetch("k"), fetch.call_count) +print(Service().fetch("k")) + +svc = Service() +with patch.object(svc, "fetch") as fm: + fm.return_value = "inst" + print(svc.fetch("z")) + fm.assert_called_once_with("z") +print(svc.fetch("z")) + +with patch.object(textwrap, "dedent", return_value="D") as d: + print(textwrap.dedent(" x")) +print(textwrap.dedent(" x")) + + +@patch("__main__.greet") +def decorated(extra, g): + g.return_value = "dec" + return use_greet() + extra + + +print(decorated("!"), use_greet()) + + +@patch.object(Service, "fetch", new=lambda self, k: "new " + k) +def decorated2(): + return Service().fetch("a") + + +print(decorated2(), Service().fetch("a")) + +config = {"a": 1} +with patch.dict(config, {"b": 2}): + print(sorted(config)) +print(sorted(config)) +with patch.dict(config, {"c": 3}, clear=True): + print(sorted(config)) +print(sorted(config)) + +p = patch("__main__.greet") +started = p.start() +print(greet("q") is started.return_value) +p.stop() +print(greet("q")) + +try: + with patch.object(Service, "missing"): + pass +except AttributeError: + print("missing attribute") +with patch.object(Service, "missing", create=True) as c: + print(hasattr(Service, "missing")) +print(hasattr(Service, "missing")) + + +class T(unittest.TestCase): + @patch("__main__.greet") + def test_patched(self, g): + g.return_value = "tc" + self.assertEqual(use_greet(), "tc") + g.assert_called_once_with("x") + + def test_mock(self): + mk = Mock() + mk.go(1) + mk.go.assert_called_once_with(1) + with self.assertRaises(AssertionError): + mk.go.assert_called_once_with(2) + + +res = unittest.TestResult() +unittest.defaultTestLoader.loadTestsFromTestCase(T).run(res) +print(res.testsRun, res.wasSuccessful()) From 904ff0cc9dd859d36823c79ac5de52652233f1f9 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 08:35:19 +0200 Subject: [PATCH 036/112] Python: threading, a blocking queue and concurrent.futures over Java threads with a global interpreter lock Co-Authored-By: Claude Opus 5.5 --- .../com/codename1/python/runtime/Gil.java | 115 ++++ .../codename1/python/runtime/JavaBridge.java | 7 +- .../com/codename1/python/runtime/PyLock.java | 84 +++ .../codename1/python/runtime/PyServer.java | 9 + .../codename1/python/runtime/StdModules.java | 85 ++- .../transpiler/api/PythonTranspiler.java | 2 + .../python/stdlib/concurrent/__init__.py | 3 + .../stdlib/concurrent/futures/__init__.py | 352 ++++++++++++ .../com/codename1/python/stdlib/queue.py | 123 ++-- .../com/codename1/python/stdlib/threading.py | 537 ++++++++++++++++++ .../test/resources/conformance/threading_.out | 24 + .../test/resources/conformance/threading_.py | 198 +++++++ 12 files changed, 1501 insertions(+), 38 deletions(-) create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/Gil.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/PyLock.java create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/concurrent/__init__.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/concurrent/futures/__init__.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/threading.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/threading_.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/threading_.py diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Gil.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Gil.java new file mode 100644 index 00000000000..a5e95f772e5 --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Gil.java @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +/// The global interpreter lock: Python code runs on one thread at a time. +/// +/// The runtime's caches, containers and module state assume a single +/// running thread, as CPython's do, and a torn update of one could hand an +/// object of the wrong class to the next instruction -- a native crash on a +/// device, where casts are not checked. Every way into Python takes this +/// lock (the program's start, a callback from Java, a `threading` thread), +/// and every operation that waits releases it for the wait (`time.sleep`, a +/// lock or condition, an HTTP request), so threads take turns at the points +/// where a Python program waits. CPU-bound threads do not run in parallel, +/// as under CPython. +/// +/// Reentrant: a callback into Python from Java code that Python called runs +/// on the same thread, which already holds the lock. +public final class Gil { + private static final Object MONITOR = new Object(); + private static Thread owner; + private static int depth; + + private Gil() { + } + + /// Takes the lock, waiting for the thread that holds it to wait or + /// leave Python. + public static void enter() { + Thread t = Thread.currentThread(); + synchronized (MONITOR) { + if (owner == t) { //NOPMD CompareObjectsWithEquals - identity is meant + depth++; + return; + } + while (owner != null) { + try { + MONITOR.wait(); + } catch (InterruptedException e) { + // keep waiting: the lock is the only way back into Python + } + } + owner = t; + depth = 1; + } + } + + /// Leaves one [#enter()]. + public static void exit() { + synchronized (MONITOR) { + if (owner != Thread.currentThread()) { //NOPMD CompareObjectsWithEquals - identity is meant + return; + } + if (--depth == 0) { + owner = null; + MONITOR.notifyAll(); + } + } + } + + /// Releases the lock entirely before a wait, answering what + /// [#restore(int)] takes back afterwards (0 when this thread did not + /// hold it). + public static int release() { + synchronized (MONITOR) { + if (owner != Thread.currentThread()) { //NOPMD CompareObjectsWithEquals - identity is meant + return 0; + } + int d = depth; + owner = null; + depth = 0; + MONITOR.notifyAll(); + return d; + } + } + + /// Takes the lock back after a wait. + public static void restore(int saved) { + if (saved > 0) { + enter(); + synchronized (MONITOR) { + depth = saved; + } + } + } + + /// Lets another thread waiting for the lock run, as `time.sleep(0)` + /// does. + public static void yieldNow() { + int saved = release(); + Thread.yield(); + restore(saved); + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/JavaBridge.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/JavaBridge.java index 1457a954667..29ee4bf2c1e 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/JavaBridge.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/JavaBridge.java @@ -234,7 +234,12 @@ public static char asChar(Object v) { /// Calls a Python function from a functional interface adapter. public static Object callPy(Object fn, Object[] args) { - return Py.obj(fn).call(args, Py.NO_KW); + Gil.enter(); + try { + return Py.obj(fn).call(args, Py.NO_KW); + } finally { + Gil.exit(); + } } /// The error for a call no overload accepts. diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyLock.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyLock.java new file mode 100644 index 00000000000..8e42cb4d2c5 --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyLock.java @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +/// `_thread.LockType`: a lock that is not owned by the thread that took it, +/// so any thread may release it. Waiting for it releases the [Gil]. +final class PyLock { + private boolean locked; + + /// Takes the lock: at once, without waiting when `blocking` is false, + /// or waiting at most `timeoutMillis` when that is not negative. + boolean acquire(boolean blocking, long timeoutMillis) { + synchronized (this) { + if (!locked) { + locked = true; + return true; + } + if (!blocking) { + return false; + } + } + int saved = Gil.release(); + try { + synchronized (this) { + long deadline = timeoutMillis < 0 ? 0 : System.currentTimeMillis() + timeoutMillis; + while (locked) { + long wait = 0; + if (timeoutMillis >= 0) { + wait = deadline - System.currentTimeMillis(); + if (wait <= 0) { + return false; + } + } + try { + wait(wait); + } catch (InterruptedException e) { + // a spurious wake-up: the loop checks again + } + } + locked = true; + return true; + } + } finally { + Gil.restore(saved); + } + } + + void release() { + synchronized (this) { + if (!locked) { + throw Py.error(Py.RUNTIME_ERROR, "release unlocked lock"); + } + locked = false; + notifyAll(); + } + } + + boolean locked() { + synchronized (this) { + return locked; + } + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyServer.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyServer.java index a5d57fcf51c..5e92a3fe08a 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyServer.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyServer.java @@ -62,6 +62,15 @@ public static Object[] request(String method, String target, String[] headers, b /// Calls `dispatch` and checks what it answered. public static Response call(Object dispatch, Object[] request) { + Gil.enter(); + try { + return call0(dispatch, request); + } finally { + Gil.exit(); + } + } + + private static Response call0(Object dispatch, Object[] request) { Object r = Py.obj(dispatch).call(request, Py.NO_KW); if (!(r instanceof PyTuple) || ((PyTuple) r).size() != 3) { throw Py.typeError("dispatch must return (status, headers, body)"); diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java index d9b823a0c6e..81a23b491aa 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java @@ -74,6 +74,10 @@ static boolean provide(String name) { PyModule.registerModule(m); return true; } + if ("_thread".equals(name)) { + PyModule.registerModule(thread()); + return true; + } if ("_cn1sql".equals(name)) { PyModule.registerModule(sql()); return true; @@ -443,6 +447,69 @@ private static Object sqlValue(Object v) { return v; } + private static final java.util.Map THREAD_IDS = new java.util.HashMap(); + private static long threadIds; + + /// The `threading` ident of the calling thread, assigned on first use. + /// Read only while the [Gil] is held, which is what guards the map. + private static long threadIdent() { + Thread t = Thread.currentThread(); + Long id = THREAD_IDS.get(t); + if (id == null) { + id = Long.valueOf(++threadIds); + THREAD_IDS.put(t, id); + } + return id.longValue(); + } + + /// `_thread`: `start_new_thread(fn, args)` runs `fn` on a new Java + /// thread that enters the [Gil] first; `allocate_lock()` answers a + /// [PyLock], driven by `lock_acquire(lock, blocking, timeout)`, + /// `lock_release` and `lock_locked`; `get_ident()`. An exception that + /// leaves a thread is reported on stderr, as CPython reports it. + private static PyModule thread() { + PyModule m = new PyModule("_thread", new String[0]); + m.setAttr("start_new_thread", PyBuiltin.positional("start_new_thread", a -> { + final Object fn = a[0]; + final Object[] args = a.length > 1 && a[1] instanceof PyTuple ? ((PyTuple) a[1]).items : new Object[0]; + Thread t = new Thread(() -> { + Gil.enter(); + try { + Py.obj(fn).call(args, Py.NO_KW); + } catch (PyException e) { + if (!e.matches(Py.SYSTEM_EXIT)) { + System.err.println("Exception in thread:"); + exitStatus(e); + } + } finally { + THREAD_IDS.remove(Thread.currentThread()); + Gil.exit(); + } + }); + t.start(); + return Py.NONE; + })); + m.setAttr("allocate_lock", PyBuiltin.positional("allocate_lock", a -> new PyJavaObject(new PyLock()))); + m.setAttr("lock_acquire", PyBuiltin.positional("lock_acquire", a -> { + double timeout = a[2] == Py.NONE ? -1 : Py.asDouble(a[2]); //NOPMD CompareObjectsWithEquals - identity is meant + return Py.bool(lock(a[0]).acquire(Py.isTrue(a[1]), timeout < 0 ? -1 : (long) (timeout * 1000))); + })); + m.setAttr("lock_release", PyBuiltin.positional("lock_release", a -> { + lock(a[0]).release(); + return Py.NONE; + })); + m.setAttr("lock_locked", PyBuiltin.positional("lock_locked", a -> Py.bool(lock(a[0]).locked()))); + m.setAttr("get_ident", PyBuiltin.positional("get_ident", a -> PyInt.of(threadIdent()))); + return m; + } + + private static PyLock lock(Object o) { + if (o instanceof PyJavaObject && ((PyJavaObject) o).value instanceof PyLock) { + return (PyLock) ((PyJavaObject) o).value; + } + throw Py.typeError("not a lock"); + } + /// `_cn1http`: `request(method, url, headers, body, timeout, redirects)` /// over [PyHttp], answering `(status, reason, url, headers, body)` with /// the headers a list of pairs. No response at all raises `OSError`. @@ -466,10 +533,16 @@ private static PyModule http() { byte[] body = a[3] == Py.NONE ? null : PyBytes.like(a[3]); //NOPMD CompareObjectsWithEquals - identity is meant int timeout = a[4] == Py.NONE ? 0 : (int) Math.ceil(Py.asDouble(a[4]) * 1000); //NOPMD CompareObjectsWithEquals - identity is meant PyHttp.Response r; + String method = Py.toStr(a[0]); + String url = Py.toStr(a[1]); + boolean redirects = Py.isTrue(a[5]); + int saved = Gil.release(); try { - r = PyHttp.get().send(Py.toStr(a[0]), Py.toStr(a[1]), headers, body, timeout, Py.isTrue(a[5])); + r = PyHttp.get().send(method, url, headers, body, timeout, redirects); } catch (java.io.IOException e) { throw Py.error(Py.OS_ERROR, String.valueOf(e.getMessage())); //NOPMD PreserveStackTrace - a Python exception has no Java cause + } finally { + Gil.restore(saved); } Object[] pairs = new Object[r.headers.length / 2]; for (int i = 0; i < pairs.length; i++) { @@ -717,10 +790,18 @@ private static PyModule time() { m.setAttr("perf_counter_ns", PyBuiltin.positional("perf_counter_ns", a -> PyInt.of(System.nanoTime()))); m.setAttr("monotonic", PyBuiltin.positional("monotonic", a -> Py.ofFloat(System.nanoTime() / 1e9))); m.setAttr("sleep", PyBuiltin.positional("sleep", a -> { + long ms = (long) (Py.asDouble(a[0]) * 1000); + int saved = Gil.release(); try { - Thread.sleep((long) (Py.asDouble(a[0]) * 1000)); + if (ms <= 0) { + Thread.yield(); + } else { + Thread.sleep(ms); + } } catch (InterruptedException e) { throw Py.error(Py.KEYBOARD_INTERRUPT, ""); //NOPMD PreserveStackTrace - a Python exception has no Java cause + } finally { + Gil.restore(saved); } return Py.NONE; })); diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java index b908d644d2a..7ea6b3ed3b3 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java @@ -524,6 +524,7 @@ private static String registry(Map classes, String mainModule, b sb.append(" public static int runMain() {\n"); sb.append(" registerAll();\n"); if (mainModule != null) { + sb.append(" com.codename1.python.runtime.Gil.enter();\n"); sb.append(" try {\n"); sb.append(" ").append(classes.get(mainModule)).append(".exec(\"__main__\");\n"); sb.append(" return 0;\n"); @@ -531,6 +532,7 @@ private static String registry(Map classes, String mainModule, b sb.append(" return com.codename1.python.runtime.StdModules.exitStatus(e);\n"); sb.append(" } finally {\n"); sb.append(" com.codename1.python.runtime.StdModules.runExitHandlers();\n"); + sb.append(" com.codename1.python.runtime.Gil.exit();\n"); sb.append(" }\n"); } else { sb.append(" return 0;\n"); diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/concurrent/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/concurrent/__init__.py new file mode 100644 index 00000000000..ba37a9a03c5 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/concurrent/__init__.py @@ -0,0 +1,3 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/concurrent/futures/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/concurrent/futures/__init__.py new file mode 100644 index 00000000000..ca97779c55f --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/concurrent/futures/__init__.py @@ -0,0 +1,352 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Futures and executors over threading. + +ProcessPoolExecutor runs its calls on threads: an application is one +process on a device, so it is a ThreadPoolExecutor under the process +pool's name, and its calls take turns at the interpreter lock. +""" + +import threading +import time +import queue as _queue + +__all__ = ["Future", "Executor", "ThreadPoolExecutor", "ProcessPoolExecutor", "wait", + "as_completed", "CancelledError", "TimeoutError", "InvalidStateError", + "BrokenExecutor", "FIRST_COMPLETED", "FIRST_EXCEPTION", "ALL_COMPLETED"] + +FIRST_COMPLETED = "FIRST_COMPLETED" +FIRST_EXCEPTION = "FIRST_EXCEPTION" +ALL_COMPLETED = "ALL_COMPLETED" + +_PENDING = "PENDING" +_RUNNING = "RUNNING" +_CANCELLED = "CANCELLED" +_FINISHED = "FINISHED" + + +class Error(Exception): + pass + + +class CancelledError(Error): + pass + + +TimeoutError = TimeoutError + + +class InvalidStateError(Error): + pass + + +class BrokenExecutor(RuntimeError): + pass + + +class Future: + def __init__(self): + self._condition = threading.Condition() + self._state = _PENDING + self._result = None + self._exception = None + self._callbacks = [] + self._waiters = [] + + def __repr__(self): + with self._condition: + if self._state == _FINISHED: + if self._exception is not None: + return "" % (id(self), type(self._exception).__name__) + return "" % (id(self), type(self._result).__name__) + return "" % (id(self), self._state.lower()) + + def _invoke_callbacks(self): + for cb in self._callbacks: + try: + cb(self) + except Exception: + pass + + def cancel(self): + with self._condition: + if self._state in (_RUNNING, _FINISHED): + return False + if self._state == _CANCELLED: + return True + self._state = _CANCELLED + self._condition.notify_all() + self._notify_waiters() + self._invoke_callbacks() + return True + + def cancelled(self): + return self._state == _CANCELLED + + def running(self): + return self._state == _RUNNING + + def done(self): + return self._state in (_CANCELLED, _FINISHED) + + def _get(self): + if self._state == _CANCELLED: + raise CancelledError() + if self._exception is not None: + raise self._exception + return self._result + + def result(self, timeout=None): + with self._condition: + if self.done(): + return self._get() + self._condition.wait(timeout) + if self.done(): + return self._get() + raise TimeoutError() + + def exception(self, timeout=None): + with self._condition: + if not self.done(): + self._condition.wait(timeout) + if self._state == _CANCELLED: + raise CancelledError() + if not self.done(): + raise TimeoutError() + return self._exception + + def add_done_callback(self, fn): + with self._condition: + if not self.done(): + self._callbacks.append(fn) + return + try: + fn(self) + except Exception: + pass + + def set_running_or_notify_cancel(self): + with self._condition: + if self._state == _CANCELLED: + return False + if self._state == _PENDING: + self._state = _RUNNING + return True + raise RuntimeError("Future in unexpected state") + + def set_result(self, result): + with self._condition: + if self.done(): + raise InvalidStateError("%s: %r" % (self._state, self)) + self._result = result + self._state = _FINISHED + self._condition.notify_all() + self._notify_waiters() + self._invoke_callbacks() + + def set_exception(self, exception): + with self._condition: + if self.done(): + raise InvalidStateError("%s: %r" % (self._state, self)) + self._exception = exception + self._state = _FINISHED + self._condition.notify_all() + self._notify_waiters() + self._invoke_callbacks() + + def _notify_waiters(self): + for w in self._waiters: + w.put(self) + + +class Executor: + def submit(self, fn, /, *args, **kwargs): + raise NotImplementedError() + + def map(self, fn, *iterables, timeout=None, chunksize=1): + end = None if timeout is None else time.monotonic() + timeout + fs = [self.submit(fn, *args) for args in zip(*iterables)] + + def results(): + try: + fs.reverse() + while fs: + f = fs.pop() + if end is None: + yield f.result() + else: + yield f.result(end - time.monotonic()) + finally: + for f in fs: + f.cancel() + return results() + + def shutdown(self, wait=True, *, cancel_futures=False): + pass + + def __enter__(self): + return self + + def __exit__(self, *exc): + self.shutdown(wait=True) + return False + + +class _WorkItem: + def __init__(self, future, fn, args, kwargs): + self.future = future + self.fn = fn + self.args = args + self.kwargs = kwargs + + def run(self): + if not self.future.set_running_or_notify_cancel(): + return + try: + result = self.fn(*self.args, **self.kwargs) + except BaseException as exc: + self.future.set_exception(exc) + else: + self.future.set_result(result) + + +class ThreadPoolExecutor(Executor): + _counter = 0 + + def __init__(self, max_workers=None, thread_name_prefix="", initializer=None, initargs=()): + if max_workers is None: + max_workers = 8 + if max_workers <= 0: + raise ValueError("max_workers must be greater than 0") + ThreadPoolExecutor._counter += 1 + self._max_workers = max_workers + self._work = _queue.SimpleQueue() + self._threads = [] + self._idle = threading.Semaphore(0) + self._shutdown = False + self._lock = threading.Lock() + self._prefix = thread_name_prefix or "ThreadPoolExecutor-%d" % (ThreadPoolExecutor._counter - 1) + self._initializer = initializer + self._initargs = initargs + + def submit(self, fn, /, *args, **kwargs): + with self._lock: + if self._shutdown: + raise RuntimeError("cannot schedule new futures after shutdown") + f = Future() + self._work.put(_WorkItem(f, fn, args, kwargs)) + self._adjust() + return f + + def _adjust(self): + if self._idle.acquire(False): + return + if len(self._threads) < self._max_workers: + t = threading.Thread(name="%s_%d" % (self._prefix, len(self._threads)), target=self._worker) + t.daemon = True + t.start() + self._threads.append(t) + + def _worker(self): + if self._initializer is not None: + self._initializer(*self._initargs) + while True: + item = self._work.get() + if item is None: + self._work.put(None) + return + item.run() + self._idle.release() + + def shutdown(self, wait=True, *, cancel_futures=False): + with self._lock: + self._shutdown = True + if cancel_futures: + while True: + try: + item = self._work.get_nowait() + except _queue.Empty: + break + if item is not None: + item.future.cancel() + self._work.put(None) + if wait: + for t in self._threads: + t.join() + + +class ProcessPoolExecutor(ThreadPoolExecutor): + def __init__(self, max_workers=None, mp_context=None, initializer=None, initargs=(), **kw): + ThreadPoolExecutor.__init__(self, max_workers, "ProcessPoolExecutor", initializer, initargs) + + +def wait(fs, timeout=None, return_when=ALL_COMPLETED): + fs = set(fs) + end = None if timeout is None else time.monotonic() + timeout + q = _queue.SimpleQueue() + for f in fs: + with f._condition: + f._waiters.append(q) + try: + while True: + done = set(f for f in fs if f.done()) + if return_when == FIRST_COMPLETED and done: + break + if return_when == FIRST_EXCEPTION and any( + not f.cancelled() and f._exception is not None for f in done): + break + if len(done) == len(fs): + break + left = None if end is None else end - time.monotonic() + if left is not None and left <= 0: + break + try: + q.get(True, left) + except _queue.Empty: + break + finally: + for f in fs: + with f._condition: + f._waiters.remove(q) + done = set(f for f in fs if f.done()) + return DoneAndNotDoneFutures(done, fs - done) + + +class DoneAndNotDoneFutures(tuple): + def __new__(cls, done, not_done): + return tuple.__new__(cls, (done, not_done)) + + @property + def done(self): + return self[0] + + @property + def not_done(self): + return self[1] + + +def as_completed(fs, timeout=None): + fs = list(dict.fromkeys(fs)) + end = None if timeout is None else time.monotonic() + timeout + q = _queue.SimpleQueue() + pending = [] + for f in fs: + with f._condition: + if f.done(): + q.put(f) + else: + f._waiters.append(q) + pending.append(f) + try: + for _ in range(len(fs)): + left = None if end is None else end - time.monotonic() + try: + yield q.get(True, left) if left is None or left > 0 else q.get(False) + except _queue.Empty: + raise TimeoutError("%d (of %d) futures unfinished" % (len(fs) - _, len(fs))) + finally: + for f in pending: + with f._condition: + if q in f._waiters: + f._waiters.remove(q) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/queue.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/queue.py index 4e40f9951d5..c0cc263d85c 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/queue.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/queue.py @@ -1,16 +1,16 @@ # Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. # Licensed under the GNU General Public License version 2 with the Classpath # exception, as the rest of the Codename One Python runtime. -"""Queues: FIFO, LIFO and priority. +"""Queues: FIFO, LIFO and priority, safe to share between threads. -A translated program runs its Python code on one thread, so nothing can -fill an empty queue while get() waits on it. A get() on an empty queue or a -put() on a full one therefore raises Empty or Full at once, where CPython -would wait for another thread (forever, without one). +get() on an empty queue and put() on a full one wait for another thread, +releasing the interpreter lock while they wait. """ from collections import deque import heapq +import threading +import time __all__ = ["Empty", "Full", "ShutDown", "Queue", "PriorityQueue", "LifoQueue", "SimpleQueue"] @@ -31,46 +31,93 @@ class Queue: def __init__(self, maxsize=0): self.maxsize = maxsize self._init(maxsize) + self.mutex = threading.Lock() + self.not_empty = threading.Condition(self.mutex) + self.not_full = threading.Condition(self.mutex) + self.all_tasks_done = threading.Condition(self.mutex) self.unfinished_tasks = 0 self.is_shutdown = False def task_done(self): - unfinished = self.unfinished_tasks - 1 - if unfinished < 0: - raise ValueError("task_done() called too many times") - self.unfinished_tasks = unfinished + with self.all_tasks_done: + unfinished = self.unfinished_tasks - 1 + if unfinished <= 0: + if unfinished < 0: + raise ValueError("task_done() called too many times") + self.all_tasks_done.notify_all() + self.unfinished_tasks = unfinished def join(self): - if self.unfinished_tasks: - raise RuntimeError("join() would wait forever: %d unfinished tasks" % self.unfinished_tasks) + with self.all_tasks_done: + while self.unfinished_tasks: + self.all_tasks_done.wait() def qsize(self): - return self._qsize() + with self.mutex: + return self._qsize() def empty(self): - return not self._qsize() + with self.mutex: + return not self._qsize() def full(self): - return 0 < self.maxsize <= self._qsize() + with self.mutex: + return 0 < self.maxsize <= self._qsize() def put(self, item, block=True, timeout=None): - if self.is_shutdown: - raise ShutDown - if block and timeout is not None and timeout < 0: - raise ValueError("'timeout' must be a non-negative number") - if 0 < self.maxsize <= self._qsize(): - raise Full - self._put(item) - self.unfinished_tasks += 1 + with self.not_full: + if self.is_shutdown: + raise ShutDown + if self.maxsize > 0: + if not block: + if self._qsize() >= self.maxsize: + raise Full + elif timeout is None: + while self._qsize() >= self.maxsize: + self.not_full.wait() + if self.is_shutdown: + raise ShutDown + elif timeout < 0: + raise ValueError("'timeout' must be a non-negative number") + else: + endtime = time.monotonic() + timeout + while self._qsize() >= self.maxsize: + remaining = endtime - time.monotonic() + if remaining <= 0.0: + raise Full + self.not_full.wait(remaining) + if self.is_shutdown: + raise ShutDown + self._put(item) + self.unfinished_tasks += 1 + self.not_empty.notify() def get(self, block=True, timeout=None): - if block and timeout is not None and timeout < 0: - raise ValueError("'timeout' must be a non-negative number") - if not self._qsize(): - if self.is_shutdown: + with self.not_empty: + if self.is_shutdown and not self._qsize(): raise ShutDown - raise Empty - return self._get() + if not block: + if not self._qsize(): + raise Empty + elif timeout is None: + while not self._qsize(): + self.not_empty.wait() + if self.is_shutdown and not self._qsize(): + raise ShutDown + elif timeout < 0: + raise ValueError("'timeout' must be a non-negative number") + else: + endtime = time.monotonic() + timeout + while not self._qsize(): + remaining = endtime - time.monotonic() + if remaining <= 0.0: + raise Empty + self.not_empty.wait(remaining) + if self.is_shutdown and not self._qsize(): + raise ShutDown + item = self._get() + self.not_full.notify() + return item def put_nowait(self, item): return self.put(item, block=False) @@ -79,12 +126,16 @@ def get_nowait(self): return self.get(block=False) def shutdown(self, immediate=False): - self.is_shutdown = True - if immediate: - while self._qsize(): - self._get() - if self.unfinished_tasks > 0: - self.unfinished_tasks -= 1 + with self.mutex: + self.is_shutdown = True + if immediate: + while self._qsize(): + self._get() + if self.unfinished_tasks > 0: + self.unfinished_tasks -= 1 + self.all_tasks_done.notify_all() + self.not_empty.notify_all() + self.not_full.notify_all() def _init(self, maxsize): self.queue = deque() @@ -130,14 +181,16 @@ def _get(self): class SimpleQueue: def __init__(self): self._queue = deque() + self._count = threading.Semaphore(0) def put(self, item, block=True, timeout=None): self._queue.append(item) + self._count.release() def get(self, block=True, timeout=None): if timeout is not None and timeout < 0: raise ValueError("'timeout' must be a non-negative number") - if not self._queue: + if not self._count.acquire(block, timeout if block else None): raise Empty return self._queue.popleft() diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/threading.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/threading.py new file mode 100644 index 00000000000..e9f11e53bc1 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/threading.py @@ -0,0 +1,537 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Threads over Java threads, with CPython's global interpreter lock. + +Python code runs on one thread at a time. A thread lets the others run +while it waits: in time.sleep(), on a lock, condition, event or queue, and +during an HTTP request. A thread that computes without waiting keeps the +others waiting, as under CPython. +""" + +import _thread +import time as _time + +__all__ = ["Thread", "Lock", "RLock", "Condition", "Semaphore", "BoundedSemaphore", + "Event", "Barrier", "BrokenBarrierError", "Timer", "local", "current_thread", + "main_thread", "active_count", "enumerate", "get_ident", "TIMEOUT_MAX", + "ThreadError", "excepthook"] + +TIMEOUT_MAX = 9223372036.0 +ThreadError = RuntimeError +get_ident = _thread.get_ident + + +class Lock: + def __init__(self): + self._lock = _thread.allocate_lock() + + def acquire(self, blocking=True, timeout=-1): + if not blocking and timeout != -1: + raise ValueError("can't specify a timeout for a non-blocking call") + return _thread.lock_acquire(self._lock, blocking, None if timeout < 0 else timeout) + + def release(self): + _thread.lock_release(self._lock) + + def locked(self): + return _thread.lock_locked(self._lock) + + __enter__ = acquire + + def __exit__(self, *exc): + self.release() + + def __repr__(self): + return "<%s _thread.lock object>" % ("locked" if self.locked() else "unlocked") + + +class RLock: + def __init__(self): + self._block = Lock() + self._owner = None + self._count = 0 + + def acquire(self, blocking=True, timeout=-1): + me = get_ident() + if self._owner == me: + self._count += 1 + return True + if not self._block.acquire(blocking, timeout): + return False + self._owner = me + self._count = 1 + return True + + def release(self): + if self._owner != get_ident(): + raise RuntimeError("cannot release un-acquired lock") + self._count -= 1 + if self._count == 0: + self._owner = None + self._block.release() + + def _is_owned(self): + return self._owner == get_ident() + + def _release_save(self): + state = (self._count, self._owner) + self._count = 0 + self._owner = None + self._block.release() + return state + + def _acquire_restore(self, state): + self._block.acquire() + self._count, self._owner = state + + __enter__ = acquire + + def __exit__(self, *exc): + self.release() + + +class Condition: + def __init__(self, lock=None): + self._lock = lock if lock is not None else RLock() + self._waiters = [] + + def __enter__(self): + return self._lock.__enter__() + + def __exit__(self, *exc): + return self._lock.__exit__(*exc) + + def acquire(self, *args): + return self._lock.acquire(*args) + + def release(self): + self._lock.release() + + def _owned(self): + if isinstance(self._lock, RLock): + return self._lock._is_owned() + return self._lock.locked() + + def wait(self, timeout=None): + if not self._owned(): + raise RuntimeError("cannot wait on un-acquired lock") + waiter = Lock() + waiter.acquire() + self._waiters.append(waiter) + if isinstance(self._lock, RLock): + saved = self._lock._release_save() + else: + saved = None + self._lock.release() + try: + if timeout is None: + got = waiter.acquire() + else: + got = waiter.acquire(True, timeout) if timeout > 0 else waiter.acquire(False) + return got + finally: + if saved is not None: + self._lock._acquire_restore(saved) + else: + self._lock.acquire() + if not got: + try: + self._waiters.remove(waiter) + except ValueError: + pass + + def wait_for(self, predicate, timeout=None): + end = None if timeout is None else _time.monotonic() + timeout + result = predicate() + while not result: + if end is not None: + left = end - _time.monotonic() + if left <= 0: + break + self.wait(left) + else: + self.wait() + result = predicate() + return result + + def notify(self, n=1): + if not self._owned(): + raise RuntimeError("cannot notify on un-acquired lock") + for w in self._waiters[:n]: + self._waiters.remove(w) + w.release() + + def notify_all(self): + self.notify(len(self._waiters)) + + notifyAll = notify_all + + +class Semaphore: + def __init__(self, value=1): + if value < 0: + raise ValueError("semaphore initial value must be >= 0") + self._cond = Condition(Lock()) + self._value = value + + def acquire(self, blocking=True, timeout=None): + if not blocking and timeout is not None: + raise ValueError("can't specify timeout for non-blocking acquire") + with self._cond: + if not blocking and self._value == 0: + return False + if not self._cond.wait_for(lambda: self._value > 0, timeout): + return False + self._value -= 1 + return True + + __enter__ = acquire + + def release(self, n=1): + if n < 1: + raise ValueError("n must be one or more") + with self._cond: + self._value += n + self._cond.notify(n) + + def __exit__(self, *exc): + self.release() + + +class BoundedSemaphore(Semaphore): + def __init__(self, value=1): + Semaphore.__init__(self, value) + self._initial_value = value + + def release(self, n=1): + if n < 1: + raise ValueError("n must be one or more") + with self._cond: + if self._value + n > self._initial_value: + raise ValueError("Semaphore released too many times") + self._value += n + self._cond.notify(n) + + +class Event: + def __init__(self): + self._cond = Condition(Lock()) + self._flag = False + + def is_set(self): + return self._flag + + isSet = is_set + + def set(self): + with self._cond: + self._flag = True + self._cond.notify_all() + + def clear(self): + with self._cond: + self._flag = False + + def wait(self, timeout=None): + with self._cond: + if not self._flag: + self._cond.wait_for(lambda: self._flag, timeout) + return self._flag + + +class BrokenBarrierError(RuntimeError): + pass + + +class Barrier: + def __init__(self, parties, action=None, timeout=None): + if parties < 1: + raise ValueError("parties must be >= 1") + self._cond = Condition(Lock()) + self._parties = parties + self._action = action + self._timeout = timeout + self._count = 0 + self._generation = 0 + self._broken = False + + @property + def parties(self): + return self._parties + + @property + def n_waiting(self): + return self._count + + @property + def broken(self): + return self._broken + + def wait(self, timeout=None): + if timeout is None: + timeout = self._timeout + with self._cond: + if self._broken: + raise BrokenBarrierError + gen = self._generation + index = self._count + self._count += 1 + if self._count == self._parties: + try: + if self._action: + self._action() + except BaseException: + self._break() + raise + self._count = 0 + self._generation += 1 + self._cond.notify_all() + return index + if not self._cond.wait_for(lambda: self._generation != gen or self._broken, timeout): + self._break() + if self._broken: + raise BrokenBarrierError + return index + + def _break(self): + self._broken = True + self._cond.notify_all() + + def abort(self): + with self._cond: + self._break() + + def reset(self): + with self._cond: + self._broken = False + self._count = 0 + self._generation += 1 + self._cond.notify_all() + + +class local: + """Attributes per thread, kept in a dictionary keyed by the thread.""" + + def __init__(self): + object.__setattr__(self, "_local__d", {}) + + def _local__ns(self): + d = object.__getattribute__(self, "_local__d") + me = get_ident() + ns = d.get(me) + if ns is None: + ns = {} + d[me] = ns + return ns + + def __getattr__(self, name): + ns = self._local__ns() + if name in ns: + return ns[name] + raise AttributeError("'_thread._local' object has no attribute '%s'" % name) + + def __setattr__(self, name, value): + self._local__ns()[name] = value + + def __delattr__(self, name): + ns = self._local__ns() + if name not in ns: + raise AttributeError(name) + del ns[name] + + +_active = {} +_counter = 0 + + +def _newname(prefix="Thread-%d"): + global _counter + _counter += 1 + return prefix % _counter + + +class ExceptHookArgs: + def __init__(self, exc_type, exc_value, exc_traceback, thread): + self.exc_type = exc_type + self.exc_value = exc_value + self.exc_traceback = exc_traceback + self.thread = thread + + +def excepthook(args): + if args.exc_type is SystemExit: + return + import sys + name = args.thread.name if args.thread is not None else get_ident() + print("Exception in thread %s:" % name, file=sys.stderr) + print("%s: %s" % (args.exc_type.__name__, args.exc_value), file=sys.stderr) + + +class Thread: + def __init__(self, group=None, target=None, name=None, args=(), kwargs=None, *, daemon=None): + self._target = target + self._args = args + self._kwargs = kwargs if kwargs is not None else {} + if name is None: + name = _newname() + if target is not None: + tname = getattr(target, "__name__", None) + if tname: + name += " (%s)" % tname + self._name = name + self._daemonic = bool(daemon) if daemon is not None else False + self._ident = None + self._started = Event() + self._done = Event() + + @property + def name(self): + return self._name + + @name.setter + def name(self, value): + self._name = str(value) + + def getName(self): + return self._name + + def setName(self, value): + self._name = str(value) + + @property + def ident(self): + return self._ident + + native_id = ident + + @property + def daemon(self): + return self._daemonic + + @daemon.setter + def daemon(self, value): + if self._started.is_set(): + raise RuntimeError("cannot set daemon status of active thread") + self._daemonic = bool(value) + + def isDaemon(self): + return self._daemonic + + def setDaemon(self, value): + self.daemon = value + + def start(self): + if self._started.is_set(): + raise RuntimeError("threads can only be started once") + _thread.start_new_thread(self._bootstrap, ()) + self._started.wait() + + def _bootstrap(self): + self._ident = get_ident() + _active[self._ident] = self + self._started.set() + try: + self.run() + except BaseException as e: + excepthook(ExceptHookArgs(type(e), e, None, self)) + finally: + _active.pop(self._ident, None) + self._done.set() + + def run(self): + try: + if self._target is not None: + self._target(*self._args, **self._kwargs) + finally: + del self._target, self._args, self._kwargs + + def join(self, timeout=None): + if not self._started.is_set(): + raise RuntimeError("cannot join thread before it is started") + if self is current_thread(): + raise RuntimeError("cannot join current thread") + self._done.wait(timeout) + + def is_alive(self): + return self._started.is_set() and not self._done.is_set() + + def __repr__(self): + status = "initial" + if self._started.is_set(): + status = "started" + if self._done.is_set(): + status = "stopped" + if self._daemonic: + status += " daemon" + if self._ident is not None: + status += " %s" % self._ident + return "<%s(%s, %s)>" % (type(self).__name__, self._name, status) + + +class _MainThread(Thread): + def __init__(self): + Thread.__init__(self, name="MainThread") + self._ident = get_ident() + self._started.set() + _active[self._ident] = self + + +class _DummyThread(Thread): + def __init__(self): + Thread.__init__(self, name=_newname("Dummy-%d"), daemon=True) + self._ident = get_ident() + self._started.set() + + def is_alive(self): + return True + + def join(self, timeout=None): + raise RuntimeError("cannot join a dummy thread") + + +_main = _MainThread() + + +def current_thread(): + t = _active.get(get_ident()) + if t is None: + t = _DummyThread() + return t + + +currentThread = current_thread + + +def main_thread(): + return _main + + +def active_count(): + return len(_active) + + +activeCount = active_count + + +def enumerate(): + return list(_active.values()) + + +class Timer(Thread): + def __init__(self, interval, function, args=None, kwargs=None): + Thread.__init__(self) + self.interval = interval + self.function = function + self.args = args if args is not None else [] + self.kwargs = kwargs if kwargs is not None else {} + self.finished = Event() + + def cancel(self): + self.finished.set() + + def run(self): + self.finished.wait(self.interval) + if not self.finished.is_set(): + self.function(*self.args, **self.kwargs) + self.finished.set() diff --git a/maven/python-transpiler/src/test/resources/conformance/threading_.out b/maven/python-transpiler/src/test/resources/conformance/threading_.out new file mode 100644 index 00000000000..449fcb46606 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/threading_.out @@ -0,0 +1,24 @@ +squares [0, 1, 4, 9, 16, 25, 36, 49, 64, 81] +alive False False consumer +counter 200 +order ['set', 'woke'] +wait timeout False +cond saw [0, 1, 2] +rlock nested ok +sem True True False +sem error Semaphore released too many times +barrier [0, 1, 2] +local [False, 'worker'] main +main True MainThread +timer [1] +map [0, 10, 20, 30, 40] +completed [1, 2, 4, 8, 16, 32] +future raised ZeroDivisionError +exception ZeroDivisionError +wait 1 0 +callback [7] True False +process pool [1, 2, 3] +lifo [2, 1, 0] +empty after timeout +simple hello +joined 0 diff --git a/maven/python-transpiler/src/test/resources/conformance/threading_.py b/maven/python-transpiler/src/test/resources/conformance/threading_.py new file mode 100644 index 00000000000..63177b1b810 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/threading_.py @@ -0,0 +1,198 @@ +import threading +import queue +import time +from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor, as_completed, wait, FIRST_COMPLETED + +# producer / consumer over a bounded queue +q = queue.Queue(maxsize=2) +got = [] + + +def producer(): + for i in range(10): + q.put(i) + q.put(None) + + +def consumer(): + while True: + item = q.get() + q.task_done() + if item is None: + break + got.append(item * item) + + +tp = threading.Thread(target=producer) +tc = threading.Thread(target=consumer, name="consumer") +tc.start() +tp.start() +tp.join() +tc.join() +print("squares", got) +print("alive", tp.is_alive(), tc.is_alive(), tc.name) + +# a lock protecting a counter across threads that sleep in between +lock = threading.Lock() +counter = [0] + + +def bump(n): + for _ in range(n): + with lock: + v = counter[0] + time.sleep(0) + counter[0] = v + 1 + + +ts = [threading.Thread(target=bump, args=(50,)) for _ in range(4)] +for t in ts: + t.start() +for t in ts: + t.join() +print("counter", counter[0]) + +# events and conditions +ev = threading.Event() +order = [] + + +def waiter(): + ev.wait() + order.append("woke") + + +w = threading.Thread(target=waiter) +w.start() +time.sleep(0.01) +order.append("set") +ev.set() +w.join() +print("order", order) +print("wait timeout", threading.Event().wait(0.01)) + +cond = threading.Condition() +items = [] + + +def cons(): + with cond: + cond.wait_for(lambda: len(items) >= 3) + print("cond saw", sorted(items)) + + +c = threading.Thread(target=cons) +c.start() +for i in range(3): + with cond: + items.append(i) + cond.notify() +c.join() + +# rlock, semaphore, barrier +r = threading.RLock() +with r: + with r: + print("rlock nested ok") +sem = threading.BoundedSemaphore(2) +print("sem", sem.acquire(), sem.acquire(), sem.acquire(False)) +sem.release() +try: + sem.release() + sem.release() +except ValueError as e: + print("sem error", e) + +b = threading.Barrier(3) +idx = [] + + +def party(): + idx.append(b.wait()) + + +ps = [threading.Thread(target=party) for _ in range(3)] +for p in ps: + p.start() +for p in ps: + p.join() +print("barrier", sorted(idx)) + +# thread-local data +data = threading.local() +data.x = "main" +seen = [] + + +def loc(): + seen.append(hasattr(data, "x")) + data.x = "worker" + seen.append(data.x) + + +t = threading.Thread(target=loc) +t.start() +t.join() +print("local", seen, data.x) +print("main", threading.current_thread() is threading.main_thread(), threading.current_thread().name) + +# timer +fired = [] +tm = threading.Timer(0.01, lambda: fired.append(1)) +tm.start() +tm.join() +tm2 = threading.Timer(10, lambda: fired.append(2)) +tm2.start() +tm2.cancel() +tm2.join() +print("timer", fired) + +# futures +with ThreadPoolExecutor(max_workers=3) as ex: + print("map", list(ex.map(lambda x: x * 10, range(5)))) + fs = [ex.submit(pow, 2, i) for i in range(6)] + print("completed", sorted(f.result() for f in as_completed(fs))) + bad = ex.submit(lambda: 1 / 0) + try: + bad.result() + except ZeroDivisionError as e: + print("future raised", type(e).__name__) + print("exception", type(bad.exception()).__name__) + done, not_done = wait([ex.submit(time.sleep, 0)], return_when=FIRST_COMPLETED) + print("wait", len(done), len(not_done)) + cb = [] + f = ex.submit(lambda: 7) + f.result() + f.add_done_callback(lambda fut: cb.append(fut.result())) + print("callback", cb, f.done(), f.cancelled()) + +if __name__ == "__main__": + with ProcessPoolExecutor(2) as px: + print("process pool", list(px.map(abs, [-1, -2, 3]))) + +# queue variants +lq = queue.LifoQueue() +for i in range(3): + lq.put(i) +print("lifo", [lq.get() for _ in range(3)]) +try: + queue.Queue().get(timeout=0.01) +except queue.Empty: + print("empty after timeout") +sq = queue.SimpleQueue() +threading.Thread(target=lambda: sq.put("hello")).start() +print("simple", sq.get()) +jq = queue.Queue() +for i in range(3): + jq.put(i) + + +def drain(): + while not jq.empty(): + jq.get() + jq.task_done() + + +threading.Thread(target=drain).start() +jq.join() +print("joined", jq.qsize()) From cf4b92ee1897f2ef3c67fdea25dc012b611dc89d Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 08:36:23 +0200 Subject: [PATCH 037/112] Python: pickle (protocols 0-5, byte-compatible with CPython), copyreg and shelve Co-Authored-By: Claude Opus 5.5 --- .../com/codename1/python/stdlib/copyreg.py | 103 ++ .../com/codename1/python/stdlib/functools.py | 11 + .../com/codename1/python/stdlib/pickle.py | 1429 +++++++++++++++++ .../com/codename1/python/stdlib/shelve.py | 237 +++ .../test/resources/conformance/pickle_.out | 233 +++ .../src/test/resources/conformance/pickle_.py | 255 +++ .../test/resources/conformance/shelve_.out | 12 + .../src/test/resources/conformance/shelve_.py | 53 + 8 files changed, 2333 insertions(+) create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/copyreg.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/pickle.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/shelve.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/pickle_.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/pickle_.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/shelve_.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/shelve_.py diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/copyreg.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/copyreg.py new file mode 100644 index 00000000000..2f556a650c2 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/copyreg.py @@ -0,0 +1,103 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Helper to provide extensibility for pickle. + +This is the registry of reduction functions (`pickle`), the reconstructors +`pickle` writes for plain objects, and the extension-code registry. + +`pickle` does not ask an instance for a default `__reduce_ex__`, because the +runtime's `object` has none: `pickle` builds the same reduction itself, and +`__newobj__`, `__newobj_ex__` and `_reconstructor` here are what it writes. +""" + +__all__ = ["pickle", "constructor", "add_extension", "remove_extension", "clear_extension_cache"] + +dispatch_table = {} + + +def pickle(ob_type, pickle_function, constructor_ob=None): + if not callable(pickle_function): + raise TypeError("reduction functions must be callable") + dispatch_table[ob_type] = pickle_function + if constructor_ob is not None: + constructor(constructor_ob) + + +def constructor(object): + if not callable(object): + raise TypeError("constructors must be callable") + + +def _reconstructor(cls, base, state): + if base is object: + obj = object.__new__(cls) + else: + obj = base.__new__(cls, state) + return obj + + +def __newobj__(cls, *args): + return cls.__new__(cls, *args) + + +def __newobj_ex__(cls, args, kwargs): + return cls.__new__(cls, *args, **kwargs) + + +def _slotnames(cls): + """Return the slot names of a class and its bases, in CPython's order.""" + names = [] + if not hasattr(cls, "__slots__"): + return names + for c in cls.__mro__: + if "__slots__" in c.__dict__: + slots = c.__dict__["__slots__"] + if isinstance(slots, str): + slots = (slots,) + for name in slots: + if name in ("__dict__", "__weakref__"): + continue + elif name.startswith("__") and not name.endswith("__"): + stripped = c.__name__.lstrip("_") + if stripped: + names.append("_%s%s" % (stripped, name)) + else: + names.append(name) + else: + names.append(name) + return names + + +_extension_registry = {} +_inverted_registry = {} +_extension_cache = {} + + +def add_extension(module, name, code): + code = int(code) + if not 1 <= code <= 0x7fffffff: + raise ValueError("code out of range") + key = (module, name) + if _extension_registry.get(key) == code and _inverted_registry.get(code) == key: + return + if key in _extension_registry: + raise ValueError("key %s is already registered with code %s" % (key, _extension_registry[key])) + if code in _inverted_registry: + raise ValueError("code %s is already in use for key %s" % (code, _inverted_registry[code])) + _extension_registry[key] = code + _inverted_registry[code] = key + + +def remove_extension(module, name, code): + key = (module, name) + if _extension_registry.get(key) != code or _inverted_registry.get(code) != key: + raise ValueError("key %s is not registered with code %s" % (key, code)) + del _extension_registry[key] + del _inverted_registry[code] + if code in _extension_cache: + del _extension_cache[code] + + +def clear_extension_cache(): + _extension_cache.clear() diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/functools.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/functools.py index 7399af601f3..ba7078a795e 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/functools.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/functools.py @@ -37,6 +37,17 @@ def __init__(self, func, *args, **keywords): def __call__(self, *args, **keywords): return self.func(*self.args, *args, **{**self.keywords, **keywords}) + def __reduce__(self): + return (type(self), (self.func,), (self.func, self.args, self.keywords or None, None)) + + def __setstate__(self, state): + func, args, keywords, namespace = state + if not callable(func) or not isinstance(args, tuple) or (keywords is not None and not isinstance(keywords, dict)): + raise TypeError("invalid partial state") + self.func = func + self.args = args + self.keywords = {} if keywords is None else dict(keywords) + def __repr__(self): parts = [repr(self.func)] parts.extend(repr(a) for a in self.args) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/pickle.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/pickle.py new file mode 100644 index 00000000000..bbc9e08da78 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/pickle.py @@ -0,0 +1,1429 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Create portable serialized representations of Python objects. + +The stream is the one CPython writes: `dumps` produces the same bytes for +the same value under protocols 0 to 5 (the default is 5), and `loads` reads +every opcode CPython emits, including the framed protocol 4 and 5 streams. + +Supported values: None, bool, int (of any size), float, str, bytes, +bytearray, tuple, list, dict, set, frozenset, complex, range, classes and +functions by name, and instances of user classes through `__reduce_ex__`, +`__reduce__`, `__getnewargs__`, `__getnewargs_ex__`, `__getstate__` and +`__setstate__`, `__slots__`, dataclasses and enums. Shared and recursive +references are kept through the memo. `persistent_id` / `persistent_load`, +`reducer_override`, `dispatch_table` and `copyreg` extension codes work. + +Not supported: out of band buffers (`PickleBuffer`, `buffer_callback`, +`buffers`) and `__reduce_ex__` of a subclass of a built-in container under +protocols 0 and 1. A class or function is found on load through +`sys.modules`, so its module has to be part of the program: a module the +program never imports cannot be named by a pickle (the program is compiled +ahead of time and nothing imports a module by a computed name). +`PickleError` subclasses are raised as in CPython; the messages carry no +"when serializing" notes. +""" + +import sys +import builtins +import io +import copyreg +import functools +from struct import pack, unpack + +__all__ = ["PickleError", "PicklingError", "UnpicklingError", "Pickler", "Unpickler", "dump", "dumps", + "load", "loads", "HIGHEST_PROTOCOL", "DEFAULT_PROTOCOL", "format_version", "compatible_formats", + "encode_long", "decode_long", "whichmodule"] + +format_version = "4.0" +compatible_formats = ["1.0", "1.1", "1.2", "1.3", "2.0", "3.0", "4.0", "5.0"] +HIGHEST_PROTOCOL = 5 +DEFAULT_PROTOCOL = 5 + + +class PickleError(Exception): + pass + + +class PicklingError(PickleError): + pass + + +class UnpicklingError(PickleError): + pass + + +PickleError.__module__ = "pickle" +PicklingError.__module__ = "pickle" +UnpicklingError.__module__ = "pickle" + +bytes_types = (bytes, bytearray) + +MARK = b'(' +STOP = b'.' +POP = b'0' +POP_MARK = b'1' +DUP = b'2' +FLOAT = b'F' +INT = b'I' +BININT = b'J' +BININT1 = b'K' +LONG = b'L' +BININT2 = b'M' +NONE = b'N' +PERSID = b'P' +BINPERSID = b'Q' +REDUCE = b'R' +STRING = b'S' +BINSTRING = b'T' +SHORT_BINSTRING = b'U' +UNICODE = b'V' +BINUNICODE = b'X' +APPEND = b'a' +BUILD = b'b' +GLOBAL = b'c' +DICT = b'd' +EMPTY_DICT = b'}' +APPENDS = b'e' +GET = b'g' +BINGET = b'h' +INST = b'i' +LONG_BINGET = b'j' +LIST = b'l' +EMPTY_LIST = b']' +OBJ = b'o' +PUT = b'p' +BINPUT = b'q' +LONG_BINPUT = b'r' +SETITEM = b's' +TUPLE = b't' +EMPTY_TUPLE = b')' +SETITEMS = b'u' +BINFLOAT = b'G' +TRUE = b'I01\n' +FALSE = b'I00\n' +PROTO = b'\x80' +NEWOBJ = b'\x81' +EXT1 = b'\x82' +EXT2 = b'\x83' +EXT4 = b'\x84' +TUPLE1 = b'\x85' +TUPLE2 = b'\x86' +TUPLE3 = b'\x87' +NEWTRUE = b'\x88' +NEWFALSE = b'\x89' +LONG1 = b'\x8a' +LONG4 = b'\x8b' +BINBYTES = b'B' +SHORT_BINBYTES = b'C' +SHORT_BINUNICODE = b'\x8c' +BINUNICODE8 = b'\x8d' +BINBYTES8 = b'\x8e' +EMPTY_SET = b'\x8f' +ADDITEMS = b'\x90' +FROZENSET = b'\x91' +NEWOBJ_EX = b'\x92' +STACK_GLOBAL = b'\x93' +MEMOIZE = b'\x94' +FRAME = b'\x95' +BYTEARRAY8 = b'\x96' +NEXT_BUFFER = b'\x97' +READONLY_BUFFER = b'\x98' + +_tuplesize2code = [EMPTY_TUPLE, TUPLE1, TUPLE2, TUPLE3] + +_NoValue = object() + +# What fix_imports maps for protocols below 3: the Python 2 module names. +_EXCEPTION_NAMES = ( + "ArithmeticError AssertionError AttributeError BaseException BufferError BytesWarning DeprecationWarning " + "EOFError EnvironmentError Exception FloatingPointError FutureWarning GeneratorExit IOError ImportError " + "ImportWarning IndentationError IndexError KeyError KeyboardInterrupt LookupError MemoryError NameError " + "NotImplementedError OSError OverflowError PendingDeprecationWarning ReferenceError RuntimeError " + "RuntimeWarning StopIteration SyntaxError SyntaxWarning SystemError SystemExit TabError TypeError " + "UnboundLocalError UnicodeDecodeError UnicodeEncodeError UnicodeError UnicodeTranslateError " + "UnicodeWarning UserWarning ValueError Warning ZeroDivisionError").split() +_OS_ERRORS = ("BrokenPipeError ChildProcessError ConnectionAbortedError ConnectionError ConnectionRefusedError " + "ConnectionResetError FileExistsError FileNotFoundError InterruptedError IsADirectoryError " + "NotADirectoryError PermissionError ProcessLookupError TimeoutError").split() +_NAME_FIX = {"range": "xrange", "chr": "unichr", "str": "unicode", "int": "long"} +_NAME_FIX_MODULE = {"zip": "itertools:izip", "map": "itertools:imap", "filter": "itertools:ifilter"} +_IMPORT_FIX = {"builtins": "__builtin__", "copyreg": "copy_reg", "queue": "Queue", "socketserver": "SocketServer", + "configparser": "ConfigParser", "reprlib": "repr", "_thread": "thread"} +_UNFIX_IMPORT = {} +for _k in _IMPORT_FIX: + _UNFIX_IMPORT[_IMPORT_FIX[_k]] = _k +_UNFIX_NAME = {} +for _k in _NAME_FIX: + _UNFIX_NAME[("__builtin__", _NAME_FIX[_k])] = ("builtins", _k) +_UNFIX_NAME[("itertools", "izip")] = ("builtins", "zip") +_UNFIX_NAME[("itertools", "imap")] = ("builtins", "map") +_UNFIX_NAME[("itertools", "ifilter")] = ("builtins", "filter") +_UNFIX_NAME[("__builtin__", "basestring")] = ("builtins", "str") +_UNFIX_NAME[("exceptions", "StandardError")] = ("builtins", "Exception") +for _k in _EXCEPTION_NAMES: + _UNFIX_NAME[("exceptions", _k)] = ("builtins", _k) + + +def _fix_global(module_name, name): + """The Python 2 name of a builtins/stdlib global, for protocols 0 to 2.""" + if module_name == "builtins": + if name in _NAME_FIX: + return "__builtin__", _NAME_FIX[name] + if name in _NAME_FIX_MODULE: + m, n = _NAME_FIX_MODULE[name].split(":") + return m, n + if name in _EXCEPTION_NAMES: + return "exceptions", name + if name in _OS_ERRORS: + return "exceptions", "OSError" + if name == "ModuleNotFoundError": + return "exceptions", "ImportError" + if module_name in _IMPORT_FIX: + return _IMPORT_FIX[module_name], name + return module_name, name + + +def _codecs_encode(obj, encoding="utf-8", errors="strict"): + """What `_codecs.encode` is for a str written by protocols 0 to 2.""" + return obj.encode(encoding, errors) + + +_codecs_encode.__module__ = "_codecs" +_codecs_encode.__name__ = "encode" +_codecs_encode.__qualname__ = "encode" + + +def encode_long(x): + """Encode a long to a two's complement little-endian binary string. + + 0 is a special case, returning an empty string, to save a byte in the + LONG1 pickling context.""" + if x == 0: + return b'' + nbytes = (x.bit_length() >> 3) + 1 + result = x.to_bytes(nbytes, byteorder='little', signed=True) + if x < 0 and nbytes > 1: + if result[-1] == 0xff and (result[-2] & 0x80) != 0: + result = result[:-1] + return result + + +def decode_long(data): + """Decode a long from a two's complement little-endian binary string.""" + return int.from_bytes(data, byteorder='little', signed=True) + + +def _getattribute(obj, dotted_path): + for subpath in dotted_path: + if subpath == '': + raise AttributeError("Can't get local attribute") + obj = getattr(obj, subpath) + return obj + + +def whichmodule(obj, name): + """Find the module an object belongs to.""" + dotted_path = name.split('.') + module_name = getattr(obj, '__module__', None) + if '' in dotted_path: + raise PicklingError("Can't pickle local object %r" % (obj,)) + if module_name is None: + builtins = sys.modules.get("builtins") + if builtins is not None: + try: + if _getattribute(builtins, dotted_path) is obj: + return "builtins" + except AttributeError: + pass + for module_name, module in list(sys.modules.items()): + if module_name == '__main__' or module is None: + continue + try: + if _getattribute(module, dotted_path) is obj: + return module_name + except AttributeError: + pass + module_name = '__main__' + module = sys.modules.get(module_name) + if module is None: + raise PicklingError("Can't pickle %r: No module named %r" % (obj, module_name)) + try: + if _getattribute(module, dotted_path) is obj: + return module_name + except AttributeError: + raise PicklingError("Can't pickle %r: it's not found as %s.%s" % (obj, module_name, name)) + raise PicklingError("Can't pickle %r: it's not the same object as %s.%s" % (obj, module_name, name)) + + +def _has_dict(cls): + """Whether instances of cls have a `__dict__` of their own (not only `__slots__`).""" + for c in cls.__mro__: + if c is object: + continue + if "__slots__" not in c.__dict__: + return True + slots = c.__dict__["__slots__"] + if slots == "__dict__" or (not isinstance(slots, str) and "__dict__" in slots): + return True + return False + + +def _T(obj): + cls = type(obj) + module = cls.__module__ + if module is None or module == 'builtins' or module == '__main__': + return cls.__qualname__ + return '%s.%s' % (module, cls.__qualname__) + + +def _batched(items, n): + out = [] + cur = [] + for x in items: + cur.append(x) + if len(cur) == n: + out.append(cur) + cur = [] + if cur: + out.append(cur) + return out + + +def _raw_unicode_escape(s): + out = bytearray() + for ch in s: + o = ord(ch) + if o < 256: + out.append(o) + elif o < 0x10000: + out += ("\\u%04x" % o).encode("ascii") + else: + out += ("\\U%08x" % o).encode("ascii") + return bytes(out) + + +def _raw_unicode_unescape(data): + s = data.decode("latin-1") + out = [] + i = 0 + n = len(s) + while i < n: + ch = s[i] + if ch == "\\" and i + 1 < n and (s[i + 1] == "u" or s[i + 1] == "U"): + width = 4 if s[i + 1] == "u" else 8 + digits = s[i + 2:i + 2 + width] + if len(digits) == width: + try: + out.append(chr(int(digits, 16))) + i += 2 + width + continue + except ValueError: + pass + raise UnpicklingError("truncated \\uXXXX escape") + out.append(ch) + i += 1 + return "".join(out) + + +def _unescape_string(raw): + """Python 2 repr() string literal escapes, as 'S' writes them.""" + out = bytearray() + i = 0 + n = len(raw) + simple = {ord("n"): 10, ord("r"): 13, ord("t"): 9, ord("a"): 7, ord("b"): 8, ord("f"): 12, ord("v"): 11, + ord("\\"): 92, ord("'"): 39, ord('"'): 34} + while i < n: + c = raw[i] + if c != 92: + out.append(c) + i += 1 + continue + i += 1 + if i >= n: + raise ValueError("Trailing \\ in string") + c = raw[i] + if c in simple: + out.append(simple[c]) + i += 1 + elif c == ord("x"): + out.append(int(raw[i + 1:i + 3].decode("ascii"), 16)) + i += 3 + elif 48 <= c <= 55: + j = i + while j < n and j < i + 3 and 48 <= raw[j] <= 55: + j += 1 + out.append(int(raw[i:j].decode("ascii"), 8) & 255) + i = j + else: + out.append(92) + out.append(c) + i += 1 + return bytes(out) + + +class _Stop(Exception): + def __init__(self, value): + Exception.__init__(self) + self.value = value + + +class _Framer: + _FRAME_SIZE_MIN = 4 + _FRAME_SIZE_TARGET = 64 * 1024 + + def __init__(self, file_write): + self.file_write = file_write + self.current_frame = None + + def start_framing(self): + self.current_frame = bytearray() + + def end_framing(self): + if self.current_frame is not None and len(self.current_frame) > 0: + self.commit_frame(True) + self.current_frame = None + + def commit_frame(self, force=False): + f = self.current_frame + if f is not None: + if len(f) >= self._FRAME_SIZE_TARGET or force: + data = bytes(f) + if len(data) >= self._FRAME_SIZE_MIN: + self.file_write(FRAME + pack("= 1 + self.fast = 0 + self.fix_imports = fix_imports and protocol < 3 + + def clear_memo(self): + self.memo.clear() + + def dump(self, obj): + """Write a pickled representation of obj to the open file.""" + if self.proto >= 2: + self.write(PROTO + pack("= 4: + self.framer.start_framing() + self.save(obj) + self.write(STOP) + self.framer.end_framing() + + def memoize(self, obj): + if self.fast: + return + idx = len(self.memo) + self.write(self.put(idx)) + self.memo[id(obj)] = (idx, obj) + + def put(self, idx): + if self.proto >= 4: + return MEMOIZE + elif self.bin: + if idx < 256: + return BINPUT + pack("= 2: + return self._reduce_newobj(obj, t) + return self._reduce_legacy(obj, t) + + def _getstate(self, obj): + t = type(obj) + getstate = getattr(obj, "__getstate__", _NoValue) + if getstate is not _NoValue: + return getstate() + state = getattr(obj, "__dict__", None) + if state is not None and len(state) == 0: + state = None + slots = copyreg._slotnames(t) + if slots: + if not _has_dict(t): + state = None + slotstate = {} + for name in slots: + value = getattr(obj, name, _NoValue) + if value is not _NoValue: + slotstate[name] = value + if slotstate: + return (state, slotstate) + return state + + def _reduce_newobj(self, obj, cls): + getnewargs_ex = getattr(obj, "__getnewargs_ex__", _NoValue) + if getnewargs_ex is not _NoValue: + args, kwargs = getnewargs_ex() + if not isinstance(args, tuple): + raise TypeError("__getnewargs_ex__ should return a tuple of (tuple, dict)") + if kwargs: + func = copyreg.__newobj_ex__ + newargs = (cls, args, kwargs) + else: + func = copyreg.__newobj__ + newargs = (cls,) + args + else: + getnewargs = getattr(obj, "__getnewargs__", _NoValue) + if getnewargs is not _NoValue: + args = getnewargs() + if not isinstance(args, tuple): + raise TypeError("__getnewargs__ should return a tuple") + else: + args = () + func = copyreg.__newobj__ + newargs = (cls,) + args + state = self._getstate(obj) + listitems = iter(obj) if isinstance(obj, list) else None + dictitems = iter(obj.items()) if isinstance(obj, dict) else None + return (func, newargs, state, listitems, dictitems) + + def _reduce_legacy(self, obj, cls): + if isinstance(obj, (list, dict, set, frozenset, bytes, bytearray, str, int, float, tuple)): + raise PicklingError("can't pickle %s with protocol %d" % (_T(obj), self.proto)) + if getattr(obj, "__getstate__", _NoValue) is _NoValue: + state = getattr(obj, "__dict__", None) + if state is not None and len(state) == 0: + state = None + slots = copyreg._slotnames(cls) + if slots: + raise PicklingError("a class that defines __slots__ without defining __getstate__ cannot be pickled with protocol %d" % self.proto) + else: + state = obj.__getstate__() + args = (cls, object, None) + if state is not None: + return (copyreg._reconstructor, args, state) + return (copyreg._reconstructor, args) + + def save_pers(self, pid): + if self.bin: + self.save(pid, False) + self.write(BINPERSID) + else: + try: + self.write(PERSID + str(pid).encode("ascii") + b'\n') + except UnicodeEncodeError: + raise PicklingError("persistent IDs in protocol 0 must be ASCII strings") + + def save_reduce(self, func, args, state=None, listitems=None, dictitems=None, state_setter=None, obj=None): + if not callable(func): + raise PicklingError("first item of the tuple returned by __reduce__ must be callable, not %s" % _T(func)) + if not isinstance(args, tuple): + raise PicklingError("second item of the tuple returned by __reduce__ must be a tuple, not %s" % _T(args)) + save = self.save + write = self.write + func_name = getattr(func, "__name__", "") + if self.proto >= 2 and func_name == "__newobj_ex__": + cls = args[0] + cargs = args[1] + kwargs = args[2] + if obj is not None and cls is not obj.__class__: + raise PicklingError("first argument to __newobj_ex__() must be %r, not %r" % (obj.__class__, cls)) + if self.proto >= 4: + save(cls) + save(cargs) + save(kwargs) + write(NEWOBJ_EX) + else: + save(_NewObjExPartial(cls, cargs, kwargs)) + save(()) + write(REDUCE) + elif self.proto >= 2 and func_name == "__newobj__": + cls = args[0] + if obj is not None and cls is not obj.__class__: + raise PicklingError("first argument to __newobj__() must be %r, not %r" % (obj.__class__, cls)) + args = args[1:] + save(cls) + save(args) + write(NEWOBJ) + else: + save(func) + save(args) + write(REDUCE) + if obj is not None: + if id(obj) in self.memo: + write(POP + self.get(self.memo[id(obj)][0])) + else: + self.memoize(obj) + if listitems is not None: + self._batch_appends(listitems, obj) + if dictitems is not None: + self._batch_setitems(dictitems, obj) + if state is not None: + if state_setter is None: + save(state) + write(BUILD) + else: + save(state_setter) + save(obj) + save(state) + write(TUPLE2) + write(REDUCE) + write(POP) + + def save_bool(self, obj): + if self.proto >= 2: + self.write(NEWTRUE if obj else NEWFALSE) + else: + self.write(TRUE if obj else FALSE) + + def save_long(self, obj): + if self.bin: + if obj >= 0: + if obj <= 0xff: + self.write(BININT1 + pack("= 2: + encoded = encode_long(obj) + n = len(encoded) + if n < 256: + self.write(LONG1 + pack("d', obj)) + else: + self.write(FLOAT + repr(obj).encode("ascii") + b'\n') + + def _save_bytes_no_memo(self, obj): + n = len(obj) + if n <= 0xff: + self.write(SHORT_BINBYTES + pack(" 0xffffffff and self.proto >= 4: + self._write_large_bytes(BINBYTES8 + pack("= self.framer._FRAME_SIZE_TARGET: + self._write_large_bytes(BINBYTES + pack("= self.framer._FRAME_SIZE_TARGET: + self._write_large_bytes(BYTEARRAY8 + pack("= 4: + self.write(SHORT_BINUNICODE + pack(" 0xffffffff and self.proto >= 4: + self._write_large_bytes(BINUNICODE8 + pack("= self.framer._FRAME_SIZE_TARGET: + self._write_large_bytes(BINUNICODE + pack("= 2: + for element in obj: + save(element) + if id(obj) in memo: + get = self.get(memo[id(obj)][0]) + self.write(POP * n + get) + else: + self.write(_tuplesize2code[n]) + self.memoize(obj) + return + write = self.write + write(MARK) + for element in obj: + save(element) + if id(obj) in memo: + get = self.get(memo[id(obj)][0]) + if self.bin: + write(POP_MARK + get) + else: + write(POP * (n + 1) + get) + return + write(TUPLE) + self.memoize(obj) + + def save_list(self, obj): + if self.bin: + self.write(EMPTY_LIST) + else: + self.write(MARK + LIST) + self.memoize(obj) + self._batch_appends(obj, obj) + + def _batch_appends(self, items, obj): + save = self.save + write = self.write + if not self.bin: + for x in items: + save(x) + write(APPEND) + return + for batch in _batched(items, self._BATCHSIZE): + if len(batch) != 1: + write(MARK) + for x in batch: + save(x) + write(APPENDS) + else: + save(batch[0]) + write(APPEND) + + def save_dict(self, obj): + if self.bin: + self.write(EMPTY_DICT) + else: + self.write(MARK + DICT) + self.memoize(obj) + self._batch_setitems(obj.items(), obj) + + def _batch_setitems(self, items, obj): + save = self.save + write = self.write + if not self.bin: + for k, v in items: + save(k) + save(v) + write(SETITEM) + return + for batch in _batched(items, self._BATCHSIZE): + if len(batch) != 1: + write(MARK) + for kv in batch: + save(kv[0]) + save(kv[1]) + write(SETITEMS) + else: + save(batch[0][0]) + save(batch[0][1]) + write(SETITEM) + + def save_set(self, obj): + save = self.save + write = self.write + if self.proto < 4: + self.save_reduce(set, (list(obj),), None, None, None, None, obj) + return + write(EMPTY_SET) + self.memoize(obj) + for batch in _batched(obj, self._BATCHSIZE): + write(MARK) + for item in batch: + save(item) + write(ADDITEMS) + + def save_frozenset(self, obj): + save = self.save + write = self.write + if self.proto < 4: + self.save_reduce(frozenset, (list(obj),), None, None, None, None, obj) + return + write(MARK) + for item in obj: + save(item) + if id(obj) in self.memo: + write(POP_MARK + self.get(self.memo[id(obj)][0])) + return + write(FROZENSET) + self.memoize(obj) + + def save_global(self, obj, name=None): + write = self.write + if obj is _codecs_encode: + module_name = "_codecs" + name = "encode" + else: + if name is None: + name = getattr(obj, '__qualname__', None) + if name is None: + name = obj.__name__ + module_name = whichmodule(obj, name) + module = sys.modules.get(module_name) + if module is None: + raise PicklingError("Can't pickle %r: No module named %r" % (obj, module_name)) + try: + obj2 = _getattribute(module, name.split('.')) + except AttributeError: + raise PicklingError("Can't pickle %r: it's not found as %s.%s" % (obj, module_name, name)) + if obj2 is not obj: + raise PicklingError("Can't pickle %r: it's not the same object as %s.%s" % (obj, module_name, name)) + self._write_global(module_name, name, obj) + + def _write_global(self, module_name, name, obj): + write = self.write + if self.proto >= 2: + code = copyreg._extension_registry.get((module_name, name), _NoValue) + if code is not _NoValue: + if code <= 0xff: + write(EXT1 + pack("= 4: + self.save(module_name) + self.save(name) + write(STACK_GLOBAL) + elif '.' in name: + dotted_path = name.split('.') + name = dotted_path.pop(0) + for attrname in dotted_path: + self.save(getattr) + if self.proto < 2: + write(MARK) + self._save_toplevel_by_name(module_name, name) + for attrname in dotted_path: + self.save(attrname) + if self.proto < 2: + write(TUPLE) + else: + write(TUPLE2) + write(REDUCE) + else: + self._save_toplevel_by_name(module_name, name) + self.memoize(obj) + + def _save_toplevel_by_name(self, module_name, name): + if self.proto >= 3: + encoding = "utf-8" + else: + if self.fix_imports: + module_name, name = _fix_global(module_name, name) + encoding = "ascii" + try: + self.write(GLOBAL + module_name.encode(encoding) + b'\n') + except UnicodeEncodeError: + raise PicklingError("can't pickle module identifier %r using pickle protocol %d" % (module_name, self.proto)) + try: + self.write(name.encode(encoding) + b'\n') + except UnicodeEncodeError: + raise PicklingError("can't pickle global identifier %r using pickle protocol %d" % (name, self.proto)) + + def save_type(self, obj): + if obj is type(None): + return self.save_reduce(type, (None,), None, None, None, None, obj) + elif obj is type(NotImplemented): + return self.save_reduce(type, (NotImplemented,), None, None, None, None, obj) + elif obj is type(...): + return self.save_reduce(type, (...,), None, None, None, None, obj) + return self.save_global(obj) + + +class _NewRef: + """Stands for `cls.__new__`, which CPython pickles as a global named `Owner.__new__`.""" + + def __init__(self, cls): + self.cls = cls + + def location(self): + for c in self.cls.__mro__: + if c is not object and "__new__" in c.__dict__: + return c.__module__, c.__qualname__ + ".__new__" + return "builtins", "object.__new__" + + +class _NewObjExPartial: + """`partial(cls.__new__, cls, *args, **kwargs)`, what protocols 2 and 3 write for `__newobj_ex__`.""" + + def __init__(self, cls, args, kwargs): + self.ref = _NewRef(cls) + self.cls = cls + self.args = args + self.kwargs = kwargs + + def __reduce__(self): + return (functools.partial, (self.ref,), (self.ref, (self.cls,) + tuple(self.args), self.kwargs or None, None)) + + +class Unpickler: + """This takes a binary file for reading a pickle data stream. + + The protocol version is detected automatically. The *file* needs read() + and readline() methods returning bytes.""" + + def __init__(self, file, *, fix_imports=True, encoding="ASCII", errors="strict", buffers=None): + if buffers is not None: + raise ValueError("buffers is not supported") + self._file_readline = file.readline + self._file_read = file.read + self.memo = {} + self.encoding = encoding + self.errors = errors + self.proto = 0 + self.fix_imports = fix_imports + self.stack = [] + self.metastack = [] + + def read(self, n): + data = self._file_read(n) + if len(data) != n: + raise UnpicklingError("pickle data was truncated") + return data + + def readline(self): + data = self._file_readline() + if not data or data[-1] != 10: + raise UnpicklingError("pickle data was truncated") + return data + + def persistent_load(self, pid): + raise UnpicklingError("unsupported persistent id encountered") + + def pop_mark(self): + items = self.stack + self.stack = self.metastack.pop() + return items + + def load(self): + """Read a pickled object representation from the open file.""" + self.metastack = [] + self.stack = [] + self.proto = 0 + while True: + key = self._file_read(1) + if not key: + raise EOFError("Ran out of input") + if self._step(key[0]): + return self.stack.pop() + + def _step(self, op): + """Run one opcode; True once STOP has been read.""" + stack = self.stack + if op == 46: # STOP + return True + elif op == 0x80: # PROTO + proto = self.read(1)[0] + if not 0 <= proto <= HIGHEST_PROTOCOL: + raise ValueError("unsupported pickle protocol: %d" % proto) + self.proto = proto + elif op == 0x95: # FRAME: the frames only wrap the stream + frame_size = unpack(' sys.maxsize: + raise ValueError("frame size > sys.maxsize: %d" % frame_size) + elif op == 78: # NONE + stack.append(None) + elif op == 0x89: + stack.append(False) + elif op == 0x88: + stack.append(True) + elif op == 73: # INT + data = self.readline() + if data == b'01\n': + stack.append(True) + elif data == b'00\n': + stack.append(False) + else: + stack.append(int(data.decode("ascii"))) + elif op == 74: # BININT + stack.append(unpack('d', self.read(8))[0]) + elif op == 83: # STRING + data = self.readline()[:-1] + if len(data) >= 2 and data[0] == data[-1] and (data[0] == 34 or data[0] == 39): + data = data[1:-1] + else: + raise UnpicklingError("the STRING opcode argument must be quoted") + stack.append(self._decode_string(_unescape_string(data))) + elif op == 84: # BINSTRING + n = unpack('= 4: + return _getattribute(mod, name.split('.')) + return getattr(mod, name) + + +def _dump(obj, file, protocol=None, *, fix_imports=True, buffer_callback=None): + Pickler(file, protocol, fix_imports=fix_imports, buffer_callback=buffer_callback).dump(obj) + + +def _dumps(obj, protocol=None, *, fix_imports=True, buffer_callback=None): + f = io.BytesIO() + Pickler(f, protocol, fix_imports=fix_imports, buffer_callback=buffer_callback).dump(obj) + return f.getvalue() + + +def _load(file, *, fix_imports=True, encoding="ASCII", errors="strict", buffers=None): + return Unpickler(file, fix_imports=fix_imports, encoding=encoding, errors=errors, buffers=buffers).load() + + +def _loads(s, *, fix_imports=True, encoding="ASCII", errors="strict", buffers=None): + if isinstance(s, str): + raise TypeError("Can't load pickle from unicode string") + file = io.BytesIO(s) + return Unpickler(file, fix_imports=fix_imports, encoding=encoding, errors=errors, buffers=buffers).load() + + +dump = _dump +dumps = _dumps +load = _load +loads = _loads diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/shelve.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/shelve.py new file mode 100644 index 00000000000..1e9818ce59b --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/shelve.py @@ -0,0 +1,237 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Manage shelves of pickled objects. + +A shelf is a persistent, dictionary-like object whose keys are strings and +whose values are anything `pickle` can write. `open()` returns one for a +file name. + +Unlike CPython, whose shelves sit on a `dbm` database, the whole shelf here +is ONE file holding a pickle of a dictionary from key to pickled value. It is +read when the shelf is opened and written back by `sync()` and `close()` (and +when a `with` block ends), never earlier. A file made by CPython's `shelve` +cannot be opened here, and the other way round. Two programs must not use +one file at the same time. + +`flag` is 'r' (read only, the file must exist), 'w' (read and write, the +file must exist), 'c' (read and write, created when missing: the default) or +'n' (read and write, always starts empty). +""" + +import io +import pickle + +__all__ = ["Shelf", "BsdDbShelf", "DbfilenameShelf", "open"] + +_open = open + + +class _ClosedDict: + def __getattr__(self, name): + raise ValueError("invalid operation on closed shelf") + + +class Shelf: + """A dictionary-like object over a mapping of key to pickled value.""" + + def __init__(self, dict, protocol=None, writeback=False, keyencoding="utf-8"): + self.dict = dict + if protocol is None: + protocol = pickle.DEFAULT_PROTOCOL + self._protocol = protocol + self.writeback = writeback + self.cache = {} + self.keyencoding = keyencoding + self._closed = False + + def _check(self): + if self._closed: + raise ValueError("invalid operation on closed shelf") + + def _key(self, key): + if not isinstance(key, str): + raise TypeError("shelf keys must be str, not %s" % type(key).__name__) + return key + + def keys(self): + self._check() + return list(self.dict.keys()) + + def __len__(self): + self._check() + return len(self.dict) + + def __contains__(self, key): + self._check() + return key in self.dict + + def get(self, key, default=None): + if key in self: + return self[key] + return default + + def __getitem__(self, key): + self._check() + key = self._key(key) + if key in self.cache: + return self.cache[key] + value = pickle.loads(self.dict[key]) + if self.writeback: + self.cache[key] = value + return value + + def __setitem__(self, key, value): + self._check() + self._write_guard() + key = self._key(key) + if self.writeback: + self.cache[key] = value + f = io.BytesIO() + pickle.Pickler(f, self._protocol).dump(value) + self.dict[key] = f.getvalue() + + def __delitem__(self, key): + self._check() + self._write_guard() + key = self._key(key) + del self.dict[key] + if key in self.cache: + del self.cache[key] + + def __iter__(self): + return iter(self.keys()) + + def items(self): + return [(k, self[k]) for k in self.keys()] + + def values(self): + return [self[k] for k in self.keys()] + + def setdefault(self, key, default=None): + if key in self: + return self[key] + self[key] = default + return default + + def pop(self, key, *default): + if len(default) > 1: + raise TypeError("pop expected at most 2 arguments, got %d" % (1 + len(default))) + if key in self: + value = self[key] + del self[key] + return value + if default: + return default[0] + raise KeyError(key) + + def popitem(self): + keys = self.keys() + if not keys: + raise KeyError("popitem(): shelf is empty") + key = keys[-1] + value = self[key] + del self[key] + return (key, value) + + def update(self, other=(), **kwargs): + if hasattr(other, "keys"): + for k in other.keys(): + self[k] = other[k] + else: + for k, v in other: + self[k] = v + for k in kwargs: + self[k] = kwargs[k] + + def clear(self): + for k in self.keys(): + del self[k] + + def _write_guard(self): + pass + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + + def close(self): + if self._closed: + return + try: + self.sync() + finally: + self._closed = True + self.cache = {} + + def sync(self): + """Write cached entries (writeback) and the file out.""" + self._check() + if self.writeback and self.cache: + self.writeback = False + for key in self.cache: + self[key] = self.cache[key] + self.writeback = True + self.cache = {} + self._flush() + + def _flush(self): + pass + + +class BsdDbShelf(Shelf): + pass + + +class DbfilenameShelf(Shelf): + """A shelf kept in the file `filename`.""" + + def __init__(self, filename, flag="c", protocol=None, writeback=False): + if flag not in ("r", "w", "c", "n"): + raise ValueError("Flag must be one of 'r', 'w', 'c', or 'n'") + self._filename = filename + self._readonly = flag == "r" + data = {} + exists = True + if flag != "n": + try: + f = _open(filename, "rb") + except FileNotFoundError: + exists = False + f = None + if f is not None: + try: + raw = f.read() + finally: + f.close() + if raw: + loaded = pickle.loads(raw) + if not isinstance(loaded, dict): + raise OSError("%s is not a shelf file" % filename) + data = loaded + if not exists and flag != "c": + raise FileNotFoundError("need 'c' or 'n' flag to open new shelf %r" % (filename,)) + Shelf.__init__(self, data, protocol, writeback) + self._dirty = flag == "n" or not exists + + def _write_guard(self): + if self._readonly: + raise OSError("shelf %r is open read only" % (self._filename,)) + self._dirty = True + + def _flush(self): + if self._readonly or not self._dirty: + return + f = _open(self._filename, "wb") + try: + f.write(pickle.dumps(self.dict, 4)) + finally: + f.close() + self._dirty = False + + +def open(filename, flag="c", protocol=None, writeback=False): + """Open a persistent dictionary for reading and writing.""" + return DbfilenameShelf(filename, flag, protocol, writeback) diff --git a/maven/python-transpiler/src/test/resources/conformance/pickle_.out b/maven/python-transpiler/src/test/resources/conformance/pickle_.out new file mode 100644 index 00000000000..16ec8a5e43d --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/pickle_.out @@ -0,0 +1,233 @@ +5 5 +80054e2e +none 0 4e2e +none 1 4e2e +none 2 80024e2e +none 3 80034e2e +none 4 80044e2e +none 5 80054e2e +bool 0 284930310a4930300a7470300a2e +bool 1 284930310a4930300a7471002e +bool 2 800288898671002e +bool 3 800388898671002e +bool 4 8004950500000000000000888986942e +bool 5 8005950500000000000000888986942e +ints 0 286c70300a49300a6149310a61493235350a61493235360a614936353533350a614936353533360a61492d310a61492d3132380a6149323134373438333634370a614c323134373438333634384c0a61492d323134373438333634380a614c2d323134373438333634394c0a612e +ints 1 5d7100284b004b014bff4d00014dffff4a000001004affffffff4a80ffffff4affffff7f4c323134373438333634384c0a4a000000804c2d323134373438333634394c0a652e +ints 2 80025d7100284b004b014bff4d00014dffff4a000001004affffffff4a80ffffff4affffff7f8a0500000080004a000000808a05ffffff7fff652e +ints 3 80035d7100284b004b014bff4d00014dffff4a000001004affffffff4a80ffffff4affffff7f8a0500000080004a000000808a05ffffff7fff652e +ints 4 80049538000000000000005d94284b004b014bff4d00014dffff4a000001004affffffff4a80ffffff4affffff7f8a0500000080004a000000808a05ffffff7fff652e +ints 5 80059538000000000000005d94284b004b014bff4d00014dffff4a000001004affffffff4a80ffffff4affffff7f8a0500000080004a000000808a05ffffff7fff652e +big 0 286c70300a4c31383434363734343037333730393535313631364c0a614c2d31383434363734343037333730393535313631364c0a614c313030303030303030303030303030303030303030303030303030303030304c0a614c2d313030303030303030303030303030303030303030303030303030303030304c0a614c3131343831333036393532373432353435323432333238333332303131373736383139383430323233313737303230383836393532303034373736343237333638323537363632363133393233373033313338353636353934383633313635303632363939313834343539363436333839383734363237373334343731313839363038363330353533333134323539333133353631363636353331383533393132393938393134353331323238303030303638383737393134383234303034343837313432383932363939303036333438363234343738313631353436333634363338383336333934373331373032363034303436363335333937303930343939363535383136323339383830383934343632393630353632333331313634393533363136343232313937303333323638313334343136383930383938343435383530353630323337393438343830373931343035383930303933343737363530303432393030323731363730363632353833303532323030383133323233363238313239313736313236373838333331373230363539383939353339363431383132373032313737393835383430343034323135393835333138333235313534303838393433333930323039313932303535343935373738333538393637323033393136303038313935373231363633303538323735353338303432353538333732363031353532383334383738363431393433323035343530383931353237353738333838323632353137353433353532383830303832323834323737303831373936353435333736323138343835313134393032393337364c0a614936353238300a61492d36353238300a614c393232333337323033363835343737353830374c0a612e +big 1 5d7100284c31383434363734343037333730393535313631364c0a4c2d31383434363734343037333730393535313631364c0a4c313030303030303030303030303030303030303030303030303030303030304c0a4c2d313030303030303030303030303030303030303030303030303030303030304c0a4c3131343831333036393532373432353435323432333238333332303131373736383139383430323233313737303230383836393532303034373736343237333638323537363632363133393233373033313338353636353934383633313635303632363939313834343539363436333839383734363237373334343731313839363038363330353533333134323539333133353631363636353331383533393132393938393134353331323238303030303638383737393134383234303034343837313432383932363939303036333438363234343738313631353436333634363338383336333934373331373032363034303436363335333937303930343939363535383136323339383830383934343632393630353632333331313634393533363136343232313937303333323638313334343136383930383938343435383530353630323337393438343830373931343035383930303933343737363530303432393030323731363730363632353833303532323030383133323233363238313239313736313236373838333331373230363539383939353339363431383132373032313737393835383430343034323135393835333138333235313534303838393433333930323039313932303535343935373738333538393637323033393136303038313935373231363633303538323735353338303432353538333732363031353532383334383738363431393433323035343530383931353237353738333838323632353137353433353532383830303832323834323737303831373936353435333736323138343835313134393032393337364c0a4d00ff4a0001ffff4c393232333337323033363835343737353830374c0a652e +big 2 80025d7100288a090000000000000000018a090000000000000000ff8a0d00000040eaed7446d09c2c9f0c8a0d000000c015128bb92f63d360f38afb00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000014d00ff4a0001ffff8a08ffffffffffffff7f652e +big 3 80035d7100288a090000000000000000018a090000000000000000ff8a0d00000040eaed7446d09c2c9f0c8a0d000000c015128bb92f63d360f38afb00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000014d00ff4a0001ffff8a08ffffffffffffff7f652e +big 4 80049548010000000000005d94288a090000000000000000018a090000000000000000ff8a0d00000040eaed7446d09c2c9f0c8a0d000000c015128bb92f63d360f38afb00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000014d00ff4a0001ffff8a08ffffffffffffff7f652e +big 5 80059548010000000000005d94288a090000000000000000018a090000000000000000ff8a0d00000040eaed7446d09c2c9f0c8a0d000000c015128bb92f63d360f38afb00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000014d00ff4a0001ffff8a08ffffffffffffff7f652e +floats 0 286c70300a46302e300a6146312e350a61462d322e32350a614631652b3330300a6146332e3134313539323635333538393739330a612e +floats 1 5d710028470000000000000000473ff800000000000047c002000000000000477e37e43c8800759c47400921fb54442d18652e +floats 2 80025d710028470000000000000000473ff800000000000047c002000000000000477e37e43c8800759c47400921fb54442d18652e +floats 3 80035d710028470000000000000000473ff800000000000047c002000000000000477e37e43c8800759c47400921fb54442d18652e +floats 4 80049532000000000000005d9428470000000000000000473ff800000000000047c002000000000000477e37e43c8800759c47400921fb54442d18652e +floats 5 80059532000000000000005d9428470000000000000000473ff800000000000047c002000000000000477e37e43c8800759c47400921fb54442d18652e +strs 0 286c70300a560a70310a61566162630a70320a6156636166e90a70330a61566c696e655c7530303061627265616b5c75303035630a70340a61565c75346532645c75363538370a70350a61565c5530303031663630300a70360a61567878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878780a70370a612e +strs 1 5d71002858000000007101580300000061626371025805000000636166c3a97103580b0000006c696e650a627265616b5c71045806000000e4b8ade6968771055804000000f09f98807106582c0100007878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787107652e +strs 2 80025d71002858000000007101580300000061626371025805000000636166c3a97103580b0000006c696e650a627265616b5c71045806000000e4b8ade6968771055804000000f09f98807106582c0100007878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787107652e +strs 3 80035d71002858000000007101580300000061626371025805000000636166c3a97103580b0000006c696e650a627265616b5c71045806000000e4b8ade6968771055804000000f09f98807106582c0100007878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787107652e +strs 4 80049566010000000000005d94288c00948c03616263948c05636166c3a9948c0b6c696e650a627265616b5c948c06e4b8ade69687948c04f09f988094582c01000078787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787894652e +strs 5 80059566010000000000005d94288c00948c03616263948c05636166c3a9948c0b6c696e650a627265616b5c948c06e4b8ade69687948c04f09f988094582c01000078787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787894652e +bytes 0 286c70300a635f5f6275696c74696e5f5f0a62797465730a70310a28745270320a61635f636f646563730a656e636f64650a70330a28566162630a70340a566c6174696e310a70350a7470360a5270370a6167330a28565c75303030300102030405060708095c75303030610b0c5c75303030640e0f101112131415161718195c75303031611b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f505152535455565758595a5b5c75303035635d5e5f606162636465666768696a6b6c6d6e6f707172737475767778797a7b7c7d7e7f808182838485868788898a8b8c8d8e8f909192939495969798999a9b9c9d9e9fa0a1a2a3a4a5a6a7a8a9aaabacadaeafb0b1b2b3b4b5b6b7b8b9babbbcbdbebfc0c1c2c3c4c5c6c7c8c9cacbcccdcecfd0d1d2d3d4d5d6d7d8d9dadbdcdddedfe0e1e2e3e4e5e6e7e8e9eaebecedeeeff0f1f2f3f4f5f6f7f8f9fafbfcfdfeff0a70380a67350a7470390a527031300a6167330a28567979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979790a7031310a67350a747031320a527031330a612e +bytes 1 5d710028635f5f6275696c74696e5f5f0a62797465730a710129527102635f636f646563730a656e636f64650a7103285803000000616263710458060000006c6174696e3171057471065271076803285880010000000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f505152535455565758595a5b5c5d5e5f606162636465666768696a6b6c6d6e6f707172737475767778797a7b7c7d7e7fc280c281c282c283c284c285c286c287c288c289c28ac28bc28cc28dc28ec28fc290c291c292c293c294c295c296c297c298c299c29ac29bc29cc29dc29ec29fc2a0c2a1c2a2c2a3c2a4c2a5c2a6c2a7c2a8c2a9c2aac2abc2acc2adc2aec2afc2b0c2b1c2b2c2b3c2b4c2b5c2b6c2b7c2b8c2b9c2bac2bbc2bcc2bdc2bec2bfc380c381c382c383c384c385c386c387c388c389c38ac38bc38cc38dc38ec38fc390c391c392c393c394c395c396c397c398c399c39ac39bc39cc39dc39ec39fc3a0c3a1c3a2c3a3c3a4c3a5c3a6c3a7c3a8c3a9c3aac3abc3acc3adc3aec3afc3b0c3b1c3b2c3b3c3b4c3b5c3b6c3b7c3b8c3b9c3bac3bbc3bcc3bdc3bec3bf7108680574710952710a680328582c010000797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979710b680574710c52710d652e +bytes 2 80025d710028635f5f6275696c74696e5f5f0a62797465730a710129527102635f636f646563730a656e636f64650a71035803000000616263710458060000006c6174696e31710586710652710768035880010000000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f505152535455565758595a5b5c5d5e5f606162636465666768696a6b6c6d6e6f707172737475767778797a7b7c7d7e7fc280c281c282c283c284c285c286c287c288c289c28ac28bc28cc28dc28ec28fc290c291c292c293c294c295c296c297c298c299c29ac29bc29cc29dc29ec29fc2a0c2a1c2a2c2a3c2a4c2a5c2a6c2a7c2a8c2a9c2aac2abc2acc2adc2aec2afc2b0c2b1c2b2c2b3c2b4c2b5c2b6c2b7c2b8c2b9c2bac2bbc2bcc2bdc2bec2bfc380c381c382c383c384c385c386c387c388c389c38ac38bc38cc38dc38ec38fc390c391c392c393c394c395c396c397c398c399c39ac39bc39cc39dc39ec39fc3a0c3a1c3a2c3a3c3a4c3a5c3a6c3a7c3a8c3a9c3aac3abc3acc3adc3aec3afc3b0c3b1c3b2c3b3c3b4c3b5c3b6c3b7c3b8c3b9c3bac3bbc3bcc3bdc3bec3bf7108680586710952710a6803582c010000797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979710b680586710c52710d652e +bytes 3 80035d71002843007101430361626371024200010000000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f505152535455565758595a5b5c5d5e5f606162636465666768696a6b6c6d6e6f707172737475767778797a7b7c7d7e7f808182838485868788898a8b8c8d8e8f909192939495969798999a9b9c9d9e9fa0a1a2a3a4a5a6a7a8a9aaabacadaeafb0b1b2b3b4b5b6b7b8b9babbbcbdbebfc0c1c2c3c4c5c6c7c8c9cacbcccdcecfd0d1d2d3d4d5d6d7d8d9dadbdcdddedfe0e1e2e3e4e5e6e7e8e9eaebecedeeeff0f1f2f3f4f5f6f7f8f9fafbfcfdfeff7103422c0100007979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797104652e +bytes 4 80049546020000000000005d94284300944303616263944200010000000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f505152535455565758595a5b5c5d5e5f606162636465666768696a6b6c6d6e6f707172737475767778797a7b7c7d7e7f808182838485868788898a8b8c8d8e8f909192939495969798999a9b9c9d9e9fa0a1a2a3a4a5a6a7a8a9aaabacadaeafb0b1b2b3b4b5b6b7b8b9babbbcbdbebfc0c1c2c3c4c5c6c7c8c9cacbcccdcecfd0d1d2d3d4d5d6d7d8d9dadbdcdddedfe0e1e2e3e4e5e6e7e8e9eaebecedeeeff0f1f2f3f4f5f6f7f8f9fafbfcfdfeff94422c01000079797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797994652e +bytes 5 80059546020000000000005d94284300944303616263944200010000000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f505152535455565758595a5b5c5d5e5f606162636465666768696a6b6c6d6e6f707172737475767778797a7b7c7d7e7f808182838485868788898a8b8c8d8e8f909192939495969798999a9b9c9d9e9fa0a1a2a3a4a5a6a7a8a9aaabacadaeafb0b1b2b3b4b5b6b7b8b9babbbcbdbebfc0c1c2c3c4c5c6c7c8c9cacbcccdcecfd0d1d2d3d4d5d6d7d8d9dadbdcdddedfe0e1e2e3e4e5e6e7e8e9eaebecedeeeff0f1f2f3f4f5f6f7f8f9fafbfcfdfeff94422c01000079797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797979797994652e +bytearray 0 286c70300a635f5f6275696c74696e5f5f0a6279746561727261790a70310a28745270320a6167310a28635f636f646563730a656e636f64650a70330a28566865790a70340a566c6174696e310a70350a7470360a5270370a7470380a5270390a612e +bytearray 1 5d710028635f5f6275696c74696e5f5f0a6279746561727261790a710129527102680128635f636f646563730a656e636f64650a7103285803000000686579710458060000006c6174696e317105747106527107747108527109652e +bytearray 2 80025d710028635f5f6275696c74696e5f5f0a6279746561727261790a7101295271026801635f636f646563730a656e636f64650a71035803000000686579710458060000006c6174696e317105867106527107857108527109652e +bytearray 3 80035d710028636275696c74696e730a6279746561727261790a710129527102680143036865797103857104527105652e +bytearray 4 8004952d000000000000005d94288c086275696c74696e73948c09627974656172726179949394295294680343036865799485945294652e +bytearray 5 8005951c000000000000005d94289600000000000000009496030000000000000068657994652e +tuples 0 286c70300a2874612849310a7470310a612849310a49320a7470320a612849310a49320a49330a7470330a612849310a49320a49330a49340a7470340a61282849340a49350a7470350a2849360a7470360a7470370a612e +tuples 1 5d71002829284b01747101284b014b02747102284b014b024b03747103284b014b024b034b0474710428284b044b05747105284b06747106747107652e +tuples 2 80025d710028294b018571014b014b028671024b014b024b03877103284b014b024b034b047471044b044b058671054b06857106867107652e +tuples 3 80035d710028294b018571014b014b028671024b014b024b03877103284b014b024b034b047471044b044b058671054b06857106867107652e +tuples 4 8004952f000000000000005d9428294b0185944b014b0286944b014b024b038794284b014b024b034b0474944b044b0586944b0685948694652e +tuples 5 8005952f000000000000005d9428294b0185944b014b0286944b014b024b038794284b014b024b034b0474944b044b0586944b0685948694652e +lists 0 286c70300a286c70310a61286c70320a49310a6161286c70330a49310a6149320a6161286c70340a49300a6149310a6149320a6149330a6149340a6149350a6149360a6149370a6149380a6149390a614931300a614931310a614931320a614931330a614931340a614931350a614931360a614931370a614931380a614931390a614932300a614932310a614932320a614932330a614932340a614932350a614932360a614932370a614932380a614932390a614933300a614933310a614933320a614933330a614933340a614933350a614933360a614933370a614933380a614933390a614934300a614934310a614934320a614934330a614934340a614934350a614934360a614934370a614934380a614934390a614935300a614935310a614935320a614935330a614935340a614935350a614935360a614935370a614935380a614935390a614936300a614936310a614936320a614936330a614936340a614936350a614936360a614936370a614936380a614936390a614937300a614937310a614937320a614937330a614937340a614937350a614937360a614937370a614937380a614937390a614938300a614938310a614938320a614938330a614938340a614938350a614938360a614938370a614938380a614938390a614939300a614939310a614939320a614939330a614939340a614939350a614939360a614939370a614939380a614939390a61493130300a61493130310a61493130320a61493130330a61493130340a61493130350a61493130360a61493130370a61493130380a61493130390a61493131300a61493131310a61493131320a61493131330a61493131340a61493131350a61493131360a61493131370a61493131380a61493131390a61493132300a61493132310a61493132320a61493132330a61493132340a61493132350a61493132360a61493132370a61493132380a61493132390a61493133300a61493133310a61493133320a61493133330a61493133340a61493133350a61493133360a61493133370a61493133380a61493133390a61493134300a61493134310a61493134320a61493134330a61493134340a61493134350a61493134360a61493134370a61493134380a61493134390a61493135300a61493135310a61493135320a61493135330a61493135340a61493135350a61493135360a61493135370a61493135380a61493135390a61493136300a61493136310a61493136320a61493136330a61493136340a61493136350a61493136360a61493136370a61493136380a61493136390a61493137300a61493137310a61493137320a61493137330a61493137340a61493137350a61493137360a61493137370a61493137380a61493137390a61493138300a61493138310a61493138320a61493138330a61493138340a61493138350a61493138360a61493138370a61493138380a61493138390a61493139300a61493139310a61493139320a61493139330a61493139340a61493139350a61493139360a61493139370a61493139380a61493139390a61493230300a61493230310a61493230320a61493230330a61493230340a61493230350a61493230360a61493230370a61493230380a61493230390a61493231300a61493231310a61493231320a61493231330a61493231340a61493231350a61493231360a61493231370a61493231380a61493231390a61493232300a61493232310a61493232320a61493232330a61493232340a61493232350a61493232360a61493232370a61493232380a61493232390a61493233300a61493233310a61493233320a61493233330a61493233340a61493233350a61493233360a61493233370a61493233380a61493233390a61493234300a61493234310a61493234320a61493234330a61493234340a61493234350a61493234360a61493234370a61493234380a61493234390a61493235300a61493235310a61493235320a61493235330a61493235340a61493235350a61493235360a61493235370a61493235380a61493235390a61493236300a61493236310a61493236320a61493236330a61493236340a61493236350a61493236360a61493236370a61493236380a61493236390a61493237300a61493237310a61493237320a61493237330a61493237340a61493237350a61493237360a61493237370a61493237380a61493237390a61493238300a61493238310a61493238320a61493238330a61493238340a61493238350a61493238360a61493238370a61493238380a61493238390a61493239300a61493239310a61493239320a61493239330a61493239340a61493239350a61493239360a61493239370a61493239380a61493239390a61493330300a61493330310a61493330320a61493330330a61493330340a61493330350a61493330360a61493330370a61493330380a61493330390a61493331300a61493331310a61493331320a61493331330a61493331340a61493331350a61493331360a61493331370a61493331380a61493331390a61493332300a61493332310a61493332320a61493332330a61493332340a61493332350a61493332360a61493332370a61493332380a61493332390a61493333300a61493333310a61493333320a61493333330a61493333340a61493333350a61493333360a61493333370a61493333380a61493333390a61493334300a61493334310a61493334320a61493334330a61493334340a61493334350a61493334360a61493334370a61493334380a61493334390a61493335300a61493335310a61493335320a61493335330a61493335340a61493335350a61493335360a61493335370a61493335380a61493335390a61493336300a61493336310a61493336320a61493336330a61493336340a61493336350a61493336360a61493336370a61493336380a61493336390a61493337300a61493337310a61493337320a61493337330a61493337340a61493337350a61493337360a61493337370a61493337380a61493337390a61493338300a61493338310a61493338320a61493338330a61493338340a61493338350a61493338360a61493338370a61493338380a61493338390a61493339300a61493339310a61493339320a61493339330a61493339340a61493339350a61493339360a61493339370a61493339380a61493339390a61493430300a61493430310a61493430320a61493430330a61493430340a61493430350a61493430360a61493430370a61493430380a61493430390a61493431300a61493431310a61493431320a61493431330a61493431340a61493431350a61493431360a61493431370a61493431380a61493431390a61493432300a61493432310a61493432320a61493432330a61493432340a61493432350a61493432360a61493432370a61493432380a61493432390a61493433300a61493433310a61493433320a61493433330a61493433340a61493433350a61493433360a61493433370a61493433380a61493433390a61493434300a61493434310a61493434320a61493434330a61493434340a61493434350a61493434360a61493434370a61493434380a61493434390a61493435300a61493435310a61493435320a61493435330a61493435340a61493435350a61493435360a61493435370a61493435380a61493435390a61493436300a61493436310a61493436320a61493436330a61493436340a61493436350a61493436360a61493436370a61493436380a61493436390a61493437300a61493437310a61493437320a61493437330a61493437340a61493437350a61493437360a61493437370a61493437380a61493437390a61493438300a61493438310a61493438320a61493438330a61493438340a61493438350a61493438360a61493438370a61493438380a61493438390a61493439300a61493439310a61493439320a61493439330a61493439340a61493439350a61493439360a61493439370a61493439380a61493439390a61493530300a61493530310a61493530320a61493530330a61493530340a61493530350a61493530360a61493530370a61493530380a61493530390a61493531300a61493531310a61493531320a61493531330a61493531340a61493531350a61493531360a61493531370a61493531380a61493531390a61493532300a61493532310a61493532320a61493532330a61493532340a61493532350a61493532360a61493532370a61493532380a61493532390a61493533300a61493533310a61493533320a61493533330a61493533340a61493533350a61493533360a61493533370a61493533380a61493533390a61493534300a61493534310a61493534320a61493534330a61493534340a61493534350a61493534360a61493534370a61493534380a61493534390a61493535300a61493535310a61493535320a61493535330a61493535340a61493535350a61493535360a61493535370a61493535380a61493535390a61493536300a61493536310a61493536320a61493536330a61493536340a61493536350a61493536360a61493536370a61493536380a61493536390a61493537300a61493537310a61493537320a61493537330a61493537340a61493537350a61493537360a61493537370a61493537380a61493537390a61493538300a61493538310a61493538320a61493538330a61493538340a61493538350a61493538360a61493538370a61493538380a61493538390a61493539300a61493539310a61493539320a61493539330a61493539340a61493539350a61493539360a61493539370a61493539380a61493539390a61493630300a61493630310a61493630320a61493630330a61493630340a61493630350a61493630360a61493630370a61493630380a61493630390a61493631300a61493631310a61493631320a61493631330a61493631340a61493631350a61493631360a61493631370a61493631380a61493631390a61493632300a61493632310a61493632320a61493632330a61493632340a61493632350a61493632360a61493632370a61493632380a61493632390a61493633300a61493633310a61493633320a61493633330a61493633340a61493633350a61493633360a61493633370a61493633380a61493633390a61493634300a61493634310a61493634320a61493634330a61493634340a61493634350a61493634360a61493634370a61493634380a61493634390a61493635300a61493635310a61493635320a61493635330a61493635340a61493635350a61493635360a61493635370a61493635380a61493635390a61493636300a61493636310a61493636320a61493636330a61493636340a61493636350a61493636360a61493636370a61493636380a61493636390a61493637300a61493637310a61493637320a61493637330a61493637340a61493637350a61493637360a61493637370a61493637380a61493637390a61493638300a61493638310a61493638320a61493638330a61493638340a61493638350a61493638360a61493638370a61493638380a61493638390a61493639300a61493639310a61493639320a61493639330a61493639340a61493639350a61493639360a61493639370a61493639380a61493639390a61493730300a61493730310a61493730320a61493730330a61493730340a61493730350a61493730360a61493730370a61493730380a61493730390a61493731300a61493731310a61493731320a61493731330a61493731340a61493731350a61493731360a61493731370a61493731380a61493731390a61493732300a61493732310a61493732320a61493732330a61493732340a61493732350a61493732360a61493732370a61493732380a61493732390a61493733300a61493733310a61493733320a61493733330a61493733340a61493733350a61493733360a61493733370a61493733380a61493733390a61493734300a61493734310a61493734320a61493734330a61493734340a61493734350a61493734360a61493734370a61493734380a61493734390a61493735300a61493735310a61493735320a61493735330a61493735340a61493735350a61493735360a61493735370a61493735380a61493735390a61493736300a61493736310a61493736320a61493736330a61493736340a61493736350a61493736360a61493736370a61493736380a61493736390a61493737300a61493737310a61493737320a61493737330a61493737340a61493737350a61493737360a61493737370a61493737380a61493737390a61493738300a61493738310a61493738320a61493738330a61493738340a61493738350a61493738360a61493738370a61493738380a61493738390a61493739300a61493739310a61493739320a61493739330a61493739340a61493739350a61493739360a61493739370a61493739380a61493739390a61493830300a61493830310a61493830320a61493830330a61493830340a61493830350a61493830360a61493830370a61493830380a61493830390a61493831300a61493831310a61493831320a61493831330a61493831340a61493831350a61493831360a61493831370a61493831380a61493831390a61493832300a61493832310a61493832320a61493832330a61493832340a61493832350a61493832360a61493832370a61493832380a61493832390a61493833300a61493833310a61493833320a61493833330a61493833340a61493833350a61493833360a61493833370a61493833380a61493833390a61493834300a61493834310a61493834320a61493834330a61493834340a61493834350a61493834360a61493834370a61493834380a61493834390a61493835300a61493835310a61493835320a61493835330a61493835340a61493835350a61493835360a61493835370a61493835380a61493835390a61493836300a61493836310a61493836320a61493836330a61493836340a61493836350a61493836360a61493836370a61493836380a61493836390a61493837300a61493837310a61493837320a61493837330a61493837340a61493837350a61493837360a61493837370a61493837380a61493837390a61493838300a61493838310a61493838320a61493838330a61493838340a61493838350a61493838360a61493838370a61493838380a61493838390a61493839300a61493839310a61493839320a61493839330a61493839340a61493839350a61493839360a61493839370a61493839380a61493839390a61493930300a61493930310a61493930320a61493930330a61493930340a61493930350a61493930360a61493930370a61493930380a61493930390a61493931300a61493931310a61493931320a61493931330a61493931340a61493931350a61493931360a61493931370a61493931380a61493931390a61493932300a61493932310a61493932320a61493932330a61493932340a61493932350a61493932360a61493932370a61493932380a61493932390a61493933300a61493933310a61493933320a61493933330a61493933340a61493933350a61493933360a61493933370a61493933380a61493933390a61493934300a61493934310a61493934320a61493934330a61493934340a61493934350a61493934360a61493934370a61493934380a61493934390a61493935300a61493935310a61493935320a61493935330a61493935340a61493935350a61493935360a61493935370a61493935380a61493935390a61493936300a61493936310a61493936320a61493936330a61493936340a61493936350a61493936360a61493936370a61493936380a61493936390a61493937300a61493937310a61493937320a61493937330a61493937340a61493937350a61493937360a61493937370a61493937380a61493937390a61493938300a61493938310a61493938320a61493938330a61493938340a61493938350a61493938360a61493938370a61493938380a61493938390a61493939300a61493939310a61493939320a61493939330a61493939340a61493939350a61493939360a61493939370a61493939380a61493939390a6149313030300a6149313030310a6149313030320a6149313030330a6149313030340a6149313030350a6149313030360a6149313030370a6149313030380a6149313030390a6149313031300a6149313031310a6149313031320a6149313031330a6149313031340a6149313031350a6149313031360a6149313031370a6149313031380a6149313031390a6149313032300a6149313032310a6149313032320a6149313032330a6149313032340a6149313032350a6149313032360a6149313032370a6149313032380a6149313032390a6149313033300a6149313033310a6149313033320a6149313033330a6149313033340a6149313033350a6149313033360a6149313033370a6149313033380a6149313033390a6149313034300a6149313034310a6149313034320a6149313034330a6149313034340a6149313034350a6149313034360a6149313034370a6149313034380a6149313034390a6149313035300a6149313035310a6149313035320a6149313035330a6149313035340a6149313035350a6149313035360a6149313035370a6149313035380a6149313035390a6149313036300a6149313036310a6149313036320a6149313036330a6149313036340a6149313036350a6149313036360a6149313036370a6149313036380a6149313036390a6149313037300a6149313037310a6149313037320a6149313037330a6149313037340a6149313037350a6149313037360a6149313037370a6149313037380a6149313037390a6149313038300a6149313038310a6149313038320a6149313038330a6149313038340a6149313038350a6149313038360a6149313038370a6149313038380a6149313038390a6149313039300a6149313039310a6149313039320a6149313039330a6149313039340a6149313039350a6149313039360a6149313039370a6149313039380a6149313039390a6149313130300a6149313130310a6149313130320a6149313130330a6149313130340a6149313130350a6149313130360a6149313130370a6149313130380a6149313130390a6149313131300a6149313131310a6149313131320a6149313131330a6149313131340a6149313131350a6149313131360a6149313131370a6149313131380a6149313131390a6149313132300a6149313132310a6149313132320a6149313132330a6149313132340a6149313132350a6149313132360a6149313132370a6149313132380a6149313132390a6149313133300a6149313133310a6149313133320a6149313133330a6149313133340a6149313133350a6149313133360a6149313133370a6149313133380a6149313133390a6149313134300a6149313134310a6149313134320a6149313134330a6149313134340a6149313134350a6149313134360a6149313134370a6149313134380a6149313134390a6149313135300a6149313135310a6149313135320a6149313135330a6149313135340a6149313135350a6149313135360a6149313135370a6149313135380a6149313135390a6149313136300a6149313136310a6149313136320a6149313136330a6149313136340a6149313136350a6149313136360a6149313136370a6149313136380a6149313136390a6149313137300a6149313137310a6149313137320a6149313137330a6149313137340a6149313137350a6149313137360a6149313137370a6149313137380a6149313137390a6149313138300a6149313138310a6149313138320a6149313138330a6149313138340a6149313138350a6149313138360a6149313138370a6149313138380a6149313138390a6149313139300a6149313139310a6149313139320a6149313139330a6149313139340a6149313139350a6149313139360a6149313139370a6149313139380a6149313139390a6161286c70350a286c70360a61286c70370a286c70380a6161612e +lists 1 5d7100285d71015d71024b01615d7103284b014b02655d7104284b004b014b024b034b044b054b064b074b084b094b0a4b0b4b0c4b0d4b0e4b0f4b104b114b124b134b144b154b164b174b184b194b1a4b1b4b1c4b1d4b1e4b1f4b204b214b224b234b244b254b264b274b284b294b2a4b2b4b2c4b2d4b2e4b2f4b304b314b324b334b344b354b364b374b384b394b3a4b3b4b3c4b3d4b3e4b3f4b404b414b424b434b444b454b464b474b484b494b4a4b4b4b4c4b4d4b4e4b4f4b504b514b524b534b544b554b564b574b584b594b5a4b5b4b5c4b5d4b5e4b5f4b604b614b624b634b644b654b664b674b684b694b6a4b6b4b6c4b6d4b6e4b6f4b704b714b724b734b744b754b764b774b784b794b7a4b7b4b7c4b7d4b7e4b7f4b804b814b824b834b844b854b864b874b884b894b8a4b8b4b8c4b8d4b8e4b8f4b904b914b924b934b944b954b964b974b984b994b9a4b9b4b9c4b9d4b9e4b9f4ba04ba14ba24ba34ba44ba54ba64ba74ba84ba94baa4bab4bac4bad4bae4baf4bb04bb14bb24bb34bb44bb54bb64bb74bb84bb94bba4bbb4bbc4bbd4bbe4bbf4bc04bc14bc24bc34bc44bc54bc64bc74bc84bc94bca4bcb4bcc4bcd4bce4bcf4bd04bd14bd24bd34bd44bd54bd64bd74bd84bd94bda4bdb4bdc4bdd4bde4bdf4be04be14be24be34be44be54be64be74be84be94bea4beb4bec4bed4bee4bef4bf04bf14bf24bf34bf44bf54bf64bf74bf84bf94bfa4bfb4bfc4bfd4bfe4bff4d00014d01014d02014d03014d04014d05014d06014d07014d08014d09014d0a014d0b014d0c014d0d014d0e014d0f014d10014d11014d12014d13014d14014d15014d16014d17014d18014d19014d1a014d1b014d1c014d1d014d1e014d1f014d20014d21014d22014d23014d24014d25014d26014d27014d28014d29014d2a014d2b014d2c014d2d014d2e014d2f014d30014d31014d32014d33014d34014d35014d36014d37014d38014d39014d3a014d3b014d3c014d3d014d3e014d3f014d40014d41014d42014d43014d44014d45014d46014d47014d48014d49014d4a014d4b014d4c014d4d014d4e014d4f014d50014d51014d52014d53014d54014d55014d56014d57014d58014d59014d5a014d5b014d5c014d5d014d5e014d5f014d60014d61014d62014d63014d64014d65014d66014d67014d68014d69014d6a014d6b014d6c014d6d014d6e014d6f014d70014d71014d72014d73014d74014d75014d76014d77014d78014d79014d7a014d7b014d7c014d7d014d7e014d7f014d80014d81014d82014d83014d84014d85014d86014d87014d88014d89014d8a014d8b014d8c014d8d014d8e014d8f014d90014d91014d92014d93014d94014d95014d96014d97014d98014d99014d9a014d9b014d9c014d9d014d9e014d9f014da0014da1014da2014da3014da4014da5014da6014da7014da8014da9014daa014dab014dac014dad014dae014daf014db0014db1014db2014db3014db4014db5014db6014db7014db8014db9014dba014dbb014dbc014dbd014dbe014dbf014dc0014dc1014dc2014dc3014dc4014dc5014dc6014dc7014dc8014dc9014dca014dcb014dcc014dcd014dce014dcf014dd0014dd1014dd2014dd3014dd4014dd5014dd6014dd7014dd8014dd9014dda014ddb014ddc014ddd014dde014ddf014de0014de1014de2014de3014de4014de5014de6014de7014de8014de9014dea014deb014dec014ded014dee014def014df0014df1014df2014df3014df4014df5014df6014df7014df8014df9014dfa014dfb014dfc014dfd014dfe014dff014d00024d01024d02024d03024d04024d05024d06024d07024d08024d09024d0a024d0b024d0c024d0d024d0e024d0f024d10024d11024d12024d13024d14024d15024d16024d17024d18024d19024d1a024d1b024d1c024d1d024d1e024d1f024d20024d21024d22024d23024d24024d25024d26024d27024d28024d29024d2a024d2b024d2c024d2d024d2e024d2f024d30024d31024d32024d33024d34024d35024d36024d37024d38024d39024d3a024d3b024d3c024d3d024d3e024d3f024d40024d41024d42024d43024d44024d45024d46024d47024d48024d49024d4a024d4b024d4c024d4d024d4e024d4f024d50024d51024d52024d53024d54024d55024d56024d57024d58024d59024d5a024d5b024d5c024d5d024d5e024d5f024d60024d61024d62024d63024d64024d65024d66024d67024d68024d69024d6a024d6b024d6c024d6d024d6e024d6f024d70024d71024d72024d73024d74024d75024d76024d77024d78024d79024d7a024d7b024d7c024d7d024d7e024d7f024d80024d81024d82024d83024d84024d85024d86024d87024d88024d89024d8a024d8b024d8c024d8d024d8e024d8f024d90024d91024d92024d93024d94024d95024d96024d97024d98024d99024d9a024d9b024d9c024d9d024d9e024d9f024da0024da1024da2024da3024da4024da5024da6024da7024da8024da9024daa024dab024dac024dad024dae024daf024db0024db1024db2024db3024db4024db5024db6024db7024db8024db9024dba024dbb024dbc024dbd024dbe024dbf024dc0024dc1024dc2024dc3024dc4024dc5024dc6024dc7024dc8024dc9024dca024dcb024dcc024dcd024dce024dcf024dd0024dd1024dd2024dd3024dd4024dd5024dd6024dd7024dd8024dd9024dda024ddb024ddc024ddd024dde024ddf024de0024de1024de2024de3024de4024de5024de6024de7024de8024de9024dea024deb024dec024ded024dee024def024df0024df1024df2024df3024df4024df5024df6024df7024df8024df9024dfa024dfb024dfc024dfd024dfe024dff024d00034d01034d02034d03034d04034d05034d06034d07034d08034d09034d0a034d0b034d0c034d0d034d0e034d0f034d10034d11034d12034d13034d14034d15034d16034d17034d18034d19034d1a034d1b034d1c034d1d034d1e034d1f034d20034d21034d22034d23034d24034d25034d26034d27034d28034d29034d2a034d2b034d2c034d2d034d2e034d2f034d30034d31034d32034d33034d34034d35034d36034d37034d38034d39034d3a034d3b034d3c034d3d034d3e034d3f034d40034d41034d42034d43034d44034d45034d46034d47034d48034d49034d4a034d4b034d4c034d4d034d4e034d4f034d50034d51034d52034d53034d54034d55034d56034d57034d58034d59034d5a034d5b034d5c034d5d034d5e034d5f034d60034d61034d62034d63034d64034d65034d66034d67034d68034d69034d6a034d6b034d6c034d6d034d6e034d6f034d70034d71034d72034d73034d74034d75034d76034d77034d78034d79034d7a034d7b034d7c034d7d034d7e034d7f034d80034d81034d82034d83034d84034d85034d86034d87034d88034d89034d8a034d8b034d8c034d8d034d8e034d8f034d90034d91034d92034d93034d94034d95034d96034d97034d98034d99034d9a034d9b034d9c034d9d034d9e034d9f034da0034da1034da2034da3034da4034da5034da6034da7034da8034da9034daa034dab034dac034dad034dae034daf034db0034db1034db2034db3034db4034db5034db6034db7034db8034db9034dba034dbb034dbc034dbd034dbe034dbf034dc0034dc1034dc2034dc3034dc4034dc5034dc6034dc7034dc8034dc9034dca034dcb034dcc034dcd034dce034dcf034dd0034dd1034dd2034dd3034dd4034dd5034dd6034dd7034dd8034dd9034dda034ddb034ddc034ddd034dde034ddf034de0034de1034de2034de3034de4034de5034de6034de70365284de8034de9034dea034deb034dec034ded034dee034def034df0034df1034df2034df3034df4034df5034df6034df7034df8034df9034dfa034dfb034dfc034dfd034dfe034dff034d00044d01044d02044d03044d04044d05044d06044d07044d08044d09044d0a044d0b044d0c044d0d044d0e044d0f044d10044d11044d12044d13044d14044d15044d16044d17044d18044d19044d1a044d1b044d1c044d1d044d1e044d1f044d20044d21044d22044d23044d24044d25044d26044d27044d28044d29044d2a044d2b044d2c044d2d044d2e044d2f044d30044d31044d32044d33044d34044d35044d36044d37044d38044d39044d3a044d3b044d3c044d3d044d3e044d3f044d40044d41044d42044d43044d44044d45044d46044d47044d48044d49044d4a044d4b044d4c044d4d044d4e044d4f044d50044d51044d52044d53044d54044d55044d56044d57044d58044d59044d5a044d5b044d5c044d5d044d5e044d5f044d60044d61044d62044d63044d64044d65044d66044d67044d68044d69044d6a044d6b044d6c044d6d044d6e044d6f044d70044d71044d72044d73044d74044d75044d76044d77044d78044d79044d7a044d7b044d7c044d7d044d7e044d7f044d80044d81044d82044d83044d84044d85044d86044d87044d88044d89044d8a044d8b044d8c044d8d044d8e044d8f044d90044d91044d92044d93044d94044d95044d96044d97044d98044d99044d9a044d9b044d9c044d9d044d9e044d9f044da0044da1044da2044da3044da4044da5044da6044da7044da8044da9044daa044dab044dac044dad044dae044daf04655d7105285d71065d71075d71086165652e +lists 2 80025d7100285d71015d71024b01615d7103284b014b02655d7104284b004b014b024b034b044b054b064b074b084b094b0a4b0b4b0c4b0d4b0e4b0f4b104b114b124b134b144b154b164b174b184b194b1a4b1b4b1c4b1d4b1e4b1f4b204b214b224b234b244b254b264b274b284b294b2a4b2b4b2c4b2d4b2e4b2f4b304b314b324b334b344b354b364b374b384b394b3a4b3b4b3c4b3d4b3e4b3f4b404b414b424b434b444b454b464b474b484b494b4a4b4b4b4c4b4d4b4e4b4f4b504b514b524b534b544b554b564b574b584b594b5a4b5b4b5c4b5d4b5e4b5f4b604b614b624b634b644b654b664b674b684b694b6a4b6b4b6c4b6d4b6e4b6f4b704b714b724b734b744b754b764b774b784b794b7a4b7b4b7c4b7d4b7e4b7f4b804b814b824b834b844b854b864b874b884b894b8a4b8b4b8c4b8d4b8e4b8f4b904b914b924b934b944b954b964b974b984b994b9a4b9b4b9c4b9d4b9e4b9f4ba04ba14ba24ba34ba44ba54ba64ba74ba84ba94baa4bab4bac4bad4bae4baf4bb04bb14bb24bb34bb44bb54bb64bb74bb84bb94bba4bbb4bbc4bbd4bbe4bbf4bc04bc14bc24bc34bc44bc54bc64bc74bc84bc94bca4bcb4bcc4bcd4bce4bcf4bd04bd14bd24bd34bd44bd54bd64bd74bd84bd94bda4bdb4bdc4bdd4bde4bdf4be04be14be24be34be44be54be64be74be84be94bea4beb4bec4bed4bee4bef4bf04bf14bf24bf34bf44bf54bf64bf74bf84bf94bfa4bfb4bfc4bfd4bfe4bff4d00014d01014d02014d03014d04014d05014d06014d07014d08014d09014d0a014d0b014d0c014d0d014d0e014d0f014d10014d11014d12014d13014d14014d15014d16014d17014d18014d19014d1a014d1b014d1c014d1d014d1e014d1f014d20014d21014d22014d23014d24014d25014d26014d27014d28014d29014d2a014d2b014d2c014d2d014d2e014d2f014d30014d31014d32014d33014d34014d35014d36014d37014d38014d39014d3a014d3b014d3c014d3d014d3e014d3f014d40014d41014d42014d43014d44014d45014d46014d47014d48014d49014d4a014d4b014d4c014d4d014d4e014d4f014d50014d51014d52014d53014d54014d55014d56014d57014d58014d59014d5a014d5b014d5c014d5d014d5e014d5f014d60014d61014d62014d63014d64014d65014d66014d67014d68014d69014d6a014d6b014d6c014d6d014d6e014d6f014d70014d71014d72014d73014d74014d75014d76014d77014d78014d79014d7a014d7b014d7c014d7d014d7e014d7f014d80014d81014d82014d83014d84014d85014d86014d87014d88014d89014d8a014d8b014d8c014d8d014d8e014d8f014d90014d91014d92014d93014d94014d95014d96014d97014d98014d99014d9a014d9b014d9c014d9d014d9e014d9f014da0014da1014da2014da3014da4014da5014da6014da7014da8014da9014daa014dab014dac014dad014dae014daf014db0014db1014db2014db3014db4014db5014db6014db7014db8014db9014dba014dbb014dbc014dbd014dbe014dbf014dc0014dc1014dc2014dc3014dc4014dc5014dc6014dc7014dc8014dc9014dca014dcb014dcc014dcd014dce014dcf014dd0014dd1014dd2014dd3014dd4014dd5014dd6014dd7014dd8014dd9014dda014ddb014ddc014ddd014dde014ddf014de0014de1014de2014de3014de4014de5014de6014de7014de8014de9014dea014deb014dec014ded014dee014def014df0014df1014df2014df3014df4014df5014df6014df7014df8014df9014dfa014dfb014dfc014dfd014dfe014dff014d00024d01024d02024d03024d04024d05024d06024d07024d08024d09024d0a024d0b024d0c024d0d024d0e024d0f024d10024d11024d12024d13024d14024d15024d16024d17024d18024d19024d1a024d1b024d1c024d1d024d1e024d1f024d20024d21024d22024d23024d24024d25024d26024d27024d28024d29024d2a024d2b024d2c024d2d024d2e024d2f024d30024d31024d32024d33024d34024d35024d36024d37024d38024d39024d3a024d3b024d3c024d3d024d3e024d3f024d40024d41024d42024d43024d44024d45024d46024d47024d48024d49024d4a024d4b024d4c024d4d024d4e024d4f024d50024d51024d52024d53024d54024d55024d56024d57024d58024d59024d5a024d5b024d5c024d5d024d5e024d5f024d60024d61024d62024d63024d64024d65024d66024d67024d68024d69024d6a024d6b024d6c024d6d024d6e024d6f024d70024d71024d72024d73024d74024d75024d76024d77024d78024d79024d7a024d7b024d7c024d7d024d7e024d7f024d80024d81024d82024d83024d84024d85024d86024d87024d88024d89024d8a024d8b024d8c024d8d024d8e024d8f024d90024d91024d92024d93024d94024d95024d96024d97024d98024d99024d9a024d9b024d9c024d9d024d9e024d9f024da0024da1024da2024da3024da4024da5024da6024da7024da8024da9024daa024dab024dac024dad024dae024daf024db0024db1024db2024db3024db4024db5024db6024db7024db8024db9024dba024dbb024dbc024dbd024dbe024dbf024dc0024dc1024dc2024dc3024dc4024dc5024dc6024dc7024dc8024dc9024dca024dcb024dcc024dcd024dce024dcf024dd0024dd1024dd2024dd3024dd4024dd5024dd6024dd7024dd8024dd9024dda024ddb024ddc024ddd024dde024ddf024de0024de1024de2024de3024de4024de5024de6024de7024de8024de9024dea024deb024dec024ded024dee024def024df0024df1024df2024df3024df4024df5024df6024df7024df8024df9024dfa024dfb024dfc024dfd024dfe024dff024d00034d01034d02034d03034d04034d05034d06034d07034d08034d09034d0a034d0b034d0c034d0d034d0e034d0f034d10034d11034d12034d13034d14034d15034d16034d17034d18034d19034d1a034d1b034d1c034d1d034d1e034d1f034d20034d21034d22034d23034d24034d25034d26034d27034d28034d29034d2a034d2b034d2c034d2d034d2e034d2f034d30034d31034d32034d33034d34034d35034d36034d37034d38034d39034d3a034d3b034d3c034d3d034d3e034d3f034d40034d41034d42034d43034d44034d45034d46034d47034d48034d49034d4a034d4b034d4c034d4d034d4e034d4f034d50034d51034d52034d53034d54034d55034d56034d57034d58034d59034d5a034d5b034d5c034d5d034d5e034d5f034d60034d61034d62034d63034d64034d65034d66034d67034d68034d69034d6a034d6b034d6c034d6d034d6e034d6f034d70034d71034d72034d73034d74034d75034d76034d77034d78034d79034d7a034d7b034d7c034d7d034d7e034d7f034d80034d81034d82034d83034d84034d85034d86034d87034d88034d89034d8a034d8b034d8c034d8d034d8e034d8f034d90034d91034d92034d93034d94034d95034d96034d97034d98034d99034d9a034d9b034d9c034d9d034d9e034d9f034da0034da1034da2034da3034da4034da5034da6034da7034da8034da9034daa034dab034dac034dad034dae034daf034db0034db1034db2034db3034db4034db5034db6034db7034db8034db9034dba034dbb034dbc034dbd034dbe034dbf034dc0034dc1034dc2034dc3034dc4034dc5034dc6034dc7034dc8034dc9034dca034dcb034dcc034dcd034dce034dcf034dd0034dd1034dd2034dd3034dd4034dd5034dd6034dd7034dd8034dd9034dda034ddb034ddc034ddd034dde034ddf034de0034de1034de2034de3034de4034de5034de6034de70365284de8034de9034dea034deb034dec034ded034dee034def034df0034df1034df2034df3034df4034df5034df6034df7034df8034df9034dfa034dfb034dfc034dfd034dfe034dff034d00044d01044d02044d03044d04044d05044d06044d07044d08044d09044d0a044d0b044d0c044d0d044d0e044d0f044d10044d11044d12044d13044d14044d15044d16044d17044d18044d19044d1a044d1b044d1c044d1d044d1e044d1f044d20044d21044d22044d23044d24044d25044d26044d27044d28044d29044d2a044d2b044d2c044d2d044d2e044d2f044d30044d31044d32044d33044d34044d35044d36044d37044d38044d39044d3a044d3b044d3c044d3d044d3e044d3f044d40044d41044d42044d43044d44044d45044d46044d47044d48044d49044d4a044d4b044d4c044d4d044d4e044d4f044d50044d51044d52044d53044d54044d55044d56044d57044d58044d59044d5a044d5b044d5c044d5d044d5e044d5f044d60044d61044d62044d63044d64044d65044d66044d67044d68044d69044d6a044d6b044d6c044d6d044d6e044d6f044d70044d71044d72044d73044d74044d75044d76044d77044d78044d79044d7a044d7b044d7c044d7d044d7e044d7f044d80044d81044d82044d83044d84044d85044d86044d87044d88044d89044d8a044d8b044d8c044d8d044d8e044d8f044d90044d91044d92044d93044d94044d95044d96044d97044d98044d99044d9a044d9b044d9c044d9d044d9e044d9f044da0044da1044da2044da3044da4044da5044da6044da7044da8044da9044daa044dab044dac044dad044dae044daf04655d7105285d71065d71075d71086165652e +lists 3 80035d7100285d71015d71024b01615d7103284b014b02655d7104284b004b014b024b034b044b054b064b074b084b094b0a4b0b4b0c4b0d4b0e4b0f4b104b114b124b134b144b154b164b174b184b194b1a4b1b4b1c4b1d4b1e4b1f4b204b214b224b234b244b254b264b274b284b294b2a4b2b4b2c4b2d4b2e4b2f4b304b314b324b334b344b354b364b374b384b394b3a4b3b4b3c4b3d4b3e4b3f4b404b414b424b434b444b454b464b474b484b494b4a4b4b4b4c4b4d4b4e4b4f4b504b514b524b534b544b554b564b574b584b594b5a4b5b4b5c4b5d4b5e4b5f4b604b614b624b634b644b654b664b674b684b694b6a4b6b4b6c4b6d4b6e4b6f4b704b714b724b734b744b754b764b774b784b794b7a4b7b4b7c4b7d4b7e4b7f4b804b814b824b834b844b854b864b874b884b894b8a4b8b4b8c4b8d4b8e4b8f4b904b914b924b934b944b954b964b974b984b994b9a4b9b4b9c4b9d4b9e4b9f4ba04ba14ba24ba34ba44ba54ba64ba74ba84ba94baa4bab4bac4bad4bae4baf4bb04bb14bb24bb34bb44bb54bb64bb74bb84bb94bba4bbb4bbc4bbd4bbe4bbf4bc04bc14bc24bc34bc44bc54bc64bc74bc84bc94bca4bcb4bcc4bcd4bce4bcf4bd04bd14bd24bd34bd44bd54bd64bd74bd84bd94bda4bdb4bdc4bdd4bde4bdf4be04be14be24be34be44be54be64be74be84be94bea4beb4bec4bed4bee4bef4bf04bf14bf24bf34bf44bf54bf64bf74bf84bf94bfa4bfb4bfc4bfd4bfe4bff4d00014d01014d02014d03014d04014d05014d06014d07014d08014d09014d0a014d0b014d0c014d0d014d0e014d0f014d10014d11014d12014d13014d14014d15014d16014d17014d18014d19014d1a014d1b014d1c014d1d014d1e014d1f014d20014d21014d22014d23014d24014d25014d26014d27014d28014d29014d2a014d2b014d2c014d2d014d2e014d2f014d30014d31014d32014d33014d34014d35014d36014d37014d38014d39014d3a014d3b014d3c014d3d014d3e014d3f014d40014d41014d42014d43014d44014d45014d46014d47014d48014d49014d4a014d4b014d4c014d4d014d4e014d4f014d50014d51014d52014d53014d54014d55014d56014d57014d58014d59014d5a014d5b014d5c014d5d014d5e014d5f014d60014d61014d62014d63014d64014d65014d66014d67014d68014d69014d6a014d6b014d6c014d6d014d6e014d6f014d70014d71014d72014d73014d74014d75014d76014d77014d78014d79014d7a014d7b014d7c014d7d014d7e014d7f014d80014d81014d82014d83014d84014d85014d86014d87014d88014d89014d8a014d8b014d8c014d8d014d8e014d8f014d90014d91014d92014d93014d94014d95014d96014d97014d98014d99014d9a014d9b014d9c014d9d014d9e014d9f014da0014da1014da2014da3014da4014da5014da6014da7014da8014da9014daa014dab014dac014dad014dae014daf014db0014db1014db2014db3014db4014db5014db6014db7014db8014db9014dba014dbb014dbc014dbd014dbe014dbf014dc0014dc1014dc2014dc3014dc4014dc5014dc6014dc7014dc8014dc9014dca014dcb014dcc014dcd014dce014dcf014dd0014dd1014dd2014dd3014dd4014dd5014dd6014dd7014dd8014dd9014dda014ddb014ddc014ddd014dde014ddf014de0014de1014de2014de3014de4014de5014de6014de7014de8014de9014dea014deb014dec014ded014dee014def014df0014df1014df2014df3014df4014df5014df6014df7014df8014df9014dfa014dfb014dfc014dfd014dfe014dff014d00024d01024d02024d03024d04024d05024d06024d07024d08024d09024d0a024d0b024d0c024d0d024d0e024d0f024d10024d11024d12024d13024d14024d15024d16024d17024d18024d19024d1a024d1b024d1c024d1d024d1e024d1f024d20024d21024d22024d23024d24024d25024d26024d27024d28024d29024d2a024d2b024d2c024d2d024d2e024d2f024d30024d31024d32024d33024d34024d35024d36024d37024d38024d39024d3a024d3b024d3c024d3d024d3e024d3f024d40024d41024d42024d43024d44024d45024d46024d47024d48024d49024d4a024d4b024d4c024d4d024d4e024d4f024d50024d51024d52024d53024d54024d55024d56024d57024d58024d59024d5a024d5b024d5c024d5d024d5e024d5f024d60024d61024d62024d63024d64024d65024d66024d67024d68024d69024d6a024d6b024d6c024d6d024d6e024d6f024d70024d71024d72024d73024d74024d75024d76024d77024d78024d79024d7a024d7b024d7c024d7d024d7e024d7f024d80024d81024d82024d83024d84024d85024d86024d87024d88024d89024d8a024d8b024d8c024d8d024d8e024d8f024d90024d91024d92024d93024d94024d95024d96024d97024d98024d99024d9a024d9b024d9c024d9d024d9e024d9f024da0024da1024da2024da3024da4024da5024da6024da7024da8024da9024daa024dab024dac024dad024dae024daf024db0024db1024db2024db3024db4024db5024db6024db7024db8024db9024dba024dbb024dbc024dbd024dbe024dbf024dc0024dc1024dc2024dc3024dc4024dc5024dc6024dc7024dc8024dc9024dca024dcb024dcc024dcd024dce024dcf024dd0024dd1024dd2024dd3024dd4024dd5024dd6024dd7024dd8024dd9024dda024ddb024ddc024ddd024dde024ddf024de0024de1024de2024de3024de4024de5024de6024de7024de8024de9024dea024deb024dec024ded024dee024def024df0024df1024df2024df3024df4024df5024df6024df7024df8024df9024dfa024dfb024dfc024dfd024dfe024dff024d00034d01034d02034d03034d04034d05034d06034d07034d08034d09034d0a034d0b034d0c034d0d034d0e034d0f034d10034d11034d12034d13034d14034d15034d16034d17034d18034d19034d1a034d1b034d1c034d1d034d1e034d1f034d20034d21034d22034d23034d24034d25034d26034d27034d28034d29034d2a034d2b034d2c034d2d034d2e034d2f034d30034d31034d32034d33034d34034d35034d36034d37034d38034d39034d3a034d3b034d3c034d3d034d3e034d3f034d40034d41034d42034d43034d44034d45034d46034d47034d48034d49034d4a034d4b034d4c034d4d034d4e034d4f034d50034d51034d52034d53034d54034d55034d56034d57034d58034d59034d5a034d5b034d5c034d5d034d5e034d5f034d60034d61034d62034d63034d64034d65034d66034d67034d68034d69034d6a034d6b034d6c034d6d034d6e034d6f034d70034d71034d72034d73034d74034d75034d76034d77034d78034d79034d7a034d7b034d7c034d7d034d7e034d7f034d80034d81034d82034d83034d84034d85034d86034d87034d88034d89034d8a034d8b034d8c034d8d034d8e034d8f034d90034d91034d92034d93034d94034d95034d96034d97034d98034d99034d9a034d9b034d9c034d9d034d9e034d9f034da0034da1034da2034da3034da4034da5034da6034da7034da8034da9034daa034dab034dac034dad034dae034daf034db0034db1034db2034db3034db4034db5034db6034db7034db8034db9034dba034dbb034dbc034dbd034dbe034dbf034dc0034dc1034dc2034dc3034dc4034dc5034dc6034dc7034dc8034dc9034dca034dcb034dcc034dcd034dce034dcf034dd0034dd1034dd2034dd3034dd4034dd5034dd6034dd7034dd8034dd9034dda034ddb034ddc034ddd034dde034ddf034de0034de1034de2034de3034de4034de5034de6034de70365284de8034de9034dea034deb034dec034ded034dee034def034df0034df1034df2034df3034df4034df5034df6034df7034df8034df9034dfa034dfb034dfc034dfd034dfe034dff034d00044d01044d02044d03044d04044d05044d06044d07044d08044d09044d0a044d0b044d0c044d0d044d0e044d0f044d10044d11044d12044d13044d14044d15044d16044d17044d18044d19044d1a044d1b044d1c044d1d044d1e044d1f044d20044d21044d22044d23044d24044d25044d26044d27044d28044d29044d2a044d2b044d2c044d2d044d2e044d2f044d30044d31044d32044d33044d34044d35044d36044d37044d38044d39044d3a044d3b044d3c044d3d044d3e044d3f044d40044d41044d42044d43044d44044d45044d46044d47044d48044d49044d4a044d4b044d4c044d4d044d4e044d4f044d50044d51044d52044d53044d54044d55044d56044d57044d58044d59044d5a044d5b044d5c044d5d044d5e044d5f044d60044d61044d62044d63044d64044d65044d66044d67044d68044d69044d6a044d6b044d6c044d6d044d6e044d6f044d70044d71044d72044d73044d74044d75044d76044d77044d78044d79044d7a044d7b044d7c044d7d044d7e044d7f044d80044d81044d82044d83044d84044d85044d86044d87044d88044d89044d8a044d8b044d8c044d8d044d8e044d8f044d90044d91044d92044d93044d94044d95044d96044d97044d98044d99044d9a044d9b044d9c044d9d044d9e044d9f044da0044da1044da2044da3044da4044da5044da6044da7044da8044da9044daa044dab044dac044dad044dae044daf04655d7105285d71065d71075d71086165652e +lists 4 800495350d0000000000005d94285d945d944b01615d94284b014b02655d94284b004b014b024b034b044b054b064b074b084b094b0a4b0b4b0c4b0d4b0e4b0f4b104b114b124b134b144b154b164b174b184b194b1a4b1b4b1c4b1d4b1e4b1f4b204b214b224b234b244b254b264b274b284b294b2a4b2b4b2c4b2d4b2e4b2f4b304b314b324b334b344b354b364b374b384b394b3a4b3b4b3c4b3d4b3e4b3f4b404b414b424b434b444b454b464b474b484b494b4a4b4b4b4c4b4d4b4e4b4f4b504b514b524b534b544b554b564b574b584b594b5a4b5b4b5c4b5d4b5e4b5f4b604b614b624b634b644b654b664b674b684b694b6a4b6b4b6c4b6d4b6e4b6f4b704b714b724b734b744b754b764b774b784b794b7a4b7b4b7c4b7d4b7e4b7f4b804b814b824b834b844b854b864b874b884b894b8a4b8b4b8c4b8d4b8e4b8f4b904b914b924b934b944b954b964b974b984b994b9a4b9b4b9c4b9d4b9e4b9f4ba04ba14ba24ba34ba44ba54ba64ba74ba84ba94baa4bab4bac4bad4bae4baf4bb04bb14bb24bb34bb44bb54bb64bb74bb84bb94bba4bbb4bbc4bbd4bbe4bbf4bc04bc14bc24bc34bc44bc54bc64bc74bc84bc94bca4bcb4bcc4bcd4bce4bcf4bd04bd14bd24bd34bd44bd54bd64bd74bd84bd94bda4bdb4bdc4bdd4bde4bdf4be04be14be24be34be44be54be64be74be84be94bea4beb4bec4bed4bee4bef4bf04bf14bf24bf34bf44bf54bf64bf74bf84bf94bfa4bfb4bfc4bfd4bfe4bff4d00014d01014d02014d03014d04014d05014d06014d07014d08014d09014d0a014d0b014d0c014d0d014d0e014d0f014d10014d11014d12014d13014d14014d15014d16014d17014d18014d19014d1a014d1b014d1c014d1d014d1e014d1f014d20014d21014d22014d23014d24014d25014d26014d27014d28014d29014d2a014d2b014d2c014d2d014d2e014d2f014d30014d31014d32014d33014d34014d35014d36014d37014d38014d39014d3a014d3b014d3c014d3d014d3e014d3f014d40014d41014d42014d43014d44014d45014d46014d47014d48014d49014d4a014d4b014d4c014d4d014d4e014d4f014d50014d51014d52014d53014d54014d55014d56014d57014d58014d59014d5a014d5b014d5c014d5d014d5e014d5f014d60014d61014d62014d63014d64014d65014d66014d67014d68014d69014d6a014d6b014d6c014d6d014d6e014d6f014d70014d71014d72014d73014d74014d75014d76014d77014d78014d79014d7a014d7b014d7c014d7d014d7e014d7f014d80014d81014d82014d83014d84014d85014d86014d87014d88014d89014d8a014d8b014d8c014d8d014d8e014d8f014d90014d91014d92014d93014d94014d95014d96014d97014d98014d99014d9a014d9b014d9c014d9d014d9e014d9f014da0014da1014da2014da3014da4014da5014da6014da7014da8014da9014daa014dab014dac014dad014dae014daf014db0014db1014db2014db3014db4014db5014db6014db7014db8014db9014dba014dbb014dbc014dbd014dbe014dbf014dc0014dc1014dc2014dc3014dc4014dc5014dc6014dc7014dc8014dc9014dca014dcb014dcc014dcd014dce014dcf014dd0014dd1014dd2014dd3014dd4014dd5014dd6014dd7014dd8014dd9014dda014ddb014ddc014ddd014dde014ddf014de0014de1014de2014de3014de4014de5014de6014de7014de8014de9014dea014deb014dec014ded014dee014def014df0014df1014df2014df3014df4014df5014df6014df7014df8014df9014dfa014dfb014dfc014dfd014dfe014dff014d00024d01024d02024d03024d04024d05024d06024d07024d08024d09024d0a024d0b024d0c024d0d024d0e024d0f024d10024d11024d12024d13024d14024d15024d16024d17024d18024d19024d1a024d1b024d1c024d1d024d1e024d1f024d20024d21024d22024d23024d24024d25024d26024d27024d28024d29024d2a024d2b024d2c024d2d024d2e024d2f024d30024d31024d32024d33024d34024d35024d36024d37024d38024d39024d3a024d3b024d3c024d3d024d3e024d3f024d40024d41024d42024d43024d44024d45024d46024d47024d48024d49024d4a024d4b024d4c024d4d024d4e024d4f024d50024d51024d52024d53024d54024d55024d56024d57024d58024d59024d5a024d5b024d5c024d5d024d5e024d5f024d60024d61024d62024d63024d64024d65024d66024d67024d68024d69024d6a024d6b024d6c024d6d024d6e024d6f024d70024d71024d72024d73024d74024d75024d76024d77024d78024d79024d7a024d7b024d7c024d7d024d7e024d7f024d80024d81024d82024d83024d84024d85024d86024d87024d88024d89024d8a024d8b024d8c024d8d024d8e024d8f024d90024d91024d92024d93024d94024d95024d96024d97024d98024d99024d9a024d9b024d9c024d9d024d9e024d9f024da0024da1024da2024da3024da4024da5024da6024da7024da8024da9024daa024dab024dac024dad024dae024daf024db0024db1024db2024db3024db4024db5024db6024db7024db8024db9024dba024dbb024dbc024dbd024dbe024dbf024dc0024dc1024dc2024dc3024dc4024dc5024dc6024dc7024dc8024dc9024dca024dcb024dcc024dcd024dce024dcf024dd0024dd1024dd2024dd3024dd4024dd5024dd6024dd7024dd8024dd9024dda024ddb024ddc024ddd024dde024ddf024de0024de1024de2024de3024de4024de5024de6024de7024de8024de9024dea024deb024dec024ded024dee024def024df0024df1024df2024df3024df4024df5024df6024df7024df8024df9024dfa024dfb024dfc024dfd024dfe024dff024d00034d01034d02034d03034d04034d05034d06034d07034d08034d09034d0a034d0b034d0c034d0d034d0e034d0f034d10034d11034d12034d13034d14034d15034d16034d17034d18034d19034d1a034d1b034d1c034d1d034d1e034d1f034d20034d21034d22034d23034d24034d25034d26034d27034d28034d29034d2a034d2b034d2c034d2d034d2e034d2f034d30034d31034d32034d33034d34034d35034d36034d37034d38034d39034d3a034d3b034d3c034d3d034d3e034d3f034d40034d41034d42034d43034d44034d45034d46034d47034d48034d49034d4a034d4b034d4c034d4d034d4e034d4f034d50034d51034d52034d53034d54034d55034d56034d57034d58034d59034d5a034d5b034d5c034d5d034d5e034d5f034d60034d61034d62034d63034d64034d65034d66034d67034d68034d69034d6a034d6b034d6c034d6d034d6e034d6f034d70034d71034d72034d73034d74034d75034d76034d77034d78034d79034d7a034d7b034d7c034d7d034d7e034d7f034d80034d81034d82034d83034d84034d85034d86034d87034d88034d89034d8a034d8b034d8c034d8d034d8e034d8f034d90034d91034d92034d93034d94034d95034d96034d97034d98034d99034d9a034d9b034d9c034d9d034d9e034d9f034da0034da1034da2034da3034da4034da5034da6034da7034da8034da9034daa034dab034dac034dad034dae034daf034db0034db1034db2034db3034db4034db5034db6034db7034db8034db9034dba034dbb034dbc034dbd034dbe034dbf034dc0034dc1034dc2034dc3034dc4034dc5034dc6034dc7034dc8034dc9034dca034dcb034dcc034dcd034dce034dcf034dd0034dd1034dd2034dd3034dd4034dd5034dd6034dd7034dd8034dd9034dda034ddb034ddc034ddd034dde034ddf034de0034de1034de2034de3034de4034de5034de6034de70365284de8034de9034dea034deb034dec034ded034dee034def034df0034df1034df2034df3034df4034df5034df6034df7034df8034df9034dfa034dfb034dfc034dfd034dfe034dff034d00044d01044d02044d03044d04044d05044d06044d07044d08044d09044d0a044d0b044d0c044d0d044d0e044d0f044d10044d11044d12044d13044d14044d15044d16044d17044d18044d19044d1a044d1b044d1c044d1d044d1e044d1f044d20044d21044d22044d23044d24044d25044d26044d27044d28044d29044d2a044d2b044d2c044d2d044d2e044d2f044d30044d31044d32044d33044d34044d35044d36044d37044d38044d39044d3a044d3b044d3c044d3d044d3e044d3f044d40044d41044d42044d43044d44044d45044d46044d47044d48044d49044d4a044d4b044d4c044d4d044d4e044d4f044d50044d51044d52044d53044d54044d55044d56044d57044d58044d59044d5a044d5b044d5c044d5d044d5e044d5f044d60044d61044d62044d63044d64044d65044d66044d67044d68044d69044d6a044d6b044d6c044d6d044d6e044d6f044d70044d71044d72044d73044d74044d75044d76044d77044d78044d79044d7a044d7b044d7c044d7d044d7e044d7f044d80044d81044d82044d83044d84044d85044d86044d87044d88044d89044d8a044d8b044d8c044d8d044d8e044d8f044d90044d91044d92044d93044d94044d95044d96044d97044d98044d99044d9a044d9b044d9c044d9d044d9e044d9f044da0044da1044da2044da3044da4044da5044da6044da7044da8044da9044daa044dab044dac044dad044dae044daf04655d94285d945d945d946165652e +lists 5 800595350d0000000000005d94285d945d944b01615d94284b014b02655d94284b004b014b024b034b044b054b064b074b084b094b0a4b0b4b0c4b0d4b0e4b0f4b104b114b124b134b144b154b164b174b184b194b1a4b1b4b1c4b1d4b1e4b1f4b204b214b224b234b244b254b264b274b284b294b2a4b2b4b2c4b2d4b2e4b2f4b304b314b324b334b344b354b364b374b384b394b3a4b3b4b3c4b3d4b3e4b3f4b404b414b424b434b444b454b464b474b484b494b4a4b4b4b4c4b4d4b4e4b4f4b504b514b524b534b544b554b564b574b584b594b5a4b5b4b5c4b5d4b5e4b5f4b604b614b624b634b644b654b664b674b684b694b6a4b6b4b6c4b6d4b6e4b6f4b704b714b724b734b744b754b764b774b784b794b7a4b7b4b7c4b7d4b7e4b7f4b804b814b824b834b844b854b864b874b884b894b8a4b8b4b8c4b8d4b8e4b8f4b904b914b924b934b944b954b964b974b984b994b9a4b9b4b9c4b9d4b9e4b9f4ba04ba14ba24ba34ba44ba54ba64ba74ba84ba94baa4bab4bac4bad4bae4baf4bb04bb14bb24bb34bb44bb54bb64bb74bb84bb94bba4bbb4bbc4bbd4bbe4bbf4bc04bc14bc24bc34bc44bc54bc64bc74bc84bc94bca4bcb4bcc4bcd4bce4bcf4bd04bd14bd24bd34bd44bd54bd64bd74bd84bd94bda4bdb4bdc4bdd4bde4bdf4be04be14be24be34be44be54be64be74be84be94bea4beb4bec4bed4bee4bef4bf04bf14bf24bf34bf44bf54bf64bf74bf84bf94bfa4bfb4bfc4bfd4bfe4bff4d00014d01014d02014d03014d04014d05014d06014d07014d08014d09014d0a014d0b014d0c014d0d014d0e014d0f014d10014d11014d12014d13014d14014d15014d16014d17014d18014d19014d1a014d1b014d1c014d1d014d1e014d1f014d20014d21014d22014d23014d24014d25014d26014d27014d28014d29014d2a014d2b014d2c014d2d014d2e014d2f014d30014d31014d32014d33014d34014d35014d36014d37014d38014d39014d3a014d3b014d3c014d3d014d3e014d3f014d40014d41014d42014d43014d44014d45014d46014d47014d48014d49014d4a014d4b014d4c014d4d014d4e014d4f014d50014d51014d52014d53014d54014d55014d56014d57014d58014d59014d5a014d5b014d5c014d5d014d5e014d5f014d60014d61014d62014d63014d64014d65014d66014d67014d68014d69014d6a014d6b014d6c014d6d014d6e014d6f014d70014d71014d72014d73014d74014d75014d76014d77014d78014d79014d7a014d7b014d7c014d7d014d7e014d7f014d80014d81014d82014d83014d84014d85014d86014d87014d88014d89014d8a014d8b014d8c014d8d014d8e014d8f014d90014d91014d92014d93014d94014d95014d96014d97014d98014d99014d9a014d9b014d9c014d9d014d9e014d9f014da0014da1014da2014da3014da4014da5014da6014da7014da8014da9014daa014dab014dac014dad014dae014daf014db0014db1014db2014db3014db4014db5014db6014db7014db8014db9014dba014dbb014dbc014dbd014dbe014dbf014dc0014dc1014dc2014dc3014dc4014dc5014dc6014dc7014dc8014dc9014dca014dcb014dcc014dcd014dce014dcf014dd0014dd1014dd2014dd3014dd4014dd5014dd6014dd7014dd8014dd9014dda014ddb014ddc014ddd014dde014ddf014de0014de1014de2014de3014de4014de5014de6014de7014de8014de9014dea014deb014dec014ded014dee014def014df0014df1014df2014df3014df4014df5014df6014df7014df8014df9014dfa014dfb014dfc014dfd014dfe014dff014d00024d01024d02024d03024d04024d05024d06024d07024d08024d09024d0a024d0b024d0c024d0d024d0e024d0f024d10024d11024d12024d13024d14024d15024d16024d17024d18024d19024d1a024d1b024d1c024d1d024d1e024d1f024d20024d21024d22024d23024d24024d25024d26024d27024d28024d29024d2a024d2b024d2c024d2d024d2e024d2f024d30024d31024d32024d33024d34024d35024d36024d37024d38024d39024d3a024d3b024d3c024d3d024d3e024d3f024d40024d41024d42024d43024d44024d45024d46024d47024d48024d49024d4a024d4b024d4c024d4d024d4e024d4f024d50024d51024d52024d53024d54024d55024d56024d57024d58024d59024d5a024d5b024d5c024d5d024d5e024d5f024d60024d61024d62024d63024d64024d65024d66024d67024d68024d69024d6a024d6b024d6c024d6d024d6e024d6f024d70024d71024d72024d73024d74024d75024d76024d77024d78024d79024d7a024d7b024d7c024d7d024d7e024d7f024d80024d81024d82024d83024d84024d85024d86024d87024d88024d89024d8a024d8b024d8c024d8d024d8e024d8f024d90024d91024d92024d93024d94024d95024d96024d97024d98024d99024d9a024d9b024d9c024d9d024d9e024d9f024da0024da1024da2024da3024da4024da5024da6024da7024da8024da9024daa024dab024dac024dad024dae024daf024db0024db1024db2024db3024db4024db5024db6024db7024db8024db9024dba024dbb024dbc024dbd024dbe024dbf024dc0024dc1024dc2024dc3024dc4024dc5024dc6024dc7024dc8024dc9024dca024dcb024dcc024dcd024dce024dcf024dd0024dd1024dd2024dd3024dd4024dd5024dd6024dd7024dd8024dd9024dda024ddb024ddc024ddd024dde024ddf024de0024de1024de2024de3024de4024de5024de6024de7024de8024de9024dea024deb024dec024ded024dee024def024df0024df1024df2024df3024df4024df5024df6024df7024df8024df9024dfa024dfb024dfc024dfd024dfe024dff024d00034d01034d02034d03034d04034d05034d06034d07034d08034d09034d0a034d0b034d0c034d0d034d0e034d0f034d10034d11034d12034d13034d14034d15034d16034d17034d18034d19034d1a034d1b034d1c034d1d034d1e034d1f034d20034d21034d22034d23034d24034d25034d26034d27034d28034d29034d2a034d2b034d2c034d2d034d2e034d2f034d30034d31034d32034d33034d34034d35034d36034d37034d38034d39034d3a034d3b034d3c034d3d034d3e034d3f034d40034d41034d42034d43034d44034d45034d46034d47034d48034d49034d4a034d4b034d4c034d4d034d4e034d4f034d50034d51034d52034d53034d54034d55034d56034d57034d58034d59034d5a034d5b034d5c034d5d034d5e034d5f034d60034d61034d62034d63034d64034d65034d66034d67034d68034d69034d6a034d6b034d6c034d6d034d6e034d6f034d70034d71034d72034d73034d74034d75034d76034d77034d78034d79034d7a034d7b034d7c034d7d034d7e034d7f034d80034d81034d82034d83034d84034d85034d86034d87034d88034d89034d8a034d8b034d8c034d8d034d8e034d8f034d90034d91034d92034d93034d94034d95034d96034d97034d98034d99034d9a034d9b034d9c034d9d034d9e034d9f034da0034da1034da2034da3034da4034da5034da6034da7034da8034da9034daa034dab034dac034dad034dae034daf034db0034db1034db2034db3034db4034db5034db6034db7034db8034db9034dba034dbb034dbc034dbd034dbe034dbf034dc0034dc1034dc2034dc3034dc4034dc5034dc6034dc7034dc8034dc9034dca034dcb034dcc034dcd034dce034dcf034dd0034dd1034dd2034dd3034dd4034dd5034dd6034dd7034dd8034dd9034dda034ddb034ddc034ddd034dde034ddf034de0034de1034de2034de3034de4034de5034de6034de70365284de8034de9034dea034deb034dec034ded034dee034def034df0034df1034df2034df3034df4034df5034df6034df7034df8034df9034dfa034dfb034dfc034dfd034dfe034dff034d00044d01044d02044d03044d04044d05044d06044d07044d08044d09044d0a044d0b044d0c044d0d044d0e044d0f044d10044d11044d12044d13044d14044d15044d16044d17044d18044d19044d1a044d1b044d1c044d1d044d1e044d1f044d20044d21044d22044d23044d24044d25044d26044d27044d28044d29044d2a044d2b044d2c044d2d044d2e044d2f044d30044d31044d32044d33044d34044d35044d36044d37044d38044d39044d3a044d3b044d3c044d3d044d3e044d3f044d40044d41044d42044d43044d44044d45044d46044d47044d48044d49044d4a044d4b044d4c044d4d044d4e044d4f044d50044d51044d52044d53044d54044d55044d56044d57044d58044d59044d5a044d5b044d5c044d5d044d5e044d5f044d60044d61044d62044d63044d64044d65044d66044d67044d68044d69044d6a044d6b044d6c044d6d044d6e044d6f044d70044d71044d72044d73044d74044d75044d76044d77044d78044d79044d7a044d7b044d7c044d7d044d7e044d7f044d80044d81044d82044d83044d84044d85044d86044d87044d88044d89044d8a044d8b044d8c044d8d044d8e044d8f044d90044d91044d92044d93044d94044d95044d96044d97044d98044d99044d9a044d9b044d9c044d9d044d9e044d9f044da0044da1044da2044da3044da4044da5044da6044da7044da8044da9044daa044dab044dac044dad044dae044daf04655d94285d945d945d946165652e +dicts 0 286c70300a286470310a61286470320a56610a70330a49310a7361286470340a67330a49310a7356620a70350a286c70360a49320a6149330a617356630a70370a286470380a56640a70390a4e73736128647031300a49300a49300a7349310a49310a7349320a49320a7349330a49330a7349340a49340a7349350a49350a7349360a49360a7349370a49370a7349380a49380a7349390a49390a734931300a4931300a734931310a4931310a734931320a4931320a734931330a4931330a734931340a4931340a734931350a4931350a734931360a4931360a734931370a4931370a734931380a4931380a734931390a4931390a734932300a4932300a734932310a4932310a734932320a4932320a734932330a4932330a734932340a4932340a734932350a4932350a734932360a4932360a734932370a4932370a734932380a4932380a734932390a4932390a734933300a4933300a734933310a4933310a734933320a4933320a734933330a4933330a734933340a4933340a734933350a4933350a734933360a4933360a734933370a4933370a734933380a4933380a734933390a4933390a734934300a4934300a734934310a4934310a734934320a4934320a734934330a4934330a734934340a4934340a734934350a4934350a734934360a4934360a734934370a4934370a734934380a4934380a734934390a4934390a734935300a4935300a734935310a4935310a734935320a4935320a734935330a4935330a734935340a4935340a734935350a4935350a734935360a4935360a734935370a4935370a734935380a4935380a734935390a4935390a734936300a4936300a734936310a4936310a734936320a4936320a734936330a4936330a734936340a4936340a734936350a4936350a734936360a4936360a734936370a4936370a734936380a4936380a734936390a4936390a734937300a4937300a734937310a4937310a734937320a4937320a734937330a4937330a734937340a4937340a734937350a4937350a734937360a4937360a734937370a4937370a734937380a4937380a734937390a4937390a734938300a4938300a734938310a4938310a734938320a4938320a734938330a4938330a734938340a4938340a734938350a4938350a734938360a4938360a734938370a4938370a734938380a4938380a734938390a4938390a734939300a4939300a734939310a4939310a734939320a4939320a734939330a4939330a734939340a4939340a734939350a4939350a734939360a4939360a734939370a4939370a734939380a4939380a734939390a4939390a73493130300a493130300a73493130310a493130310a73493130320a493130320a73493130330a493130330a73493130340a493130340a73493130350a493130350a73493130360a493130360a73493130370a493130370a73493130380a493130380a73493130390a493130390a73493131300a493131300a73493131310a493131310a73493131320a493131320a73493131330a493131330a73493131340a493131340a73493131350a493131350a73493131360a493131360a73493131370a493131370a73493131380a493131380a73493131390a493131390a73493132300a493132300a73493132310a493132310a73493132320a493132320a73493132330a493132330a73493132340a493132340a73493132350a493132350a73493132360a493132360a73493132370a493132370a73493132380a493132380a73493132390a493132390a73493133300a493133300a73493133310a493133310a73493133320a493133320a73493133330a493133330a73493133340a493133340a73493133350a493133350a73493133360a493133360a73493133370a493133370a73493133380a493133380a73493133390a493133390a73493134300a493134300a73493134310a493134310a73493134320a493134320a73493134330a493134330a73493134340a493134340a73493134350a493134350a73493134360a493134360a73493134370a493134370a73493134380a493134380a73493134390a493134390a73493135300a493135300a73493135310a493135310a73493135320a493135320a73493135330a493135330a73493135340a493135340a73493135350a493135350a73493135360a493135360a73493135370a493135370a73493135380a493135380a73493135390a493135390a73493136300a493136300a73493136310a493136310a73493136320a493136320a73493136330a493136330a73493136340a493136340a73493136350a493136350a73493136360a493136360a73493136370a493136370a73493136380a493136380a73493136390a493136390a73493137300a493137300a73493137310a493137310a73493137320a493137320a73493137330a493137330a73493137340a493137340a73493137350a493137350a73493137360a493137360a73493137370a493137370a73493137380a493137380a73493137390a493137390a73493138300a493138300a73493138310a493138310a73493138320a493138320a73493138330a493138330a73493138340a493138340a73493138350a493138350a73493138360a493138360a73493138370a493138370a73493138380a493138380a73493138390a493138390a73493139300a493139300a73493139310a493139310a73493139320a493139320a73493139330a493139330a73493139340a493139340a73493139350a493139350a73493139360a493139360a73493139370a493139370a73493139380a493139380a73493139390a493139390a73493230300a493230300a73493230310a493230310a73493230320a493230320a73493230330a493230330a73493230340a493230340a73493230350a493230350a73493230360a493230360a73493230370a493230370a73493230380a493230380a73493230390a493230390a73493231300a493231300a73493231310a493231310a73493231320a493231320a73493231330a493231330a73493231340a493231340a73493231350a493231350a73493231360a493231360a73493231370a493231370a73493231380a493231380a73493231390a493231390a73493232300a493232300a73493232310a493232310a73493232320a493232320a73493232330a493232330a73493232340a493232340a73493232350a493232350a73493232360a493232360a73493232370a493232370a73493232380a493232380a73493232390a493232390a73493233300a493233300a73493233310a493233310a73493233320a493233320a73493233330a493233330a73493233340a493233340a73493233350a493233350a73493233360a493233360a73493233370a493233370a73493233380a493233380a73493233390a493233390a73493234300a493234300a73493234310a493234310a73493234320a493234320a73493234330a493234330a73493234340a493234340a73493234350a493234350a73493234360a493234360a73493234370a493234370a73493234380a493234380a73493234390a493234390a73493235300a493235300a73493235310a493235310a73493235320a493235320a73493235330a493235330a73493235340a493235340a73493235350a493235350a73493235360a493235360a73493235370a493235370a73493235380a493235380a73493235390a493235390a73493236300a493236300a73493236310a493236310a73493236320a493236320a73493236330a493236330a73493236340a493236340a73493236350a493236350a73493236360a493236360a73493236370a493236370a73493236380a493236380a73493236390a493236390a73493237300a493237300a73493237310a493237310a73493237320a493237320a73493237330a493237330a73493237340a493237340a73493237350a493237350a73493237360a493237360a73493237370a493237370a73493237380a493237380a73493237390a493237390a73493238300a493238300a73493238310a493238310a73493238320a493238320a73493238330a493238330a73493238340a493238340a73493238350a493238350a73493238360a493238360a73493238370a493238370a73493238380a493238380a73493238390a493238390a73493239300a493239300a73493239310a493239310a73493239320a493239320a73493239330a493239330a73493239340a493239340a73493239350a493239350a73493239360a493239360a73493239370a493239370a73493239380a493239380a73493239390a493239390a73493330300a493330300a73493330310a493330310a73493330320a493330320a73493330330a493330330a73493330340a493330340a73493330350a493330350a73493330360a493330360a73493330370a493330370a73493330380a493330380a73493330390a493330390a73493331300a493331300a73493331310a493331310a73493331320a493331320a73493331330a493331330a73493331340a493331340a73493331350a493331350a73493331360a493331360a73493331370a493331370a73493331380a493331380a73493331390a493331390a73493332300a493332300a73493332310a493332310a73493332320a493332320a73493332330a493332330a73493332340a493332340a73493332350a493332350a73493332360a493332360a73493332370a493332370a73493332380a493332380a73493332390a493332390a73493333300a493333300a73493333310a493333310a73493333320a493333320a73493333330a493333330a73493333340a493333340a73493333350a493333350a73493333360a493333360a73493333370a493333370a73493333380a493333380a73493333390a493333390a73493334300a493334300a73493334310a493334310a73493334320a493334320a73493334330a493334330a73493334340a493334340a73493334350a493334350a73493334360a493334360a73493334370a493334370a73493334380a493334380a73493334390a493334390a73493335300a493335300a73493335310a493335310a73493335320a493335320a73493335330a493335330a73493335340a493335340a73493335350a493335350a73493335360a493335360a73493335370a493335370a73493335380a493335380a73493335390a493335390a73493336300a493336300a73493336310a493336310a73493336320a493336320a73493336330a493336330a73493336340a493336340a73493336350a493336350a73493336360a493336360a73493336370a493336370a73493336380a493336380a73493336390a493336390a73493337300a493337300a73493337310a493337310a73493337320a493337320a73493337330a493337330a73493337340a493337340a73493337350a493337350a73493337360a493337360a73493337370a493337370a73493337380a493337380a73493337390a493337390a73493338300a493338300a73493338310a493338310a73493338320a493338320a73493338330a493338330a73493338340a493338340a73493338350a493338350a73493338360a493338360a73493338370a493338370a73493338380a493338380a73493338390a493338390a73493339300a493339300a73493339310a493339310a73493339320a493339320a73493339330a493339330a73493339340a493339340a73493339350a493339350a73493339360a493339360a73493339370a493339370a73493339380a493339380a73493339390a493339390a73493430300a493430300a73493430310a493430310a73493430320a493430320a73493430330a493430330a73493430340a493430340a73493430350a493430350a73493430360a493430360a73493430370a493430370a73493430380a493430380a73493430390a493430390a73493431300a493431300a73493431310a493431310a73493431320a493431320a73493431330a493431330a73493431340a493431340a73493431350a493431350a73493431360a493431360a73493431370a493431370a73493431380a493431380a73493431390a493431390a73493432300a493432300a73493432310a493432310a73493432320a493432320a73493432330a493432330a73493432340a493432340a73493432350a493432350a73493432360a493432360a73493432370a493432370a73493432380a493432380a73493432390a493432390a73493433300a493433300a73493433310a493433310a73493433320a493433320a73493433330a493433330a73493433340a493433340a73493433350a493433350a73493433360a493433360a73493433370a493433370a73493433380a493433380a73493433390a493433390a73493434300a493434300a73493434310a493434310a73493434320a493434320a73493434330a493434330a73493434340a493434340a73493434350a493434350a73493434360a493434360a73493434370a493434370a73493434380a493434380a73493434390a493434390a73493435300a493435300a73493435310a493435310a73493435320a493435320a73493435330a493435330a73493435340a493435340a73493435350a493435350a73493435360a493435360a73493435370a493435370a73493435380a493435380a73493435390a493435390a73493436300a493436300a73493436310a493436310a73493436320a493436320a73493436330a493436330a73493436340a493436340a73493436350a493436350a73493436360a493436360a73493436370a493436370a73493436380a493436380a73493436390a493436390a73493437300a493437300a73493437310a493437310a73493437320a493437320a73493437330a493437330a73493437340a493437340a73493437350a493437350a73493437360a493437360a73493437370a493437370a73493437380a493437380a73493437390a493437390a73493438300a493438300a73493438310a493438310a73493438320a493438320a73493438330a493438330a73493438340a493438340a73493438350a493438350a73493438360a493438360a73493438370a493438370a73493438380a493438380a73493438390a493438390a73493439300a493439300a73493439310a493439310a73493439320a493439320a73493439330a493439330a73493439340a493439340a73493439350a493439350a73493439360a493439360a73493439370a493439370a73493439380a493439380a73493439390a493439390a73493530300a493530300a73493530310a493530310a73493530320a493530320a73493530330a493530330a73493530340a493530340a73493530350a493530350a73493530360a493530360a73493530370a493530370a73493530380a493530380a73493530390a493530390a73493531300a493531300a73493531310a493531310a73493531320a493531320a73493531330a493531330a73493531340a493531340a73493531350a493531350a73493531360a493531360a73493531370a493531370a73493531380a493531380a73493531390a493531390a73493532300a493532300a73493532310a493532310a73493532320a493532320a73493532330a493532330a73493532340a493532340a73493532350a493532350a73493532360a493532360a73493532370a493532370a73493532380a493532380a73493532390a493532390a73493533300a493533300a73493533310a493533310a73493533320a493533320a73493533330a493533330a73493533340a493533340a73493533350a493533350a73493533360a493533360a73493533370a493533370a73493533380a493533380a73493533390a493533390a73493534300a493534300a73493534310a493534310a73493534320a493534320a73493534330a493534330a73493534340a493534340a73493534350a493534350a73493534360a493534360a73493534370a493534370a73493534380a493534380a73493534390a493534390a73493535300a493535300a73493535310a493535310a73493535320a493535320a73493535330a493535330a73493535340a493535340a73493535350a493535350a73493535360a493535360a73493535370a493535370a73493535380a493535380a73493535390a493535390a73493536300a493536300a73493536310a493536310a73493536320a493536320a73493536330a493536330a73493536340a493536340a73493536350a493536350a73493536360a493536360a73493536370a493536370a73493536380a493536380a73493536390a493536390a73493537300a493537300a73493537310a493537310a73493537320a493537320a73493537330a493537330a73493537340a493537340a73493537350a493537350a73493537360a493537360a73493537370a493537370a73493537380a493537380a73493537390a493537390a73493538300a493538300a73493538310a493538310a73493538320a493538320a73493538330a493538330a73493538340a493538340a73493538350a493538350a73493538360a493538360a73493538370a493538370a73493538380a493538380a73493538390a493538390a73493539300a493539300a73493539310a493539310a73493539320a493539320a73493539330a493539330a73493539340a493539340a73493539350a493539350a73493539360a493539360a73493539370a493539370a73493539380a493539380a73493539390a493539390a73493630300a493630300a73493630310a493630310a73493630320a493630320a73493630330a493630330a73493630340a493630340a73493630350a493630350a73493630360a493630360a73493630370a493630370a73493630380a493630380a73493630390a493630390a73493631300a493631300a73493631310a493631310a73493631320a493631320a73493631330a493631330a73493631340a493631340a73493631350a493631350a73493631360a493631360a73493631370a493631370a73493631380a493631380a73493631390a493631390a73493632300a493632300a73493632310a493632310a73493632320a493632320a73493632330a493632330a73493632340a493632340a73493632350a493632350a73493632360a493632360a73493632370a493632370a73493632380a493632380a73493632390a493632390a73493633300a493633300a73493633310a493633310a73493633320a493633320a73493633330a493633330a73493633340a493633340a73493633350a493633350a73493633360a493633360a73493633370a493633370a73493633380a493633380a73493633390a493633390a73493634300a493634300a73493634310a493634310a73493634320a493634320a73493634330a493634330a73493634340a493634340a73493634350a493634350a73493634360a493634360a73493634370a493634370a73493634380a493634380a73493634390a493634390a73493635300a493635300a73493635310a493635310a73493635320a493635320a73493635330a493635330a73493635340a493635340a73493635350a493635350a73493635360a493635360a73493635370a493635370a73493635380a493635380a73493635390a493635390a73493636300a493636300a73493636310a493636310a73493636320a493636320a73493636330a493636330a73493636340a493636340a73493636350a493636350a73493636360a493636360a73493636370a493636370a73493636380a493636380a73493636390a493636390a73493637300a493637300a73493637310a493637310a73493637320a493637320a73493637330a493637330a73493637340a493637340a73493637350a493637350a73493637360a493637360a73493637370a493637370a73493637380a493637380a73493637390a493637390a73493638300a493638300a73493638310a493638310a73493638320a493638320a73493638330a493638330a73493638340a493638340a73493638350a493638350a73493638360a493638360a73493638370a493638370a73493638380a493638380a73493638390a493638390a73493639300a493639300a73493639310a493639310a73493639320a493639320a73493639330a493639330a73493639340a493639340a73493639350a493639350a73493639360a493639360a73493639370a493639370a73493639380a493639380a73493639390a493639390a73493730300a493730300a73493730310a493730310a73493730320a493730320a73493730330a493730330a73493730340a493730340a73493730350a493730350a73493730360a493730360a73493730370a493730370a73493730380a493730380a73493730390a493730390a73493731300a493731300a73493731310a493731310a73493731320a493731320a73493731330a493731330a73493731340a493731340a73493731350a493731350a73493731360a493731360a73493731370a493731370a73493731380a493731380a73493731390a493731390a73493732300a493732300a73493732310a493732310a73493732320a493732320a73493732330a493732330a73493732340a493732340a73493732350a493732350a73493732360a493732360a73493732370a493732370a73493732380a493732380a73493732390a493732390a73493733300a493733300a73493733310a493733310a73493733320a493733320a73493733330a493733330a73493733340a493733340a73493733350a493733350a73493733360a493733360a73493733370a493733370a73493733380a493733380a73493733390a493733390a73493734300a493734300a73493734310a493734310a73493734320a493734320a73493734330a493734330a73493734340a493734340a73493734350a493734350a73493734360a493734360a73493734370a493734370a73493734380a493734380a73493734390a493734390a73493735300a493735300a73493735310a493735310a73493735320a493735320a73493735330a493735330a73493735340a493735340a73493735350a493735350a73493735360a493735360a73493735370a493735370a73493735380a493735380a73493735390a493735390a73493736300a493736300a73493736310a493736310a73493736320a493736320a73493736330a493736330a73493736340a493736340a73493736350a493736350a73493736360a493736360a73493736370a493736370a73493736380a493736380a73493736390a493736390a73493737300a493737300a73493737310a493737310a73493737320a493737320a73493737330a493737330a73493737340a493737340a73493737350a493737350a73493737360a493737360a73493737370a493737370a73493737380a493737380a73493737390a493737390a73493738300a493738300a73493738310a493738310a73493738320a493738320a73493738330a493738330a73493738340a493738340a73493738350a493738350a73493738360a493738360a73493738370a493738370a73493738380a493738380a73493738390a493738390a73493739300a493739300a73493739310a493739310a73493739320a493739320a73493739330a493739330a73493739340a493739340a73493739350a493739350a73493739360a493739360a73493739370a493739370a73493739380a493739380a73493739390a493739390a73493830300a493830300a73493830310a493830310a73493830320a493830320a73493830330a493830330a73493830340a493830340a73493830350a493830350a73493830360a493830360a73493830370a493830370a73493830380a493830380a73493830390a493830390a73493831300a493831300a73493831310a493831310a73493831320a493831320a73493831330a493831330a73493831340a493831340a73493831350a493831350a73493831360a493831360a73493831370a493831370a73493831380a493831380a73493831390a493831390a73493832300a493832300a73493832310a493832310a73493832320a493832320a73493832330a493832330a73493832340a493832340a73493832350a493832350a73493832360a493832360a73493832370a493832370a73493832380a493832380a73493832390a493832390a73493833300a493833300a73493833310a493833310a73493833320a493833320a73493833330a493833330a73493833340a493833340a73493833350a493833350a73493833360a493833360a73493833370a493833370a73493833380a493833380a73493833390a493833390a73493834300a493834300a73493834310a493834310a73493834320a493834320a73493834330a493834330a73493834340a493834340a73493834350a493834350a73493834360a493834360a73493834370a493834370a73493834380a493834380a73493834390a493834390a73493835300a493835300a73493835310a493835310a73493835320a493835320a73493835330a493835330a73493835340a493835340a73493835350a493835350a73493835360a493835360a73493835370a493835370a73493835380a493835380a73493835390a493835390a73493836300a493836300a73493836310a493836310a73493836320a493836320a73493836330a493836330a73493836340a493836340a73493836350a493836350a73493836360a493836360a73493836370a493836370a73493836380a493836380a73493836390a493836390a73493837300a493837300a73493837310a493837310a73493837320a493837320a73493837330a493837330a73493837340a493837340a73493837350a493837350a73493837360a493837360a73493837370a493837370a73493837380a493837380a73493837390a493837390a73493838300a493838300a73493838310a493838310a73493838320a493838320a73493838330a493838330a73493838340a493838340a73493838350a493838350a73493838360a493838360a73493838370a493838370a73493838380a493838380a73493838390a493838390a73493839300a493839300a73493839310a493839310a73493839320a493839320a73493839330a493839330a73493839340a493839340a73493839350a493839350a73493839360a493839360a73493839370a493839370a73493839380a493839380a73493839390a493839390a73493930300a493930300a73493930310a493930310a73493930320a493930320a73493930330a493930330a73493930340a493930340a73493930350a493930350a73493930360a493930360a73493930370a493930370a73493930380a493930380a73493930390a493930390a73493931300a493931300a73493931310a493931310a73493931320a493931320a73493931330a493931330a73493931340a493931340a73493931350a493931350a73493931360a493931360a73493931370a493931370a73493931380a493931380a73493931390a493931390a73493932300a493932300a73493932310a493932310a73493932320a493932320a73493932330a493932330a73493932340a493932340a73493932350a493932350a73493932360a493932360a73493932370a493932370a73493932380a493932380a73493932390a493932390a73493933300a493933300a73493933310a493933310a73493933320a493933320a73493933330a493933330a73493933340a493933340a73493933350a493933350a73493933360a493933360a73493933370a493933370a73493933380a493933380a73493933390a493933390a73493934300a493934300a73493934310a493934310a73493934320a493934320a73493934330a493934330a73493934340a493934340a73493934350a493934350a73493934360a493934360a73493934370a493934370a73493934380a493934380a73493934390a493934390a73493935300a493935300a73493935310a493935310a73493935320a493935320a73493935330a493935330a73493935340a493935340a73493935350a493935350a73493935360a493935360a73493935370a493935370a73493935380a493935380a73493935390a493935390a73493936300a493936300a73493936310a493936310a73493936320a493936320a73493936330a493936330a73493936340a493936340a73493936350a493936350a73493936360a493936360a73493936370a493936370a73493936380a493936380a73493936390a493936390a73493937300a493937300a73493937310a493937310a73493937320a493937320a73493937330a493937330a73493937340a493937340a73493937350a493937350a73493937360a493937360a73493937370a493937370a73493937380a493937380a73493937390a493937390a73493938300a493938300a73493938310a493938310a73493938320a493938320a73493938330a493938330a73493938340a493938340a73493938350a493938350a73493938360a493938360a73493938370a493938370a73493938380a493938380a73493938390a493938390a73493939300a493939300a73493939310a493939310a73493939320a493939320a73493939330a493939330a73493939340a493939340a73493939350a493939350a73493939360a493939360a73493939370a493939370a73493939380a493939380a73493939390a493939390a7349313030300a49313030300a7349313030310a49313030310a7349313030320a49313030320a7349313030330a49313030330a7349313030340a49313030340a7349313030350a49313030350a7349313030360a49313030360a7349313030370a49313030370a7349313030380a49313030380a7349313030390a49313030390a7349313031300a49313031300a7349313031310a49313031310a7349313031320a49313031320a7349313031330a49313031330a7349313031340a49313031340a7349313031350a49313031350a7349313031360a49313031360a7349313031370a49313031370a7349313031380a49313031380a7349313031390a49313031390a7349313032300a49313032300a7349313032310a49313032310a7349313032320a49313032320a7349313032330a49313032330a7349313032340a49313032340a7349313032350a49313032350a7349313032360a49313032360a7349313032370a49313032370a7349313032380a49313032380a7349313032390a49313032390a7349313033300a49313033300a7349313033310a49313033310a7349313033320a49313033320a7349313033330a49313033330a7349313033340a49313033340a7349313033350a49313033350a7349313033360a49313033360a7349313033370a49313033370a7349313033380a49313033380a7349313033390a49313033390a7349313034300a49313034300a7349313034310a49313034310a7349313034320a49313034320a7349313034330a49313034330a7349313034340a49313034340a7349313034350a49313034350a7349313034360a49313034360a7349313034370a49313034370a7349313034380a49313034380a7349313034390a49313034390a7349313035300a49313035300a7349313035310a49313035310a7349313035320a49313035320a7349313035330a49313035330a7349313035340a49313035340a7349313035350a49313035350a7349313035360a49313035360a7349313035370a49313035370a7349313035380a49313035380a7349313035390a49313035390a7349313036300a49313036300a7349313036310a49313036310a7349313036320a49313036320a7349313036330a49313036330a7349313036340a49313036340a7349313036350a49313036350a7349313036360a49313036360a7349313036370a49313036370a7349313036380a49313036380a7349313036390a49313036390a7349313037300a49313037300a7349313037310a49313037310a7349313037320a49313037320a7349313037330a49313037330a7349313037340a49313037340a7349313037350a49313037350a7349313037360a49313037360a7349313037370a49313037370a7349313037380a49313037380a7349313037390a49313037390a7349313038300a49313038300a7349313038310a49313038310a7349313038320a49313038320a7349313038330a49313038330a7349313038340a49313038340a7349313038350a49313038350a7349313038360a49313038360a7349313038370a49313038370a7349313038380a49313038380a7349313038390a49313038390a7349313039300a49313039300a7349313039310a49313039310a7349313039320a49313039320a7349313039330a49313039330a7349313039340a49313039340a7349313039350a49313039350a7349313039360a49313039360a7349313039370a49313039370a7349313039380a49313039380a7349313039390a49313039390a73612e +dicts 1 5d7100287d71017d710258010000006171034b01737d71042868034b0158010000006271055d7106284b024b036558010000006371077d710858010000006471094e73757d710a284b004b004b014b014b024b024b034b034b044b044b054b054b064b064b074b074b084b084b094b094b0a4b0a4b0b4b0b4b0c4b0c4b0d4b0d4b0e4b0e4b0f4b0f4b104b104b114b114b124b124b134b134b144b144b154b154b164b164b174b174b184b184b194b194b1a4b1a4b1b4b1b4b1c4b1c4b1d4b1d4b1e4b1e4b1f4b1f4b204b204b214b214b224b224b234b234b244b244b254b254b264b264b274b274b284b284b294b294b2a4b2a4b2b4b2b4b2c4b2c4b2d4b2d4b2e4b2e4b2f4b2f4b304b304b314b314b324b324b334b334b344b344b354b354b364b364b374b374b384b384b394b394b3a4b3a4b3b4b3b4b3c4b3c4b3d4b3d4b3e4b3e4b3f4b3f4b404b404b414b414b424b424b434b434b444b444b454b454b464b464b474b474b484b484b494b494b4a4b4a4b4b4b4b4b4c4b4c4b4d4b4d4b4e4b4e4b4f4b4f4b504b504b514b514b524b524b534b534b544b544b554b554b564b564b574b574b584b584b594b594b5a4b5a4b5b4b5b4b5c4b5c4b5d4b5d4b5e4b5e4b5f4b5f4b604b604b614b614b624b624b634b634b644b644b654b654b664b664b674b674b684b684b694b694b6a4b6a4b6b4b6b4b6c4b6c4b6d4b6d4b6e4b6e4b6f4b6f4b704b704b714b714b724b724b734b734b744b744b754b754b764b764b774b774b784b784b794b794b7a4b7a4b7b4b7b4b7c4b7c4b7d4b7d4b7e4b7e4b7f4b7f4b804b804b814b814b824b824b834b834b844b844b854b854b864b864b874b874b884b884b894b894b8a4b8a4b8b4b8b4b8c4b8c4b8d4b8d4b8e4b8e4b8f4b8f4b904b904b914b914b924b924b934b934b944b944b954b954b964b964b974b974b984b984b994b994b9a4b9a4b9b4b9b4b9c4b9c4b9d4b9d4b9e4b9e4b9f4b9f4ba04ba04ba14ba14ba24ba24ba34ba34ba44ba44ba54ba54ba64ba64ba74ba74ba84ba84ba94ba94baa4baa4bab4bab4bac4bac4bad4bad4bae4bae4baf4baf4bb04bb04bb14bb14bb24bb24bb34bb34bb44bb44bb54bb54bb64bb64bb74bb74bb84bb84bb94bb94bba4bba4bbb4bbb4bbc4bbc4bbd4bbd4bbe4bbe4bbf4bbf4bc04bc04bc14bc14bc24bc24bc34bc34bc44bc44bc54bc54bc64bc64bc74bc74bc84bc84bc94bc94bca4bca4bcb4bcb4bcc4bcc4bcd4bcd4bce4bce4bcf4bcf4bd04bd04bd14bd14bd24bd24bd34bd34bd44bd44bd54bd54bd64bd64bd74bd74bd84bd84bd94bd94bda4bda4bdb4bdb4bdc4bdc4bdd4bdd4bde4bde4bdf4bdf4be04be04be14be14be24be24be34be34be44be44be54be54be64be64be74be74be84be84be94be94bea4bea4beb4beb4bec4bec4bed4bed4bee4bee4bef4bef4bf04bf04bf14bf14bf24bf24bf34bf34bf44bf44bf54bf54bf64bf64bf74bf74bf84bf84bf94bf94bfa4bfa4bfb4bfb4bfc4bfc4bfd4bfd4bfe4bfe4bff4bff4d00014d00014d01014d01014d02014d02014d03014d03014d04014d04014d05014d05014d06014d06014d07014d07014d08014d08014d09014d09014d0a014d0a014d0b014d0b014d0c014d0c014d0d014d0d014d0e014d0e014d0f014d0f014d10014d10014d11014d11014d12014d12014d13014d13014d14014d14014d15014d15014d16014d16014d17014d17014d18014d18014d19014d19014d1a014d1a014d1b014d1b014d1c014d1c014d1d014d1d014d1e014d1e014d1f014d1f014d20014d20014d21014d21014d22014d22014d23014d23014d24014d24014d25014d25014d26014d26014d27014d27014d28014d28014d29014d29014d2a014d2a014d2b014d2b014d2c014d2c014d2d014d2d014d2e014d2e014d2f014d2f014d30014d30014d31014d31014d32014d32014d33014d33014d34014d34014d35014d35014d36014d36014d37014d37014d38014d38014d39014d39014d3a014d3a014d3b014d3b014d3c014d3c014d3d014d3d014d3e014d3e014d3f014d3f014d40014d40014d41014d41014d42014d42014d43014d43014d44014d44014d45014d45014d46014d46014d47014d47014d48014d48014d49014d49014d4a014d4a014d4b014d4b014d4c014d4c014d4d014d4d014d4e014d4e014d4f014d4f014d50014d50014d51014d51014d52014d52014d53014d53014d54014d54014d55014d55014d56014d56014d57014d57014d58014d58014d59014d59014d5a014d5a014d5b014d5b014d5c014d5c014d5d014d5d014d5e014d5e014d5f014d5f014d60014d60014d61014d61014d62014d62014d63014d63014d64014d64014d65014d65014d66014d66014d67014d67014d68014d68014d69014d69014d6a014d6a014d6b014d6b014d6c014d6c014d6d014d6d014d6e014d6e014d6f014d6f014d70014d70014d71014d71014d72014d72014d73014d73014d74014d74014d75014d75014d76014d76014d77014d77014d78014d78014d79014d79014d7a014d7a014d7b014d7b014d7c014d7c014d7d014d7d014d7e014d7e014d7f014d7f014d80014d80014d81014d81014d82014d82014d83014d83014d84014d84014d85014d85014d86014d86014d87014d87014d88014d88014d89014d89014d8a014d8a014d8b014d8b014d8c014d8c014d8d014d8d014d8e014d8e014d8f014d8f014d90014d90014d91014d91014d92014d92014d93014d93014d94014d94014d95014d95014d96014d96014d97014d97014d98014d98014d99014d99014d9a014d9a014d9b014d9b014d9c014d9c014d9d014d9d014d9e014d9e014d9f014d9f014da0014da0014da1014da1014da2014da2014da3014da3014da4014da4014da5014da5014da6014da6014da7014da7014da8014da8014da9014da9014daa014daa014dab014dab014dac014dac014dad014dad014dae014dae014daf014daf014db0014db0014db1014db1014db2014db2014db3014db3014db4014db4014db5014db5014db6014db6014db7014db7014db8014db8014db9014db9014dba014dba014dbb014dbb014dbc014dbc014dbd014dbd014dbe014dbe014dbf014dbf014dc0014dc0014dc1014dc1014dc2014dc2014dc3014dc3014dc4014dc4014dc5014dc5014dc6014dc6014dc7014dc7014dc8014dc8014dc9014dc9014dca014dca014dcb014dcb014dcc014dcc014dcd014dcd014dce014dce014dcf014dcf014dd0014dd0014dd1014dd1014dd2014dd2014dd3014dd3014dd4014dd4014dd5014dd5014dd6014dd6014dd7014dd7014dd8014dd8014dd9014dd9014dda014dda014ddb014ddb014ddc014ddc014ddd014ddd014dde014dde014ddf014ddf014de0014de0014de1014de1014de2014de2014de3014de3014de4014de4014de5014de5014de6014de6014de7014de7014de8014de8014de9014de9014dea014dea014deb014deb014dec014dec014ded014ded014dee014dee014def014def014df0014df0014df1014df1014df2014df2014df3014df3014df4014df4014df5014df5014df6014df6014df7014df7014df8014df8014df9014df9014dfa014dfa014dfb014dfb014dfc014dfc014dfd014dfd014dfe014dfe014dff014dff014d00024d00024d01024d01024d02024d02024d03024d03024d04024d04024d05024d05024d06024d06024d07024d07024d08024d08024d09024d09024d0a024d0a024d0b024d0b024d0c024d0c024d0d024d0d024d0e024d0e024d0f024d0f024d10024d10024d11024d11024d12024d12024d13024d13024d14024d14024d15024d15024d16024d16024d17024d17024d18024d18024d19024d19024d1a024d1a024d1b024d1b024d1c024d1c024d1d024d1d024d1e024d1e024d1f024d1f024d20024d20024d21024d21024d22024d22024d23024d23024d24024d24024d25024d25024d26024d26024d27024d27024d28024d28024d29024d29024d2a024d2a024d2b024d2b024d2c024d2c024d2d024d2d024d2e024d2e024d2f024d2f024d30024d30024d31024d31024d32024d32024d33024d33024d34024d34024d35024d35024d36024d36024d37024d37024d38024d38024d39024d39024d3a024d3a024d3b024d3b024d3c024d3c024d3d024d3d024d3e024d3e024d3f024d3f024d40024d40024d41024d41024d42024d42024d43024d43024d44024d44024d45024d45024d46024d46024d47024d47024d48024d48024d49024d49024d4a024d4a024d4b024d4b024d4c024d4c024d4d024d4d024d4e024d4e024d4f024d4f024d50024d50024d51024d51024d52024d52024d53024d53024d54024d54024d55024d55024d56024d56024d57024d57024d58024d58024d59024d59024d5a024d5a024d5b024d5b024d5c024d5c024d5d024d5d024d5e024d5e024d5f024d5f024d60024d60024d61024d61024d62024d62024d63024d63024d64024d64024d65024d65024d66024d66024d67024d67024d68024d68024d69024d69024d6a024d6a024d6b024d6b024d6c024d6c024d6d024d6d024d6e024d6e024d6f024d6f024d70024d70024d71024d71024d72024d72024d73024d73024d74024d74024d75024d75024d76024d76024d77024d77024d78024d78024d79024d79024d7a024d7a024d7b024d7b024d7c024d7c024d7d024d7d024d7e024d7e024d7f024d7f024d80024d80024d81024d81024d82024d82024d83024d83024d84024d84024d85024d85024d86024d86024d87024d87024d88024d88024d89024d89024d8a024d8a024d8b024d8b024d8c024d8c024d8d024d8d024d8e024d8e024d8f024d8f024d90024d90024d91024d91024d92024d92024d93024d93024d94024d94024d95024d95024d96024d96024d97024d97024d98024d98024d99024d99024d9a024d9a024d9b024d9b024d9c024d9c024d9d024d9d024d9e024d9e024d9f024d9f024da0024da0024da1024da1024da2024da2024da3024da3024da4024da4024da5024da5024da6024da6024da7024da7024da8024da8024da9024da9024daa024daa024dab024dab024dac024dac024dad024dad024dae024dae024daf024daf024db0024db0024db1024db1024db2024db2024db3024db3024db4024db4024db5024db5024db6024db6024db7024db7024db8024db8024db9024db9024dba024dba024dbb024dbb024dbc024dbc024dbd024dbd024dbe024dbe024dbf024dbf024dc0024dc0024dc1024dc1024dc2024dc2024dc3024dc3024dc4024dc4024dc5024dc5024dc6024dc6024dc7024dc7024dc8024dc8024dc9024dc9024dca024dca024dcb024dcb024dcc024dcc024dcd024dcd024dce024dce024dcf024dcf024dd0024dd0024dd1024dd1024dd2024dd2024dd3024dd3024dd4024dd4024dd5024dd5024dd6024dd6024dd7024dd7024dd8024dd8024dd9024dd9024dda024dda024ddb024ddb024ddc024ddc024ddd024ddd024dde024dde024ddf024ddf024de0024de0024de1024de1024de2024de2024de3024de3024de4024de4024de5024de5024de6024de6024de7024de7024de8024de8024de9024de9024dea024dea024deb024deb024dec024dec024ded024ded024dee024dee024def024def024df0024df0024df1024df1024df2024df2024df3024df3024df4024df4024df5024df5024df6024df6024df7024df7024df8024df8024df9024df9024dfa024dfa024dfb024dfb024dfc024dfc024dfd024dfd024dfe024dfe024dff024dff024d00034d00034d01034d01034d02034d02034d03034d03034d04034d04034d05034d05034d06034d06034d07034d07034d08034d08034d09034d09034d0a034d0a034d0b034d0b034d0c034d0c034d0d034d0d034d0e034d0e034d0f034d0f034d10034d10034d11034d11034d12034d12034d13034d13034d14034d14034d15034d15034d16034d16034d17034d17034d18034d18034d19034d19034d1a034d1a034d1b034d1b034d1c034d1c034d1d034d1d034d1e034d1e034d1f034d1f034d20034d20034d21034d21034d22034d22034d23034d23034d24034d24034d25034d25034d26034d26034d27034d27034d28034d28034d29034d29034d2a034d2a034d2b034d2b034d2c034d2c034d2d034d2d034d2e034d2e034d2f034d2f034d30034d30034d31034d31034d32034d32034d33034d33034d34034d34034d35034d35034d36034d36034d37034d37034d38034d38034d39034d39034d3a034d3a034d3b034d3b034d3c034d3c034d3d034d3d034d3e034d3e034d3f034d3f034d40034d40034d41034d41034d42034d42034d43034d43034d44034d44034d45034d45034d46034d46034d47034d47034d48034d48034d49034d49034d4a034d4a034d4b034d4b034d4c034d4c034d4d034d4d034d4e034d4e034d4f034d4f034d50034d50034d51034d51034d52034d52034d53034d53034d54034d54034d55034d55034d56034d56034d57034d57034d58034d58034d59034d59034d5a034d5a034d5b034d5b034d5c034d5c034d5d034d5d034d5e034d5e034d5f034d5f034d60034d60034d61034d61034d62034d62034d63034d63034d64034d64034d65034d65034d66034d66034d67034d67034d68034d68034d69034d69034d6a034d6a034d6b034d6b034d6c034d6c034d6d034d6d034d6e034d6e034d6f034d6f034d70034d70034d71034d71034d72034d72034d73034d73034d74034d74034d75034d75034d76034d76034d77034d77034d78034d78034d79034d79034d7a034d7a034d7b034d7b034d7c034d7c034d7d034d7d034d7e034d7e034d7f034d7f034d80034d80034d81034d81034d82034d82034d83034d83034d84034d84034d85034d85034d86034d86034d87034d87034d88034d88034d89034d89034d8a034d8a034d8b034d8b034d8c034d8c034d8d034d8d034d8e034d8e034d8f034d8f034d90034d90034d91034d91034d92034d92034d93034d93034d94034d94034d95034d95034d96034d96034d97034d97034d98034d98034d99034d99034d9a034d9a034d9b034d9b034d9c034d9c034d9d034d9d034d9e034d9e034d9f034d9f034da0034da0034da1034da1034da2034da2034da3034da3034da4034da4034da5034da5034da6034da6034da7034da7034da8034da8034da9034da9034daa034daa034dab034dab034dac034dac034dad034dad034dae034dae034daf034daf034db0034db0034db1034db1034db2034db2034db3034db3034db4034db4034db5034db5034db6034db6034db7034db7034db8034db8034db9034db9034dba034dba034dbb034dbb034dbc034dbc034dbd034dbd034dbe034dbe034dbf034dbf034dc0034dc0034dc1034dc1034dc2034dc2034dc3034dc3034dc4034dc4034dc5034dc5034dc6034dc6034dc7034dc7034dc8034dc8034dc9034dc9034dca034dca034dcb034dcb034dcc034dcc034dcd034dcd034dce034dce034dcf034dcf034dd0034dd0034dd1034dd1034dd2034dd2034dd3034dd3034dd4034dd4034dd5034dd5034dd6034dd6034dd7034dd7034dd8034dd8034dd9034dd9034dda034dda034ddb034ddb034ddc034ddc034ddd034ddd034dde034dde034ddf034ddf034de0034de0034de1034de1034de2034de2034de3034de3034de4034de4034de5034de5034de6034de6034de7034de70375284de8034de8034de9034de9034dea034dea034deb034deb034dec034dec034ded034ded034dee034dee034def034def034df0034df0034df1034df1034df2034df2034df3034df3034df4034df4034df5034df5034df6034df6034df7034df7034df8034df8034df9034df9034dfa034dfa034dfb034dfb034dfc034dfc034dfd034dfd034dfe034dfe034dff034dff034d00044d00044d01044d01044d02044d02044d03044d03044d04044d04044d05044d05044d06044d06044d07044d07044d08044d08044d09044d09044d0a044d0a044d0b044d0b044d0c044d0c044d0d044d0d044d0e044d0e044d0f044d0f044d10044d10044d11044d11044d12044d12044d13044d13044d14044d14044d15044d15044d16044d16044d17044d17044d18044d18044d19044d19044d1a044d1a044d1b044d1b044d1c044d1c044d1d044d1d044d1e044d1e044d1f044d1f044d20044d20044d21044d21044d22044d22044d23044d23044d24044d24044d25044d25044d26044d26044d27044d27044d28044d28044d29044d29044d2a044d2a044d2b044d2b044d2c044d2c044d2d044d2d044d2e044d2e044d2f044d2f044d30044d30044d31044d31044d32044d32044d33044d33044d34044d34044d35044d35044d36044d36044d37044d37044d38044d38044d39044d39044d3a044d3a044d3b044d3b044d3c044d3c044d3d044d3d044d3e044d3e044d3f044d3f044d40044d40044d41044d41044d42044d42044d43044d43044d44044d44044d45044d45044d46044d46044d47044d47044d48044d48044d49044d49044d4a044d4a044d4b044d4b0475652e +dicts 2 80025d7100287d71017d710258010000006171034b01737d71042868034b0158010000006271055d7106284b024b036558010000006371077d710858010000006471094e73757d710a284b004b004b014b014b024b024b034b034b044b044b054b054b064b064b074b074b084b084b094b094b0a4b0a4b0b4b0b4b0c4b0c4b0d4b0d4b0e4b0e4b0f4b0f4b104b104b114b114b124b124b134b134b144b144b154b154b164b164b174b174b184b184b194b194b1a4b1a4b1b4b1b4b1c4b1c4b1d4b1d4b1e4b1e4b1f4b1f4b204b204b214b214b224b224b234b234b244b244b254b254b264b264b274b274b284b284b294b294b2a4b2a4b2b4b2b4b2c4b2c4b2d4b2d4b2e4b2e4b2f4b2f4b304b304b314b314b324b324b334b334b344b344b354b354b364b364b374b374b384b384b394b394b3a4b3a4b3b4b3b4b3c4b3c4b3d4b3d4b3e4b3e4b3f4b3f4b404b404b414b414b424b424b434b434b444b444b454b454b464b464b474b474b484b484b494b494b4a4b4a4b4b4b4b4b4c4b4c4b4d4b4d4b4e4b4e4b4f4b4f4b504b504b514b514b524b524b534b534b544b544b554b554b564b564b574b574b584b584b594b594b5a4b5a4b5b4b5b4b5c4b5c4b5d4b5d4b5e4b5e4b5f4b5f4b604b604b614b614b624b624b634b634b644b644b654b654b664b664b674b674b684b684b694b694b6a4b6a4b6b4b6b4b6c4b6c4b6d4b6d4b6e4b6e4b6f4b6f4b704b704b714b714b724b724b734b734b744b744b754b754b764b764b774b774b784b784b794b794b7a4b7a4b7b4b7b4b7c4b7c4b7d4b7d4b7e4b7e4b7f4b7f4b804b804b814b814b824b824b834b834b844b844b854b854b864b864b874b874b884b884b894b894b8a4b8a4b8b4b8b4b8c4b8c4b8d4b8d4b8e4b8e4b8f4b8f4b904b904b914b914b924b924b934b934b944b944b954b954b964b964b974b974b984b984b994b994b9a4b9a4b9b4b9b4b9c4b9c4b9d4b9d4b9e4b9e4b9f4b9f4ba04ba04ba14ba14ba24ba24ba34ba34ba44ba44ba54ba54ba64ba64ba74ba74ba84ba84ba94ba94baa4baa4bab4bab4bac4bac4bad4bad4bae4bae4baf4baf4bb04bb04bb14bb14bb24bb24bb34bb34bb44bb44bb54bb54bb64bb64bb74bb74bb84bb84bb94bb94bba4bba4bbb4bbb4bbc4bbc4bbd4bbd4bbe4bbe4bbf4bbf4bc04bc04bc14bc14bc24bc24bc34bc34bc44bc44bc54bc54bc64bc64bc74bc74bc84bc84bc94bc94bca4bca4bcb4bcb4bcc4bcc4bcd4bcd4bce4bce4bcf4bcf4bd04bd04bd14bd14bd24bd24bd34bd34bd44bd44bd54bd54bd64bd64bd74bd74bd84bd84bd94bd94bda4bda4bdb4bdb4bdc4bdc4bdd4bdd4bde4bde4bdf4bdf4be04be04be14be14be24be24be34be34be44be44be54be54be64be64be74be74be84be84be94be94bea4bea4beb4beb4bec4bec4bed4bed4bee4bee4bef4bef4bf04bf04bf14bf14bf24bf24bf34bf34bf44bf44bf54bf54bf64bf64bf74bf74bf84bf84bf94bf94bfa4bfa4bfb4bfb4bfc4bfc4bfd4bfd4bfe4bfe4bff4bff4d00014d00014d01014d01014d02014d02014d03014d03014d04014d04014d05014d05014d06014d06014d07014d07014d08014d08014d09014d09014d0a014d0a014d0b014d0b014d0c014d0c014d0d014d0d014d0e014d0e014d0f014d0f014d10014d10014d11014d11014d12014d12014d13014d13014d14014d14014d15014d15014d16014d16014d17014d17014d18014d18014d19014d19014d1a014d1a014d1b014d1b014d1c014d1c014d1d014d1d014d1e014d1e014d1f014d1f014d20014d20014d21014d21014d22014d22014d23014d23014d24014d24014d25014d25014d26014d26014d27014d27014d28014d28014d29014d29014d2a014d2a014d2b014d2b014d2c014d2c014d2d014d2d014d2e014d2e014d2f014d2f014d30014d30014d31014d31014d32014d32014d33014d33014d34014d34014d35014d35014d36014d36014d37014d37014d38014d38014d39014d39014d3a014d3a014d3b014d3b014d3c014d3c014d3d014d3d014d3e014d3e014d3f014d3f014d40014d40014d41014d41014d42014d42014d43014d43014d44014d44014d45014d45014d46014d46014d47014d47014d48014d48014d49014d49014d4a014d4a014d4b014d4b014d4c014d4c014d4d014d4d014d4e014d4e014d4f014d4f014d50014d50014d51014d51014d52014d52014d53014d53014d54014d54014d55014d55014d56014d56014d57014d57014d58014d58014d59014d59014d5a014d5a014d5b014d5b014d5c014d5c014d5d014d5d014d5e014d5e014d5f014d5f014d60014d60014d61014d61014d62014d62014d63014d63014d64014d64014d65014d65014d66014d66014d67014d67014d68014d68014d69014d69014d6a014d6a014d6b014d6b014d6c014d6c014d6d014d6d014d6e014d6e014d6f014d6f014d70014d70014d71014d71014d72014d72014d73014d73014d74014d74014d75014d75014d76014d76014d77014d77014d78014d78014d79014d79014d7a014d7a014d7b014d7b014d7c014d7c014d7d014d7d014d7e014d7e014d7f014d7f014d80014d80014d81014d81014d82014d82014d83014d83014d84014d84014d85014d85014d86014d86014d87014d87014d88014d88014d89014d89014d8a014d8a014d8b014d8b014d8c014d8c014d8d014d8d014d8e014d8e014d8f014d8f014d90014d90014d91014d91014d92014d92014d93014d93014d94014d94014d95014d95014d96014d96014d97014d97014d98014d98014d99014d99014d9a014d9a014d9b014d9b014d9c014d9c014d9d014d9d014d9e014d9e014d9f014d9f014da0014da0014da1014da1014da2014da2014da3014da3014da4014da4014da5014da5014da6014da6014da7014da7014da8014da8014da9014da9014daa014daa014dab014dab014dac014dac014dad014dad014dae014dae014daf014daf014db0014db0014db1014db1014db2014db2014db3014db3014db4014db4014db5014db5014db6014db6014db7014db7014db8014db8014db9014db9014dba014dba014dbb014dbb014dbc014dbc014dbd014dbd014dbe014dbe014dbf014dbf014dc0014dc0014dc1014dc1014dc2014dc2014dc3014dc3014dc4014dc4014dc5014dc5014dc6014dc6014dc7014dc7014dc8014dc8014dc9014dc9014dca014dca014dcb014dcb014dcc014dcc014dcd014dcd014dce014dce014dcf014dcf014dd0014dd0014dd1014dd1014dd2014dd2014dd3014dd3014dd4014dd4014dd5014dd5014dd6014dd6014dd7014dd7014dd8014dd8014dd9014dd9014dda014dda014ddb014ddb014ddc014ddc014ddd014ddd014dde014dde014ddf014ddf014de0014de0014de1014de1014de2014de2014de3014de3014de4014de4014de5014de5014de6014de6014de7014de7014de8014de8014de9014de9014dea014dea014deb014deb014dec014dec014ded014ded014dee014dee014def014def014df0014df0014df1014df1014df2014df2014df3014df3014df4014df4014df5014df5014df6014df6014df7014df7014df8014df8014df9014df9014dfa014dfa014dfb014dfb014dfc014dfc014dfd014dfd014dfe014dfe014dff014dff014d00024d00024d01024d01024d02024d02024d03024d03024d04024d04024d05024d05024d06024d06024d07024d07024d08024d08024d09024d09024d0a024d0a024d0b024d0b024d0c024d0c024d0d024d0d024d0e024d0e024d0f024d0f024d10024d10024d11024d11024d12024d12024d13024d13024d14024d14024d15024d15024d16024d16024d17024d17024d18024d18024d19024d19024d1a024d1a024d1b024d1b024d1c024d1c024d1d024d1d024d1e024d1e024d1f024d1f024d20024d20024d21024d21024d22024d22024d23024d23024d24024d24024d25024d25024d26024d26024d27024d27024d28024d28024d29024d29024d2a024d2a024d2b024d2b024d2c024d2c024d2d024d2d024d2e024d2e024d2f024d2f024d30024d30024d31024d31024d32024d32024d33024d33024d34024d34024d35024d35024d36024d36024d37024d37024d38024d38024d39024d39024d3a024d3a024d3b024d3b024d3c024d3c024d3d024d3d024d3e024d3e024d3f024d3f024d40024d40024d41024d41024d42024d42024d43024d43024d44024d44024d45024d45024d46024d46024d47024d47024d48024d48024d49024d49024d4a024d4a024d4b024d4b024d4c024d4c024d4d024d4d024d4e024d4e024d4f024d4f024d50024d50024d51024d51024d52024d52024d53024d53024d54024d54024d55024d55024d56024d56024d57024d57024d58024d58024d59024d59024d5a024d5a024d5b024d5b024d5c024d5c024d5d024d5d024d5e024d5e024d5f024d5f024d60024d60024d61024d61024d62024d62024d63024d63024d64024d64024d65024d65024d66024d66024d67024d67024d68024d68024d69024d69024d6a024d6a024d6b024d6b024d6c024d6c024d6d024d6d024d6e024d6e024d6f024d6f024d70024d70024d71024d71024d72024d72024d73024d73024d74024d74024d75024d75024d76024d76024d77024d77024d78024d78024d79024d79024d7a024d7a024d7b024d7b024d7c024d7c024d7d024d7d024d7e024d7e024d7f024d7f024d80024d80024d81024d81024d82024d82024d83024d83024d84024d84024d85024d85024d86024d86024d87024d87024d88024d88024d89024d89024d8a024d8a024d8b024d8b024d8c024d8c024d8d024d8d024d8e024d8e024d8f024d8f024d90024d90024d91024d91024d92024d92024d93024d93024d94024d94024d95024d95024d96024d96024d97024d97024d98024d98024d99024d99024d9a024d9a024d9b024d9b024d9c024d9c024d9d024d9d024d9e024d9e024d9f024d9f024da0024da0024da1024da1024da2024da2024da3024da3024da4024da4024da5024da5024da6024da6024da7024da7024da8024da8024da9024da9024daa024daa024dab024dab024dac024dac024dad024dad024dae024dae024daf024daf024db0024db0024db1024db1024db2024db2024db3024db3024db4024db4024db5024db5024db6024db6024db7024db7024db8024db8024db9024db9024dba024dba024dbb024dbb024dbc024dbc024dbd024dbd024dbe024dbe024dbf024dbf024dc0024dc0024dc1024dc1024dc2024dc2024dc3024dc3024dc4024dc4024dc5024dc5024dc6024dc6024dc7024dc7024dc8024dc8024dc9024dc9024dca024dca024dcb024dcb024dcc024dcc024dcd024dcd024dce024dce024dcf024dcf024dd0024dd0024dd1024dd1024dd2024dd2024dd3024dd3024dd4024dd4024dd5024dd5024dd6024dd6024dd7024dd7024dd8024dd8024dd9024dd9024dda024dda024ddb024ddb024ddc024ddc024ddd024ddd024dde024dde024ddf024ddf024de0024de0024de1024de1024de2024de2024de3024de3024de4024de4024de5024de5024de6024de6024de7024de7024de8024de8024de9024de9024dea024dea024deb024deb024dec024dec024ded024ded024dee024dee024def024def024df0024df0024df1024df1024df2024df2024df3024df3024df4024df4024df5024df5024df6024df6024df7024df7024df8024df8024df9024df9024dfa024dfa024dfb024dfb024dfc024dfc024dfd024dfd024dfe024dfe024dff024dff024d00034d00034d01034d01034d02034d02034d03034d03034d04034d04034d05034d05034d06034d06034d07034d07034d08034d08034d09034d09034d0a034d0a034d0b034d0b034d0c034d0c034d0d034d0d034d0e034d0e034d0f034d0f034d10034d10034d11034d11034d12034d12034d13034d13034d14034d14034d15034d15034d16034d16034d17034d17034d18034d18034d19034d19034d1a034d1a034d1b034d1b034d1c034d1c034d1d034d1d034d1e034d1e034d1f034d1f034d20034d20034d21034d21034d22034d22034d23034d23034d24034d24034d25034d25034d26034d26034d27034d27034d28034d28034d29034d29034d2a034d2a034d2b034d2b034d2c034d2c034d2d034d2d034d2e034d2e034d2f034d2f034d30034d30034d31034d31034d32034d32034d33034d33034d34034d34034d35034d35034d36034d36034d37034d37034d38034d38034d39034d39034d3a034d3a034d3b034d3b034d3c034d3c034d3d034d3d034d3e034d3e034d3f034d3f034d40034d40034d41034d41034d42034d42034d43034d43034d44034d44034d45034d45034d46034d46034d47034d47034d48034d48034d49034d49034d4a034d4a034d4b034d4b034d4c034d4c034d4d034d4d034d4e034d4e034d4f034d4f034d50034d50034d51034d51034d52034d52034d53034d53034d54034d54034d55034d55034d56034d56034d57034d57034d58034d58034d59034d59034d5a034d5a034d5b034d5b034d5c034d5c034d5d034d5d034d5e034d5e034d5f034d5f034d60034d60034d61034d61034d62034d62034d63034d63034d64034d64034d65034d65034d66034d66034d67034d67034d68034d68034d69034d69034d6a034d6a034d6b034d6b034d6c034d6c034d6d034d6d034d6e034d6e034d6f034d6f034d70034d70034d71034d71034d72034d72034d73034d73034d74034d74034d75034d75034d76034d76034d77034d77034d78034d78034d79034d79034d7a034d7a034d7b034d7b034d7c034d7c034d7d034d7d034d7e034d7e034d7f034d7f034d80034d80034d81034d81034d82034d82034d83034d83034d84034d84034d85034d85034d86034d86034d87034d87034d88034d88034d89034d89034d8a034d8a034d8b034d8b034d8c034d8c034d8d034d8d034d8e034d8e034d8f034d8f034d90034d90034d91034d91034d92034d92034d93034d93034d94034d94034d95034d95034d96034d96034d97034d97034d98034d98034d99034d99034d9a034d9a034d9b034d9b034d9c034d9c034d9d034d9d034d9e034d9e034d9f034d9f034da0034da0034da1034da1034da2034da2034da3034da3034da4034da4034da5034da5034da6034da6034da7034da7034da8034da8034da9034da9034daa034daa034dab034dab034dac034dac034dad034dad034dae034dae034daf034daf034db0034db0034db1034db1034db2034db2034db3034db3034db4034db4034db5034db5034db6034db6034db7034db7034db8034db8034db9034db9034dba034dba034dbb034dbb034dbc034dbc034dbd034dbd034dbe034dbe034dbf034dbf034dc0034dc0034dc1034dc1034dc2034dc2034dc3034dc3034dc4034dc4034dc5034dc5034dc6034dc6034dc7034dc7034dc8034dc8034dc9034dc9034dca034dca034dcb034dcb034dcc034dcc034dcd034dcd034dce034dce034dcf034dcf034dd0034dd0034dd1034dd1034dd2034dd2034dd3034dd3034dd4034dd4034dd5034dd5034dd6034dd6034dd7034dd7034dd8034dd8034dd9034dd9034dda034dda034ddb034ddb034ddc034ddc034ddd034ddd034dde034dde034ddf034ddf034de0034de0034de1034de1034de2034de2034de3034de3034de4034de4034de5034de5034de6034de6034de7034de70375284de8034de8034de9034de9034dea034dea034deb034deb034dec034dec034ded034ded034dee034dee034def034def034df0034df0034df1034df1034df2034df2034df3034df3034df4034df4034df5034df5034df6034df6034df7034df7034df8034df8034df9034df9034dfa034dfa034dfb034dfb034dfc034dfc034dfd034dfd034dfe034dfe034dff034dff034d00044d00044d01044d01044d02044d02044d03044d03044d04044d04044d05044d05044d06044d06044d07044d07044d08044d08044d09044d09044d0a044d0a044d0b044d0b044d0c044d0c044d0d044d0d044d0e044d0e044d0f044d0f044d10044d10044d11044d11044d12044d12044d13044d13044d14044d14044d15044d15044d16044d16044d17044d17044d18044d18044d19044d19044d1a044d1a044d1b044d1b044d1c044d1c044d1d044d1d044d1e044d1e044d1f044d1f044d20044d20044d21044d21044d22044d22044d23044d23044d24044d24044d25044d25044d26044d26044d27044d27044d28044d28044d29044d29044d2a044d2a044d2b044d2b044d2c044d2c044d2d044d2d044d2e044d2e044d2f044d2f044d30044d30044d31044d31044d32044d32044d33044d33044d34044d34044d35044d35044d36044d36044d37044d37044d38044d38044d39044d39044d3a044d3a044d3b044d3b044d3c044d3c044d3d044d3d044d3e044d3e044d3f044d3f044d40044d40044d41044d41044d42044d42044d43044d43044d44044d44044d45044d45044d46044d46044d47044d47044d48044d48044d49044d49044d4a044d4a044d4b044d4b0475652e +dicts 3 80035d7100287d71017d710258010000006171034b01737d71042868034b0158010000006271055d7106284b024b036558010000006371077d710858010000006471094e73757d710a284b004b004b014b014b024b024b034b034b044b044b054b054b064b064b074b074b084b084b094b094b0a4b0a4b0b4b0b4b0c4b0c4b0d4b0d4b0e4b0e4b0f4b0f4b104b104b114b114b124b124b134b134b144b144b154b154b164b164b174b174b184b184b194b194b1a4b1a4b1b4b1b4b1c4b1c4b1d4b1d4b1e4b1e4b1f4b1f4b204b204b214b214b224b224b234b234b244b244b254b254b264b264b274b274b284b284b294b294b2a4b2a4b2b4b2b4b2c4b2c4b2d4b2d4b2e4b2e4b2f4b2f4b304b304b314b314b324b324b334b334b344b344b354b354b364b364b374b374b384b384b394b394b3a4b3a4b3b4b3b4b3c4b3c4b3d4b3d4b3e4b3e4b3f4b3f4b404b404b414b414b424b424b434b434b444b444b454b454b464b464b474b474b484b484b494b494b4a4b4a4b4b4b4b4b4c4b4c4b4d4b4d4b4e4b4e4b4f4b4f4b504b504b514b514b524b524b534b534b544b544b554b554b564b564b574b574b584b584b594b594b5a4b5a4b5b4b5b4b5c4b5c4b5d4b5d4b5e4b5e4b5f4b5f4b604b604b614b614b624b624b634b634b644b644b654b654b664b664b674b674b684b684b694b694b6a4b6a4b6b4b6b4b6c4b6c4b6d4b6d4b6e4b6e4b6f4b6f4b704b704b714b714b724b724b734b734b744b744b754b754b764b764b774b774b784b784b794b794b7a4b7a4b7b4b7b4b7c4b7c4b7d4b7d4b7e4b7e4b7f4b7f4b804b804b814b814b824b824b834b834b844b844b854b854b864b864b874b874b884b884b894b894b8a4b8a4b8b4b8b4b8c4b8c4b8d4b8d4b8e4b8e4b8f4b8f4b904b904b914b914b924b924b934b934b944b944b954b954b964b964b974b974b984b984b994b994b9a4b9a4b9b4b9b4b9c4b9c4b9d4b9d4b9e4b9e4b9f4b9f4ba04ba04ba14ba14ba24ba24ba34ba34ba44ba44ba54ba54ba64ba64ba74ba74ba84ba84ba94ba94baa4baa4bab4bab4bac4bac4bad4bad4bae4bae4baf4baf4bb04bb04bb14bb14bb24bb24bb34bb34bb44bb44bb54bb54bb64bb64bb74bb74bb84bb84bb94bb94bba4bba4bbb4bbb4bbc4bbc4bbd4bbd4bbe4bbe4bbf4bbf4bc04bc04bc14bc14bc24bc24bc34bc34bc44bc44bc54bc54bc64bc64bc74bc74bc84bc84bc94bc94bca4bca4bcb4bcb4bcc4bcc4bcd4bcd4bce4bce4bcf4bcf4bd04bd04bd14bd14bd24bd24bd34bd34bd44bd44bd54bd54bd64bd64bd74bd74bd84bd84bd94bd94bda4bda4bdb4bdb4bdc4bdc4bdd4bdd4bde4bde4bdf4bdf4be04be04be14be14be24be24be34be34be44be44be54be54be64be64be74be74be84be84be94be94bea4bea4beb4beb4bec4bec4bed4bed4bee4bee4bef4bef4bf04bf04bf14bf14bf24bf24bf34bf34bf44bf44bf54bf54bf64bf64bf74bf74bf84bf84bf94bf94bfa4bfa4bfb4bfb4bfc4bfc4bfd4bfd4bfe4bfe4bff4bff4d00014d00014d01014d01014d02014d02014d03014d03014d04014d04014d05014d05014d06014d06014d07014d07014d08014d08014d09014d09014d0a014d0a014d0b014d0b014d0c014d0c014d0d014d0d014d0e014d0e014d0f014d0f014d10014d10014d11014d11014d12014d12014d13014d13014d14014d14014d15014d15014d16014d16014d17014d17014d18014d18014d19014d19014d1a014d1a014d1b014d1b014d1c014d1c014d1d014d1d014d1e014d1e014d1f014d1f014d20014d20014d21014d21014d22014d22014d23014d23014d24014d24014d25014d25014d26014d26014d27014d27014d28014d28014d29014d29014d2a014d2a014d2b014d2b014d2c014d2c014d2d014d2d014d2e014d2e014d2f014d2f014d30014d30014d31014d31014d32014d32014d33014d33014d34014d34014d35014d35014d36014d36014d37014d37014d38014d38014d39014d39014d3a014d3a014d3b014d3b014d3c014d3c014d3d014d3d014d3e014d3e014d3f014d3f014d40014d40014d41014d41014d42014d42014d43014d43014d44014d44014d45014d45014d46014d46014d47014d47014d48014d48014d49014d49014d4a014d4a014d4b014d4b014d4c014d4c014d4d014d4d014d4e014d4e014d4f014d4f014d50014d50014d51014d51014d52014d52014d53014d53014d54014d54014d55014d55014d56014d56014d57014d57014d58014d58014d59014d59014d5a014d5a014d5b014d5b014d5c014d5c014d5d014d5d014d5e014d5e014d5f014d5f014d60014d60014d61014d61014d62014d62014d63014d63014d64014d64014d65014d65014d66014d66014d67014d67014d68014d68014d69014d69014d6a014d6a014d6b014d6b014d6c014d6c014d6d014d6d014d6e014d6e014d6f014d6f014d70014d70014d71014d71014d72014d72014d73014d73014d74014d74014d75014d75014d76014d76014d77014d77014d78014d78014d79014d79014d7a014d7a014d7b014d7b014d7c014d7c014d7d014d7d014d7e014d7e014d7f014d7f014d80014d80014d81014d81014d82014d82014d83014d83014d84014d84014d85014d85014d86014d86014d87014d87014d88014d88014d89014d89014d8a014d8a014d8b014d8b014d8c014d8c014d8d014d8d014d8e014d8e014d8f014d8f014d90014d90014d91014d91014d92014d92014d93014d93014d94014d94014d95014d95014d96014d96014d97014d97014d98014d98014d99014d99014d9a014d9a014d9b014d9b014d9c014d9c014d9d014d9d014d9e014d9e014d9f014d9f014da0014da0014da1014da1014da2014da2014da3014da3014da4014da4014da5014da5014da6014da6014da7014da7014da8014da8014da9014da9014daa014daa014dab014dab014dac014dac014dad014dad014dae014dae014daf014daf014db0014db0014db1014db1014db2014db2014db3014db3014db4014db4014db5014db5014db6014db6014db7014db7014db8014db8014db9014db9014dba014dba014dbb014dbb014dbc014dbc014dbd014dbd014dbe014dbe014dbf014dbf014dc0014dc0014dc1014dc1014dc2014dc2014dc3014dc3014dc4014dc4014dc5014dc5014dc6014dc6014dc7014dc7014dc8014dc8014dc9014dc9014dca014dca014dcb014dcb014dcc014dcc014dcd014dcd014dce014dce014dcf014dcf014dd0014dd0014dd1014dd1014dd2014dd2014dd3014dd3014dd4014dd4014dd5014dd5014dd6014dd6014dd7014dd7014dd8014dd8014dd9014dd9014dda014dda014ddb014ddb014ddc014ddc014ddd014ddd014dde014dde014ddf014ddf014de0014de0014de1014de1014de2014de2014de3014de3014de4014de4014de5014de5014de6014de6014de7014de7014de8014de8014de9014de9014dea014dea014deb014deb014dec014dec014ded014ded014dee014dee014def014def014df0014df0014df1014df1014df2014df2014df3014df3014df4014df4014df5014df5014df6014df6014df7014df7014df8014df8014df9014df9014dfa014dfa014dfb014dfb014dfc014dfc014dfd014dfd014dfe014dfe014dff014dff014d00024d00024d01024d01024d02024d02024d03024d03024d04024d04024d05024d05024d06024d06024d07024d07024d08024d08024d09024d09024d0a024d0a024d0b024d0b024d0c024d0c024d0d024d0d024d0e024d0e024d0f024d0f024d10024d10024d11024d11024d12024d12024d13024d13024d14024d14024d15024d15024d16024d16024d17024d17024d18024d18024d19024d19024d1a024d1a024d1b024d1b024d1c024d1c024d1d024d1d024d1e024d1e024d1f024d1f024d20024d20024d21024d21024d22024d22024d23024d23024d24024d24024d25024d25024d26024d26024d27024d27024d28024d28024d29024d29024d2a024d2a024d2b024d2b024d2c024d2c024d2d024d2d024d2e024d2e024d2f024d2f024d30024d30024d31024d31024d32024d32024d33024d33024d34024d34024d35024d35024d36024d36024d37024d37024d38024d38024d39024d39024d3a024d3a024d3b024d3b024d3c024d3c024d3d024d3d024d3e024d3e024d3f024d3f024d40024d40024d41024d41024d42024d42024d43024d43024d44024d44024d45024d45024d46024d46024d47024d47024d48024d48024d49024d49024d4a024d4a024d4b024d4b024d4c024d4c024d4d024d4d024d4e024d4e024d4f024d4f024d50024d50024d51024d51024d52024d52024d53024d53024d54024d54024d55024d55024d56024d56024d57024d57024d58024d58024d59024d59024d5a024d5a024d5b024d5b024d5c024d5c024d5d024d5d024d5e024d5e024d5f024d5f024d60024d60024d61024d61024d62024d62024d63024d63024d64024d64024d65024d65024d66024d66024d67024d67024d68024d68024d69024d69024d6a024d6a024d6b024d6b024d6c024d6c024d6d024d6d024d6e024d6e024d6f024d6f024d70024d70024d71024d71024d72024d72024d73024d73024d74024d74024d75024d75024d76024d76024d77024d77024d78024d78024d79024d79024d7a024d7a024d7b024d7b024d7c024d7c024d7d024d7d024d7e024d7e024d7f024d7f024d80024d80024d81024d81024d82024d82024d83024d83024d84024d84024d85024d85024d86024d86024d87024d87024d88024d88024d89024d89024d8a024d8a024d8b024d8b024d8c024d8c024d8d024d8d024d8e024d8e024d8f024d8f024d90024d90024d91024d91024d92024d92024d93024d93024d94024d94024d95024d95024d96024d96024d97024d97024d98024d98024d99024d99024d9a024d9a024d9b024d9b024d9c024d9c024d9d024d9d024d9e024d9e024d9f024d9f024da0024da0024da1024da1024da2024da2024da3024da3024da4024da4024da5024da5024da6024da6024da7024da7024da8024da8024da9024da9024daa024daa024dab024dab024dac024dac024dad024dad024dae024dae024daf024daf024db0024db0024db1024db1024db2024db2024db3024db3024db4024db4024db5024db5024db6024db6024db7024db7024db8024db8024db9024db9024dba024dba024dbb024dbb024dbc024dbc024dbd024dbd024dbe024dbe024dbf024dbf024dc0024dc0024dc1024dc1024dc2024dc2024dc3024dc3024dc4024dc4024dc5024dc5024dc6024dc6024dc7024dc7024dc8024dc8024dc9024dc9024dca024dca024dcb024dcb024dcc024dcc024dcd024dcd024dce024dce024dcf024dcf024dd0024dd0024dd1024dd1024dd2024dd2024dd3024dd3024dd4024dd4024dd5024dd5024dd6024dd6024dd7024dd7024dd8024dd8024dd9024dd9024dda024dda024ddb024ddb024ddc024ddc024ddd024ddd024dde024dde024ddf024ddf024de0024de0024de1024de1024de2024de2024de3024de3024de4024de4024de5024de5024de6024de6024de7024de7024de8024de8024de9024de9024dea024dea024deb024deb024dec024dec024ded024ded024dee024dee024def024def024df0024df0024df1024df1024df2024df2024df3024df3024df4024df4024df5024df5024df6024df6024df7024df7024df8024df8024df9024df9024dfa024dfa024dfb024dfb024dfc024dfc024dfd024dfd024dfe024dfe024dff024dff024d00034d00034d01034d01034d02034d02034d03034d03034d04034d04034d05034d05034d06034d06034d07034d07034d08034d08034d09034d09034d0a034d0a034d0b034d0b034d0c034d0c034d0d034d0d034d0e034d0e034d0f034d0f034d10034d10034d11034d11034d12034d12034d13034d13034d14034d14034d15034d15034d16034d16034d17034d17034d18034d18034d19034d19034d1a034d1a034d1b034d1b034d1c034d1c034d1d034d1d034d1e034d1e034d1f034d1f034d20034d20034d21034d21034d22034d22034d23034d23034d24034d24034d25034d25034d26034d26034d27034d27034d28034d28034d29034d29034d2a034d2a034d2b034d2b034d2c034d2c034d2d034d2d034d2e034d2e034d2f034d2f034d30034d30034d31034d31034d32034d32034d33034d33034d34034d34034d35034d35034d36034d36034d37034d37034d38034d38034d39034d39034d3a034d3a034d3b034d3b034d3c034d3c034d3d034d3d034d3e034d3e034d3f034d3f034d40034d40034d41034d41034d42034d42034d43034d43034d44034d44034d45034d45034d46034d46034d47034d47034d48034d48034d49034d49034d4a034d4a034d4b034d4b034d4c034d4c034d4d034d4d034d4e034d4e034d4f034d4f034d50034d50034d51034d51034d52034d52034d53034d53034d54034d54034d55034d55034d56034d56034d57034d57034d58034d58034d59034d59034d5a034d5a034d5b034d5b034d5c034d5c034d5d034d5d034d5e034d5e034d5f034d5f034d60034d60034d61034d61034d62034d62034d63034d63034d64034d64034d65034d65034d66034d66034d67034d67034d68034d68034d69034d69034d6a034d6a034d6b034d6b034d6c034d6c034d6d034d6d034d6e034d6e034d6f034d6f034d70034d70034d71034d71034d72034d72034d73034d73034d74034d74034d75034d75034d76034d76034d77034d77034d78034d78034d79034d79034d7a034d7a034d7b034d7b034d7c034d7c034d7d034d7d034d7e034d7e034d7f034d7f034d80034d80034d81034d81034d82034d82034d83034d83034d84034d84034d85034d85034d86034d86034d87034d87034d88034d88034d89034d89034d8a034d8a034d8b034d8b034d8c034d8c034d8d034d8d034d8e034d8e034d8f034d8f034d90034d90034d91034d91034d92034d92034d93034d93034d94034d94034d95034d95034d96034d96034d97034d97034d98034d98034d99034d99034d9a034d9a034d9b034d9b034d9c034d9c034d9d034d9d034d9e034d9e034d9f034d9f034da0034da0034da1034da1034da2034da2034da3034da3034da4034da4034da5034da5034da6034da6034da7034da7034da8034da8034da9034da9034daa034daa034dab034dab034dac034dac034dad034dad034dae034dae034daf034daf034db0034db0034db1034db1034db2034db2034db3034db3034db4034db4034db5034db5034db6034db6034db7034db7034db8034db8034db9034db9034dba034dba034dbb034dbb034dbc034dbc034dbd034dbd034dbe034dbe034dbf034dbf034dc0034dc0034dc1034dc1034dc2034dc2034dc3034dc3034dc4034dc4034dc5034dc5034dc6034dc6034dc7034dc7034dc8034dc8034dc9034dc9034dca034dca034dcb034dcb034dcc034dcc034dcd034dcd034dce034dce034dcf034dcf034dd0034dd0034dd1034dd1034dd2034dd2034dd3034dd3034dd4034dd4034dd5034dd5034dd6034dd6034dd7034dd7034dd8034dd8034dd9034dd9034dda034dda034ddb034ddb034ddc034ddc034ddd034ddd034dde034dde034ddf034ddf034de0034de0034de1034de1034de2034de2034de3034de3034de4034de4034de5034de5034de6034de6034de7034de70375284de8034de8034de9034de9034dea034dea034deb034deb034dec034dec034ded034ded034dee034dee034def034def034df0034df0034df1034df1034df2034df2034df3034df3034df4034df4034df5034df5034df6034df6034df7034df7034df8034df8034df9034df9034dfa034dfa034dfb034dfb034dfc034dfc034dfd034dfd034dfe034dfe034dff034dff034d00044d00044d01044d01044d02044d02044d03044d03044d04044d04044d05044d05044d06044d06044d07044d07044d08044d08044d09044d09044d0a044d0a044d0b044d0b044d0c044d0c044d0d044d0d044d0e044d0e044d0f044d0f044d10044d10044d11044d11044d12044d12044d13044d13044d14044d14044d15044d15044d16044d16044d17044d17044d18044d18044d19044d19044d1a044d1a044d1b044d1b044d1c044d1c044d1d044d1d044d1e044d1e044d1f044d1f044d20044d20044d21044d21044d22044d22044d23044d23044d24044d24044d25044d25044d26044d26044d27044d27044d28044d28044d29044d29044d2a044d2a044d2b044d2b044d2c044d2c044d2d044d2d044d2e044d2e044d2f044d2f044d30044d30044d31044d31044d32044d32044d33044d33044d34044d34044d35044d35044d36044d36044d37044d37044d38044d38044d39044d39044d3a044d3a044d3b044d3b044d3c044d3c044d3d044d3d044d3e044d3e044d3f044d3f044d40044d40044d41044d41044d42044d42044d43044d43044d44044d44044d45044d45044d46044d46044d47044d47044d48044d48044d49044d49044d4a044d4a044d4b044d4b0475652e +dicts 4 800495fe170000000000005d94287d947d948c0161944b01737d942868034b018c0162945d94284b024b03658c0163947d948c0164944e73757d94284b004b004b014b014b024b024b034b034b044b044b054b054b064b064b074b074b084b084b094b094b0a4b0a4b0b4b0b4b0c4b0c4b0d4b0d4b0e4b0e4b0f4b0f4b104b104b114b114b124b124b134b134b144b144b154b154b164b164b174b174b184b184b194b194b1a4b1a4b1b4b1b4b1c4b1c4b1d4b1d4b1e4b1e4b1f4b1f4b204b204b214b214b224b224b234b234b244b244b254b254b264b264b274b274b284b284b294b294b2a4b2a4b2b4b2b4b2c4b2c4b2d4b2d4b2e4b2e4b2f4b2f4b304b304b314b314b324b324b334b334b344b344b354b354b364b364b374b374b384b384b394b394b3a4b3a4b3b4b3b4b3c4b3c4b3d4b3d4b3e4b3e4b3f4b3f4b404b404b414b414b424b424b434b434b444b444b454b454b464b464b474b474b484b484b494b494b4a4b4a4b4b4b4b4b4c4b4c4b4d4b4d4b4e4b4e4b4f4b4f4b504b504b514b514b524b524b534b534b544b544b554b554b564b564b574b574b584b584b594b594b5a4b5a4b5b4b5b4b5c4b5c4b5d4b5d4b5e4b5e4b5f4b5f4b604b604b614b614b624b624b634b634b644b644b654b654b664b664b674b674b684b684b694b694b6a4b6a4b6b4b6b4b6c4b6c4b6d4b6d4b6e4b6e4b6f4b6f4b704b704b714b714b724b724b734b734b744b744b754b754b764b764b774b774b784b784b794b794b7a4b7a4b7b4b7b4b7c4b7c4b7d4b7d4b7e4b7e4b7f4b7f4b804b804b814b814b824b824b834b834b844b844b854b854b864b864b874b874b884b884b894b894b8a4b8a4b8b4b8b4b8c4b8c4b8d4b8d4b8e4b8e4b8f4b8f4b904b904b914b914b924b924b934b934b944b944b954b954b964b964b974b974b984b984b994b994b9a4b9a4b9b4b9b4b9c4b9c4b9d4b9d4b9e4b9e4b9f4b9f4ba04ba04ba14ba14ba24ba24ba34ba34ba44ba44ba54ba54ba64ba64ba74ba74ba84ba84ba94ba94baa4baa4bab4bab4bac4bac4bad4bad4bae4bae4baf4baf4bb04bb04bb14bb14bb24bb24bb34bb34bb44bb44bb54bb54bb64bb64bb74bb74bb84bb84bb94bb94bba4bba4bbb4bbb4bbc4bbc4bbd4bbd4bbe4bbe4bbf4bbf4bc04bc04bc14bc14bc24bc24bc34bc34bc44bc44bc54bc54bc64bc64bc74bc74bc84bc84bc94bc94bca4bca4bcb4bcb4bcc4bcc4bcd4bcd4bce4bce4bcf4bcf4bd04bd04bd14bd14bd24bd24bd34bd34bd44bd44bd54bd54bd64bd64bd74bd74bd84bd84bd94bd94bda4bda4bdb4bdb4bdc4bdc4bdd4bdd4bde4bde4bdf4bdf4be04be04be14be14be24be24be34be34be44be44be54be54be64be64be74be74be84be84be94be94bea4bea4beb4beb4bec4bec4bed4bed4bee4bee4bef4bef4bf04bf04bf14bf14bf24bf24bf34bf34bf44bf44bf54bf54bf64bf64bf74bf74bf84bf84bf94bf94bfa4bfa4bfb4bfb4bfc4bfc4bfd4bfd4bfe4bfe4bff4bff4d00014d00014d01014d01014d02014d02014d03014d03014d04014d04014d05014d05014d06014d06014d07014d07014d08014d08014d09014d09014d0a014d0a014d0b014d0b014d0c014d0c014d0d014d0d014d0e014d0e014d0f014d0f014d10014d10014d11014d11014d12014d12014d13014d13014d14014d14014d15014d15014d16014d16014d17014d17014d18014d18014d19014d19014d1a014d1a014d1b014d1b014d1c014d1c014d1d014d1d014d1e014d1e014d1f014d1f014d20014d20014d21014d21014d22014d22014d23014d23014d24014d24014d25014d25014d26014d26014d27014d27014d28014d28014d29014d29014d2a014d2a014d2b014d2b014d2c014d2c014d2d014d2d014d2e014d2e014d2f014d2f014d30014d30014d31014d31014d32014d32014d33014d33014d34014d34014d35014d35014d36014d36014d37014d37014d38014d38014d39014d39014d3a014d3a014d3b014d3b014d3c014d3c014d3d014d3d014d3e014d3e014d3f014d3f014d40014d40014d41014d41014d42014d42014d43014d43014d44014d44014d45014d45014d46014d46014d47014d47014d48014d48014d49014d49014d4a014d4a014d4b014d4b014d4c014d4c014d4d014d4d014d4e014d4e014d4f014d4f014d50014d50014d51014d51014d52014d52014d53014d53014d54014d54014d55014d55014d56014d56014d57014d57014d58014d58014d59014d59014d5a014d5a014d5b014d5b014d5c014d5c014d5d014d5d014d5e014d5e014d5f014d5f014d60014d60014d61014d61014d62014d62014d63014d63014d64014d64014d65014d65014d66014d66014d67014d67014d68014d68014d69014d69014d6a014d6a014d6b014d6b014d6c014d6c014d6d014d6d014d6e014d6e014d6f014d6f014d70014d70014d71014d71014d72014d72014d73014d73014d74014d74014d75014d75014d76014d76014d77014d77014d78014d78014d79014d79014d7a014d7a014d7b014d7b014d7c014d7c014d7d014d7d014d7e014d7e014d7f014d7f014d80014d80014d81014d81014d82014d82014d83014d83014d84014d84014d85014d85014d86014d86014d87014d87014d88014d88014d89014d89014d8a014d8a014d8b014d8b014d8c014d8c014d8d014d8d014d8e014d8e014d8f014d8f014d90014d90014d91014d91014d92014d92014d93014d93014d94014d94014d95014d95014d96014d96014d97014d97014d98014d98014d99014d99014d9a014d9a014d9b014d9b014d9c014d9c014d9d014d9d014d9e014d9e014d9f014d9f014da0014da0014da1014da1014da2014da2014da3014da3014da4014da4014da5014da5014da6014da6014da7014da7014da8014da8014da9014da9014daa014daa014dab014dab014dac014dac014dad014dad014dae014dae014daf014daf014db0014db0014db1014db1014db2014db2014db3014db3014db4014db4014db5014db5014db6014db6014db7014db7014db8014db8014db9014db9014dba014dba014dbb014dbb014dbc014dbc014dbd014dbd014dbe014dbe014dbf014dbf014dc0014dc0014dc1014dc1014dc2014dc2014dc3014dc3014dc4014dc4014dc5014dc5014dc6014dc6014dc7014dc7014dc8014dc8014dc9014dc9014dca014dca014dcb014dcb014dcc014dcc014dcd014dcd014dce014dce014dcf014dcf014dd0014dd0014dd1014dd1014dd2014dd2014dd3014dd3014dd4014dd4014dd5014dd5014dd6014dd6014dd7014dd7014dd8014dd8014dd9014dd9014dda014dda014ddb014ddb014ddc014ddc014ddd014ddd014dde014dde014ddf014ddf014de0014de0014de1014de1014de2014de2014de3014de3014de4014de4014de5014de5014de6014de6014de7014de7014de8014de8014de9014de9014dea014dea014deb014deb014dec014dec014ded014ded014dee014dee014def014def014df0014df0014df1014df1014df2014df2014df3014df3014df4014df4014df5014df5014df6014df6014df7014df7014df8014df8014df9014df9014dfa014dfa014dfb014dfb014dfc014dfc014dfd014dfd014dfe014dfe014dff014dff014d00024d00024d01024d01024d02024d02024d03024d03024d04024d04024d05024d05024d06024d06024d07024d07024d08024d08024d09024d09024d0a024d0a024d0b024d0b024d0c024d0c024d0d024d0d024d0e024d0e024d0f024d0f024d10024d10024d11024d11024d12024d12024d13024d13024d14024d14024d15024d15024d16024d16024d17024d17024d18024d18024d19024d19024d1a024d1a024d1b024d1b024d1c024d1c024d1d024d1d024d1e024d1e024d1f024d1f024d20024d20024d21024d21024d22024d22024d23024d23024d24024d24024d25024d25024d26024d26024d27024d27024d28024d28024d29024d29024d2a024d2a024d2b024d2b024d2c024d2c024d2d024d2d024d2e024d2e024d2f024d2f024d30024d30024d31024d31024d32024d32024d33024d33024d34024d34024d35024d35024d36024d36024d37024d37024d38024d38024d39024d39024d3a024d3a024d3b024d3b024d3c024d3c024d3d024d3d024d3e024d3e024d3f024d3f024d40024d40024d41024d41024d42024d42024d43024d43024d44024d44024d45024d45024d46024d46024d47024d47024d48024d48024d49024d49024d4a024d4a024d4b024d4b024d4c024d4c024d4d024d4d024d4e024d4e024d4f024d4f024d50024d50024d51024d51024d52024d52024d53024d53024d54024d54024d55024d55024d56024d56024d57024d57024d58024d58024d59024d59024d5a024d5a024d5b024d5b024d5c024d5c024d5d024d5d024d5e024d5e024d5f024d5f024d60024d60024d61024d61024d62024d62024d63024d63024d64024d64024d65024d65024d66024d66024d67024d67024d68024d68024d69024d69024d6a024d6a024d6b024d6b024d6c024d6c024d6d024d6d024d6e024d6e024d6f024d6f024d70024d70024d71024d71024d72024d72024d73024d73024d74024d74024d75024d75024d76024d76024d77024d77024d78024d78024d79024d79024d7a024d7a024d7b024d7b024d7c024d7c024d7d024d7d024d7e024d7e024d7f024d7f024d80024d80024d81024d81024d82024d82024d83024d83024d84024d84024d85024d85024d86024d86024d87024d87024d88024d88024d89024d89024d8a024d8a024d8b024d8b024d8c024d8c024d8d024d8d024d8e024d8e024d8f024d8f024d90024d90024d91024d91024d92024d92024d93024d93024d94024d94024d95024d95024d96024d96024d97024d97024d98024d98024d99024d99024d9a024d9a024d9b024d9b024d9c024d9c024d9d024d9d024d9e024d9e024d9f024d9f024da0024da0024da1024da1024da2024da2024da3024da3024da4024da4024da5024da5024da6024da6024da7024da7024da8024da8024da9024da9024daa024daa024dab024dab024dac024dac024dad024dad024dae024dae024daf024daf024db0024db0024db1024db1024db2024db2024db3024db3024db4024db4024db5024db5024db6024db6024db7024db7024db8024db8024db9024db9024dba024dba024dbb024dbb024dbc024dbc024dbd024dbd024dbe024dbe024dbf024dbf024dc0024dc0024dc1024dc1024dc2024dc2024dc3024dc3024dc4024dc4024dc5024dc5024dc6024dc6024dc7024dc7024dc8024dc8024dc9024dc9024dca024dca024dcb024dcb024dcc024dcc024dcd024dcd024dce024dce024dcf024dcf024dd0024dd0024dd1024dd1024dd2024dd2024dd3024dd3024dd4024dd4024dd5024dd5024dd6024dd6024dd7024dd7024dd8024dd8024dd9024dd9024dda024dda024ddb024ddb024ddc024ddc024ddd024ddd024dde024dde024ddf024ddf024de0024de0024de1024de1024de2024de2024de3024de3024de4024de4024de5024de5024de6024de6024de7024de7024de8024de8024de9024de9024dea024dea024deb024deb024dec024dec024ded024ded024dee024dee024def024def024df0024df0024df1024df1024df2024df2024df3024df3024df4024df4024df5024df5024df6024df6024df7024df7024df8024df8024df9024df9024dfa024dfa024dfb024dfb024dfc024dfc024dfd024dfd024dfe024dfe024dff024dff024d00034d00034d01034d01034d02034d02034d03034d03034d04034d04034d05034d05034d06034d06034d07034d07034d08034d08034d09034d09034d0a034d0a034d0b034d0b034d0c034d0c034d0d034d0d034d0e034d0e034d0f034d0f034d10034d10034d11034d11034d12034d12034d13034d13034d14034d14034d15034d15034d16034d16034d17034d17034d18034d18034d19034d19034d1a034d1a034d1b034d1b034d1c034d1c034d1d034d1d034d1e034d1e034d1f034d1f034d20034d20034d21034d21034d22034d22034d23034d23034d24034d24034d25034d25034d26034d26034d27034d27034d28034d28034d29034d29034d2a034d2a034d2b034d2b034d2c034d2c034d2d034d2d034d2e034d2e034d2f034d2f034d30034d30034d31034d31034d32034d32034d33034d33034d34034d34034d35034d35034d36034d36034d37034d37034d38034d38034d39034d39034d3a034d3a034d3b034d3b034d3c034d3c034d3d034d3d034d3e034d3e034d3f034d3f034d40034d40034d41034d41034d42034d42034d43034d43034d44034d44034d45034d45034d46034d46034d47034d47034d48034d48034d49034d49034d4a034d4a034d4b034d4b034d4c034d4c034d4d034d4d034d4e034d4e034d4f034d4f034d50034d50034d51034d51034d52034d52034d53034d53034d54034d54034d55034d55034d56034d56034d57034d57034d58034d58034d59034d59034d5a034d5a034d5b034d5b034d5c034d5c034d5d034d5d034d5e034d5e034d5f034d5f034d60034d60034d61034d61034d62034d62034d63034d63034d64034d64034d65034d65034d66034d66034d67034d67034d68034d68034d69034d69034d6a034d6a034d6b034d6b034d6c034d6c034d6d034d6d034d6e034d6e034d6f034d6f034d70034d70034d71034d71034d72034d72034d73034d73034d74034d74034d75034d75034d76034d76034d77034d77034d78034d78034d79034d79034d7a034d7a034d7b034d7b034d7c034d7c034d7d034d7d034d7e034d7e034d7f034d7f034d80034d80034d81034d81034d82034d82034d83034d83034d84034d84034d85034d85034d86034d86034d87034d87034d88034d88034d89034d89034d8a034d8a034d8b034d8b034d8c034d8c034d8d034d8d034d8e034d8e034d8f034d8f034d90034d90034d91034d91034d92034d92034d93034d93034d94034d94034d95034d95034d96034d96034d97034d97034d98034d98034d99034d99034d9a034d9a034d9b034d9b034d9c034d9c034d9d034d9d034d9e034d9e034d9f034d9f034da0034da0034da1034da1034da2034da2034da3034da3034da4034da4034da5034da5034da6034da6034da7034da7034da8034da8034da9034da9034daa034daa034dab034dab034dac034dac034dad034dad034dae034dae034daf034daf034db0034db0034db1034db1034db2034db2034db3034db3034db4034db4034db5034db5034db6034db6034db7034db7034db8034db8034db9034db9034dba034dba034dbb034dbb034dbc034dbc034dbd034dbd034dbe034dbe034dbf034dbf034dc0034dc0034dc1034dc1034dc2034dc2034dc3034dc3034dc4034dc4034dc5034dc5034dc6034dc6034dc7034dc7034dc8034dc8034dc9034dc9034dca034dca034dcb034dcb034dcc034dcc034dcd034dcd034dce034dce034dcf034dcf034dd0034dd0034dd1034dd1034dd2034dd2034dd3034dd3034dd4034dd4034dd5034dd5034dd6034dd6034dd7034dd7034dd8034dd8034dd9034dd9034dda034dda034ddb034ddb034ddc034ddc034ddd034ddd034dde034dde034ddf034ddf034de0034de0034de1034de1034de2034de2034de3034de3034de4034de4034de5034de5034de6034de6034de7034de70375284de8034de8034de9034de9034dea034dea034deb034deb034dec034dec034ded034ded034dee034dee034def034def034df0034df0034df1034df1034df2034df2034df3034df3034df4034df4034df5034df5034df6034df6034df7034df7034df8034df8034df9034df9034dfa034dfa034dfb034dfb034dfc034dfc034dfd034dfd034dfe034dfe034dff034dff034d00044d00044d01044d01044d02044d02044d03044d03044d04044d04044d05044d05044d06044d06044d07044d07044d08044d08044d09044d09044d0a044d0a044d0b044d0b044d0c044d0c044d0d044d0d044d0e044d0e044d0f044d0f044d10044d10044d11044d11044d12044d12044d13044d13044d14044d14044d15044d15044d16044d16044d17044d17044d18044d18044d19044d19044d1a044d1a044d1b044d1b044d1c044d1c044d1d044d1d044d1e044d1e044d1f044d1f044d20044d20044d21044d21044d22044d22044d23044d23044d24044d24044d25044d25044d26044d26044d27044d27044d28044d28044d29044d29044d2a044d2a044d2b044d2b044d2c044d2c044d2d044d2d044d2e044d2e044d2f044d2f044d30044d30044d31044d31044d32044d32044d33044d33044d34044d34044d35044d35044d36044d36044d37044d37044d38044d38044d39044d39044d3a044d3a044d3b044d3b044d3c044d3c044d3d044d3d044d3e044d3e044d3f044d3f044d40044d40044d41044d41044d42044d42044d43044d43044d44044d44044d45044d45044d46044d46044d47044d47044d48044d48044d49044d49044d4a044d4a044d4b044d4b0475652e +dicts 5 800595fe170000000000005d94287d947d948c0161944b01737d942868034b018c0162945d94284b024b03658c0163947d948c0164944e73757d94284b004b004b014b014b024b024b034b034b044b044b054b054b064b064b074b074b084b084b094b094b0a4b0a4b0b4b0b4b0c4b0c4b0d4b0d4b0e4b0e4b0f4b0f4b104b104b114b114b124b124b134b134b144b144b154b154b164b164b174b174b184b184b194b194b1a4b1a4b1b4b1b4b1c4b1c4b1d4b1d4b1e4b1e4b1f4b1f4b204b204b214b214b224b224b234b234b244b244b254b254b264b264b274b274b284b284b294b294b2a4b2a4b2b4b2b4b2c4b2c4b2d4b2d4b2e4b2e4b2f4b2f4b304b304b314b314b324b324b334b334b344b344b354b354b364b364b374b374b384b384b394b394b3a4b3a4b3b4b3b4b3c4b3c4b3d4b3d4b3e4b3e4b3f4b3f4b404b404b414b414b424b424b434b434b444b444b454b454b464b464b474b474b484b484b494b494b4a4b4a4b4b4b4b4b4c4b4c4b4d4b4d4b4e4b4e4b4f4b4f4b504b504b514b514b524b524b534b534b544b544b554b554b564b564b574b574b584b584b594b594b5a4b5a4b5b4b5b4b5c4b5c4b5d4b5d4b5e4b5e4b5f4b5f4b604b604b614b614b624b624b634b634b644b644b654b654b664b664b674b674b684b684b694b694b6a4b6a4b6b4b6b4b6c4b6c4b6d4b6d4b6e4b6e4b6f4b6f4b704b704b714b714b724b724b734b734b744b744b754b754b764b764b774b774b784b784b794b794b7a4b7a4b7b4b7b4b7c4b7c4b7d4b7d4b7e4b7e4b7f4b7f4b804b804b814b814b824b824b834b834b844b844b854b854b864b864b874b874b884b884b894b894b8a4b8a4b8b4b8b4b8c4b8c4b8d4b8d4b8e4b8e4b8f4b8f4b904b904b914b914b924b924b934b934b944b944b954b954b964b964b974b974b984b984b994b994b9a4b9a4b9b4b9b4b9c4b9c4b9d4b9d4b9e4b9e4b9f4b9f4ba04ba04ba14ba14ba24ba24ba34ba34ba44ba44ba54ba54ba64ba64ba74ba74ba84ba84ba94ba94baa4baa4bab4bab4bac4bac4bad4bad4bae4bae4baf4baf4bb04bb04bb14bb14bb24bb24bb34bb34bb44bb44bb54bb54bb64bb64bb74bb74bb84bb84bb94bb94bba4bba4bbb4bbb4bbc4bbc4bbd4bbd4bbe4bbe4bbf4bbf4bc04bc04bc14bc14bc24bc24bc34bc34bc44bc44bc54bc54bc64bc64bc74bc74bc84bc84bc94bc94bca4bca4bcb4bcb4bcc4bcc4bcd4bcd4bce4bce4bcf4bcf4bd04bd04bd14bd14bd24bd24bd34bd34bd44bd44bd54bd54bd64bd64bd74bd74bd84bd84bd94bd94bda4bda4bdb4bdb4bdc4bdc4bdd4bdd4bde4bde4bdf4bdf4be04be04be14be14be24be24be34be34be44be44be54be54be64be64be74be74be84be84be94be94bea4bea4beb4beb4bec4bec4bed4bed4bee4bee4bef4bef4bf04bf04bf14bf14bf24bf24bf34bf34bf44bf44bf54bf54bf64bf64bf74bf74bf84bf84bf94bf94bfa4bfa4bfb4bfb4bfc4bfc4bfd4bfd4bfe4bfe4bff4bff4d00014d00014d01014d01014d02014d02014d03014d03014d04014d04014d05014d05014d06014d06014d07014d07014d08014d08014d09014d09014d0a014d0a014d0b014d0b014d0c014d0c014d0d014d0d014d0e014d0e014d0f014d0f014d10014d10014d11014d11014d12014d12014d13014d13014d14014d14014d15014d15014d16014d16014d17014d17014d18014d18014d19014d19014d1a014d1a014d1b014d1b014d1c014d1c014d1d014d1d014d1e014d1e014d1f014d1f014d20014d20014d21014d21014d22014d22014d23014d23014d24014d24014d25014d25014d26014d26014d27014d27014d28014d28014d29014d29014d2a014d2a014d2b014d2b014d2c014d2c014d2d014d2d014d2e014d2e014d2f014d2f014d30014d30014d31014d31014d32014d32014d33014d33014d34014d34014d35014d35014d36014d36014d37014d37014d38014d38014d39014d39014d3a014d3a014d3b014d3b014d3c014d3c014d3d014d3d014d3e014d3e014d3f014d3f014d40014d40014d41014d41014d42014d42014d43014d43014d44014d44014d45014d45014d46014d46014d47014d47014d48014d48014d49014d49014d4a014d4a014d4b014d4b014d4c014d4c014d4d014d4d014d4e014d4e014d4f014d4f014d50014d50014d51014d51014d52014d52014d53014d53014d54014d54014d55014d55014d56014d56014d57014d57014d58014d58014d59014d59014d5a014d5a014d5b014d5b014d5c014d5c014d5d014d5d014d5e014d5e014d5f014d5f014d60014d60014d61014d61014d62014d62014d63014d63014d64014d64014d65014d65014d66014d66014d67014d67014d68014d68014d69014d69014d6a014d6a014d6b014d6b014d6c014d6c014d6d014d6d014d6e014d6e014d6f014d6f014d70014d70014d71014d71014d72014d72014d73014d73014d74014d74014d75014d75014d76014d76014d77014d77014d78014d78014d79014d79014d7a014d7a014d7b014d7b014d7c014d7c014d7d014d7d014d7e014d7e014d7f014d7f014d80014d80014d81014d81014d82014d82014d83014d83014d84014d84014d85014d85014d86014d86014d87014d87014d88014d88014d89014d89014d8a014d8a014d8b014d8b014d8c014d8c014d8d014d8d014d8e014d8e014d8f014d8f014d90014d90014d91014d91014d92014d92014d93014d93014d94014d94014d95014d95014d96014d96014d97014d97014d98014d98014d99014d99014d9a014d9a014d9b014d9b014d9c014d9c014d9d014d9d014d9e014d9e014d9f014d9f014da0014da0014da1014da1014da2014da2014da3014da3014da4014da4014da5014da5014da6014da6014da7014da7014da8014da8014da9014da9014daa014daa014dab014dab014dac014dac014dad014dad014dae014dae014daf014daf014db0014db0014db1014db1014db2014db2014db3014db3014db4014db4014db5014db5014db6014db6014db7014db7014db8014db8014db9014db9014dba014dba014dbb014dbb014dbc014dbc014dbd014dbd014dbe014dbe014dbf014dbf014dc0014dc0014dc1014dc1014dc2014dc2014dc3014dc3014dc4014dc4014dc5014dc5014dc6014dc6014dc7014dc7014dc8014dc8014dc9014dc9014dca014dca014dcb014dcb014dcc014dcc014dcd014dcd014dce014dce014dcf014dcf014dd0014dd0014dd1014dd1014dd2014dd2014dd3014dd3014dd4014dd4014dd5014dd5014dd6014dd6014dd7014dd7014dd8014dd8014dd9014dd9014dda014dda014ddb014ddb014ddc014ddc014ddd014ddd014dde014dde014ddf014ddf014de0014de0014de1014de1014de2014de2014de3014de3014de4014de4014de5014de5014de6014de6014de7014de7014de8014de8014de9014de9014dea014dea014deb014deb014dec014dec014ded014ded014dee014dee014def014def014df0014df0014df1014df1014df2014df2014df3014df3014df4014df4014df5014df5014df6014df6014df7014df7014df8014df8014df9014df9014dfa014dfa014dfb014dfb014dfc014dfc014dfd014dfd014dfe014dfe014dff014dff014d00024d00024d01024d01024d02024d02024d03024d03024d04024d04024d05024d05024d06024d06024d07024d07024d08024d08024d09024d09024d0a024d0a024d0b024d0b024d0c024d0c024d0d024d0d024d0e024d0e024d0f024d0f024d10024d10024d11024d11024d12024d12024d13024d13024d14024d14024d15024d15024d16024d16024d17024d17024d18024d18024d19024d19024d1a024d1a024d1b024d1b024d1c024d1c024d1d024d1d024d1e024d1e024d1f024d1f024d20024d20024d21024d21024d22024d22024d23024d23024d24024d24024d25024d25024d26024d26024d27024d27024d28024d28024d29024d29024d2a024d2a024d2b024d2b024d2c024d2c024d2d024d2d024d2e024d2e024d2f024d2f024d30024d30024d31024d31024d32024d32024d33024d33024d34024d34024d35024d35024d36024d36024d37024d37024d38024d38024d39024d39024d3a024d3a024d3b024d3b024d3c024d3c024d3d024d3d024d3e024d3e024d3f024d3f024d40024d40024d41024d41024d42024d42024d43024d43024d44024d44024d45024d45024d46024d46024d47024d47024d48024d48024d49024d49024d4a024d4a024d4b024d4b024d4c024d4c024d4d024d4d024d4e024d4e024d4f024d4f024d50024d50024d51024d51024d52024d52024d53024d53024d54024d54024d55024d55024d56024d56024d57024d57024d58024d58024d59024d59024d5a024d5a024d5b024d5b024d5c024d5c024d5d024d5d024d5e024d5e024d5f024d5f024d60024d60024d61024d61024d62024d62024d63024d63024d64024d64024d65024d65024d66024d66024d67024d67024d68024d68024d69024d69024d6a024d6a024d6b024d6b024d6c024d6c024d6d024d6d024d6e024d6e024d6f024d6f024d70024d70024d71024d71024d72024d72024d73024d73024d74024d74024d75024d75024d76024d76024d77024d77024d78024d78024d79024d79024d7a024d7a024d7b024d7b024d7c024d7c024d7d024d7d024d7e024d7e024d7f024d7f024d80024d80024d81024d81024d82024d82024d83024d83024d84024d84024d85024d85024d86024d86024d87024d87024d88024d88024d89024d89024d8a024d8a024d8b024d8b024d8c024d8c024d8d024d8d024d8e024d8e024d8f024d8f024d90024d90024d91024d91024d92024d92024d93024d93024d94024d94024d95024d95024d96024d96024d97024d97024d98024d98024d99024d99024d9a024d9a024d9b024d9b024d9c024d9c024d9d024d9d024d9e024d9e024d9f024d9f024da0024da0024da1024da1024da2024da2024da3024da3024da4024da4024da5024da5024da6024da6024da7024da7024da8024da8024da9024da9024daa024daa024dab024dab024dac024dac024dad024dad024dae024dae024daf024daf024db0024db0024db1024db1024db2024db2024db3024db3024db4024db4024db5024db5024db6024db6024db7024db7024db8024db8024db9024db9024dba024dba024dbb024dbb024dbc024dbc024dbd024dbd024dbe024dbe024dbf024dbf024dc0024dc0024dc1024dc1024dc2024dc2024dc3024dc3024dc4024dc4024dc5024dc5024dc6024dc6024dc7024dc7024dc8024dc8024dc9024dc9024dca024dca024dcb024dcb024dcc024dcc024dcd024dcd024dce024dce024dcf024dcf024dd0024dd0024dd1024dd1024dd2024dd2024dd3024dd3024dd4024dd4024dd5024dd5024dd6024dd6024dd7024dd7024dd8024dd8024dd9024dd9024dda024dda024ddb024ddb024ddc024ddc024ddd024ddd024dde024dde024ddf024ddf024de0024de0024de1024de1024de2024de2024de3024de3024de4024de4024de5024de5024de6024de6024de7024de7024de8024de8024de9024de9024dea024dea024deb024deb024dec024dec024ded024ded024dee024dee024def024def024df0024df0024df1024df1024df2024df2024df3024df3024df4024df4024df5024df5024df6024df6024df7024df7024df8024df8024df9024df9024dfa024dfa024dfb024dfb024dfc024dfc024dfd024dfd024dfe024dfe024dff024dff024d00034d00034d01034d01034d02034d02034d03034d03034d04034d04034d05034d05034d06034d06034d07034d07034d08034d08034d09034d09034d0a034d0a034d0b034d0b034d0c034d0c034d0d034d0d034d0e034d0e034d0f034d0f034d10034d10034d11034d11034d12034d12034d13034d13034d14034d14034d15034d15034d16034d16034d17034d17034d18034d18034d19034d19034d1a034d1a034d1b034d1b034d1c034d1c034d1d034d1d034d1e034d1e034d1f034d1f034d20034d20034d21034d21034d22034d22034d23034d23034d24034d24034d25034d25034d26034d26034d27034d27034d28034d28034d29034d29034d2a034d2a034d2b034d2b034d2c034d2c034d2d034d2d034d2e034d2e034d2f034d2f034d30034d30034d31034d31034d32034d32034d33034d33034d34034d34034d35034d35034d36034d36034d37034d37034d38034d38034d39034d39034d3a034d3a034d3b034d3b034d3c034d3c034d3d034d3d034d3e034d3e034d3f034d3f034d40034d40034d41034d41034d42034d42034d43034d43034d44034d44034d45034d45034d46034d46034d47034d47034d48034d48034d49034d49034d4a034d4a034d4b034d4b034d4c034d4c034d4d034d4d034d4e034d4e034d4f034d4f034d50034d50034d51034d51034d52034d52034d53034d53034d54034d54034d55034d55034d56034d56034d57034d57034d58034d58034d59034d59034d5a034d5a034d5b034d5b034d5c034d5c034d5d034d5d034d5e034d5e034d5f034d5f034d60034d60034d61034d61034d62034d62034d63034d63034d64034d64034d65034d65034d66034d66034d67034d67034d68034d68034d69034d69034d6a034d6a034d6b034d6b034d6c034d6c034d6d034d6d034d6e034d6e034d6f034d6f034d70034d70034d71034d71034d72034d72034d73034d73034d74034d74034d75034d75034d76034d76034d77034d77034d78034d78034d79034d79034d7a034d7a034d7b034d7b034d7c034d7c034d7d034d7d034d7e034d7e034d7f034d7f034d80034d80034d81034d81034d82034d82034d83034d83034d84034d84034d85034d85034d86034d86034d87034d87034d88034d88034d89034d89034d8a034d8a034d8b034d8b034d8c034d8c034d8d034d8d034d8e034d8e034d8f034d8f034d90034d90034d91034d91034d92034d92034d93034d93034d94034d94034d95034d95034d96034d96034d97034d97034d98034d98034d99034d99034d9a034d9a034d9b034d9b034d9c034d9c034d9d034d9d034d9e034d9e034d9f034d9f034da0034da0034da1034da1034da2034da2034da3034da3034da4034da4034da5034da5034da6034da6034da7034da7034da8034da8034da9034da9034daa034daa034dab034dab034dac034dac034dad034dad034dae034dae034daf034daf034db0034db0034db1034db1034db2034db2034db3034db3034db4034db4034db5034db5034db6034db6034db7034db7034db8034db8034db9034db9034dba034dba034dbb034dbb034dbc034dbc034dbd034dbd034dbe034dbe034dbf034dbf034dc0034dc0034dc1034dc1034dc2034dc2034dc3034dc3034dc4034dc4034dc5034dc5034dc6034dc6034dc7034dc7034dc8034dc8034dc9034dc9034dca034dca034dcb034dcb034dcc034dcc034dcd034dcd034dce034dce034dcf034dcf034dd0034dd0034dd1034dd1034dd2034dd2034dd3034dd3034dd4034dd4034dd5034dd5034dd6034dd6034dd7034dd7034dd8034dd8034dd9034dd9034dda034dda034ddb034ddb034ddc034ddc034ddd034ddd034dde034dde034ddf034ddf034de0034de0034de1034de1034de2034de2034de3034de3034de4034de4034de5034de5034de6034de6034de7034de70375284de8034de8034de9034de9034dea034dea034deb034deb034dec034dec034ded034ded034dee034dee034def034def034df0034df0034df1034df1034df2034df2034df3034df3034df4034df4034df5034df5034df6034df6034df7034df7034df8034df8034df9034df9034dfa034dfa034dfb034dfb034dfc034dfc034dfd034dfd034dfe034dfe034dff034dff034d00044d00044d01044d01044d02044d02044d03044d03044d04044d04044d05044d05044d06044d06044d07044d07044d08044d08044d09044d09044d0a044d0a044d0b044d0b044d0c044d0c044d0d044d0d044d0e044d0e044d0f044d0f044d10044d10044d11044d11044d12044d12044d13044d13044d14044d14044d15044d15044d16044d16044d17044d17044d18044d18044d19044d19044d1a044d1a044d1b044d1b044d1c044d1c044d1d044d1d044d1e044d1e044d1f044d1f044d20044d20044d21044d21044d22044d22044d23044d23044d24044d24044d25044d25044d26044d26044d27044d27044d28044d28044d29044d29044d2a044d2a044d2b044d2b044d2c044d2c044d2d044d2d044d2e044d2e044d2f044d2f044d30044d30044d31044d31044d32044d32044d33044d33044d34044d34044d35044d35044d36044d36044d37044d37044d38044d38044d39044d39044d3a044d3a044d3b044d3b044d3c044d3c044d3d044d3d044d3e044d3e044d3f044d3f044d40044d40044d41044d41044d42044d42044d43044d43044d44044d44044d45044d45044d46044d46044d47044d47044d48044d48044d49044d49044d4a044d4a044d4b044d4b0475652e +sets 0 286c70300a635f5f6275696c74696e5f5f0a7365740a70310a28286c70320a7470330a5270340a6167310a28286c70350a49310a617470360a5270370a6167310a28286c70380a49310a6149320a6149330a617470390a527031300a61635f5f6275696c74696e5f5f0a66726f7a656e7365740a7031310a28286c7031320a747031330a527031340a616731310a28286c7031350a49310a6149320a6149330a61747031360a527031370a612e +sets 1 5d710028635f5f6275696c74696e5f5f0a7365740a7101285d71027471035271046801285d71054b01617471065271076801285d7108284b014b024b036574710952710a635f5f6275696c74696e5f5f0a66726f7a656e7365740a710b285d710c74710d52710e680b285d710f284b014b024b0365747110527111652e +sets 2 80025d710028635f5f6275696c74696e5f5f0a7365740a71015d710285710352710468015d71054b016185710652710768015d7108284b014b024b036585710952710a635f5f6275696c74696e5f5f0a66726f7a656e7365740a710b5d710c85710d52710e680b5d710f284b014b024b0365857110527111652e +sets 3 80035d710028636275696c74696e730a7365740a71015d710285710352710468015d71054b016185710652710768015d7108284b014b024b036585710952710a636275696c74696e730a66726f7a656e7365740a710b5d710c85710d52710e680b5d710f284b014b024b0365857110527111652e +sets 4 80049523000000000000005d94288f948f94284b01908f94284b014b024b0390289194284b014b024b039194652e +sets 5 80059523000000000000005d94288f948f94284b01908f94284b014b024b0390289194284b014b024b039194652e +complex 0 635f5f6275696c74696e5f5f0a636f6d706c65780a70300a2846312e300a46322e300a7470310a5270320a2e +complex 1 635f5f6275696c74696e5f5f0a636f6d706c65780a710028473ff00000000000004740000000000000007471015271022e +complex 2 8002635f5f6275696c74696e5f5f0a636f6d706c65780a7100473ff00000000000004740000000000000008671015271022e +complex 3 8003636275696c74696e730a636f6d706c65780a7100473ff00000000000004740000000000000008671015271022e +complex 4 8004952e000000000000008c086275696c74696e73948c07636f6d706c6578949394473ff0000000000000474000000000000000869452942e +complex 5 8005952e000000000000008c086275696c74696e73948c07636f6d706c6578949394473ff0000000000000474000000000000000869452942e +range 0 635f5f6275696c74696e5f5f0a7872616e67650a70300a2849310a4931300a49320a7470310a5270320a2e +range 1 635f5f6275696c74696e5f5f0a7872616e67650a7100284b014b0a4b027471015271022e +range 2 8002635f5f6275696c74696e5f5f0a7872616e67650a71004b014b0a4b028771015271022e +range 3 8003636275696c74696e730a72616e67650a71004b014b0a4b028771015271022e +range 4 80049520000000000000008c086275696c74696e73948c0572616e67659493944b014b0a4b02879452942e +range 5 80059520000000000000008c086275696c74696e73948c0572616e67659493944b014b0a4b02879452942e +type 0 286c70300a635f5f6275696c74696e5f5f0a6c6f6e670a70310a61635f5f6275696c74696e5f5f0a756e69636f64650a70320a61635f5f6275696c74696e5f5f0a646963740a70330a612e +type 1 5d710028635f5f6275696c74696e5f5f0a6c6f6e670a7101635f5f6275696c74696e5f5f0a756e69636f64650a7102635f5f6275696c74696e5f5f0a646963740a7103652e +type 2 80025d710028635f5f6275696c74696e5f5f0a6c6f6e670a7101635f5f6275696c74696e5f5f0a756e69636f64650a7102635f5f6275696c74696e5f5f0a646963740a7103652e +type 3 80035d710028636275696c74696e730a696e740a7101636275696c74696e730a7374720a7102636275696c74696e730a646963740a7103652e +type 4 8004952d000000000000005d94288c086275696c74696e73948c03696e7494939468018c0373747294939468018c0464696374949394652e +type 5 8005952d000000000000005d94288c086275696c74696e73948c03696e7494939468018c0373747294939468018c0464696374949394652e +shared 0 286c70300a286c70310a49310a6149320a616167310a612867310a67310a7470320a612e +shared 1 5d7100285d7101284b014b026568012868016801747102652e +shared 2 80025d7100285d7101284b014b0265680168016801867102652e +shared 3 80035d7100285d7101284b014b0265680168016801867102652e +shared 4 80049515000000000000005d94285d94284b014b02656801680168018694652e +shared 5 80059515000000000000005d94285d94284b014b02656801680168018694652e +recursive 0 286c70300a67300a612e +recursive 1 5d71006800612e +recursive 2 80025d71006800612e +recursive 3 80035d71006800612e +recursive 4 80049506000000000000005d946800612e +recursive 5 80059506000000000000005d946800612e +recdict 0 286470300a5673656c660a70310a67300a732e +recdict 1 7d7100580400000073656c6671016800732e +recdict 2 80027d7100580400000073656c6671016800732e +recdict 3 80037d7100580400000073656c6671016800732e +recdict 4 8004950d000000000000007d948c0473656c66946800732e +recdict 5 8005950d000000000000007d948c0473656c66946800732e +strmemo 0 286c70300a28566162630a70310a67310a7470320a6167320a6167310a612e +strmemo 1 5d7100282858030000006162637101680174710268026801652e +strmemo 2 80025d71002858030000006162637101680186710268026801652e +strmemo 3 80035d71002858030000006162637101680186710268026801652e +strmemo 4 80049513000000000000005d94288c03616263946801869468026801652e +strmemo 5 80059513000000000000005d94288c03616263946801869468026801652e +plain 0 63636f70795f7265670a5f7265636f6e7374727563746f720a70300a28635f5f6d61696e5f5f0a506c61696e0a70310a635f5f6275696c74696e5f5f0a6f626a6563740a70320a4e7470330a5270340a286470350a56610a70360a49310a7356620a70370a56730a70380a73622e +plain 1 63636f70795f7265670a5f7265636f6e7374727563746f720a710028635f5f6d61696e5f5f0a506c61696e0a7101635f5f6275696c74696e5f5f0a6f626a6563740a71024e7471035271047d71052858010000006171064b015801000000627107580100000073710875622e +plain 2 8002635f5f6d61696e5f5f0a506c61696e0a7100298171017d71022858010000006171034b015801000000627104580100000073710575622e +plain 3 8003635f5f6d61696e5f5f0a506c61696e0a7100298171017d71022858010000006171034b015801000000627104580100000073710575622e +plain 4 8004952c000000000000008c085f5f6d61696e5f5f948c05506c61696e9493942981947d94288c0161944b018c0162948c01739475622e +plain 5 8005952c000000000000008c085f5f6d61696e5f5f948c05506c61696e9493942981947d94288c0161944b018c0162948c01739475622e +empty 0 63636f70795f7265670a5f7265636f6e7374727563746f720a70300a28635f5f6d61696e5f5f0a456d7074790a70310a635f5f6275696c74696e5f5f0a6f626a6563740a70320a4e7470330a5270340a2e +empty 1 63636f70795f7265670a5f7265636f6e7374727563746f720a710028635f5f6d61696e5f5f0a456d7074790a7101635f5f6275696c74696e5f5f0a6f626a6563740a71024e7471035271042e +empty 2 8002635f5f6d61696e5f5f0a456d7074790a7100298171012e +empty 3 8003635f5f6d61696e5f5f0a456d7074790a7100298171012e +empty 4 80049519000000000000008c085f5f6d61696e5f5f948c05456d7074799493942981942e +empty 5 80059519000000000000008c085f5f6d61696e5f5f948c05456d7074799493942981942e +state 0 63636f70795f7265670a5f7265636f6e7374727563746f720a70300a28635f5f6d61696e5f5f0a5769746853746174650a70310a635f5f6275696c74696e5f5f0a6f626a6563740a70320a4e7470330a5270340a286470350a56780a70360a49310a73622e +state 1 63636f70795f7265670a5f7265636f6e7374727563746f720a710028635f5f6d61696e5f5f0a5769746853746174650a7101635f5f6275696c74696e5f5f0a6f626a6563740a71024e7471035271047d710558010000007871064b0173622e +state 2 8002635f5f6d61696e5f5f0a5769746853746174650a7100298171017d710258010000007871034b0173622e +state 3 8003635f5f6d61696e5f5f0a5769746853746174650a7100298171017d710258010000007871034b0173622e +state 4 80049527000000000000008c085f5f6d61696e5f5f948c095769746853746174659493942981947d948c0178944b0173622e +state 5 80059527000000000000008c085f5f6d61696e5f5f948c095769746853746174659493942981947d948c0178944b0173622e +newargs 0 63636f70795f7265670a5f7265636f6e7374727563746f720a70300a28635f5f6d61696e5f5f0a4e6577417267730a70310a635f5f6275696c74696e5f5f0a6f626a6563740a70320a4e7470330a5270340a286470350a56610a70360a49310a7356620a70370a49320a73622e +newargs 1 63636f70795f7265670a5f7265636f6e7374727563746f720a710028635f5f6d61696e5f5f0a4e6577417267730a7101635f5f6275696c74696e5f5f0a6f626a6563740a71024e7471035271047d71052858010000006171064b0158010000006271074b0275622e +newargs 2 8002635f5f6d61696e5f5f0a4e6577417267730a71004b014b028671018171027d71032858010000006171044b0158010000006271054b0275622e +newargs 3 8003635f5f6d61696e5f5f0a4e6577417267730a71004b014b028671018171027d71032858010000006171044b0158010000006271054b0275622e +newargs 4 80049531000000000000008c085f5f6d61696e5f5f948c074e6577417267739493944b014b02869481947d94288c0161944b018c0162944b0275622e +newargs 5 80059531000000000000008c085f5f6d61696e5f5f948c074e6577417267739493944b014b02869481947d94288c0161944b018c0162944b0275622e +reduced 0 635f5f6d61696e5f5f0a526564756365640a70300a2849350a7470310a5270320a2e +reduced 1 635f5f6d61696e5f5f0a526564756365640a7100284b057471015271022e +reduced 2 8002635f5f6d61696e5f5f0a526564756365640a71004b058571015271022e +reduced 3 8003635f5f6d61696e5f5f0a526564756365640a71004b058571015271022e +reduced 4 8004951e000000000000008c085f5f6d61696e5f5f948c07526564756365649493944b05859452942e +reduced 5 8005951e000000000000008c085f5f6d61696e5f5f948c07526564756365649493944b05859452942e +dataclass 0 63636f70795f7265670a5f7265636f6e7374727563746f720a70300a28635f5f6d61696e5f5f0a506f696e740a70310a635f5f6275696c74696e5f5f0a6f626a6563740a70320a4e7470330a5270340a286470350a56780a70360a49330a7356790a70370a49320a73622e +dataclass 1 63636f70795f7265670a5f7265636f6e7374727563746f720a710028635f5f6d61696e5f5f0a506f696e740a7101635f5f6275696c74696e5f5f0a6f626a6563740a71024e7471035271047d71052858010000007871064b0358010000007971074b0275622e +dataclass 2 8002635f5f6d61696e5f5f0a506f696e740a7100298171017d71022858010000007871034b0358010000007971044b0275622e +dataclass 3 8003635f5f6d61696e5f5f0a506f696e740a7100298171017d71022858010000007871034b0358010000007971044b0275622e +dataclass 4 8004952a000000000000008c085f5f6d61696e5f5f948c05506f696e749493942981947d94288c0178944b038c0179944b0275622e +dataclass 5 8005952a000000000000008c085f5f6d61696e5f5f948c05506f696e749493942981947d94288c0178944b038c0179944b0275622e +enum 0 635f5f6d61696e5f5f0a436f6c6f720a70300a2849320a7470310a5270320a2e +enum 1 635f5f6d61696e5f5f0a436f6c6f720a7100284b027471015271022e +enum 2 8002635f5f6d61696e5f5f0a436f6c6f720a71004b028571015271022e +enum 3 8003635f5f6d61696e5f5f0a436f6c6f720a71004b028571015271022e +enum 4 8004951c000000000000008c085f5f6d61696e5f5f948c05436f6c6f729493944b02859452942e +enum 5 8005951c000000000000008c085f5f6d61696e5f5f948c05436f6c6f729493944b02859452942e +func 0 635f5f6d61696e5f5f0a66756e630a70300a2e +func 1 635f5f6d61696e5f5f0a66756e630a71002e +func 2 8002635f5f6d61696e5f5f0a66756e630a71002e +func 3 8003635f5f6d61696e5f5f0a66756e630a71002e +func 4 80049515000000000000008c085f5f6d61696e5f5f948c0466756e639493942e +func 5 80059515000000000000008c085f5f6d61696e5f5f948c0466756e639493942e +class 0 635f5f6d61696e5f5f0a506c61696e0a70300a2e +class 1 635f5f6d61696e5f5f0a506c61696e0a71002e +class 2 8002635f5f6d61696e5f5f0a506c61696e0a71002e +class 3 8003635f5f6d61696e5f5f0a506c61696e0a71002e +class 4 80049516000000000000008c085f5f6d61696e5f5f948c05506c61696e9493942e +class 5 80059516000000000000008c085f5f6d61696e5f5f948c05506c61696e9493942e +shareinst 0 286c70300a63636f70795f7265670a5f7265636f6e7374727563746f720a70310a28635f5f6d61696e5f5f0a506c61696e0a70320a635f5f6275696c74696e5f5f0a6f626a6563740a70330a4e7470340a5270350a286470360a56610a70370a49310a7356620a70380a49320a73626167350a612e +shareinst 1 5d71002863636f70795f7265670a5f7265636f6e7374727563746f720a710128635f5f6d61696e5f5f0a506c61696e0a7102635f5f6275696c74696e5f5f0a6f626a6563740a71034e7471045271057d71062858010000006171074b0158010000006271084b0275626805652e +shareinst 2 80025d710028635f5f6d61696e5f5f0a506c61696e0a7101298171027d71032858010000006171044b0158010000006271054b0275626802652e +shareinst 3 80035d710028635f5f6d61696e5f5f0a506c61696e0a7101298171027d71032858010000006171044b0158010000006271054b0275626802652e +shareinst 4 80049530000000000000005d94288c085f5f6d61696e5f5f948c05506c61696e9493942981947d94288c0161944b018c0162944b0275626804652e +shareinst 5 80059530000000000000005d94288c085f5f6d61696e5f5f948c05506c61696e9493942981947d94288c0161944b018c0162944b0275626804652e +newargsex 2 80026366756e63746f6f6c730a7061727469616c0a7100635f5f6275696c74696e5f5f0a676574617474720a7101635f5f6d61696e5f5f0a4e65774172677345780a58070000005f5f6e65775f5f710286527103857104527105286803635f5f6d61696e5f5f0a4e65774172677345780a71064b078671077d710858010000006b71094b09734e74710a622952710b7d710c28580100000061710d4b0768094b0975622e +slotted 2 8002635f5f6d61696e5f5f0a536c6f747465640a7100298171014e7d71022858010000007071034b015801000000717104580300000074776f710575867106622e +mylist 2 8002635f5f6d61696e5f5f0a4d794c6973740a710029817101284b014b02652e +mydict 2 8002635f5f6d61696e5f5f0a4d79446963740a71002981710158010000006171024b01732e +newargsex 3 80036366756e63746f6f6c730a7061727469616c0a7100636275696c74696e730a676574617474720a7101635f5f6d61696e5f5f0a4e65774172677345780a58070000005f5f6e65775f5f710286527103857104527105286803635f5f6d61696e5f5f0a4e65774172677345780a71064b078671077d710858010000006b71094b09734e74710a622952710b7d710c28580100000061710d4b0768094b0975622e +slotted 3 8003635f5f6d61696e5f5f0a536c6f747465640a7100298171014e7d71022858010000007071034b015801000000717104580300000074776f710575867106622e +mylist 3 8003635f5f6d61696e5f5f0a4d794c6973740a710029817101284b014b02652e +mydict 3 8003635f5f6d61696e5f5f0a4d79446963740a71002981710158010000006171024b01732e +newargsex 4 80049538000000000000008c085f5f6d61696e5f5f948c094e65774172677345789493944b0785947d948c016b944b097392947d94288c0161944b0768054b0975622e +slotted 4 80049533000000000000008c085f5f6d61696e5f5f948c07536c6f747465649493942981944e7d94288c0170944b018c0171948c0374776f94758694622e +mylist 4 80049520000000000000008c085f5f6d61696e5f5f948c064d794c697374949394298194284b014b02652e +mydict 4 80049521000000000000008c085f5f6d61696e5f5f948c064d79446963749493942981948c0161944b01732e +newargsex 5 80059538000000000000008c085f5f6d61696e5f5f948c094e65774172677345789493944b0785947d948c016b944b097392947d94288c0161944b0768054b0975622e +slotted 5 80059533000000000000008c085f5f6d61696e5f5f948c07536c6f747465649493942981944e7d94288c0170944b018c0171948c0374776f94758694622e +mylist 5 80059520000000000000008c085f5f6d61696e5f5f948c064d794c697374949394298194284b014b02652e +mydict 5 80059521000000000000008c085f5f6d61696e5f5f948c064d79446963749493942981948c0161944b01732e +exc ['63657863657074696f6e730a56616c75654572726f720a70300a28566261640a70310a49310a7470320a5270330a2e', '63657863657074696f6e730a56616c75654572726f720a710028580300000062616471014b017471025271032e', '800263657863657074696f6e730a56616c75654572726f720a7100580300000062616471014b018671025271032e', '8003636275696c74696e730a56616c75654572726f720a7100580300000062616471014b018671025271032e', '80049527000000000000008c086275696c74696e73948c0a56616c75654572726f729493948c03626164944b01869452942e', '80059527000000000000008c086275696c74696e73948c0a56616c75654572726f729493948c03626164944b01869452942e'] +[None, True, 1, -5, 1267650600228229401496703205376, 1.5, 's', b'b', bytearray(b'ba'), (1, 2), [3], {'k': 'v'}, {4, 5}] +frozenset({1, 2}) (1+2j) [1, 4, 7] 中😀 + +[1] {2: 3} +1 restored +1 2 +1 2 +1 two +MyList [1, 2] +MyDict {'a': 1} +Point(x=5, y=6) True +True True +9 Empty +ValueError ('bad', 1) +True +True +True +0 True +1 True +2 True +3 True +4 True +5 True +True +True +True +True +True +[1, 2, 3.5, 'abc', '中文'] +[1, 2.5, 'abc'] +[1, 2] two +EOF Ran out of input +UnpicklingError +PicklingError +True True +80049511000000000000005d94288c06656d7074792194514b01652e +[('loaded', 'empty!'), 1] +forbidden Plain +80049519000000000000008c085f5f6d61696e5f5f948c05456d7074799493942952942e +ff00 255 00ff diff --git a/maven/python-transpiler/src/test/resources/conformance/pickle_.py b/maven/python-transpiler/src/test/resources/conformance/pickle_.py new file mode 100644 index 00000000000..aef4da72ea8 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/pickle_.py @@ -0,0 +1,255 @@ +import pickle +import copyreg +import dataclasses +import enum + + +def show(label, value): + for proto in range(6): + print(label, proto, pickle.dumps(value, proto).hex()) + + +print(pickle.DEFAULT_PROTOCOL, pickle.HIGHEST_PROTOCOL) +print(pickle.dumps(None).hex()) +show("none", None) +show("bool", (True, False)) +show("ints", [0, 1, 255, 256, 65535, 65536, -1, -128, 2147483647, 2147483648, -2147483648, -2147483649]) +show("big", [2 ** 64, -2 ** 64, 10 ** 30, -10 ** 30, 2 ** 2000, 255 << 8, -(255 << 8), 2 ** 63 - 1]) +show("floats", [0.0, 1.5, -2.25, 1e300, 3.141592653589793]) +show("strs", ["", "abc", "café", "line\nbreak\\", "中文", "\U0001f600", "x" * 300]) +show("bytes", [b"", b"abc", bytes(range(256)), b"y" * 300]) +show("bytearray", [bytearray(), bytearray(b"hey")]) +show("tuples", [(), (1,), (1, 2), (1, 2, 3), (1, 2, 3, 4), ((4, 5), (6,))]) +show("lists", [[], [1], [1, 2], list(range(1200)), [[], [[]]]]) +show("dicts", [{}, {"a": 1}, {"a": 1, "b": [2, 3], "c": {"d": None}}, {i: i for i in range(1100)}]) +show("sets", [set(), {1}, {1, 2, 3}, frozenset(), frozenset([1, 2, 3])]) +show("complex", 1 + 2j) +show("range", range(1, 10, 2)) +show("type", [int, str, dict]) + +shared = [1, 2] +show("shared", [shared, shared, (shared, shared)]) +rec = [] +rec.append(rec) +show("recursive", rec) +d = {} +d["self"] = d +show("recdict", d) +t = ("abc", "abc") +show("strmemo", [t, t, "abc"]) + + +class Plain: + def __init__(self, a, b): + self.a = a + self.b = b + + def __eq__(self, other): + return type(other) is Plain and (self.a, self.b) == (other.a, other.b) + + +class Empty: + pass + + +class WithState: + def __init__(self): + self.x = 1 + self.cache = "skip" + + def __getstate__(self): + return {"x": self.x} + + def __setstate__(self, state): + self.x = state["x"] + self.cache = "restored" + + +class NewArgs: + def __new__(cls, a, b): + self = object.__new__(cls) + self.a = a + self.b = b + return self + + def __getnewargs__(self): + return (self.a, self.b) + + +class NewArgsEx: + def __new__(cls, a, *, k=0): + self = object.__new__(cls) + self.a = a + self.k = k + return self + + def __getnewargs_ex__(self): + return ((self.a,), {"k": self.k}) + + +class Reduced: + def __init__(self, v): + self.v = v + + def __reduce__(self): + return (Reduced, (self.v,)) + + +class Slotted: + __slots__ = ("p", "q") + + def __init__(self): + self.p = 1 + self.q = "two" + + +class MyList(list): + pass + + +class MyDict(dict): + pass + + +@dataclasses.dataclass +class Point: + x: int + y: int = 2 + + +class Color(enum.Enum): + RED = 1 + GREEN = 2 + + +def func(x): + return x + 1 + + +show("plain", Plain(1, "s")) +show("empty", Empty()) +show("state", WithState()) +show("newargs", NewArgs(1, 2)) +show("reduced", Reduced(5)) +show("dataclass", Point(3)) +show("enum", Color.GREEN) +show("func", func) +show("class", Plain) +show("shareinst", [Plain(1, 2)] * 2) +for proto in range(2, 6): + for label, value in (("newargsex", NewArgsEx(7, k=9)), ("slotted", Slotted()), ("mylist", MyList([1, 2])), + ("mydict", MyDict(a=1))): + print(label, proto, pickle.dumps(value, proto).hex()) +print("exc", [pickle.dumps(ValueError("bad", 1), p).hex() for p in range(6)]) + + +def round_trip(value, **kw): + for proto in range(6): + out = pickle.loads(pickle.dumps(value, proto)) + if out != value: + print("MISMATCH", proto, value, out) + return pickle.loads(pickle.dumps(value, 5)) + + +print(round_trip([None, True, 1, -5, 2 ** 100, 1.5, "s", b"b", bytearray(b"ba"), (1, 2), [3], {"k": "v"}, {4, 5}])) +print(round_trip(frozenset([1, 2])), round_trip(1 + 2j), list(pickle.loads(pickle.dumps(range(1, 8, 3)))), round_trip("中\U0001f600\n")) +out = round_trip(Plain([1], {2: 3})) +print(out.a, out.b) +x = pickle.loads(pickle.dumps(WithState())) +print(x.x, x.cache) +x = pickle.loads(pickle.dumps(NewArgs(1, 2))) +print(x.a, x.b) +x = pickle.loads(pickle.dumps(NewArgsEx(1, k=2))) +print(x.a, x.k) +x = pickle.loads(pickle.dumps(Slotted())) +print(x.p, x.q) +x = pickle.loads(pickle.dumps(MyList([1, 2]))) +print(type(x).__name__, x) +x = pickle.loads(pickle.dumps(MyDict(a=1))) +print(type(x).__name__, x) +print(pickle.loads(pickle.dumps(Point(5, 6))), pickle.loads(pickle.dumps(Color.RED)) is Color.RED) +print(pickle.loads(pickle.dumps(func)) is func, pickle.loads(pickle.dumps(Plain)) is Plain) +print(pickle.loads(pickle.dumps(Reduced(9), 0)).v, type(pickle.loads(pickle.dumps(Empty(), 1))).__name__) +e = pickle.loads(pickle.dumps(ValueError("bad", 1))) +print(type(e).__name__, e.args) +r = pickle.loads(pickle.dumps(rec)) +print(r[0] is r) +d2 = pickle.loads(pickle.dumps(d)) +print(d2["self"] is d2) +s = pickle.loads(pickle.dumps([shared, shared])) +print(s[0] is s[1]) + +# Bytes CPython wrote (protocols 0 to 5 of one value). +value = {"n": [1, 2.5, None], "t": (True, "x"), "b": b"by", "s": {7}} +for proto in range(6): + data = pickle.dumps(value, proto) + print(proto, pickle.loads(data) == value) +print(pickle.loads(bytes.fromhex("286470300a566e0a70310a286c70320a49310a6146322e350a614e617356740a70330a284930310a56780a70340a7470350a7356620a70360a635f636f646563730a656e636f64650a70370a285662790a70380a566c6174696e310a70390a747031300a527031310a7356730a7031320a635f5f6275696c74696e5f5f0a7365740a7031330a28286c7031340a49370a61747031350a527031360a732e")) == value) +print(pickle.loads(bytes.fromhex("80027d71002858010000006e71015d7102284b014740040000000000004e6558010000007471038858010000007871048671055801000000627106635f636f646563730a656e636f64650a710758020000006279710858060000006c6174696e31710986710a52710b580100000073710c635f5f6275696c74696e5f5f0a7365740a710d5d710e4b076185710f527110752e")) == value) +print(pickle.loads(bytes.fromhex("80037d71002858010000006e71015d7102284b014740040000000000004e65580100000074710388580100000078710486710558010000006271064302627971075801000000737108636275696c74696e730a7365740a71095d710a4b076185710b52710c752e")) == value) +print(pickle.loads(bytes.fromhex("80049537000000000000007d94288c016e945d94284b014740040000000000004e658c017494888c01789486948c01629443026279948c0173948f94284b0790752e")) == value) +print(pickle.loads(bytes.fromhex("80059537000000000000007d94288c016e945d94284b014740040000000000004e658c017494888c01789486948c01629443026279948c0173948f94284b0790752e")) == value) +print(pickle.loads(b"(lp0\nI1\naI2\naF3.5\naS'abc'\np1\naV\\u4e2d\\u6587\np2\na.")) +print(pickle.loads(b"\x80\x02]q\x00(K\x01G@\x04\x00\x00\x00\x00\x00\x00X\x03\x00\x00\x00abcq\x01e.")) + +# Pickler and Unpickler +import io +buf = io.BytesIO() +p = pickle.Pickler(buf, 2) +p.dump([1, 2]) +p.dump("two") +buf.seek(0) +u = pickle.Unpickler(buf) +print(u.load(), u.load()) +try: + u.load() +except EOFError as ex: + print("EOF", ex) +try: + pickle.loads(b"\x80\x04\x95") +except Exception as ex: + print(type(ex).__name__) +try: + pickle.dumps(lambda: 1) +except pickle.PicklingError as ex: + print("PicklingError") +try: + pickle.dumps(Plain.__init__) +except pickle.PicklingError as ex: + print("PicklingError2") +print(issubclass(pickle.PicklingError, pickle.PickleError), issubclass(pickle.UnpicklingError, pickle.PickleError)) + + +class Persist(pickle.Pickler): + def persistent_id(self, obj): + if isinstance(obj, Empty): + return "empty!" + return None + + +class PersistLoad(pickle.Unpickler): + def persistent_load(self, pid): + return ("loaded", pid) + + +buf = io.BytesIO() +Persist(buf, 4).dump([Empty(), 1]) +print(buf.getvalue().hex()) +print(PersistLoad(io.BytesIO(buf.getvalue())).load()) + + +class Restricted(pickle.Unpickler): + def find_class(self, module, name): + if name == "Plain": + raise pickle.UnpicklingError("forbidden " + name) + return pickle.Unpickler.find_class(self, module, name) + + +try: + Restricted(io.BytesIO(pickle.dumps(Plain(1, 2)))).load() +except pickle.UnpicklingError as ex: + print(ex) + +copyreg.pickle(Empty, lambda obj: (Empty, ())) +print(pickle.dumps(Empty(), 4).hex()) +print(pickle.encode_long(255).hex(), pickle.decode_long(b"\xff\x00"), pickle.encode_long(-256).hex()) diff --git a/maven/python-transpiler/src/test/resources/conformance/shelve_.out b/maven/python-transpiler/src/test/resources/conformance/shelve_.out new file mode 100644 index 00000000000..f661d017e76 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/shelve_.out @@ -0,0 +1,12 @@ +['a', 'item', 'list'] 3 True False +[1, 2, {'k': (3, 4)}] Item(7) dflt +[1, 2, {'k': (3, 4)}] +x x None +True +['a', 'item', 'list'] 1 [1, 2, {'k': (3, 4)}] Item(7) +[('b', 2), ('c', 3), ('item', Item(7)), ('list', [1, 2, {'k': (3, 4)}])] +closed: invalid operation on closed shelf +['b', 'c', 'item', 'list'] +read only +[1, 2, 3] +0 diff --git a/maven/python-transpiler/src/test/resources/conformance/shelve_.py b/maven/python-transpiler/src/test/resources/conformance/shelve_.py new file mode 100644 index 00000000000..f619aed403a --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/shelve_.py @@ -0,0 +1,53 @@ +import os +import shelve + +path = "db.shelf" + + +class Item: + def __init__(self, n): + self.n = n + + def __repr__(self): + return "Item(%d)" % self.n + + +with shelve.open(path) as db: + db["a"] = 1 + db["list"] = [1, 2, {"k": (3, 4)}] + db["item"] = Item(7) + print(sorted(db.keys()), len(db), "a" in db, "z" in db) + print(db["list"], db["item"], db.get("z", "dflt")) + db["list"].append(5) + print(db["list"]) + print(db.setdefault("s", "x"), db.pop("s"), db.pop("s", None)) + +print(os.path.exists(path)) +db = shelve.open(path) +print(sorted(db), db["a"], db["list"], db["item"]) +del db["a"] +db.update({"b": 2}, c=3) +print(sorted(db.items())) +db.close() +try: + db["b"] +except ValueError as e: + print("closed:", e) + +db = shelve.open(path, "r") +print(sorted(db.keys())) +try: + db["x"] = 1 +except OSError: + print("read only") +db.close() + +with shelve.open(path, writeback=True) as db: + db["wb"] = [1] + db["wb"].append(2) + db["wb"].append(3) +with shelve.open(path, "r") as db: + print(db["wb"]) + +with shelve.open(path, "n") as db: + print(len(db)) From 79582760c517cba94e6347390d8c5a928b26c204 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 08:38:44 +0200 Subject: [PATCH 038/112] Python: urllib.request, urllib.error, http.client and http.HTTPStatus over the platform HTTP client; email.message Co-Authored-By: Claude Opus 5.5 --- .../codename1/python/stdlib/email/__init__.py | 4 + .../codename1/python/stdlib/email/message.py | 221 +++++++++++ .../codename1/python/stdlib/http/__init__.py | 68 ++++ .../codename1/python/stdlib/http/client.py | 373 +++++++++++++++++ .../python/stdlib/urllib/__init__.py | 2 +- .../codename1/python/stdlib/urllib/error.py | 78 ++++ .../codename1/python/stdlib/urllib/request.py | 375 ++++++++++++++++++ .../resources/conformance/http_urllib.out | 16 + .../test/resources/conformance/http_urllib.py | 73 ++++ 9 files changed, 1209 insertions(+), 1 deletion(-) create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/email/__init__.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/email/message.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/http/__init__.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/http/client.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/urllib/error.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/urllib/request.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/http_urllib.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/http_urllib.py diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/email/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/email/__init__.py new file mode 100644 index 00000000000..a82c539a52b --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/email/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Email and MIME handling: the message object HTTP headers are carried in.""" diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/email/message.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/email/message.py new file mode 100644 index 00000000000..ef38c0265a1 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/email/message.py @@ -0,0 +1,221 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""email.message.Message: headers in order, looked up without regard to case. + +Covers what headers and simple payloads need (HTTP responses, form parts); +MIME multipart trees, policies and encoders are not provided. +""" + +__all__ = ["Message", "EmailMessage"] + + +def _split_params(value): + parts = [] + cur = [] + quoted = False + for ch in value: + if ch == '"': + quoted = not quoted + cur.append(ch) + elif ch == ";" and not quoted: + parts.append("".join(cur).strip()) + cur = [] + else: + cur.append(ch) + parts.append("".join(cur).strip()) + return parts + + +def _unquote(v): + if len(v) > 1 and v[0] == v[-1] == '"': + return v[1:-1].replace('\\"', '"').replace("\\\\", "\\") + return v + + +class Message: + def __init__(self, policy=None): + self._headers = [] + self._payload = None + self._charset = None + self.preamble = None + self.epilogue = None + self.defects = [] + + def __len__(self): + return len(self._headers) + + def __contains__(self, name): + n = name.lower() + return any(k.lower() == n for k, _ in self._headers) + + def __iter__(self): + return iter([k for k, _ in self._headers]) + + def __getitem__(self, name): + return self.get(name) + + def __setitem__(self, name, val): + self._headers.append((name, val)) + + def __delitem__(self, name): + n = name.lower() + self._headers = [(k, v) for k, v in self._headers if k.lower() != n] + + def keys(self): + return [k for k, _ in self._headers] + + def values(self): + return [v for _, v in self._headers] + + def items(self): + return list(self._headers) + + def get(self, name, failobj=None): + n = name.lower() + for k, v in self._headers: + if k.lower() == n: + return v + return failobj + + def get_all(self, name, failobj=None): + n = name.lower() + values = [v for k, v in self._headers if k.lower() == n] + return values if values else failobj + + def add_header(self, _name, _value, **_params): + parts = [] if _value is None else [_value] + for k, v in _params.items(): + k = k.replace("_", "-") + if v is None: + parts.append(k) + else: + parts.append('%s="%s"' % (k, str(v).replace("\\", "\\\\").replace('"', '\\"'))) + self._headers.append((_name, "; ".join(parts))) + + def replace_header(self, _name, _value): + n = _name.lower() + for i, (k, _) in enumerate(self._headers): + if k.lower() == n: + self._headers[i] = (k, _value) + return + raise KeyError(_name) + + def get_params(self, failobj=None, header="content-type", unquote=True): + value = self.get(header) + if value is None: + return failobj + result = [] + for i, p in enumerate(_split_params(value)): + if not p and i: + continue + if "=" in p and i: + k, v = p.split("=", 1) + result.append((k.strip().lower(), _unquote(v.strip()) if unquote else v.strip())) + else: + result.append((p.strip() if i == 0 else p.strip().lower(), "")) + return result + + def get_param(self, param, failobj=None, header="content-type", unquote=True): + params = self.get_params(None, header, unquote) + if params is None: + return failobj + for k, v in params[1:]: + if k == param.lower(): + return v + return failobj + + def get_content_type(self): + value = self.get("content-type") + if value is None: + return self.get_default_type() + ctype = _split_params(value)[0].lower() + if ctype.count("/") != 1: + return "text/plain" + return ctype + + def get_content_maintype(self): + return self.get_content_type().split("/")[0] + + def get_content_subtype(self): + return self.get_content_type().split("/")[1] + + def get_default_type(self): + return "text/plain" + + def get_content_charset(self, failobj=None): + cs = self.get_param("charset") + return cs.lower() if cs else failobj + + def get_charsets(self, failobj=None): + return [self.get_content_charset(failobj)] + + def get_filename(self, failobj=None): + fn = self.get_param("filename", None, "content-disposition") + if fn is None: + fn = self.get_param("name", None, "content-type") + return failobj if fn is None else fn + + def get_boundary(self, failobj=None): + return self.get_param("boundary", failobj) + + def is_multipart(self): + return isinstance(self._payload, list) + + def set_payload(self, payload, charset=None): + self._payload = payload + if charset is not None: + self.set_charset(charset) + + def set_charset(self, charset): + self._charset = charset + if "content-type" not in self: + self["Content-Type"] = 'text/plain; charset="%s"' % charset + + def get_charset(self): + return self._charset + + def get_payload(self, i=None, decode=False): + if i is not None: + return self._payload[i] + if decode and isinstance(self._payload, str): + return self._payload.encode(self.get_content_charset("utf-8"), "surrogateescape") + return self._payload + + def attach(self, payload): + if self._payload is None: + self._payload = [payload] + else: + self._payload.append(payload) + + def walk(self): + yield self + if self.is_multipart(): + for p in self._payload: + yield from p.walk() + + def as_string(self, unixfrom=False, maxheaderlen=0, policy=None): + out = "".join("%s: %s\n" % (k, v) for k, v in self._headers) + "\n" + if isinstance(self._payload, str): + out += self._payload + return out + + def as_bytes(self, unixfrom=False, policy=None): + return self.as_string().encode("utf-8") + + def __str__(self): + return self.as_string() + + +class EmailMessage(Message): + def get_content(self): + p = self.get_payload(decode=True) + if self.get_content_maintype() == "text" and isinstance(p, bytes): + return p.decode(self.get_content_charset("utf-8")) + return p + + def set_content(self, content, subtype="plain", charset="utf-8"): + del self["Content-Type"] + if isinstance(content, str): + self["Content-Type"] = "text/%s; charset=\"%s\"" % (subtype, charset) + self._payload = content diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/http/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/http/__init__.py new file mode 100644 index 00000000000..5ed674893ee --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/http/__init__.py @@ -0,0 +1,68 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""HTTP status codes and methods.""" + +from enum import IntEnum, StrEnum + +__all__ = ["HTTPStatus", "HTTPMethod"] + +_CODES = [ + (100, "CONTINUE", "Continue"), (101, "SWITCHING_PROTOCOLS", "Switching Protocols"), + (102, "PROCESSING", "Processing"), (103, "EARLY_HINTS", "Early Hints"), + (200, "OK", "OK"), (201, "CREATED", "Created"), (202, "ACCEPTED", "Accepted"), + (203, "NON_AUTHORITATIVE_INFORMATION", "Non-Authoritative Information"), + (204, "NO_CONTENT", "No Content"), (205, "RESET_CONTENT", "Reset Content"), + (206, "PARTIAL_CONTENT", "Partial Content"), (207, "MULTI_STATUS", "Multi-Status"), + (208, "ALREADY_REPORTED", "Already Reported"), (226, "IM_USED", "IM Used"), + (300, "MULTIPLE_CHOICES", "Multiple Choices"), (301, "MOVED_PERMANENTLY", "Moved Permanently"), + (302, "FOUND", "Found"), (303, "SEE_OTHER", "See Other"), (304, "NOT_MODIFIED", "Not Modified"), + (305, "USE_PROXY", "Use Proxy"), (307, "TEMPORARY_REDIRECT", "Temporary Redirect"), + (308, "PERMANENT_REDIRECT", "Permanent Redirect"), + (400, "BAD_REQUEST", "Bad Request"), (401, "UNAUTHORIZED", "Unauthorized"), + (402, "PAYMENT_REQUIRED", "Payment Required"), (403, "FORBIDDEN", "Forbidden"), + (404, "NOT_FOUND", "Not Found"), (405, "METHOD_NOT_ALLOWED", "Method Not Allowed"), + (406, "NOT_ACCEPTABLE", "Not Acceptable"), + (407, "PROXY_AUTHENTICATION_REQUIRED", "Proxy Authentication Required"), + (408, "REQUEST_TIMEOUT", "Request Timeout"), (409, "CONFLICT", "Conflict"), (410, "GONE", "Gone"), + (411, "LENGTH_REQUIRED", "Length Required"), (412, "PRECONDITION_FAILED", "Precondition Failed"), + (413, "CONTENT_TOO_LARGE", "Content Too Large"), (414, "URI_TOO_LONG", "URI Too Long"), + (415, "UNSUPPORTED_MEDIA_TYPE", "Unsupported Media Type"), + (416, "RANGE_NOT_SATISFIABLE", "Range Not Satisfiable"), + (417, "EXPECTATION_FAILED", "Expectation Failed"), (418, "IM_A_TEAPOT", "I'm a Teapot"), + (421, "MISDIRECTED_REQUEST", "Misdirected Request"), + (422, "UNPROCESSABLE_CONTENT", "Unprocessable Content"), (423, "LOCKED", "Locked"), + (424, "FAILED_DEPENDENCY", "Failed Dependency"), (425, "TOO_EARLY", "Too Early"), + (426, "UPGRADE_REQUIRED", "Upgrade Required"), (428, "PRECONDITION_REQUIRED", "Precondition Required"), + (429, "TOO_MANY_REQUESTS", "Too Many Requests"), + (431, "REQUEST_HEADER_FIELDS_TOO_LARGE", "Request Header Fields Too Large"), + (451, "UNAVAILABLE_FOR_LEGAL_REASONS", "Unavailable For Legal Reasons"), + (500, "INTERNAL_SERVER_ERROR", "Internal Server Error"), (501, "NOT_IMPLEMENTED", "Not Implemented"), + (502, "BAD_GATEWAY", "Bad Gateway"), (503, "SERVICE_UNAVAILABLE", "Service Unavailable"), + (504, "GATEWAY_TIMEOUT", "Gateway Timeout"), + (505, "HTTP_VERSION_NOT_SUPPORTED", "HTTP Version Not Supported"), + (506, "VARIANT_ALSO_NEGOTIATES", "Variant Also Negotiates"), + (507, "INSUFFICIENT_STORAGE", "Insufficient Storage"), (508, "LOOP_DETECTED", "Loop Detected"), + (510, "NOT_EXTENDED", "Not Extended"), + (511, "NETWORK_AUTHENTICATION_REQUIRED", "Network Authentication Required"), +] + +_PHRASES = {code: phrase for code, _, phrase in _CODES} + +HTTPStatus = IntEnum("HTTPStatus", [(name, code) for code, name, _ in _CODES]) + + +def _phrase(self): + return _PHRASES.get(self.value, "") + + +HTTPStatus.phrase = property(_phrase) +HTTPStatus.is_informational = property(lambda self: 100 <= self.value < 200) +HTTPStatus.is_success = property(lambda self: 200 <= self.value < 300) +HTTPStatus.is_redirection = property(lambda self: 300 <= self.value < 400) +HTTPStatus.is_client_error = property(lambda self: 400 <= self.value < 500) +HTTPStatus.is_server_error = property(lambda self: 500 <= self.value < 600) + +HTTPMethod = StrEnum("HTTPMethod", [(m, m) for m in + ("CONNECT", "DELETE", "GET", "HEAD", "OPTIONS", "PATCH", "POST", "PUT", + "TRACE")]) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/http/client.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/http/client.py new file mode 100644 index 00000000000..0410b317896 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/http/client.py @@ -0,0 +1,373 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""HTTP client: HTTPConnection over the platform's HTTP client. + +A request is sent whole when getresponse() is called, through the same +transport as urllib.request and requests (Codename One's ConnectionRequest +in an application), so certificates and proxies follow the platform. The +response body has been read by then, which is what read() returns from. +""" + +import _cn1http +import email.message as _email_message +from http import HTTPStatus + +__all__ = ["HTTPResponse", "HTTPConnection", "HTTPSConnection", "HTTPException", "NotConnected", + "InvalidURL", "UnknownProtocol", "IncompleteRead", "ImproperConnectionState", + "CannotSendRequest", "CannotSendHeader", "ResponseNotReady", "BadStatusLine", + "LineTooLong", "RemoteDisconnected", "error", "responses", "HTTPMessage", + "HTTP_PORT", "HTTPS_PORT"] + +HTTP_PORT = 80 +HTTPS_PORT = 443 + +responses = {s.value: s.phrase for s in HTTPStatus} +CONTINUE = HTTPStatus.CONTINUE +SWITCHING_PROTOCOLS = HTTPStatus.SWITCHING_PROTOCOLS +PROCESSING = HTTPStatus.PROCESSING +EARLY_HINTS = HTTPStatus.EARLY_HINTS +OK = HTTPStatus.OK +CREATED = HTTPStatus.CREATED +ACCEPTED = HTTPStatus.ACCEPTED +NON_AUTHORITATIVE_INFORMATION = HTTPStatus.NON_AUTHORITATIVE_INFORMATION +NO_CONTENT = HTTPStatus.NO_CONTENT +RESET_CONTENT = HTTPStatus.RESET_CONTENT +PARTIAL_CONTENT = HTTPStatus.PARTIAL_CONTENT +MULTI_STATUS = HTTPStatus.MULTI_STATUS +ALREADY_REPORTED = HTTPStatus.ALREADY_REPORTED +IM_USED = HTTPStatus.IM_USED +MULTIPLE_CHOICES = HTTPStatus.MULTIPLE_CHOICES +MOVED_PERMANENTLY = HTTPStatus.MOVED_PERMANENTLY +FOUND = HTTPStatus.FOUND +SEE_OTHER = HTTPStatus.SEE_OTHER +NOT_MODIFIED = HTTPStatus.NOT_MODIFIED +USE_PROXY = HTTPStatus.USE_PROXY +TEMPORARY_REDIRECT = HTTPStatus.TEMPORARY_REDIRECT +PERMANENT_REDIRECT = HTTPStatus.PERMANENT_REDIRECT +BAD_REQUEST = HTTPStatus.BAD_REQUEST +UNAUTHORIZED = HTTPStatus.UNAUTHORIZED +PAYMENT_REQUIRED = HTTPStatus.PAYMENT_REQUIRED +FORBIDDEN = HTTPStatus.FORBIDDEN +NOT_FOUND = HTTPStatus.NOT_FOUND +METHOD_NOT_ALLOWED = HTTPStatus.METHOD_NOT_ALLOWED +NOT_ACCEPTABLE = HTTPStatus.NOT_ACCEPTABLE +PROXY_AUTHENTICATION_REQUIRED = HTTPStatus.PROXY_AUTHENTICATION_REQUIRED +REQUEST_TIMEOUT = HTTPStatus.REQUEST_TIMEOUT +CONFLICT = HTTPStatus.CONFLICT +GONE = HTTPStatus.GONE +LENGTH_REQUIRED = HTTPStatus.LENGTH_REQUIRED +PRECONDITION_FAILED = HTTPStatus.PRECONDITION_FAILED +CONTENT_TOO_LARGE = HTTPStatus.CONTENT_TOO_LARGE +URI_TOO_LONG = HTTPStatus.URI_TOO_LONG +UNSUPPORTED_MEDIA_TYPE = HTTPStatus.UNSUPPORTED_MEDIA_TYPE +RANGE_NOT_SATISFIABLE = HTTPStatus.RANGE_NOT_SATISFIABLE +EXPECTATION_FAILED = HTTPStatus.EXPECTATION_FAILED +IM_A_TEAPOT = HTTPStatus.IM_A_TEAPOT +MISDIRECTED_REQUEST = HTTPStatus.MISDIRECTED_REQUEST +UNPROCESSABLE_CONTENT = HTTPStatus.UNPROCESSABLE_CONTENT +LOCKED = HTTPStatus.LOCKED +FAILED_DEPENDENCY = HTTPStatus.FAILED_DEPENDENCY +TOO_EARLY = HTTPStatus.TOO_EARLY +UPGRADE_REQUIRED = HTTPStatus.UPGRADE_REQUIRED +PRECONDITION_REQUIRED = HTTPStatus.PRECONDITION_REQUIRED +TOO_MANY_REQUESTS = HTTPStatus.TOO_MANY_REQUESTS +REQUEST_HEADER_FIELDS_TOO_LARGE = HTTPStatus.REQUEST_HEADER_FIELDS_TOO_LARGE +UNAVAILABLE_FOR_LEGAL_REASONS = HTTPStatus.UNAVAILABLE_FOR_LEGAL_REASONS +INTERNAL_SERVER_ERROR = HTTPStatus.INTERNAL_SERVER_ERROR +NOT_IMPLEMENTED = HTTPStatus.NOT_IMPLEMENTED +BAD_GATEWAY = HTTPStatus.BAD_GATEWAY +SERVICE_UNAVAILABLE = HTTPStatus.SERVICE_UNAVAILABLE +GATEWAY_TIMEOUT = HTTPStatus.GATEWAY_TIMEOUT +HTTP_VERSION_NOT_SUPPORTED = HTTPStatus.HTTP_VERSION_NOT_SUPPORTED +VARIANT_ALSO_NEGOTIATES = HTTPStatus.VARIANT_ALSO_NEGOTIATES +INSUFFICIENT_STORAGE = HTTPStatus.INSUFFICIENT_STORAGE +LOOP_DETECTED = HTTPStatus.LOOP_DETECTED +NOT_EXTENDED = HTTPStatus.NOT_EXTENDED +NETWORK_AUTHENTICATION_REQUIRED = HTTPStatus.NETWORK_AUTHENTICATION_REQUIRED + + +class HTTPException(Exception): + pass + + +error = HTTPException + + +class NotConnected(HTTPException): + pass + + +class InvalidURL(HTTPException): + pass + + +class UnknownProtocol(HTTPException): + pass + + +class IncompleteRead(HTTPException): + def __init__(self, partial, expected=None): + HTTPException.__init__(self, partial) + self.partial = partial + self.expected = expected + + +class ImproperConnectionState(HTTPException): + pass + + +class CannotSendRequest(ImproperConnectionState): + pass + + +class CannotSendHeader(ImproperConnectionState): + pass + + +class ResponseNotReady(ImproperConnectionState): + pass + + +class BadStatusLine(HTTPException): + pass + + +class LineTooLong(HTTPException): + pass + + +class RemoteDisconnected(ConnectionResetError, BadStatusLine): + pass + + +class HTTPMessage(_email_message.Message): + def getallmatchingheaders(self, name): + return [k + ": " + v for k, v in self.items() if k.lower() == name.lower()] + + +def _message(pairs): + m = HTTPMessage() + for k, v in pairs: + m[k] = v + return m + + +class HTTPResponse: + def __init__(self, status, reason, url, pairs, body, method="GET"): + self.status = status + self.code = status + self.reason = reason + self.url = url + self.headers = self.msg = _message(pairs) + self.version = 11 + self._body = body if method != "HEAD" else b"" + self._pos = 0 + self.closed = False + self.length = len(self._body) - self._pos + self.chunked = False + self.will_close = True + + def read(self, amt=None): + if self.closed: + return b"" + if amt is None or amt < 0: + data = self._body[self._pos:] + self._pos = len(self._body) + else: + data = self._body[self._pos:self._pos + amt] + self._pos += len(data) + self.length = len(self._body) - self._pos + return data + + read1 = read + + def readinto(self, b): + data = self.read(len(b)) + b[:len(data)] = data + return len(data) + + def readline(self, limit=-1): + end = self._body.find(b"\n", self._pos) + end = len(self._body) if end < 0 else end + 1 + if limit is not None and limit >= 0: + end = min(end, self._pos + limit) + data = self._body[self._pos:end] + self._pos = end + self.length = len(self._body) - self._pos + return data + + def readlines(self, hint=-1): + lines = [] + while True: + line = self.readline() + if not line: + return lines + lines.append(line) + + def __iter__(self): + return iter(self.readlines()) + + def peek(self, n=0): + return self._body[self._pos:] + + def getheader(self, name, default=None): + values = self.headers.get_all(name) + if not values: + return default + return ", ".join(values) + + def getheaders(self): + return list(self.headers.items()) + + def geturl(self): + return self.url + + def info(self): + return self.headers + + def getcode(self): + return self.status + + def isclosed(self): + return self.closed + + def readable(self): + return True + + def fileno(self): + raise OSError("no file descriptor") + + def close(self): + self.closed = True + + def __enter__(self): + return self + + def __exit__(self, *exc): + self.close() + + +class HTTPConnection: + default_port = HTTP_PORT + _scheme = "http" + debuglevel = 0 + + def __init__(self, host, port=None, timeout=None, source_address=None, blocksize=8192): + if port is None: + i = host.rfind(":") + j = host.rfind("]") + if i > j: + try: + port = int(host[i + 1:]) + except ValueError: + if host[i + 1:] == "": + port = self.default_port + else: + raise InvalidURL("nonnumeric port: '%s'" % host[i + 1:]) + host = host[:i] + else: + port = self.default_port + if host and host[0] == "[" and host[-1] == "]": + host = host[1:-1] + self.host = host + self.port = port + self.timeout = timeout + self._method = None + self._url = None + self._headers = [] + self._body = [] + self._response = None + self._tunnel_host = None + + def set_debuglevel(self, level): + self.debuglevel = level + + def set_tunnel(self, host, port=None, headers=None): + self._tunnel_host = host + + def connect(self): + pass + + def close(self): + self._method = None + self._response = None + + def putrequest(self, method, url, skip_host=False, skip_accept_encoding=False): + if self._method is not None: + raise CannotSendRequest("Request-sent") + self._method = method + self._url = url or "/" + self._headers = [] + self._body = [] + + def putheader(self, header, *values): + if self._method is None: + raise CannotSendHeader() + self._headers.append((str(header), "\r\n\t".join(str(v) for v in values))) + + def endheaders(self, message_body=None, *, encode_chunked=False): + if message_body is not None: + self.send(message_body) + + def send(self, data): + if isinstance(data, str): + data = data.encode("iso-8859-1") + elif hasattr(data, "read"): + data = data.read() + if isinstance(data, str): + data = data.encode("iso-8859-1") + elif not isinstance(data, (bytes, bytearray, memoryview)): + data = b"".join(data) + self._body.append(bytes(data)) + + def request(self, method, url, body=None, headers={}, *, encode_chunked=False): + self.putrequest(method, url) + names = {k.lower() for k in headers} + for k, v in headers.items(): + self.putheader(k, v) + if body is not None and isinstance(body, str): + body = body.encode("iso-8859-1") + if body is not None and "content-length" not in names and isinstance(body, (bytes, bytearray)): + self.putheader("Content-Length", str(len(body))) + self.endheaders(body) + + def _full_url(self): + if self._url.startswith("http://") or self._url.startswith("https://"): + return self._url + host = self.host + if ":" in host: + host = "[" + host + "]" + port = "" if self.port == self.default_port else ":%d" % self.port + return "%s://%s%s%s" % (self._scheme, host, port, self._url) + + def getresponse(self): + if self._method is None: + raise ResponseNotReady("Idle") + method = self._method + body = b"".join(self._body) if self._body else None + if body is None and method in ("POST", "PUT", "PATCH"): + body = b"" + try: + status, reason, url, pairs, content = _cn1http.request( + method, self._full_url(), self._headers, body, self.timeout, False) + except OSError as e: + m = str(e).lower() + if "timed out" in m or "timeout" in m: + raise TimeoutError(str(e)) from None + raise ConnectionError(str(e)) from None + finally: + self._method = None + if not reason: + reason = responses.get(status, "") + self._response = HTTPResponse(status, reason, url, pairs, content, method) + return self._response + + +class HTTPSConnection(HTTPConnection): + default_port = HTTPS_PORT + _scheme = "https" + + def __init__(self, host, port=None, *, timeout=None, source_address=None, context=None, blocksize=8192): + HTTPConnection.__init__(self, host, port, timeout, source_address, blocksize) + self.context = context diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/urllib/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/urllib/__init__.py index a4e554a6034..b8026362e68 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/urllib/__init__.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/urllib/__init__.py @@ -1,4 +1,4 @@ # Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. # Licensed under the GNU General Public License version 2 with the Classpath # exception, as the rest of the Codename One Python runtime. -"""URL handling: only urllib.parse is provided; network access goes through Codename One.""" +"""URL handling: parse, request and error.""" diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/urllib/error.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/urllib/error.py new file mode 100644 index 00000000000..2cbb5af003f --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/urllib/error.py @@ -0,0 +1,78 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Exceptions raised by urllib.request.""" + +import io +from http.client import HTTPResponse + +__all__ = ["URLError", "HTTPError", "ContentTooShortError"] + + +class URLError(OSError): + def __init__(self, reason, filename=None): + self.args = (reason,) + self.reason = reason + if filename is not None: + self.filename = filename + + def __str__(self): + return "" % self.reason + + +class HTTPError(URLError): + """A response with an error status; also usable as the response.""" + + def __init__(self, url, code, msg, hdrs, fp): + self.code = code + self.msg = msg + self.hdrs = hdrs + self.fp = fp + self.filename = url + self.args = (url, code, msg, hdrs, fp) + + def __str__(self): + return "HTTP Error %s: %s" % (self.code, self.msg) + + def __repr__(self): + return "" % (self.code, self.msg) + + @property + def reason(self): + return self.msg + + @property + def headers(self): + return self.hdrs + + @property + def status(self): + return self.code + + def getcode(self): + return self.code + + def geturl(self): + return self.filename + + def info(self): + return self.hdrs + + def read(self, amt=None): + return self.fp.read(amt) if self.fp is not None else b"" + + def close(self): + if self.fp is not None: + self.fp.close() + + def __enter__(self): + return self + + def __exit__(self, *exc): + self.close() + + +class ContentTooShortError(URLError): + def __init__(self, message, content): + URLError.__init__(self, message) + self.content = content diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/urllib/request.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/urllib/request.py new file mode 100644 index 00000000000..2e01a179cfa --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/urllib/request.py @@ -0,0 +1,375 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""urlopen and Request over the platform's HTTP client. + +Requests go through the same transport as http.client and requests +(Codename One's ConnectionRequest in an application). Redirects are +followed for GET and HEAD, and for a POST answered with 301, 302 or 303, +which is re-sent as a GET, as CPython does. Handlers and openers are kept +for the code that builds them; the ones that change a request (headers, +basic authentication, cookies through http.cookiejar) are applied, while +proxy and FTP handlers are accepted and do nothing. +""" + +import _cn1http +import base64 +import os +from http.client import HTTPResponse, responses +from urllib.error import URLError, HTTPError +from urllib.parse import urlsplit, urljoin, quote, unquote, urlencode + +__all__ = ["Request", "urlopen", "build_opener", "install_opener", "OpenerDirector", + "BaseHandler", "HTTPHandler", "HTTPSHandler", "HTTPRedirectHandler", + "HTTPErrorProcessor", "HTTPDefaultErrorHandler", "HTTPBasicAuthHandler", + "HTTPPasswordMgr", "HTTPPasswordMgrWithDefaultRealm", "HTTPCookieProcessor", + "ProxyHandler", "UnknownHandler", "urlretrieve", "pathname2url", "url2pathname", + "getproxies", "quote", "unquote", "urlencode"] + + +class Request: + def __init__(self, url, data=None, headers={}, origin_req_host=None, unverifiable=False, method=None): + self.full_url = url + self.headers = {} + self.unredirected_hdrs = {} + self.data = data + for k, v in headers.items(): + self.add_header(k, v) + self.origin_req_host = origin_req_host + self.unverifiable = unverifiable + if method: + self.method = method + self.timeout = None + + @property + def full_url(self): + if self.fragment: + return "%s#%s" % (self._full_url, self.fragment) + return self._full_url + + @full_url.setter + def full_url(self, url): + self._full_url, _, self.fragment = url.partition("#") + parts = urlsplit(self._full_url) + if not parts.scheme: + raise ValueError("unknown url type: %r" % self._full_url) + self.type = parts.scheme + self.host = parts.netloc + self.selector = parts.path + ("?" + parts.query if parts.query else "") + if not self.selector: + self.selector = "/" + + def get_method(self): + default = "POST" if self.data is not None else "GET" + return getattr(self, "method", default) + + def get_full_url(self): + return self.full_url + + def add_header(self, key, val): + self.headers[key.capitalize()] = val + + def add_unredirected_header(self, key, val): + self.unredirected_hdrs[key.capitalize()] = val + + def has_header(self, header_name): + return header_name in self.headers or header_name in self.unredirected_hdrs + + def get_header(self, header_name, default=None): + return self.headers.get(header_name, self.unredirected_hdrs.get(header_name, default)) + + def remove_header(self, header_name): + self.headers.pop(header_name, None) + self.unredirected_hdrs.pop(header_name, None) + + def header_items(self): + hdrs = dict(self.unredirected_hdrs) + hdrs.update(self.headers) + return list(hdrs.items()) + + def set_proxy(self, host, type): + pass + + def has_proxy(self): + return False + + +class BaseHandler: + handler_order = 500 + + def add_parent(self, parent): + self.parent = parent + + def close(self): + pass + + +class HTTPHandler(BaseHandler): + pass + + +class HTTPSHandler(BaseHandler): + def __init__(self, debuglevel=None, context=None, check_hostname=None): + self.context = context + + +class HTTPRedirectHandler(BaseHandler): + max_repeats = 4 + max_redirections = 10 + + +class HTTPErrorProcessor(BaseHandler): + pass + + +class HTTPDefaultErrorHandler(BaseHandler): + pass + + +class UnknownHandler(BaseHandler): + pass + + +class ProxyHandler(BaseHandler): + def __init__(self, proxies=None): + self.proxies = proxies + + +class HTTPPasswordMgr: + def __init__(self): + self.passwd = {} + + def add_password(self, realm, uri, user, passwd): + if isinstance(uri, str): + uri = [uri] + for u in uri: + self.passwd[(realm, self._reduce(u))] = (user, passwd) + + def _reduce(self, uri): + parts = urlsplit(uri) + return parts.netloc if parts.scheme else uri + + def find_user_password(self, realm, authuri): + host = self._reduce(authuri) + for (r, u), creds in self.passwd.items(): + if r == realm and u == host: + return creds + return None, None + + +class HTTPPasswordMgrWithDefaultRealm(HTTPPasswordMgr): + def find_user_password(self, realm, authuri): + user, password = HTTPPasswordMgr.find_user_password(self, realm, authuri) + if user is not None: + return user, password + return HTTPPasswordMgr.find_user_password(self, None, authuri) + + +class HTTPBasicAuthHandler(BaseHandler): + def __init__(self, password_mgr=None): + self.passwd = password_mgr if password_mgr is not None else HTTPPasswordMgr() + self.add_password = self.passwd.add_password + + def _authorize(self, req, realm): + user, pw = self.passwd.find_user_password(realm, req.full_url) + if user is None: + return False + raw = ("%s:%s" % (user, pw)).encode("utf-8") + req.add_unredirected_header("Authorization", "Basic " + base64.b64encode(raw).decode("ascii")) + return True + + +class HTTPCookieProcessor(BaseHandler): + def __init__(self, cookiejar=None): + self.cookiejar = cookiejar if cookiejar is not None else _SimpleCookieJar() + + +class _SimpleCookieJar: + """Name=value cookies per host, for an HTTPCookieProcessor given none.""" + + def __init__(self): + self._cookies = {} + + def __iter__(self): + return iter([(h, n, v) for h, d in self._cookies.items() for n, v in d.items()]) + + def __len__(self): + return sum(len(d) for d in self._cookies.values()) + + def add_cookie_header(self, req): + d = self._cookies.get(urlsplit(req.full_url).hostname) + if d: + req.add_unredirected_header("Cookie", "; ".join("%s=%s" % kv for kv in d.items())) + + def extract_cookies(self, response, req): + host = urlsplit(req.full_url).hostname + for value in response.headers.get_all("Set-Cookie") or []: + pair = value.split(";", 1)[0] + if "=" in pair: + n, v = pair.split("=", 1) + self._cookies.setdefault(host, {})[n.strip()] = v.strip() + + def clear(self): + self._cookies = {} + + +def _realm(headers): + value = headers.get("WWW-Authenticate") or "" + if not value.lower().startswith("basic"): + return None + i = value.find("realm=") + if i < 0: + return "" + r = value[i + 6:].split(",")[0].strip() + return r[1:-1] if len(r) > 1 and r[0] == r[-1] == '"' else r + + +class OpenerDirector: + def __init__(self): + self.addheaders = [("User-agent", "Python-urllib/3.14")] + self.handlers = [] + + def add_handler(self, handler): + handler.add_parent(self) + self.handlers.append(handler) + + def _find(self, kind): + for h in self.handlers: + if isinstance(h, kind): + return h + return None + + def close(self): + self.handlers = [] + + def open(self, fullurl, data=None, timeout=None): + req = fullurl if isinstance(fullurl, Request) else Request(fullurl, data) + if isinstance(fullurl, Request) and data is not None: + req.data = data + req.timeout = timeout + if req.type not in ("http", "https"): + if req.type == "file": + return _open_file(req) + raise URLError("unknown url type: %s" % req.type) + if isinstance(req.data, str): + raise TypeError("POST data should be bytes, an iterable of bytes, or a file object. " + "It cannot be of type str.") + cookies = self._find(HTTPCookieProcessor) + auth = self._find(HTTPBasicAuthHandler) + visited = 0 + tried_auth = False + while True: + headers = [(k, v) for k, v in self.addheaders if not req.has_header(k.capitalize())] + if not req.has_header("Host"): + headers.append(("Host", req.host)) + headers.extend(req.header_items()) + if req.data is not None and not req.has_header("Content-type"): + headers.append(("Content-Type", "application/x-www-form-urlencoded")) + if cookies is not None: + cookies.cookiejar.add_cookie_header(req) + c = req.unredirected_hdrs.get("Cookie") + if c: + headers.append(("Cookie", c)) + body = req.data + if body is not None and not isinstance(body, (bytes, bytearray)): + body = body.read() if hasattr(body, "read") else b"".join(body) + method = req.get_method() + try: + status, reason, url, pairs, content = _cn1http.request( + method, req.full_url, headers, body, timeout, False) + except OSError as e: + raise URLError(str(e)) from None + if not reason: + reason = responses.get(status, "") + resp = HTTPResponse(status, reason, url, pairs, content, method) + if cookies is not None: + cookies.cookiejar.extract_cookies(resp, req) + loc = resp.headers.get("Location") or resp.headers.get("URI") + if status in (301, 302, 303, 307, 308) and loc: + if visited >= HTTPRedirectHandler.max_redirections: + raise HTTPError(req.full_url, status, + "The HTTP server returned a redirect error that would lead to an " + "infinite loop.\nThe last 30x error message was:\n" + reason, + resp.headers, resp) + if not (method in ("GET", "HEAD") or (status in (301, 302, 303) and method == "POST")): + raise HTTPError(req.full_url, status, reason, resp.headers, resp) + visited += 1 + newurl = urljoin(req.full_url, loc) + kept = {k: v for k, v in req.headers.items() if k.lower() not in ("content-length", "content-type")} + req = Request(newurl, headers=kept, origin_req_host=req.origin_req_host, unverifiable=True, + method="HEAD" if method == "HEAD" else "GET") + continue + if status == 401 and auth is not None and not tried_auth: + tried_auth = True + realm = _realm(resp.headers) + if realm is not None and auth._authorize(req, realm): + continue + if not 200 <= status < 300: + raise HTTPError(req.full_url, status, reason, resp.headers, resp) + return resp + + +def _open_file(req): + path = url2pathname(urlsplit(req.full_url).path) + try: + with open(path, "rb") as f: + data = f.read() + except OSError as e: + raise URLError(e) from None + ext = path.rsplit(".", 1)[-1].lower() if "." in path else "" + ctype = _TYPES.get(ext, "text/plain") + pairs = [("Content-type", ctype), ("Content-length", str(len(data)))] + return HTTPResponse(200, "OK", req.full_url, pairs, data) + + +_TYPES = {"html": "text/html", "htm": "text/html", "txt": "text/plain", "json": "application/json", + "css": "text/css", "js": "text/javascript", "png": "image/png", "jpg": "image/jpeg", + "jpeg": "image/jpeg", "gif": "image/gif", "svg": "image/svg+xml", "pdf": "application/pdf", + "xml": "application/xml", "csv": "text/csv", "zip": "application/zip"} + + +def build_opener(*handlers): + opener = OpenerDirector() + for h in handlers: + if isinstance(h, type): + h = h() + opener.add_handler(h) + return opener + + +_opener = None + + +def install_opener(opener): + global _opener + _opener = opener + + +def urlopen(url, data=None, timeout=None, *, context=None): + opener = _opener if _opener is not None else build_opener() + return opener.open(url, data, timeout) + + +def urlretrieve(url, filename=None, reporthook=None, data=None): + with urlopen(url, data) as resp: + content = resp.read() + headers = resp.info() + if filename is None: + filename = os.path.basename(urlsplit(resp.url).path) or "index.html" + with open(filename, "wb") as f: + f.write(content) + if reporthook is not None: + reporthook(1, len(content), len(content)) + return filename, headers + + +def pathname2url(pathname): + return quote(pathname) + + +def url2pathname(url): + return unquote(url) + + +def getproxies(): + return {} diff --git a/maven/python-transpiler/src/test/resources/conformance/http_urllib.out b/maven/python-transpiler/src/test/resources/conformance/http_urllib.out new file mode 100644 index 00000000000..962b7b782b5 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/http_urllib.out @@ -0,0 +1,16 @@ +200 OK 200 application/json True +GET {'q': 'a b'} +PUT True /anything True +PUT {"k": [1, 2]} application/json {'x-mine': '1'} +POST name=py+thon&n=3 application/x-www-form-urlencoded +HTTPError 404 Not Found HTTP Error 404: Not Found True True +redirect 200 BASE/get?redirected=1 {'redirected': '1'} +['x-one', 'x-two'] +URLError unknown url type: ftp-nope +auth needed 401 +no challenge, no retry 401 +auth 200 b'{"authenticated": true}' +200 OK application/json raw +503 Service Unavailable b'statu' Service Unavailable +404 Created True True +404 diff --git a/maven/python-transpiler/src/test/resources/conformance/http_urllib.py b/maven/python-transpiler/src/test/resources/conformance/http_urllib.py new file mode 100644 index 00000000000..427e5e5049d --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/http_urllib.py @@ -0,0 +1,73 @@ +# urllib.request and http.client against HttpTestServer (see ConformanceTest); +# the expected output is CPython's against the same server. +import json +import urllib.request +import urllib.error +import urllib.parse +import http.client +from http import HTTPStatus +from _testbase import BASE + +with urllib.request.urlopen(BASE + "/get?q=" + urllib.parse.quote("a b")) as r: + print(r.status, r.reason, r.getcode(), r.headers.get_content_type(), r.url == BASE + "/get?q=a%20b") + d = json.loads(r.read()) + print(d["method"], d["args"]) + +req = urllib.request.Request(BASE + "/anything", data=json.dumps({"k": [1, 2]}).encode(), + headers={"Content-Type": "application/json", "X-Mine": "1"}, method="PUT") +print(req.get_method(), req.host == BASE[7:], req.selector, req.has_header("X-mine")) +with urllib.request.urlopen(req) as r: + d = json.loads(r.read().decode("utf-8")) + print(d["method"], d["body"], d["content_type"], d["headers"]) + +form = urllib.parse.urlencode({"name": "py thon", "n": 3}).encode("ascii") +with urllib.request.urlopen(BASE + "/anything", data=form) as r: + d = json.load(r) + print(d["method"], d["body"], d["content_type"]) + +try: + urllib.request.urlopen(BASE + "/status/404") +except urllib.error.HTTPError as e: + print("HTTPError", e.code, e.reason, str(e), isinstance(e, urllib.error.URLError), isinstance(e, OSError)) + e.close() + +with urllib.request.urlopen(BASE + "/redirect") as r: + print("redirect", r.status, r.url.replace(BASE, "BASE"), json.loads(r.read())["args"]) + +with urllib.request.urlopen(BASE + "/headers-out") as r: + print(sorted(k.lower() for k in r.headers.keys() if k.lower().startswith("x-"))) + +try: + urllib.request.urlopen("ftp-nope://example") +except urllib.error.URLError as e: + print("URLError", e.reason) + +try: + urllib.request.urlopen(BASE + "/basic-auth") +except urllib.error.HTTPError as e: + print("auth needed", e.code) +mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm() +mgr.add_password(None, BASE, "user", "pass") +opener = urllib.request.build_opener(urllib.request.HTTPBasicAuthHandler(mgr)) +try: + opener.open(BASE + "/basic-auth") +except urllib.error.HTTPError as e: + print("no challenge, no retry", e.code) +import base64 +req = urllib.request.Request(BASE + "/basic-auth") +req.add_header("Authorization", "Basic " + base64.b64encode(b"user:pass").decode()) +with urllib.request.urlopen(req) as r: + print("auth", r.status, r.read()) + +host = BASE[7:] +c = http.client.HTTPConnection(host, timeout=10) +c.request("POST", "/anything?x=1", body=b"raw", headers={"Content-Type": "text/plain"}) +r = c.getresponse() +print(r.status, r.reason, r.getheader("Content-Type"), json.loads(r.read())["body"]) +c.close() +c = http.client.HTTPConnection(host.split(":")[0], int(host.split(":")[1])) +c.request("GET", "/status/503") +r = c.getresponse() +print(r.status, r.reason, r.read(5), http.client.responses[503]) +print(HTTPStatus.NOT_FOUND, HTTPStatus(201).phrase, HTTPStatus.OK == 200, HTTPStatus.IM_A_TEAPOT.is_client_error) +print(http.client.NOT_FOUND, repr(HTTPStatus.OK)) From 80c452e7953cf1e51ba412614986ae1c439099e4 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 08:47:45 +0200 Subject: [PATCH 039/112] Python: zlib over JZlib, gzip, zipfile, stat, zoneinfo over the platform's zones Also: local offsets are read from the wall time the calendar shows, which named the wrong offset in the hour that repeats when daylight time ends; TextIOWrapper writes through any binary stream; JDK members are resolved as Java 17 declares them (ct.sym), since the output compiles at release 17. Co-Authored-By: Claude Opus 5.5 --- .../com/codename1/python/runtime/PyZlib.java | 201 ++++++ .../codename1/python/runtime/StdModules.java | 69 +- .../transpiler/codegen/ClassPathReader.java | 55 +- .../com/codename1/python/stdlib/gzip.py | 298 ++++++++ .../com/codename1/python/stdlib/io.py | 13 +- .../com/codename1/python/stdlib/stat.py | 117 +++ .../com/codename1/python/stdlib/zipfile.py | 665 ++++++++++++++++++ .../com/codename1/python/stdlib/zlib.py | 171 +++++ .../com/codename1/python/stdlib/zoneinfo.py | 189 +++++ .../test/resources/conformance/compress_.out | 37 + .../test/resources/conformance/compress_.py | 120 ++++ .../test/resources/conformance/zoneinfo_.out | 31 + .../test/resources/conformance/zoneinfo_.py | 39 + 13 files changed, 1994 insertions(+), 11 deletions(-) create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/PyZlib.java create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/gzip.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/stat.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/zipfile.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/zlib.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/zoneinfo.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/compress_.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/compress_.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/zoneinfo_.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/zoneinfo_.py diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyZlib.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyZlib.java new file mode 100644 index 00000000000..dea201179fd --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyZlib.java @@ -0,0 +1,201 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +import com.codename1.io.gzip.Adler32; +import com.codename1.io.gzip.CRC32; +import com.codename1.io.gzip.JZlib; +import com.codename1.io.gzip.ZStream; +import java.io.ByteArrayOutputStream; + +/// `_cn1zlib`: the deflate streams `zlib`, `gzip` and `zipfile` are built +/// on, over the core's JZlib (`com.codename1.io.gzip`), a Java port of +/// zlib. Window bits follow zlib: 9..15 a zlib stream, -9..-15 raw +/// deflate, 25..31 gzip, and for inflating 40..47 either of the wrapped +/// two. A failure raises `OSError` with zlib's own message, which +/// `zlib.py` turns into `zlib.error`. +final class PyZlib { + private static final int CHUNK = 16384; + + private PyZlib() { + } + + static PyModule module() { + PyModule m = new PyModule("_cn1zlib", new String[0]); + m.setAttr("deflate_new", PyBuiltin.positional("deflate_new", a -> { + ZStream z = new ZStream(); + int level = (int) Py.asLong(a[0]); + int wbits = (int) Py.asLong(a[1]); + if (wbits == 8 || wbits == -8 || wbits == 24) { + // zlib widens a window of 8 to 9 for deflate + wbits += wbits < 0 ? -1 : 1; + } + int err = z.deflateInit(level, wbits, (int) Py.asLong(a[2])); + if (err != JZlib.Z_OK) { + throw Py.error(Py.VALUE_ERROR, "Invalid initialization option"); + } + int strategy = (int) Py.asLong(a[3]); + if (strategy != JZlib.Z_DEFAULT_STRATEGY) { + z.deflateParams(level, strategy); + } + if (a[4] != Py.NONE) { //NOPMD CompareObjectsWithEquals - identity is meant + byte[] dict = PyBytes.like(a[4]); + z.deflateSetDictionary(dict, dict.length); + } + return new PyJavaObject(z); + })); + m.setAttr("deflate", PyBuiltin.positional("deflate", a -> { + ZStream z = stream(a[0]); + byte[] in = PyBytes.like(a[1]); + int flush = (int) Py.asLong(a[2]); + return new PyBytes(deflate(z, in, flush)); + })); + m.setAttr("inflate_new", PyBuiltin.positional("inflate_new", a -> { + ZStream z = new ZStream(); + int err = z.inflateInit((int) Py.asLong(a[0])); + if (err != JZlib.Z_OK) { + throw Py.error(Py.VALUE_ERROR, "Invalid initialization option"); + } + return new PyJavaObject(new InflateState(z, a[1] == Py.NONE ? null : PyBytes.like(a[1]))); //NOPMD CompareObjectsWithEquals - identity is meant + })); + m.setAttr("inflate", PyBuiltin.positional("inflate", a -> { + InflateState s = inflateState(a[0]); + byte[] in = PyBytes.like(a[1]); + long max = Py.asLong(a[2]); + return s.inflate(in, max <= 0 ? Long.MAX_VALUE : max, Py.isTrue(a[3])); + })); + m.setAttr("end", PyBuiltin.positional("end", a -> { + Object v = a[0] instanceof PyJavaObject ? ((PyJavaObject) a[0]).value : null; + if (v instanceof ZStream) { + ((ZStream) v).deflateEnd(); + } else if (v instanceof InflateState) { + ((InflateState) v).z.inflateEnd(); + } + return Py.NONE; + })); + m.setAttr("crc32", PyBuiltin.positional("crc32", a -> { + byte[] b = PyBytes.like(a[0]); + CRC32 c = new CRC32(); + c.reset(Py.asLong(a[1]) & 0xffffffffL); + c.update(b, 0, b.length); + return PyInt.of(c.getValue() & 0xffffffffL); + })); + m.setAttr("adler32", PyBuiltin.positional("adler32", a -> { + byte[] b = PyBytes.like(a[0]); + Adler32 c = new Adler32(); + c.reset(Py.asLong(a[1]) & 0xffffffffL); + c.update(b, 0, b.length); + return PyInt.of(c.getValue() & 0xffffffffL); + })); + return m; + } + + private static ZStream stream(Object o) { + if (o instanceof PyJavaObject && ((PyJavaObject) o).value instanceof ZStream) { + return (ZStream) ((PyJavaObject) o).value; + } + throw Py.typeError("not a deflate stream"); + } + + private static InflateState inflateState(Object o) { + if (o instanceof PyJavaObject && ((PyJavaObject) o).value instanceof InflateState) { + return (InflateState) ((PyJavaObject) o).value; + } + throw Py.typeError("not an inflate stream"); + } + + private static byte[] deflate(ZStream z, byte[] in, int flush) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + byte[] buf = new byte[CHUNK]; + z.setInput(in, 0, in.length, false); + while (true) { + z.setOutput(buf, 0, buf.length); + int err = z.deflate(flush); + if (err != JZlib.Z_OK && err != JZlib.Z_STREAM_END && err != JZlib.Z_BUF_ERROR) { + throw Py.error(Py.OS_ERROR, "Error " + err + " while compressing data" + detail(z)); + } + out.write(buf, 0, buf.length - z.availOut); + if (err == JZlib.Z_STREAM_END || (z.availOut != 0 && z.availIn == 0)) { + break; + } + } + return out.toByteArray(); + } + + private static String detail(ZStream z) { + return z.msg == null ? "" : ": " + z.msg; + } + + /// An inflate stream and the dictionary it asks for, if any. + static final class InflateState { + final ZStream z; + private final byte[] dict; + + InflateState(ZStream z, byte[] dict) { + this.z = z; + this.dict = dict; + } + + /// Inflates `in`, producing at most `max` bytes; answers + /// `(output, bytes of in consumed, end of stream reached)`. + Object inflate(byte[] in, long max, boolean finish) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + byte[] buf = new byte[CHUNK]; + z.setInput(in, 0, in.length, false); + boolean eof = false; + while (true) { + long room = max - out.size(); + if (room <= 0) { + break; + } + int len = (int) Math.min(buf.length, room); + z.setOutput(buf, 0, len); + int err = z.inflate(finish ? JZlib.Z_FINISH : JZlib.Z_NO_FLUSH); + if (err == JZlib.Z_NEED_DICT) { + if (dict == null || z.inflateSetDictionary(dict, dict.length) != JZlib.Z_OK) { + throw Py.error(Py.OS_ERROR, "Error 2 while decompressing data"); + } + continue; + } + int produced = len - z.availOut; + out.write(buf, 0, produced); + if (err == JZlib.Z_STREAM_END) { + eof = true; + break; + } + if (err == JZlib.Z_BUF_ERROR) { + break; + } + if (err != JZlib.Z_OK) { + throw Py.error(Py.OS_ERROR, "Error " + err + " while decompressing data" + detail(z)); + } + if (z.availIn == 0 && produced < len) { + break; + } + } + int consumed = in.length - z.availIn; + return new PyTuple(new Object[] {new PyBytes(out.toByteArray()), PyInt.of(consumed), Py.bool(eof)}); + } + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java index 81a23b491aa..bcda3d3002d 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java @@ -78,6 +78,14 @@ static boolean provide(String name) { PyModule.registerModule(thread()); return true; } + if ("_cn1zlib".equals(name)) { + PyModule.registerModule(PyZlib.module()); + return true; + } + if ("_cn1tz".equals(name)) { + PyModule.registerModule(tz()); + return true; + } if ("_cn1sql".equals(name)) { PyModule.registerModule(sql()); return true; @@ -237,13 +245,36 @@ private static PyModule weakref() { /// Seconds east of UTC of the device's zone at instant `t` (seconds /// since the epoch), daylight saving included. private static int utcOffset(double t) { - java.util.TimeZone z = java.util.TimeZone.getDefault(); + return utcOffset(java.util.TimeZone.getDefault(), t); + } + + /// The offset of `z` from UTC, in seconds, at the instant `t` (seconds + /// since the epoch): the local wall time the calendar shows for that + /// instant, read as if it were UTC, less the instant. Handing those + /// local fields to `getOffset(era, ...)` instead -- the only offset + /// query every device class library has -- reads them as local + /// *standard* time, which names the wrong offset for the hour that + /// repeats when daylight time ends. + static int utcOffset(java.util.TimeZone z, double t) { + long ms = (long) Math.floor(t * 1000); java.util.Calendar c = java.util.Calendar.getInstance(z); - c.setTime(new java.util.Date((long) Math.floor(t * 1000))); - int ms = ((c.get(java.util.Calendar.HOUR_OF_DAY) * 60 + c.get(java.util.Calendar.MINUTE)) * 60 - + c.get(java.util.Calendar.SECOND)) * 1000 + c.get(java.util.Calendar.MILLISECOND); - return z.getOffset(1, c.get(java.util.Calendar.YEAR), c.get(java.util.Calendar.MONTH), - c.get(java.util.Calendar.DAY_OF_MONTH), c.get(java.util.Calendar.DAY_OF_WEEK), ms) / 1000; + c.setTime(new java.util.Date(ms)); + long days = daysFromCivil(c.get(java.util.Calendar.YEAR), c.get(java.util.Calendar.MONTH) + 1, + c.get(java.util.Calendar.DAY_OF_MONTH)); + long wall = ((days * 24 + c.get(java.util.Calendar.HOUR_OF_DAY)) * 60 + c.get(java.util.Calendar.MINUTE)) * 60 + + c.get(java.util.Calendar.SECOND); + long secs = ms >= 0 ? ms / 1000 : -((999 - ms) / 1000); + return (int) (wall - secs); + } + + /// Days from 1970-01-01 to the proleptic Gregorian date `y-m-d`. + private static long daysFromCivil(long y, int m, int d) { + long yy = m <= 2 ? y - 1 : y; + long era = (yy >= 0 ? yy : yy - 399) / 400; + long yoe = yy - era * 400; + long doy = (153L * (m + (m > 2 ? -3 : 9)) + 2) / 5 + d - 1; + long doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; + return era * 146097 + doe - 719468; } private interface FsOp { @@ -447,6 +478,32 @@ private static Object sqlValue(Object v) { return v; } + /// `_cn1tz`: `offset(key, t)` answers the zone's offset from UTC in + /// seconds at the instant `t`, `raw(key)` its standard offset, + /// `known(key)` whether the platform knows the zone (an unknown ID + /// resolves to GMT), and `available()` the IDs the platform lists. + private static PyModule tz() { + PyModule m = new PyModule("_cn1tz", new String[0]); + m.setAttr("offset", PyBuiltin.positional("offset", a -> + PyInt.of(utcOffset(java.util.TimeZone.getTimeZone(Py.toStr(a[0])), Py.asDouble(a[1]))))); + m.setAttr("raw", PyBuiltin.positional("raw", a -> + PyInt.of(java.util.TimeZone.getTimeZone(Py.toStr(a[0])).getRawOffset() / 1000))); + m.setAttr("known", PyBuiltin.positional("known", a -> { + String key = Py.toStr(a[0]); + String id = java.util.TimeZone.getTimeZone(key).getID(); + return Py.bool(id != null && (id.equals(key) || !"GMT".equals(id))); + })); + m.setAttr("available", PyBuiltin.positional("available", a -> { + String[] ids = java.util.TimeZone.getAvailableIDs(); + Object[] out = new Object[ids.length]; + for (int i = 0; i < ids.length; i++) { + out[i] = Py.str(ids[i]); + } + return new PyList(out); + })); + return m; + } + private static final java.util.Map THREAD_IDS = new java.util.HashMap(); private static long threadIds; diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/ClassPathReader.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/ClassPathReader.java index 6b81a7c7f63..d83731a5420 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/ClassPathReader.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/ClassPathReader.java @@ -101,6 +101,13 @@ public boolean is(int flag) { private final Map cache = new HashMap(); private FileSystem jrt; private ZipFile rtJar; + private ZipFile ctSym; + /// `java/lang/Math.class` to its Java 17 entry in `ct.sym`. + private Map release17; + + /// The release the generated Java is compiled for (`javac --release + /// 17`), as `ct.sym` spells releases: one base-36 digit each. + private static final char RELEASE = Character.toUpperCase(Character.forDigit(17, 36)); public ClassPathReader(List classPath) throws IOException { for (File f : classPath) { @@ -162,15 +169,34 @@ private byte[] fromRoots(String path) throws IOException { return null; } - /// A JDK class: from the module image on JDK 9 and newer, from `rt.jar` - /// on JDK 8. + /// A JDK class, as Java 17 declares it. The generated Java is compiled + /// with `--release 17`, which checks it against the JDK's own record of + /// each release's API (`lib/ct.sym`); a member read from a newer + /// running JDK instead (`Math.TAU`, from 19) would be bridged and then + /// fail to compile. Without `ct.sym`, from the module image on JDK 9 and + /// newer and from `rt.jar` on JDK 8. private byte[] jdkClass(String path) throws IOException { if (jrt == null && rtJar == null) { - File rt = new File(System.getProperty("java.home"), "lib/rt.jar"); + File home = new File(System.getProperty("java.home")); + File rt = new File(home, "lib/rt.jar"); + File ct = new File(home, "lib/ct.sym"); if (rt.isFile()) { rtJar = new ZipFile(rt); } else { jrt = FileSystems.getFileSystem(URI.create("jrt:/")); + if (ct.isFile()) { + ctSym = new ZipFile(ct); + release17 = indexRelease(ctSym); + } + } + } + if (release17 != null && !release17.isEmpty()) { + ZipEntry e = release17.get(path); + if (e == null) { + return null; + } + try (InputStream in = ctSym.getInputStream(e)) { + return readAll(in); } } if (rtJar != null) { @@ -186,6 +212,26 @@ private byte[] jdkClass(String path) throws IOException { return Files.isRegularFile(p) ? Files.readAllBytes(p) : null; } + /// The `java.base` classes of Java 17 in `ct.sym`, whose entries are + /// `/java.base/java/lang/Math.sig`: the directory names every + /// release that class file holds for. + private static Map indexRelease(ZipFile ct) { + Map out = new HashMap(); + java.util.Enumeration en = ct.entries(); + String module = "/java.base/"; + while (en.hasMoreElements()) { + ZipEntry e = en.nextElement(); + String n = e.getName(); + int slash = n.indexOf('/'); + if (slash < 0 || !n.endsWith(".sig") || !n.startsWith(module, slash) + || n.substring(0, slash).indexOf(RELEASE) < 0) { + continue; + } + out.put(n.substring(slash + module.length(), n.length() - 4) + ".class", e); + } + return out; + } + private static byte[] readAll(InputStream in) throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); byte[] buf = new byte[8192]; @@ -275,5 +321,8 @@ public void close() throws IOException { if (rtJar != null) { rtJar.close(); } + if (ctSym != null) { + ctSym.close(); + } } } diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/gzip.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/gzip.py new file mode 100644 index 00000000000..d78cc71ca4d --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/gzip.py @@ -0,0 +1,298 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""gzip files and streams, framed here over zlib's raw deflate as CPython does. + +A file opened for reading is decompressed whole when it is opened (every +member, as CPython concatenates them) and read from memory, so it can be +seeked freely. Writing compresses as it goes. +""" + +import builtins +import io +import os +import struct +import time +import zlib + +__all__ = ["BadGzipFile", "GzipFile", "open", "compress", "decompress"] + +READ = "rb" +WRITE = "wb" +FTEXT, FHCRC, FEXTRA, FNAME, FCOMMENT = 1, 2, 4, 8, 16 +_COMPRESS_LEVEL_FAST = 1 +_COMPRESS_LEVEL_TRADEOFF = 6 +_COMPRESS_LEVEL_BEST = 9 + + +class BadGzipFile(OSError): + pass + + +def open(filename, mode="rb", compresslevel=_COMPRESS_LEVEL_BEST, encoding=None, errors=None, newline=None): + if "t" in mode: + if "b" in mode: + raise ValueError("Invalid mode: %r" % (mode,)) + else: + if encoding is not None: + raise ValueError("Argument 'encoding' not supported in binary mode") + if errors is not None: + raise ValueError("Argument 'errors' not supported in binary mode") + if newline is not None: + raise ValueError("Argument 'newline' not supported in binary mode") + gz_mode = mode.replace("t", "") + if isinstance(filename, (str, bytes, os.PathLike)): + binary_file = GzipFile(filename, gz_mode, compresslevel) + elif hasattr(filename, "read") or hasattr(filename, "write"): + binary_file = GzipFile(None, gz_mode, compresslevel, filename) + else: + raise TypeError("filename must be a str or bytes object, or a file") + if "t" in mode: + encoding = io.text_encoding(encoding) + return io.TextIOWrapper(binary_file, encoding, errors, newline) + return binary_file + + +def _header(mtime, level, name=b""): + flags = FNAME if name else 0 + if level == _COMPRESS_LEVEL_BEST: + xfl = 2 + elif level == _COMPRESS_LEVEL_FAST: + xfl = 4 + else: + xfl = 0 + out = b"\037\213\010" + bytes([flags]) + struct.pack("" + + def _check_not_closed(self): + if self.closed: + raise ValueError("I/O operation on closed file.") + + def write(self, data): + self._check_not_closed() + if self.mode != WRITE: + raise OSError(9, "write() on read-only GzipFile object") + if isinstance(data, (bytes, bytearray)): + length = len(data) + else: + data = memoryview(data) + length = data.nbytes + if length > 0: + self.fileobj.write(self._compress.compress(data)) + self._size += length + self._crc = zlib.crc32(data, self._crc) + return length + + def _need_read(self): + self._check_not_closed() + if self.mode != READ: + raise OSError(9, "read() on write-only GzipFile object") + + def read(self, size=-1): + self._need_read() + return self._buffer.read(size) + + def read1(self, size=-1): + return self.read(size) + + def readinto(self, b): + data = self.read(len(b)) + b[:len(data)] = data + return len(data) + + def peek(self, n): + self._need_read() + pos = self._buffer.tell() + data = self._buffer.read(max(n, 1)) + self._buffer.seek(pos) + return data + + def readline(self, size=-1): + self._need_read() + return self._buffer.readline(size) + + def __next__(self): + line = self.readline() + if not line: + raise StopIteration + return line + + def __iter__(self): + self._check_not_closed() + return self + + def readlines(self, hint=-1): + out = [] + for line in self: + out.append(line) + return out + + def close(self): + if self.closed: + return + self.closed = True + fileobj = self.fileobj + if fileobj is None: + return + self.fileobj = None + try: + if self.mode == WRITE: + fileobj.write(self._compress.flush()) + fileobj.write(struct.pack("= 0: + if i + 22 <= len(data): + clen = struct.unpack("= 0: + filename = filename[:null] + if os.sep != "/": + filename = filename.replace(os.sep, "/") + self.filename = filename + self.date_time = date_time + if date_time[0] < 1980: + raise ValueError("ZIP does not support timestamps before 1980") + self.compress_type = ZIP_STORED + self.compress_level = None + self.comment = b"" + self.extra = b"" + self.create_system = 3 + self.create_version = 20 + self.extract_version = 20 + self.reserved = 0 + self.flag_bits = 0 + self.volume = 0 + self.internal_attr = 0 + self.external_attr = 0 + self.header_offset = 0 + self.CRC = 0 + self.compress_size = 0 + self.file_size = 0 + self._end_offset = None + + def __repr__(self): + result = ["<%s filename=%r" % (self.__class__.__name__, self.filename)] + if self.compress_type != ZIP_STORED: + result.append(" compress_type=%s" % {8: "deflate", 12: "bzip2", 14: "lzma"}.get( + self.compress_type, self.compress_type)) + hi = self.external_attr >> 16 + lo = self.external_attr & 0xFFFF + if hi: + import stat + result.append(" filemode=%r" % stat.filemode(hi)) + if lo: + result.append(" external_attr=%#x" % lo) + isdir = self.is_dir() + if not isdir or self.file_size: + result.append(" file_size=%r" % self.file_size) + if ((not isdir or self.compress_size) and + (self.compress_type != ZIP_STORED or self.file_size != self.compress_size)): + result.append(" compress_size=%r" % self.compress_size) + result.append(">") + return "".join(result) + + def is_dir(self): + return self.filename.endswith("/") + + @classmethod + def from_file(cls, filename, arcname=None, *, strict_timestamps=True): + st = os.stat(filename) + isdir = os.path.isdir(filename) + mtime = time.localtime(st.st_mtime) + date_time = mtime[0:6] + if not strict_timestamps and date_time[0] < 1980: + date_time = (1980, 1, 1, 0, 0, 0) + if arcname is None: + arcname = filename + arcname = os.path.normpath(os.path.splitdrive(arcname)[1]) + while arcname[0] in (os.sep, os.altsep): + arcname = arcname[1:] + if isdir: + arcname += "/" + zinfo = cls(arcname, date_time) + zinfo.external_attr = (st.st_mode & 0xFFFF) << 16 + if isdir: + zinfo.file_size = 0 + zinfo.external_attr |= 0x10 + else: + zinfo.file_size = st.st_size + return zinfo + + def _dos(self): + dt = self.date_time + dosdate = (dt[0] - 1980) << 9 | dt[1] << 5 | dt[2] + dostime = dt[3] << 11 | dt[4] << 5 | (dt[5] // 2) + return dosdate, dostime + + def _encoded_name(self): + try: + return self.filename.encode("ascii"), self.flag_bits + except UnicodeEncodeError: + return self.filename.encode("utf-8"), self.flag_bits | 0x800 + + +def _zip64_extra(extra, info, need_size, need_csize, need_offset): + i = 0 + while i + 4 <= len(extra): + tp, ln = struct.unpack("") + return "".join(result) + + def _read_directory(self): + data = self._data + end = _find_end(data) + if end is None: + raise BadZipFile("File is not a zip file") + (sig, disk, diskcd, count_disk, count, size_cd, offset_cd, clen) = struct.unpack( + "<4sHHHHLLH", data[end:end + 22]) + self._comment = data[end + 22:end + 22 + clen] + loc = end - 20 + if loc >= 0 and data[loc:loc + 4] == _END64_LOCATOR: + off64 = struct.unpack("= 0 and data[loc:loc + 4] == _END64_LOCATOR: + concat -= 56 + 20 + self.start_dir = offset_cd + concat + pos = self.start_dir + stop = pos + size_cd + while pos < stop: + if data[pos:pos + 4] != _CENTRAL: + raise BadZipFile("Bad magic number for central directory") + (sig, cver, csys, ever, esys, flags, method, t, d, crc, csize, usize, nlen, xlen, mlen, + disk, iattr, eattr, hoff) = struct.unpack("<4sBBBBHHHHLLLHHHHHLL", data[pos:pos + 46]) + pos += 46 + raw = data[pos:pos + nlen] + pos += nlen + if flags & 0x800: + name = raw.decode("utf-8") + elif self.metadata_encoding: + name = raw.decode(self.metadata_encoding) + else: + name = _cp437(raw) + zi = ZipInfo(name) + zi.extra = data[pos:pos + xlen] + pos += xlen + zi.comment = data[pos:pos + mlen] + pos += mlen + zi.header_offset = hoff + zi.create_version, zi.create_system = cver, csys + zi.extract_version, zi.reserved = ever, esys + zi.flag_bits, zi.compress_type, zi.CRC = flags, method, crc + zi.compress_size, zi.file_size = csize, usize + zi.volume, zi.internal_attr, zi.external_attr = disk, iattr, eattr + _zip64_extra(zi.extra, zi, usize == 0xFFFFFFFF, csize == 0xFFFFFFFF, hoff == 0xFFFFFFFF) + zi.header_offset += concat + zi.date_time = ((d >> 9) + 1980, (d >> 5) & 0xF, d & 0x1F, t >> 11, (t >> 5) & 0x3F, + (t & 0x1F) * 2) + self.filelist.append(zi) + self.NameToInfo[zi.filename] = zi + + @property + def comment(self): + return self._comment + + @comment.setter + def comment(self, comment): + if not isinstance(comment, bytes): + raise TypeError("comment: expected bytes, got %s" % type(comment).__name__) + self._comment = comment[:ZIP_MAX_COMMENT] + self._modified = True + + def namelist(self): + return [zi.filename for zi in self.filelist] + + def infolist(self): + return list(self.filelist) + + def printdir(self, file=None): + print("%-46s %19s %12s" % ("File Name", "Modified ", "Size"), file=file) + for zinfo in self.filelist: + date = "%d-%02d-%02d %02d:%02d:%02d" % zinfo.date_time[:6] + print("%-46s %s %12d" % (zinfo.filename, date, zinfo.file_size), file=file) + + def getinfo(self, name): + info = self.NameToInfo.get(name) + if info is None: + raise KeyError("There is no item named %r in the archive" % name) + return info + + def setpassword(self, pwd): + self.pwd = pwd + + def _source(self): + return self._data if self.mode == "r" else bytes(self._out) + + def _read_entry(self, zinfo): + data = self._source() + off = zinfo.header_offset + if data[off:off + 4] != _LOCAL: + raise BadZipFile("Bad magic number for file header") + nlen, xlen = struct.unpack(" ZIP64_LIMIT or zinfo.header_offset > ZIP64_LIMIT: + if not self._allowZip64: + raise LargeZipFile("Filesize would require ZIP64 extensions") + name, flags = zinfo._encoded_name() + zinfo.flag_bits = flags + dosdate, dostime = zinfo._dos() + self._out += struct.pack("<4sHHHHHLLLHH", _LOCAL, zinfo.extract_version, zinfo.flag_bits, + zinfo.compress_type, dostime, dosdate, zinfo.CRC, zinfo.compress_size, + zinfo.file_size, len(name), len(zinfo.extra)) + self._out += name + zinfo.extra + comp + self.filelist.append(zinfo) + self.NameToInfo[zinfo.filename] = zinfo + self._modified = True + + def write(self, filename, arcname=None, compress_type=None, compresslevel=None): + if self.mode == "r": + raise ValueError("write() requires mode 'w', 'x', or 'a'") + zinfo = ZipInfo.from_file(filename, arcname, strict_timestamps=self._strict_timestamps) + if zinfo.is_dir(): + zinfo.compress_size = 0 + zinfo.CRC = 0 + self._write_entry(zinfo, b"") + return + zinfo.compress_type = compress_type if compress_type is not None else self.compression + zinfo.compress_level = compresslevel if compresslevel is not None else self.compresslevel + with open(filename, "rb") as f: + self._write_entry(zinfo, f.read()) + + def writestr(self, zinfo_or_arcname, data, compress_type=None, compresslevel=None): + if self.mode == "r": + raise ValueError("write() requires mode 'w', 'x', or 'a'") + if isinstance(data, str): + data = data.encode("utf-8") + if not isinstance(zinfo_or_arcname, ZipInfo): + zinfo = ZipInfo(zinfo_or_arcname, time.localtime(time.time())[:6]) + zinfo.compress_type = self.compression + zinfo.compress_level = self.compresslevel + if zinfo.is_dir(): + zinfo.external_attr = 0o40775 << 16 + zinfo.external_attr |= 0x10 + else: + zinfo.external_attr = 0o600 << 16 + else: + zinfo = zinfo_or_arcname + if compress_type is not None: + zinfo.compress_type = compress_type + if compresslevel is not None: + zinfo.compress_level = compresslevel + self._write_entry(zinfo, bytes(data)) + + def mkdir(self, zinfo_or_directory_name, mode=511): + if isinstance(zinfo_or_directory_name, ZipInfo): + zinfo = zinfo_or_directory_name + else: + name = zinfo_or_directory_name + if not name.endswith("/"): + name += "/" + zinfo = ZipInfo(name, time.localtime(time.time())[:6]) + zinfo.external_attr = ((0o40000 | mode) & 0xFFFF) << 16 + zinfo.external_attr |= 0x10 + self._write_entry(zinfo, b"") + + def testzip(self): + for zinfo in self.filelist: + try: + self._read_entry(zinfo) + except BadZipFile: + return zinfo.filename + return None + + def extract(self, member, path=None, pwd=None): + if path is None: + path = os.getcwd() + else: + path = os.fspath(path) + return self._extract_member(member, path) + + def extractall(self, path=None, members=None, pwd=None): + if members is None: + members = self.namelist() + if path is None: + path = os.getcwd() + else: + path = os.fspath(path) + for zipinfo in members: + self._extract_member(zipinfo, path) + + def _extract_member(self, member, targetpath): + if not isinstance(member, ZipInfo): + member = self.getinfo(member) + arcname = member.filename.replace("/", os.path.sep) + arcname = os.path.splitdrive(arcname)[1] + invalid = ("", os.path.curdir, os.path.pardir) + arcname = os.path.sep.join(x for x in arcname.split(os.path.sep) if x not in invalid) + targetpath = os.path.join(targetpath, arcname) + targetpath = os.path.normpath(targetpath) + upperdirs = os.path.dirname(targetpath) + if upperdirs and not os.path.exists(upperdirs): + os.makedirs(upperdirs) + if member.is_dir(): + if not os.path.isdir(targetpath): + os.mkdir(targetpath) + return targetpath + with open(targetpath, "wb") as target: + target.write(self._read_entry(member)) + return targetpath + + def _central(self): + out = bytearray() + for zinfo in self.filelist: + name, flags = zinfo._encoded_name() + dosdate, dostime = zinfo._dos() + out += struct.pack("<4sBBBBHHHHLLLHHHHHLL", _CENTRAL, zinfo.create_version, zinfo.create_system, + zinfo.extract_version, zinfo.reserved, flags, zinfo.compress_type, dostime, + dosdate, zinfo.CRC, zinfo.compress_size, zinfo.file_size, len(name), + len(zinfo.extra), len(zinfo.comment), 0, zinfo.internal_attr, + zinfo.external_attr, zinfo.header_offset) + out += name + zinfo.extra + zinfo.comment + return bytes(out) + + def close(self): + if self._data is None: + return + if self._writing: + raise ValueError("Can't close the ZIP file while there is an open writing handle on it. " + "Close the writing handle before closing the zip.") + try: + if self.mode in ("w", "x", "a") and self._modified: + start = len(self._out) + central = self._central() + count = len(self.filelist) + self._out += central + self._out += struct.pack("<4sHHHHLLH", _END, 0, 0, count, count, len(central), start, + len(self._comment)) + self._out += self._comment + if self._filePassed: + self.fp.seek(0) + self.fp.write(bytes(self._out)) + if hasattr(self.fp, "truncate"): + self.fp.truncate() + else: + with open(self.filename, "wb") as f: + f.write(bytes(self._out)) + finally: + self._data = None + + +def _check_compression(compression): + if compression in (ZIP_STORED, ZIP_DEFLATED): + return + if compression in (ZIP_BZIP2, ZIP_LZMA): + raise RuntimeError("Compression requires the (missing) %s module" % + ("bz2" if compression == ZIP_BZIP2 else "lzma")) + raise NotImplementedError("That compression method is not supported") + + +class Path: + """A path inside an archive, in the manner of pathlib.""" + + def __init__(self, root, at=""): + self.root = root if isinstance(root, ZipFile) else ZipFile(root) + self.at = at + + @property + def name(self): + return self.at.rstrip("/").split("/")[-1] + + def _names(self): + return self.root.namelist() + + def is_dir(self): + return not self.at or self.at.endswith("/") or any(n.startswith(self.at + "/") for n in self._names()) + + def is_file(self): + return self.at in self._names() and not self.at.endswith("/") + + def exists(self): + return self.is_dir() or self.is_file() + + def iterdir(self): + prefix = self.at if not self.at or self.at.endswith("/") else self.at + "/" + seen = [] + for n in self._names(): + if n.startswith(prefix) and n != prefix: + rest = n[len(prefix):] + child = prefix + rest.split("/")[0] + ("/" if "/" in rest else "") + if child not in seen: + seen.append(child) + return iter([Path(self.root, c) for c in seen]) + + def __truediv__(self, add): + base = self.at if not self.at or self.at.endswith("/") else self.at + "/" + return Path(self.root, base + add) + + joinpath = __truediv__ + + def read_bytes(self): + return self.root.read(self.at) + + def read_text(self, encoding="utf-8", *args, **kwargs): + return self.read_bytes().decode(encoding) + + def open(self, mode="r", *args, **kwargs): + f = self.root.open(self.at) + return io.TextIOWrapper(f, *args, **kwargs) if mode == "r" else f + + def __str__(self): + return (self.root.filename or "") + "/" + self.at + + def __repr__(self): + return "Path(%r, %r)" % (self.root.filename, self.at) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/zlib.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/zlib.py new file mode 100644 index 00000000000..09294ef24fa --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/zlib.py @@ -0,0 +1,171 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""zlib over Codename One's JZlib, a Java port of zlib. + +Streams are interchangeable with CPython's: what one compresses the other +decompresses, in zlib, raw deflate and gzip framing. +""" + +import _cn1zlib + +__all__ = ["compress", "decompress", "compressobj", "decompressobj", "crc32", "adler32", "error", + "MAX_WBITS", "DEFLATED", "DEF_MEM_LEVEL", "DEF_BUF_SIZE", "Z_NO_COMPRESSION", "Z_BEST_SPEED", + "Z_BEST_COMPRESSION", "Z_DEFAULT_COMPRESSION", "Z_FILTERED", "Z_HUFFMAN_ONLY", "Z_RLE", + "Z_FIXED", "Z_DEFAULT_STRATEGY", "Z_NO_FLUSH", "Z_PARTIAL_FLUSH", "Z_SYNC_FLUSH", + "Z_FULL_FLUSH", "Z_FINISH", "Z_BLOCK", "Z_TREES", "ZLIB_VERSION", "ZLIB_RUNTIME_VERSION"] + +MAX_WBITS = 15 +DEFLATED = 8 +DEF_MEM_LEVEL = 8 +DEF_BUF_SIZE = 16384 +Z_NO_COMPRESSION = 0 +Z_BEST_SPEED = 1 +Z_BEST_COMPRESSION = 9 +Z_DEFAULT_COMPRESSION = -1 +Z_FILTERED = 1 +Z_HUFFMAN_ONLY = 2 +Z_RLE = 3 +Z_FIXED = 4 +Z_DEFAULT_STRATEGY = 0 +Z_NO_FLUSH = 0 +Z_PARTIAL_FLUSH = 1 +Z_SYNC_FLUSH = 2 +Z_FULL_FLUSH = 3 +Z_FINISH = 4 +Z_BLOCK = 5 +Z_TREES = 6 +ZLIB_VERSION = "1.2.3" +ZLIB_RUNTIME_VERSION = "1.2.3" + + +class error(Exception): + pass + + +def _raise(e, action): + msg = str(e) + if msg.startswith("Error -5"): + raise error("Error -5 while %s data: incomplete or truncated stream" % action) from None + raise error(msg) from None + + +def _check_level(level): + if level < -1 or level > 9: + raise error("Bad compression level") + + +def crc32(data, value=0): + return _cn1zlib.crc32(data, value) + + +def adler32(data, value=1): + return _cn1zlib.adler32(data, value) + + +class _Compress: + def __init__(self, level, method, wbits, memLevel, strategy, zdict): + _check_level(level) + if method != DEFLATED: + raise ValueError("Invalid initialization option") + try: + self._z = _cn1zlib.deflate_new(level, wbits, memLevel, strategy, zdict) + except OSError as e: + _raise(e, "compressing") + self._done = False + + def compress(self, data): + if self._done: + raise error("Error -2 while compressing data: inconsistent stream state") + try: + return _cn1zlib.deflate(self._z, data, Z_NO_FLUSH) + except OSError as e: + _raise(e, "compressing") + + def flush(self, mode=Z_FINISH): + if mode == Z_NO_FLUSH: + return b"" + if self._done: + raise error("Error -2 while flushing: inconsistent stream state") + try: + out = _cn1zlib.deflate(self._z, b"", mode) + except OSError as e: + _raise(e, "flushing") + if mode == Z_FINISH: + self._done = True + _cn1zlib.end(self._z) + return out + + def copy(self): + raise ValueError("Inconsistent stream state") + + +class _Decompress: + def __init__(self, wbits, zdict): + try: + self._z = _cn1zlib.inflate_new(wbits, zdict) + except OSError as e: + _raise(e, "decompressing") + self.unused_data = b"" + self.unconsumed_tail = b"" + self.eof = False + + def decompress(self, data, max_length=0): + if max_length < 0: + raise ValueError("max_length must be non-negative") + data = bytes(data) + if self.eof: + self.unused_data += data + return b"" + try: + out, used, eof = _cn1zlib.inflate(self._z, data, max_length, False) + except OSError as e: + _raise(e, "decompressing") + if eof: + self.eof = True + self.unused_data += data[used:] + self.unconsumed_tail = b"" + else: + self.unconsumed_tail = data[used:] + return out + + def flush(self, length=DEF_BUF_SIZE): + if length <= 0: + raise ValueError("length must be greater than zero") + if self.eof or not self.unconsumed_tail: + return b"" + data = self.unconsumed_tail + out, used, eof = _cn1zlib.inflate(self._z, data, 0, True) + if eof: + self.eof = True + self.unused_data += data[used:] + self.unconsumed_tail = b"" + return out + + def copy(self): + raise ValueError("Inconsistent stream state") + + +def compressobj(level=Z_DEFAULT_COMPRESSION, method=DEFLATED, wbits=MAX_WBITS, memLevel=DEF_MEM_LEVEL, + strategy=Z_DEFAULT_STRATEGY, zdict=None): + return _Compress(level, method, wbits, memLevel, strategy, zdict) + + +def decompressobj(wbits=MAX_WBITS, zdict=None): + return _Decompress(wbits, zdict) + + +def compress(data, /, level=Z_DEFAULT_COMPRESSION, wbits=MAX_WBITS): + c = _Compress(level, DEFLATED, wbits, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY, None) + return c.compress(data) + c.flush() + + +def decompress(data, /, wbits=MAX_WBITS, bufsize=DEF_BUF_SIZE): + d = _Decompress(wbits, None) + try: + out, used, eof = _cn1zlib.inflate(d._z, bytes(data), 0, True) + except OSError as e: + _raise(e, "decompressing") + if not eof: + raise error("Error -5 while decompressing data: incomplete or truncated stream") + return out diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/zoneinfo.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/zoneinfo.py new file mode 100644 index 00000000000..2d5fbf1385c --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/zoneinfo.py @@ -0,0 +1,189 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""IANA time zones from the platform's own time zone database. + +Offsets come from the device (java.util.TimeZone, which is the operating +system's zone data on iOS and Android), so no tzdata package is shipped. +Abbreviations are not part of what every platform reports: the common +zones' are known here, and any other zone is named by its offset ("+03"), +as tzdata itself names zones that have no abbreviation. +""" + +import _cn1tz +from datetime import datetime, timedelta, timezone, tzinfo + +__all__ = ["ZoneInfo", "ZoneInfoNotFoundError", "InvalidTZPathError", "reset_tzpath", "available_timezones", + "TZPATH"] + +TZPATH = () + + +class ZoneInfoNotFoundError(KeyError): + pass + + +class InvalidTZPathError(ValueError): + pass + + +def reset_tzpath(to=None): + pass + + +def available_timezones(): + return set(k for k in _cn1tz.available() if "/" in k or k == "UTC") + + +# (standard, daylight) abbreviations of common zones +_ABBREV = { + "UTC": ("UTC", "UTC"), "Etc/UTC": ("UTC", "UTC"), "GMT": ("GMT", "GMT"), "Etc/GMT": ("GMT", "GMT"), + "America/New_York": ("EST", "EDT"), "America/Detroit": ("EST", "EDT"), "America/Toronto": ("EST", "EDT"), + "America/Chicago": ("CST", "CDT"), "America/Winnipeg": ("CST", "CDT"), "America/Mexico_City": ("CST", "CDT"), + "America/Denver": ("MST", "MDT"), "America/Edmonton": ("MST", "MDT"), "America/Phoenix": ("MST", "MST"), + "America/Los_Angeles": ("PST", "PDT"), "America/Vancouver": ("PST", "PDT"), "America/Tijuana": ("PST", "PDT"), + "America/Anchorage": ("AKST", "AKDT"), "Pacific/Honolulu": ("HST", "HST"), "America/Halifax": ("AST", "ADT"), + "America/St_Johns": ("NST", "NDT"), "America/Puerto_Rico": ("AST", "AST"), + "Europe/London": ("GMT", "BST"), "Europe/Dublin": ("GMT", "IST"), "Europe/Lisbon": ("WET", "WEST"), + "Atlantic/Canary": ("WET", "WEST"), + "Europe/Paris": ("CET", "CEST"), "Europe/Berlin": ("CET", "CEST"), "Europe/Madrid": ("CET", "CEST"), + "Europe/Rome": ("CET", "CEST"), "Europe/Amsterdam": ("CET", "CEST"), "Europe/Brussels": ("CET", "CEST"), + "Europe/Vienna": ("CET", "CEST"), "Europe/Zurich": ("CET", "CEST"), "Europe/Stockholm": ("CET", "CEST"), + "Europe/Oslo": ("CET", "CEST"), "Europe/Copenhagen": ("CET", "CEST"), "Europe/Prague": ("CET", "CEST"), + "Europe/Warsaw": ("CET", "CEST"), "Europe/Budapest": ("CET", "CEST"), "Europe/Belgrade": ("CET", "CEST"), + "Africa/Lagos": ("WAT", "WAT"), + "Europe/Athens": ("EET", "EEST"), "Europe/Helsinki": ("EET", "EEST"), "Europe/Kiev": ("EET", "EEST"), + "Europe/Kyiv": ("EET", "EEST"), "Europe/Bucharest": ("EET", "EEST"), "Europe/Sofia": ("EET", "EEST"), + "Africa/Cairo": ("EET", "EEST"), "Asia/Jerusalem": ("IST", "IDT"), "Europe/Moscow": ("MSK", "MSK"), + "Africa/Johannesburg": ("SAST", "SAST"), "Africa/Nairobi": ("EAT", "EAT"), + "Asia/Kolkata": ("IST", "IST"), "Asia/Calcutta": ("IST", "IST"), "Asia/Karachi": ("PKT", "PKT"), + "Asia/Shanghai": ("CST", "CST"), "Asia/Hong_Kong": ("HKT", "HKT"), "Asia/Taipei": ("CST", "CST"), + "Asia/Manila": ("PST", "PST"), "Asia/Jakarta": ("WIB", "WIB"), + "Asia/Tokyo": ("JST", "JST"), "Asia/Seoul": ("KST", "KST"), + "Australia/Sydney": ("AEST", "AEDT"), "Australia/Melbourne": ("AEST", "AEDT"), + "Australia/Hobart": ("AEST", "AEDT"), "Australia/Brisbane": ("AEST", "AEST"), + "Australia/Adelaide": ("ACST", "ACDT"), "Australia/Darwin": ("ACST", "ACST"), + "Australia/Perth": ("AWST", "AWST"), "Pacific/Auckland": ("NZST", "NZDT"), +} + +_EPOCH = datetime(1970, 1, 1) +_DAY = 86400 + + +def _offset_name(seconds): + sign = "+" if seconds >= 0 else "-" + seconds = abs(seconds) + h, rem = divmod(seconds, 3600) + m = rem // 60 + return "%s%02d" % (sign, h) if m == 0 else "%s%02d%02d" % (sign, h, m) + + +class ZoneInfo(tzinfo): + _cache = {} + + def __new__(cls, key): + z = cls._cache.get(key) + if z is None: + z = cls.no_cache(key) + cls._cache[key] = z + return z + + @classmethod + def no_cache(cls, key): + if not isinstance(key, str): + raise TypeError("key must be a str") + if not key or key.startswith("/") or ".." in key.split("/") or not _cn1tz.known(key): + raise ZoneInfoNotFoundError("No time zone found with key %s" % key) + self = object.__new__(cls) + self._key = key + self._raw = _cn1tz.raw(key) + return self + + @classmethod + def clear_cache(cls, *, only_keys=None): + if only_keys is None: + cls._cache.clear() + else: + for k in only_keys: + cls._cache.pop(k, None) + + @classmethod + def from_file(cls, fobj, key=None): + raise ValueError("zone files are not read: the platform's zone data is used") + + @property + def key(self): + return self._key + + def __str__(self): + return self._key + + def __repr__(self): + return "zoneinfo.ZoneInfo(key=%r)" % self._key + + def __reduce__(self): + return (self.__class__, (self._key,)) + + def _at(self, utc_seconds): + return _cn1tz.offset(self._key, utc_seconds) + + def _wall_offset(self, dt): + """The offset in seconds at the wall time dt, honouring dt.fold.""" + naive = dt.replace(tzinfo=None, fold=0) + w = (naive - _EPOCH) // timedelta(seconds=1) + before = self._at(w - _DAY) + after = self._at(w + _DAY) + if before == after: + return self._at(w - before) + ok_before = self._at(w - before) == before + ok_after = self._at(w - after) == after + if ok_before and ok_after: + return after if dt.fold else before + if ok_before: + return before + if ok_after: + return after + return after if dt.fold else before + + def utcoffset(self, dt): + if dt is None: + return None + return timedelta(seconds=self._wall_offset(dt)) + + def dst(self, dt): + if dt is None: + return None + off = self._wall_offset(dt) + return timedelta(seconds=max(0, off - self._std_near(dt, off))) + + def _std_near(self, dt, off): + # the standard offset is the smaller of the year's two offsets + naive = dt.replace(tzinfo=None, fold=0) + w = (naive - _EPOCH) // timedelta(seconds=1) + jan = self._at(w - (naive.timetuple().tm_yday - 1) * _DAY) + jul = self._at(w + (182 - naive.timetuple().tm_yday) * _DAY) + return min(jan, jul, off) + + def tzname(self, dt): + if dt is None: + return None + off = self._wall_offset(dt) + names = _ABBREV.get(self._key) + if names is not None: + return names[1] if off > self._std_near(dt, off) else names[0] + return _offset_name(off) + + def fromutc(self, dt): + if not isinstance(dt, datetime): + raise TypeError("fromutc() requires a datetime argument") + if dt.tzinfo is not self: + raise ValueError("dt.tzinfo is not self") + naive = dt.replace(tzinfo=None) + u = (naive - _EPOCH) // timedelta(seconds=1) + off = self._at(u) + local = dt + timedelta(seconds=off) + # the second of two wall times that repeat is fold=1 + earlier = self._at(u - _DAY) + if earlier > off and self._at(u - (earlier - off)) == earlier: + local = local.replace(fold=1) + return local diff --git a/maven/python-transpiler/src/test/resources/conformance/compress_.out b/maven/python-transpiler/src/test/resources/conformance/compress_.out new file mode 100644 index 00000000000..23d717565ad --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/compress_.out @@ -0,0 +1,37 @@ +True True True True True +level 0 True 78 True +level 1 True 78 True +level 6 True 78 True +level 9 True 78 True +level -1 True 78 True +0x6f087e52 0x5fe8e499 1821799577 1 0 +dict True True +stream True True +max_length 100 True False +rest True True b'extra' +error Error -3 while decompressing data: incorrect header check +error Error -5 while decompressing data: incomplete or truncated stream +15 9 8 +gzip True 1f8b08000000000002ff 1f8b08000500000004ff True +members True +text ['line one\n', 'line two\n'] +binary b'line' b' one\nline two\n' +header 1f8b0808 b'f.txt\x00' +fileobj 1f8b08000000000002ff b'abcdef' +bad Not a gzipped file (b'pl') +eof Compressed file ended before the end-of-stream marker was reached +['a.txt', 'dir/b.bin'] [(2020, 1, 2, 3, 4, 6), (2021, 5, 6, 7, 8, 10)] [0, 0] +True True None +600 0x6f087e52 False +b'\x00\x01\x02' +KeyError "There is no item named 'missing' in the archive" +is_zipfile True False +['x.txt', 'stored.bin', 'w.txt', 'empty/', 'nested/f.gz', 'appended.txt'] b'a comment' +True b'written through open' b'more' True +[('x.txt', 8, False), ('stored.bin', 0, False), ('w.txt', 8, False), ('empty/', 0, True), ('nested/f.gz', 8, False), ('appended.txt', 0, False)] +(2022, 2, 3, 4, 5, 6) 2 +['appended.txt', 'empty', 'nested', 'stored.bin', 'w.txt', 'x.txt'] ['f.gz'] True +['appended.txt', 'empty', 'nested', 'stored.bin', 'w.txt', 'x.txt'] written through open True +written through open +BadZipFile File is not a zip file +NotImplementedError That compression method is not supported diff --git a/maven/python-transpiler/src/test/resources/conformance/compress_.py b/maven/python-transpiler/src/test/resources/conformance/compress_.py new file mode 100644 index 00000000000..c120b60a911 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/compress_.py @@ -0,0 +1,120 @@ +import zlib, gzip, io, os, zipfile, struct +T = b'hello hello hello compression ' * 20 +ZL=bytes.fromhex('789ccb48cdc9c957c8402293f3730b8a528b8b33f3f350c447654765899705005fe8e499') +RAW=bytes.fromhex('cb48cdc9c957c8402293f3730b8a528b8b33f3f350c44765476589970500') +GZ=bytes.fromhex('1f8b08000000000002ffcb48cdc9c957c8402293f3730b8a528b8b33f3f350c44765476589970500527e086f58020000') +DICT=bytes.fromhex('78bb3ca206e7cb000b209348920aa3b2a3b2e4c902005fe8e499') +ZIP=bytes.fromhex('504b030414000000000083182250527e086f580200005802000005000000612e74787468656c6c6f2068656c6c6f2068656c6c6f20636f6d7072657373696f6e2068656c6c6f2068656c6c6f2068656c6c6f20636f6d7072657373696f6e2068656c6c6f2068656c6c6f2068656c6c6f20636f6d7072657373696f6e2068656c6c6f2068656c6c6f2068656c6c6f20636f6d7072657373696f6e2068656c6c6f2068656c6c6f2068656c6c6f20636f6d7072657373696f6e2068656c6c6f2068656c6c6f2068656c6c6f20636f6d7072657373696f6e2068656c6c6f2068656c6c6f2068656c6c6f20636f6d7072657373696f6e2068656c6c6f2068656c6c6f2068656c6c6f20636f6d7072657373696f6e2068656c6c6f2068656c6c6f2068656c6c6f20636f6d7072657373696f6e2068656c6c6f2068656c6c6f2068656c6c6f20636f6d7072657373696f6e2068656c6c6f2068656c6c6f2068656c6c6f20636f6d7072657373696f6e2068656c6c6f2068656c6c6f2068656c6c6f20636f6d7072657373696f6e2068656c6c6f2068656c6c6f2068656c6c6f20636f6d7072657373696f6e2068656c6c6f2068656c6c6f2068656c6c6f20636f6d7072657373696f6e2068656c6c6f2068656c6c6f2068656c6c6f20636f6d7072657373696f6e2068656c6c6f2068656c6c6f2068656c6c6f20636f6d7072657373696f6e2068656c6c6f2068656c6c6f2068656c6c6f20636f6d7072657373696f6e2068656c6c6f2068656c6c6f2068656c6c6f20636f6d7072657373696f6e2068656c6c6f2068656c6c6f2068656c6c6f20636f6d7072657373696f6e2068656c6c6f2068656c6c6f2068656c6c6f20636f6d7072657373696f6e20504b03041400000000000539a652738c05290001000000010000090000006469722f622e62696e000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f505152535455565758595a5b5c5d5e5f606162636465666768696a6b6c6d6e6f707172737475767778797a7b7c7d7e7f808182838485868788898a8b8c8d8e8f909192939495969798999a9b9c9d9e9fa0a1a2a3a4a5a6a7a8a9aaabacadaeafb0b1b2b3b4b5b6b7b8b9babbbcbdbebfc0c1c2c3c4c5c6c7c8c9cacbcccdcecfd0d1d2d3d4d5d6d7d8d9dadbdcdddedfe0e1e2e3e4e5e6e7e8e9eaebecedeeeff0f1f2f3f4f5f6f7f8f9fafbfcfdfeff504b0102140314000000000083182250527e086f5802000058020000050000000000000000000000800100000000612e747874504b010214031400000000000539a652738c0529000100000001000009000000000000000000000080017b0200006469722f622e62696e504b050600000000020002006a000000a20300000000') + +# zlib: decompress what CPython's zlib produced, and round-trip our own +print(zlib.decompress(ZL) == T, zlib.decompress(RAW, -15) == T, zlib.decompress(GZ, 31) == T, + zlib.decompress(GZ, 47) == T, zlib.decompress(ZL, 47) == T) +for level in (0, 1, 6, 9, -1): + c = zlib.compress(T, level) + print("level", level, zlib.decompress(c) == T, c[:1].hex(), len(c) < len(T) or level == 0) +print(hex(zlib.crc32(T)), hex(zlib.adler32(T)), zlib.crc32(b"abc", zlib.crc32(b"xyz")), + zlib.adler32(b""), zlib.crc32(b"")) +d = zlib.decompressobj(zdict=b"hello compression") +print("dict", d.decompress(DICT) == T, d.eof) + +# streaming, max_length, unused data +c = zlib.compressobj(9, zlib.DEFLATED, -15) +parts = [c.compress(T[i:i + 50]) for i in range(0, len(T), 50)] +parts.append(c.flush(zlib.Z_SYNC_FLUSH)) +parts.append(c.compress(b"tail")) +parts.append(c.flush()) +stream = b"".join(parts) +d = zlib.decompressobj(-15) +out = b"" +for i in range(0, len(stream), 7): + out += d.decompress(stream[i:i + 7]) +print("stream", out == T + b"tail", d.eof) +d = zlib.decompressobj() +first = d.decompress(ZL + b"extra", 100) +print("max_length", len(first), len(d.unconsumed_tail) > 0, d.eof) +rest = d.decompress(d.unconsumed_tail) +print("rest", first + rest == T, d.eof, d.unused_data) +try: + zlib.decompress(b"not zlib data") +except zlib.error as e: + print("error", e) +try: + zlib.decompress(ZL[:20]) +except zlib.error as e: + print("error", e) +print(zlib.MAX_WBITS, zlib.Z_BEST_COMPRESSION, zlib.DEFLATED) + +# gzip +print("gzip", gzip.decompress(GZ) == T, gzip.compress(T, mtime=0)[:10].hex(), + gzip.compress(T, 1, mtime=5)[:10].hex(), gzip.decompress(gzip.compress(T)) == T) +print("members", gzip.decompress(GZ + gzip.compress(b"!", mtime=0)) == T + b"!") +os.makedirs("gz", exist_ok=True) +with gzip.open("gz/f.txt.gz", "wt", encoding="utf-8") as f: + f.write("line one\nline two\n") +with gzip.open("gz/f.txt.gz", "rt", encoding="utf-8") as f: + print("text", f.readlines()) +with gzip.open("gz/f.txt.gz", "rb") as f: + print("binary", f.read(4), f.read()) +with open("gz/f.txt.gz", "rb") as f: + raw = f.read() +print("header", raw[:4].hex(), raw[10:16]) +buf = io.BytesIO() +with gzip.GzipFile(fileobj=buf, mode="wb", mtime=0) as g: + g.write(b"abc") + g.write(memoryview(b"def")) +print("fileobj", buf.getvalue()[:10].hex(), gzip.decompress(buf.getvalue())) +try: + gzip.decompress(b"plain") +except gzip.BadGzipFile as e: + print("bad", e) +try: + gzip.decompress(GZ[:-3]) +except EOFError as e: + print("eof", e) + +# zipfile: read CPython's archive +z = zipfile.ZipFile(io.BytesIO(ZIP)) +print(z.namelist(), [i.date_time for i in z.infolist()], [i.compress_type for i in z.infolist()]) +print(z.read("a.txt") == T, z.read("dir/b.bin") == bytes(range(256)), z.testzip()) +info = z.getinfo("a.txt") +print(info.file_size, hex(info.CRC), info.is_dir()) +with z.open("dir/b.bin") as f: + print(f.read(3)) +try: + z.getinfo("missing") +except KeyError as e: + print("KeyError", e) +z.close() + +# write, append and extract +with zipfile.ZipFile("gz/out.zip", "w", zipfile.ZIP_DEFLATED) as z: + z.writestr("x.txt", "text " * 50) + z.writestr(zipfile.ZipInfo("stored.bin", (2022, 2, 3, 4, 5, 6)), b"\x00\x01", zipfile.ZIP_STORED) + with z.open("w.txt", "w") as w: + w.write(b"written ") + w.write(b"through open") + z.mkdir("empty") + z.write("gz/f.txt.gz", "nested/f.gz") + z.comment = b"a comment" +with zipfile.ZipFile("gz/out.zip", "a") as z: + z.writestr("appended.txt", b"more") +print("is_zipfile", zipfile.is_zipfile("gz/out.zip"), zipfile.is_zipfile("gz/f.txt.gz")) +with zipfile.ZipFile("gz/out.zip") as z: + print(z.namelist(), z.comment) + print(z.read("x.txt") == b"text " * 50, z.read("w.txt"), z.read("appended.txt"), z.read("nested/f.gz") == raw) + print([(i.filename, i.compress_type, i.is_dir()) for i in z.infolist()]) + print(z.getinfo("stored.bin").date_time, z.getinfo("stored.bin").compress_size) + z.extractall("gz/ex") + print(sorted(os.listdir("gz/ex")), sorted(os.listdir("gz/ex/nested")), os.path.isdir("gz/ex/empty")) + p = zipfile.Path(z) + print(sorted(c.name for c in p.iterdir()), (p / "w.txt").read_text(), (p / "nested").is_dir()) +with open("gz/ex/w.txt") as f: + print(f.read()) +try: + zipfile.ZipFile(io.BytesIO(b"nope")) +except zipfile.BadZipFile as e: + print("BadZipFile", e) +try: + zipfile.ZipFile(io.BytesIO(), "w", zipfile.ZIP_LZMA + 100) +except NotImplementedError as e: + print("NotImplementedError", e) diff --git a/maven/python-transpiler/src/test/resources/conformance/zoneinfo_.out b/maven/python-transpiler/src/test/resources/conformance/zoneinfo_.out new file mode 100644 index 00000000000..04cd806f3b0 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/zoneinfo_.out @@ -0,0 +1,31 @@ +America/New_York zoneinfo.ZoneInfo(key='America/New_York') America/New_York True +America/New_York 2024-01-15T12:00:00-05:00 -1 day, 19:00:00 0:00:00 EST +America/New_York 2024-07-15T12:00:00-04:00 -1 day, 20:00:00 1:00:00 EDT +Europe/London 2024-01-15T12:00:00+00:00 0:00:00 0:00:00 GMT +Europe/London 2024-07-15T12:00:00+01:00 1:00:00 1:00:00 BST +Europe/Paris 2024-01-15T12:00:00+01:00 1:00:00 0:00:00 CET +Europe/Paris 2024-07-15T12:00:00+02:00 2:00:00 1:00:00 CEST +Asia/Tokyo 2024-01-15T12:00:00+09:00 9:00:00 0:00:00 JST +Asia/Tokyo 2024-07-15T12:00:00+09:00 9:00:00 0:00:00 JST +Australia/Sydney 2024-01-15T12:00:00+11:00 11:00:00 1:00:00 AEDT +Australia/Sydney 2024-07-15T12:00:00+10:00 10:00:00 0:00:00 AEST +Asia/Kolkata 2024-01-15T12:00:00+05:30 5:30:00 0:00:00 IST +Asia/Kolkata 2024-07-15T12:00:00+05:30 5:30:00 0:00:00 IST +America/Los_Angeles 2024-01-15T12:00:00-08:00 -1 day, 16:00:00 0:00:00 PST +America/Los_Angeles 2024-07-15T12:00:00-07:00 -1 day, 17:00:00 1:00:00 PDT +UTC 2024-01-15T12:00:00+00:00 0:00:00 0:00:00 UTC +UTC 2024-07-15T12:00:00+00:00 0:00:00 0:00:00 UTC +spring 2024-03-10T01:30:00-05:00 EST 0 +spring 2024-03-10T03:30:00-04:00 EDT 0 +spring 2024-03-10T04:30:00-04:00 EDT 0 +spring 2024-03-10T05:30:00-04:00 EDT 0 +fall 2024-11-03T00:30:00-04:00 EDT 0 +fall 2024-11-03T01:30:00-04:00 EDT 0 +fall 2024-11-03T01:30:00-05:00 EST 1 +fall 2024-11-03T02:30:00-05:00 EST 0 +ambiguous -1 day, 20:00:00 -1 day, 19:00:00 EDT EST +gap -1 day, 19:00:00 -1 day, 20:00:00 +to utc 2024-11-03 05:30:00+00:00 2024-11-03 06:30:00+00:00 +0:00:00 False 2024-06-01 16:00:00+09:00 +1704067200.0 +not found ZoneInfoNotFoundError True diff --git a/maven/python-transpiler/src/test/resources/conformance/zoneinfo_.py b/maven/python-transpiler/src/test/resources/conformance/zoneinfo_.py new file mode 100644 index 00000000000..847822113db --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/zoneinfo_.py @@ -0,0 +1,39 @@ +from datetime import datetime, timedelta, timezone +from zoneinfo import ZoneInfo, ZoneInfoNotFoundError + +ny = ZoneInfo("America/New_York") +print(ny, repr(ny), ny.key, ZoneInfo("America/New_York") is ny) +for zone in ("America/New_York", "Europe/London", "Europe/Paris", "Asia/Tokyo", "Australia/Sydney", + "Asia/Kolkata", "America/Los_Angeles", "UTC"): + z = ZoneInfo(zone) + for when in (datetime(2024, 1, 15, 12, 0), datetime(2024, 7, 15, 12, 0)): + dt = when.replace(tzinfo=z) + print(zone, dt.isoformat(), dt.utcoffset(), dt.dst(), dt.tzname()) + +# conversion from UTC across a transition +utc = datetime(2024, 3, 10, 6, 30, tzinfo=timezone.utc) +for h in range(4): + t = (utc + timedelta(hours=h)).astimezone(ny) + print("spring", t.isoformat(), t.tzname(), t.fold) +utc = datetime(2024, 11, 3, 4, 30, tzinfo=timezone.utc) +for h in range(4): + t = (utc + timedelta(hours=h)).astimezone(ny) + print("fall", t.isoformat(), t.tzname(), t.fold) + +# ambiguous and missing wall times +amb = datetime(2024, 11, 3, 1, 30, tzinfo=ny) +print("ambiguous", amb.utcoffset(), amb.replace(fold=1).utcoffset(), amb.tzname(), amb.replace(fold=1).tzname()) +gap = datetime(2024, 3, 10, 2, 30, tzinfo=ny) +print("gap", gap.utcoffset(), gap.replace(fold=1).utcoffset()) +print("to utc", amb.astimezone(timezone.utc), amb.replace(fold=1).astimezone(timezone.utc)) + +# arithmetic and comparison +a = datetime(2024, 6, 1, 9, 0, tzinfo=ZoneInfo("Europe/Paris")) +b = datetime(2024, 6, 1, 16, 0, tzinfo=ZoneInfo("Asia/Tokyo")) +print(b - a, a < b, a.astimezone(ZoneInfo("Asia/Tokyo"))) +print(datetime(2024, 1, 1, tzinfo=ZoneInfo("UTC")).timestamp()) + +try: + ZoneInfo("Not/AZone") +except ZoneInfoNotFoundError as e: + print("not found", type(e).__name__, isinstance(e, KeyError)) From 1c7efad14356ccba3f5cfc95c082bf2fbebbd6ee Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 09:10:06 +0200 Subject: [PATCH 040/112] Python: resolve PyPI requirements on import and replace C extension modules with generated Java cn1:import-python-project now resolves the project's requirements (PEP 508/440, markers for CPython 3.13 on POSIX) against PyPI's JSON API, verifies each file's SHA-256, vendors the Python into src/main/python-packages with a python.lock, and skips packages the runtime provides. A compiled module found in a wheel or sdist gets a generated Java class (com.codename1.python.ext) with a stub per name the package uses, built on the new NativeModule API; modules whose imports all fall back on ImportError need nothing. The build transpiles the vendored packages and installs the native modules before any module runs. Also clears the SpotBugs and PMD findings in python-runtime and build-engine. Co-Authored-By: Claude Opus 5.5 --- .../codename1/maven/PythonExtensionStubs.java | 301 +++++++++ .../com/codename1/maven/PythonLineMapper.java | 11 +- .../maven/PythonPackageResolver.java | 637 ++++++++++++++++++ .../codename1/maven/PythonProjectBuilder.java | 13 +- .../maven/PythonProjectImporter.java | 45 +- .../codename1/maven/PythonRequirement.java | 515 ++++++++++++++ .../maven/PythonPackageResolverTest.java | 223 ++++++ .../maven/ImportPythonProjectMojo.java | 43 +- .../codename1/python/runtime/Builtins.java | 2 +- .../com/codename1/python/runtime/Cn1Sql.java | 5 +- .../com/codename1/python/runtime/Gil.java | 3 +- .../python/runtime/NativeModule.java | 169 +++++ .../codename1/python/runtime/StdModules.java | 6 +- .../transpiler/api/PythonTranspiler.java | 35 +- .../python/transpiler/ConformanceTest.java | 32 +- .../src/test/resources/conformance/cext_.out | 6 + .../conformance/cext_/FastlibCore.java | 46 ++ .../conformance/cext_/fastlib/__init__.py | 20 + .../conformance/cext_/fastlib/_core.py | 21 + .../test/resources/conformance/cext_/main.py | 12 + .../conformance/cext_/natives.properties | 1 + 21 files changed, 2121 insertions(+), 25 deletions(-) create mode 100644 maven/build-engine/src/main/java/com/codename1/maven/PythonExtensionStubs.java create mode 100644 maven/build-engine/src/main/java/com/codename1/maven/PythonPackageResolver.java create mode 100644 maven/build-engine/src/main/java/com/codename1/maven/PythonRequirement.java create mode 100644 maven/build-engine/src/test/java/com/codename1/maven/PythonPackageResolverTest.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/NativeModule.java create mode 100644 maven/python-transpiler/src/test/resources/conformance/cext_.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/cext_/FastlibCore.java create mode 100644 maven/python-transpiler/src/test/resources/conformance/cext_/fastlib/__init__.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/cext_/fastlib/_core.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/cext_/main.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/cext_/natives.properties diff --git a/maven/build-engine/src/main/java/com/codename1/maven/PythonExtensionStubs.java b/maven/build-engine/src/main/java/com/codename1/maven/PythonExtensionStubs.java new file mode 100644 index 00000000000..a8db3ca1a2e --- /dev/null +++ b/maven/build-engine/src/main/java/com/codename1/maven/PythonExtensionStubs.java @@ -0,0 +1,301 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.maven; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.Map; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/// Writes the Java that stands in for a package's compiled modules. +/// +/// A C extension is supported by replacing its compiled part with Java: the +/// package's Python is transpiled as it is, and each compiled module it +/// imports is a Java class that registers the same names through +/// `com.codename1.python.runtime.NativeModule`. Most compiled modules are +/// small accelerators over the platform's own facilities, so the class is +/// usually a few calls into Codename One. +/// +/// This class finds the names the package's Python takes from each compiled +/// module -- `from ._speedups import escape`, `_speedups.escape(...)` -- and +/// writes a class with a stub for each, raising `NotImplementedError` until +/// it is filled in. A stub that exists is never overwritten. A module every +/// import of which sits in a `try` that falls back on `ImportError` needs +/// nothing at all: the package's pure-Python path runs instead. +final class PythonExtensionStubs { + /// The file, beside the packages, that maps each compiled module to its + /// Java class for the build. + static final String INDEX = "cn1-native-modules.properties"; + static final String PACKAGE = "com.codename1.python.ext"; + + private static final Pattern FROM = Pattern.compile( + "^([ \\t]*)from[ \\t]+(\\.*)([\\w.]*)[ \\t]+import[ \\t]+\\(?([^#)\\n]*)", Pattern.MULTILINE); + private static final Pattern IMPORT = Pattern.compile( + "^([ \\t]*)import[ \\t]+([\\w.]+)(?:[ \\t]+as[ \\t]+(\\w+))?", Pattern.MULTILINE); + + private final Map sources; + + /// `sources` maps each Python file's path to its text. + PythonExtensionStubs(Map sources) { + this.sources = sources; + } + + /// `pkg.sub._speedups` becomes `PkgSubSpeedups`. + String className(String module) { + return PACKAGE + "." + simpleName(module); + } + + static String simpleName(String module) { + StringBuilder sb = new StringBuilder(); + boolean up = true; + for (int i = 0; i < module.length(); i++) { + char c = module.charAt(i); + if (c == '.' || c == '_') { + up = true; + } else if (up) { + sb.append(Character.toUpperCase(c)); + up = false; + } else { + sb.append(c); + } + } + if (sb.length() == 0 || !Character.isJavaIdentifierStart(sb.charAt(0))) { + sb.insert(0, 'M'); + } + return sb.toString(); + } + + /// Whether every import of `module` falls back when it is missing. + boolean isOptional(String module) { + boolean any = false; + for (Map.Entry e : sources.entrySet()) { + String pkg = packageOf(e.getKey()); + String text = e.getValue(); + for (int[] at : imports(text, pkg, module, null)) { + any = true; + if (!guarded(text, at[0], at[1])) { + return false; + } + } + } + return any; + } + + /// The names the package takes from `module`. + TreeSet usedNames(String module) { + TreeSet names = new TreeSet(); + for (Map.Entry e : sources.entrySet()) { + imports(e.getValue(), packageOf(e.getKey()), module, names); + } + names.remove("*"); + return names; + } + + /// Writes the stub for `module` under `javaDir`; false when one exists. + boolean write(String module, String distribution, File javaDir) throws IOException { + String simple = simpleName(module); + File f = new File(javaDir, PACKAGE.replace('.', '/') + "/" + simple + ".java"); + if (f.exists()) { + return false; + } + StringBuilder sb = new StringBuilder(); + sb.append("package ").append(PACKAGE).append(";\n\n"); + sb.append("import com.codename1.python.runtime.NativeModule;\n\n"); + sb.append("/// `").append(module).append("`, which ").append(distribution) + .append(" ships compiled, written in Java.\n"); + sb.append("///\n"); + sb.append("/// Generated with a stub for each name the package's Python takes from the\n"); + sb.append("/// module. Replace each stub's body with Java -- usually a call into Codename\n"); + sb.append("/// One -- over plain Java values: a `Number` for an int or float, `String`,\n"); + sb.append("/// `byte[]`, `java.util.List`, `java.util.Map`, `null` for `None`. This file\n"); + sb.append("/// is yours: importing the project again does not overwrite it.\n"); + sb.append("public final class ").append(simple).append(" {\n"); + sb.append(" private ").append(simple).append("() {\n }\n\n"); + sb.append(" /// Registers the module; the generated program calls this at start.\n"); + sb.append(" public static void install() {\n"); + sb.append(" new NativeModule(\"").append(module).append("\")\n"); + for (String n : usedNames(module)) { + sb.append(" .function(\"").append(n).append("\", args -> {\n"); + sb.append(" throw NativeModule.notImplemented(\"").append(module).append('.') + .append(n).append("\");\n"); + sb.append(" })\n"); + } + sb.append(" .install();\n"); + sb.append(" }\n}\n"); + Files.createDirectories(f.getParentFile().toPath()); + Files.write(f.toPath(), sb.toString().getBytes(StandardCharsets.UTF_8)); + return true; + } + + static void writeIndex(File file, Map modules) throws IOException { + if (modules.isEmpty()) { + if (file.isFile()) { + Files.delete(file.toPath()); + } + return; + } + StringBuilder sb = new StringBuilder("# Compiled Python modules and the Java classes that replace them\n"); + for (Map.Entry e : new TreeMap(modules).entrySet()) { + sb.append(e.getKey()).append('=').append(e.getValue()).append('\n'); + } + Files.createDirectories(file.getParentFile().toPath()); + Files.write(file.toPath(), sb.toString().getBytes(StandardCharsets.UTF_8)); + } + + /// Reads [#INDEX] back. + static Map readIndex(File file) throws IOException { + Map out = new TreeMap(); + if (!file.isFile()) { + return out; + } + for (String line : Files.readAllLines(file.toPath(), StandardCharsets.UTF_8)) { + int eq = line.indexOf('='); + if (!line.startsWith("#") && eq > 0) { + out.put(line.substring(0, eq).trim(), line.substring(eq + 1).trim()); + } + } + return out; + } + + private static String packageOf(String path) { + String p = path.endsWith("/__init__.py") ? path.substring(0, path.length() - 12) + : path.contains("/") ? path.substring(0, path.lastIndexOf('/')) : ""; + return p.replace('/', '.'); + } + + /// Resolves `from .x import` relative to the importing file's package. + private static String absolute(String pkg, String dots, String rest) { + if (dots.isEmpty()) { + return rest; + } + String base = pkg; + for (int i = 1; i < dots.length(); i++) { + int d = base.lastIndexOf('.'); + base = d < 0 ? "" : base.substring(0, d); + } + return rest.isEmpty() ? base : base.isEmpty() ? rest : base + "." + rest; + } + + /// The start offset and indent of every import of `module` in `text`, + /// adding the names taken from it to `names` when that is given. + private static java.util.List imports(String text, String pkg, String module, TreeSet names) { + java.util.List at = new java.util.ArrayList(); + String leaf = module.substring(module.lastIndexOf('.') + 1); + Matcher m = FROM.matcher(text); + while (m.find()) { + String from = absolute(pkg, m.group(2), m.group(3)); + if (from.equals(module)) { + at.add(new int[] {m.start(), m.group(1).length()}); + if (names != null) { + for (String part : m.group(4).split(",")) { + String n = part.trim().split("\\s+")[0]; + if (!n.isEmpty()) { + names.add(n); + } + } + } + } else if (module.equals(from.isEmpty() ? leaf : from + "." + leaf)) { + // from pkg import _speedups [as alias] + for (String part : m.group(4).split(",")) { + String[] w = part.trim().split("\\s+"); + if (w[0].equals(leaf)) { + at.add(new int[] {m.start(), m.group(1).length()}); + if (names != null) { + attributes(text, w.length == 3 ? w[2] : leaf, names); + } + } + } + } + } + m = IMPORT.matcher(text); + while (m.find()) { + if (m.group(2).equals(module)) { + at.add(new int[] {m.start(), m.group(1).length()}); + if (names != null) { + attributes(text, m.group(3) != null ? m.group(3) : module, names); + } + } + } + return at; + } + + private static void attributes(String text, String alias, TreeSet names) { + Matcher m = Pattern.compile("(?= 0; i--) { + String l = before[i]; + if (l.trim().isEmpty() || l.trim().startsWith("#")) { + continue; + } + int ind = indentOf(l); + if (ind < indent) { + if (l.trim().startsWith("try:")) { + tryIndent = ind; + tryLine = i; + } + break; + } + } + if (tryLine < 0) { + return false; + } + String[] after = text.substring(offset).split("\n"); + for (String l : after) { + String t = l.trim(); + if (t.isEmpty() || t.startsWith("#") || indentOf(l) > tryIndent) { + continue; + } + if (!t.startsWith("except")) { + return false; + } + if (t.startsWith("except:") || t.contains("ImportError") || t.contains("ModuleNotFoundError") + || t.matches("except\\s*\\(?\\s*(Base)?Exception\\b.*")) { + return true; + } + } + return false; + } + + private static int indentOf(String l) { + int i = 0; + while (i < l.length() && (l.charAt(i) == ' ' || l.charAt(i) == '\t')) { + i++; + } + return i; + } +} diff --git a/maven/build-engine/src/main/java/com/codename1/maven/PythonLineMapper.java b/maven/build-engine/src/main/java/com/codename1/maven/PythonLineMapper.java index eaaa5a16fb5..3fe0a75ad41 100644 --- a/maven/build-engine/src/main/java/com/codename1/maven/PythonLineMapper.java +++ b/maven/build-engine/src/main/java/com/codename1/maven/PythonLineMapper.java @@ -92,15 +92,12 @@ public static int apply(File classesDir) throws IOException { continue; } String stem = n.substring(0, n.length() - ".class".length()); - Table t = null; + // The class's own table, else its nearest enclosing class's. String outer = stem; - while (t == null) { - t = tables.get(outer); - int cut = outer.lastIndexOf('$'); - if (t != null || cut < 0) { - break; - } + Table t = tables.get(outer); + for (int cut = outer.lastIndexOf('$'); t == null && cut >= 0; cut = outer.lastIndexOf('$')) { outer = outer.substring(0, cut); + t = tables.get(outer); } if (t != null) { Files.write(f.toPath(), rewrite(Files.readAllBytes(f.toPath()), t)); diff --git a/maven/build-engine/src/main/java/com/codename1/maven/PythonPackageResolver.java b/maven/build-engine/src/main/java/com/codename1/maven/PythonPackageResolver.java new file mode 100644 index 00000000000..be4f0cef406 --- /dev/null +++ b/maven/build-engine/src/main/java/com/codename1/maven/PythonPackageResolver.java @@ -0,0 +1,637 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.maven; + +import com.codename1.build.Log; +import com.codename1.builders.BuildException; +import com.codename1.python.transpiler.api.PythonTranspiler; +import org.json.JSONArray; +import org.json.JSONObject; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.net.HttpURLConnection; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Deque; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.zip.GZIPInputStream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + +/// Provides a Python project's PyPI requirements as source, for +/// `cn1:import-python-project`. +/// +/// Every requirement, and every requirement of those, is resolved against +/// PyPI's JSON API to the newest release its specifiers accept, downloaded, +/// checked against the SHA-256 PyPI publishes, and unpacked into +/// `src/main/python-packages`, which the build transpiles beside +/// `src/main/python`. A package the runtime provides itself (the standard +/// library, and the packages Codename One maps: `requests`, `numpy`, +/// `flask` and the rest) is not downloaded. +/// +/// What was chosen is written to `python.lock` in that directory -- name, +/// version, file and hash -- and a later import installs exactly those, +/// so the packages change only when the lock is deleted or a requirement +/// no longer accepts its pinned version. +/// +/// **C extensions.** A package that ships compiled modules is supported +/// the same way: its Python is transpiled, and each compiled module is +/// replaced by Java. [PythonExtensionStubs] finds the modules and the names +/// the package uses from each, and writes a Java class per module with a +/// stub for each name, to be filled in. +public final class PythonPackageResolver { + /// The directory, beside `src/main/python`, the packages are unpacked into. + public static final String PACKAGES_DIR = "python-packages"; + /// The lock file in [#PACKAGES_DIR]. + public static final String LOCK = "python.lock"; + + /// Fetches a URL's bytes; replaced in tests. + public interface Fetcher { + byte[] fetch(String url) throws IOException; + } + + /// One distribution as installed. + static final class Pinned { + final String name; + final String version; + final String file; + final String url; + final String sha256; + + Pinned(String name, String version, String file, String url, String sha256) { + this.name = name; + this.version = version; + this.file = file; + this.url = url; + this.sha256 = sha256; + } + } + + /// What a resolution installed. + public static final class Result { + /// `name==version` of every distribution installed. + public final List installed = new ArrayList(); + /// Requirements the runtime provides, by name. + public final List provided = new ArrayList(); + /// The compiled modules found, by dotted name, each with the Java + /// class that stands in for it. + public final Map nativeModules = new TreeMap(); + /// Compiled modules every import of which falls back to Python + /// when they are missing: nothing has to be written for these. + public final List optionalNativeModules = new ArrayList(); + /// Stub classes written by this run (an existing one is never + /// overwritten). + public final List stubsWritten = new ArrayList(); + } + + private final Log log; + private final Fetcher fetcher; + private String index = "https://pypi.org/pypi/"; + + public PythonPackageResolver(Log log) { + this(log, PythonPackageResolver::download); + } + + PythonPackageResolver(Log log, Fetcher fetcher) { + this.log = log; + this.fetcher = fetcher; + } + + /// Uses another index that speaks PyPI's JSON API (a mirror). + public PythonPackageResolver index(String url) { + this.index = url.endsWith("/") ? url : url + "/"; + return this; + } + + /// Resolves `requirements` (PEP 508 lines) into `commonDir`'s + /// `src/main/python-packages`, writing stubs for compiled modules under + /// `src/main/java`. + public Result resolve(List requirements, File commonDir) throws BuildException { + File target = new File(commonDir, "src/main/" + PACKAGES_DIR); + File javaDir = new File(commonDir, "src/main/java"); + Map locked = readLock(new File(target, LOCK)); + Result result = new Result(); + Map chosen = new LinkedHashMap(); + Map sources = new TreeMap(); + Map> nativeByDist = new LinkedHashMap>(); + Deque queue = new ArrayDeque(); + for (String line : requirements) { + PythonRequirement r = PythonRequirement.parse(line); + if (r != null && r.applies(Collections.emptyList())) { + queue.add(new Object[] {r, "the project"}); + } + } + while (!queue.isEmpty()) { + Object[] item = queue.poll(); + PythonRequirement r = (PythonRequirement) item[0]; + if (providedByRuntime(r.name)) { + if (!result.provided.contains(r.name)) { + result.provided.add(r.name); + } + continue; + } + Pinned have = chosen.get(r.name); + if (have != null) { + if (!r.accepts(have.version)) { + throw new BuildException(item[1] + " requires " + r + ", but " + have.name + " " + + have.version + " was already chosen"); + } + continue; + } + Pinned lock = locked.get(r.name); + Pinned p = lock != null && r.accepts(lock.version) ? lock : choose(r); + chosen.put(r.name, p); + byte[] archive = fetchVerified(p); + Unpacked u = unpack(p, archive); + sources.putAll(u.sources); + nativeByDist.put(p.name, u.nativeModules); + result.installed.add(p.name + "==" + p.version); + for (String dep : u.requires) { + PythonRequirement d = PythonRequirement.parse(dep); + if (d != null && d.applies(r.extras)) { + queue.add(new Object[] {d, p.name + " " + p.version}); + } + } + } + try { + writeTree(target, sources); + writeLock(new File(target, LOCK), chosen.values()); + PythonExtensionStubs stubs = new PythonExtensionStubs(sources); + for (Map.Entry> e : nativeByDist.entrySet()) { + for (String module : e.getValue()) { + if (stubs.isOptional(module)) { + result.optionalNativeModules.add(module); + continue; + } + String cls = stubs.className(module); + result.nativeModules.put(module, cls); + if (stubs.write(module, e.getKey(), javaDir)) { + result.stubsWritten.add(cls); + } + } + } + PythonExtensionStubs.writeIndex(new File(target, PythonExtensionStubs.INDEX), result.nativeModules); + } catch (IOException e) { + throw new BuildException("Cannot write " + target + ": " + e.getMessage(), e); + } + return result; + } + + /// Whether the runtime provides the distribution `name`: its import + /// name is a standard or mapped module. + static boolean providedByRuntime(String name) { + String n = PythonRequirement.normalize(name); + String module; + switch (n) { + case "pyyaml": + module = "yaml"; + break; + case "pillow": + module = "PIL"; + break; + case "python-dateutil": + module = "dateutil"; + break; + case "attrs": + module = "attr"; + break; + case "typing-extensions": + module = "typing_extensions"; + break; + default: + module = n.replace('-', '_'); + } + return PythonTranspiler.provides(module); + } + + /// The newest release `r` accepts, and the file to install from it: a + /// pure-Python wheel, else a CPython wheel (whose Python is the same, + /// and whose compiled modules name what has to be written in Java), + /// else the source distribution. + private Pinned choose(PythonRequirement r) throws BuildException { + JSONObject meta; + try { + meta = new JSONObject(new String(fetcher.fetch(index + r.name + "/json"), StandardCharsets.UTF_8)); + } catch (IOException e) { + throw new BuildException("Cannot look up " + r.name + " on " + index + ": " + e.getMessage(), e); + } + JSONObject releases = meta.optJSONObject("releases"); + if (releases == null) { + throw new BuildException(r.name + ": the index answered no releases"); + } + String best = null; + for (String v : releases.keySet()) { + JSONArray files = releases.getJSONArray(v); + if (files.length() == 0 || allYanked(files) || !r.accepts(v)) { + continue; + } + if (best == null || PythonRequirement.compare(v, best) > 0) { + best = v; + } + } + if (best == null) { + throw new BuildException("No release of " + r.name + " matches " + r); + } + JSONArray files = releases.getJSONArray(best); + JSONObject pick = null; + int rank = Integer.MAX_VALUE; + for (int i = 0; i < files.length(); i++) { + JSONObject f = files.getJSONObject(i); + if (f.optBoolean("yanked", false)) { + continue; + } + int k = rank(f.getString("filename")); + if (k < rank || (k == rank && pick != null && f.optLong("size") < pick.optLong("size"))) { + rank = k; + pick = f; + } + } + if (pick == null || rank == Integer.MAX_VALUE) { + throw new BuildException(r.name + " " + best + " has no wheel or source distribution to install"); + } + String sha = pick.getJSONObject("digests").optString("sha256", null); + if (sha == null) { + throw new BuildException(r.name + " " + best + ": the index publishes no SHA-256 for " + + pick.getString("filename")); + } + return new Pinned(r.name, best, pick.getString("filename"), pick.getString("url"), sha); + } + + private static boolean allYanked(JSONArray files) { + for (int i = 0; i < files.length(); i++) { + if (!files.getJSONObject(i).optBoolean("yanked", false)) { + return false; + } + } + return true; + } + + /// Lower is better: a pure wheel, a CPython 3 wheel for Linux, any + /// CPython 3 wheel, the source distribution. + static int rank(String filename) { + String f = filename.toLowerCase(Locale.ROOT); + if (f.endsWith(".whl")) { + if (f.endsWith("-none-any.whl") && (f.contains("-py3-") || f.contains(".py3-") || f.contains("-py2.py3-"))) { + return 0; + } + if (f.contains("-cp3") && (f.contains("manylinux") || f.contains("musllinux"))) { + return 1; + } + if (f.contains("-cp3") || f.contains("-abi3-")) { + return 2; + } + return Integer.MAX_VALUE; + } + if (f.endsWith(".tar.gz") || f.endsWith(".zip")) { + return 3; + } + return Integer.MAX_VALUE; + } + + private byte[] fetchVerified(Pinned p) throws BuildException { + byte[] b; + try { + b = fetcher.fetch(p.url); + } catch (IOException e) { + throw new BuildException("Cannot download " + p.file + ": " + e.getMessage(), e); + } + String got = sha256(b); + if (!got.equalsIgnoreCase(p.sha256)) { + throw new BuildException(p.file + " does not match its SHA-256: expected " + p.sha256 + ", got " + got); + } + return b; + } + + static String sha256(byte[] b) { + try { + byte[] d = MessageDigest.getInstance("SHA-256").digest(b); + StringBuilder sb = new StringBuilder(); + for (byte x : d) { + sb.append(Character.forDigit((x >> 4) & 0xf, 16)).append(Character.forDigit(x & 0xf, 16)); + } + return sb.toString(); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException(e); + } + } + + /// The Python sources, the compiled modules and the requirements of + /// one archive. + static final class Unpacked { + final Map sources = new TreeMap(); + final List nativeModules = new ArrayList(); + final List requires = new ArrayList(); + } + + static Unpacked unpack(Pinned p, byte[] archive) throws BuildException { + try { + Map entries = p.file.endsWith(".tar.gz") ? untar(archive) : unzip(archive); + return p.file.endsWith(".whl") ? fromWheel(entries) : fromSdist(entries); + } catch (IOException e) { + throw new BuildException("Cannot read " + p.file + ": " + e.getMessage(), e); + } + } + + private static Unpacked fromWheel(Map entries) { + Unpacked u = new Unpacked(); + for (Map.Entry e : entries.entrySet()) { + String path = e.getKey(); + String top = path.contains("/") ? path.substring(0, path.indexOf('/')) : ""; + if (top.endsWith(".dist-info")) { + if (path.endsWith("/METADATA")) { + requiresDist(new String(e.getValue(), StandardCharsets.UTF_8), u.requires); + } + continue; + } + if (top.endsWith(".data")) { + // purelib/platlib trees install into site-packages as well + String[] parts = path.split("/", 3); + if (parts.length == 3 && ("purelib".equals(parts[1]) || "platlib".equals(parts[1]))) { + path = parts[2]; + } else { + continue; + } + } + take(path, e.getValue(), u); + } + return u; + } + + private static Unpacked fromSdist(Map entries) { + Unpacked u = new Unpacked(); + String root = null; + for (String path : entries.keySet()) { + int slash = path.indexOf('/'); + root = slash < 0 ? "" : path.substring(0, slash + 1); + break; + } + boolean srcLayout = false; + for (String path : entries.keySet()) { + if (path.startsWith(root + "src/") && path.endsWith("/__init__.py")) { + srcLayout = true; + } + } + String base = srcLayout ? root + "src/" : root; + for (Map.Entry e : entries.entrySet()) { + String path = e.getKey(); + if (path.equals(root + "PKG-INFO")) { + requiresDist(new String(e.getValue(), StandardCharsets.UTF_8), u.requires); + } + if (!path.startsWith(base)) { + continue; + } + String rel = path.substring(base.length()); + String first = rel.contains("/") ? rel.substring(0, rel.indexOf('/')) : rel; + if (!rel.contains("/") && !rel.endsWith(".py") || "setup.py".equals(rel) || "conftest.py".equals(rel) + || first.equals("tests") || first.equals("test") || first.equals("docs") + || first.equals("examples") || first.equals("benchmarks") || first.equals("build")) { + continue; + } + take(rel, e.getValue(), u); + } + return u; + } + + /// Keeps a `.py` file, and notes a compiled module (`.so`, `.pyd`) or + /// the source of one (`.pyx`, `.c` beside a package) by dotted name. + private static void take(String path, byte[] data, Unpacked u) { + String name = path.substring(path.lastIndexOf('/') + 1); + String dir = path.substring(0, path.length() - name.length()); + if (path.endsWith(".py")) { + u.sources.put(path, new String(data, StandardCharsets.UTF_8)); + return; + } + String module = null; + if (name.endsWith(".so") || name.endsWith(".pyd")) { + module = name.substring(0, name.indexOf('.')); + } else if (name.endsWith(".pyx")) { + module = name.substring(0, name.length() - 4); + } + if (module != null) { + String dotted = (dir + module).replace('/', '.'); + if (!u.nativeModules.contains(dotted)) { + u.nativeModules.add(dotted); + } + } + } + + private static void requiresDist(String metadata, List into) { + for (String line : metadata.split("\n")) { + if (line.isEmpty() || line.equals("\r")) { + break; + } + if (line.regionMatches(true, 0, "Requires-Dist:", 0, 14)) { + into.add(line.substring(14).trim()); + } + } + } + + private static Map unzip(byte[] archive) throws IOException { + Map out = new TreeMap(); + try (ZipInputStream z = new ZipInputStream(new ByteArrayInputStream(archive))) { + ZipEntry e; + while ((e = z.getNextEntry()) != null) { + if (!e.isDirectory() && safe(e.getName())) { + out.put(e.getName(), readAll(z)); + } + } + } + return out; + } + + /// A ustar/pax reader: enough for source distributions. + private static Map untar(byte[] archive) throws IOException { + Map out = new TreeMap(); + byte[] tar; + try (InputStream in = new GZIPInputStream(new ByteArrayInputStream(archive))) { + tar = readAll(in); + } + int pos = 0; + String longName = null; + while (pos + 512 <= tar.length) { + if (tar[pos] == 0) { + break; + } + String name = field(tar, pos, 100); + String prefix = field(tar, pos + 345, 155); + long size = Long.parseLong(field(tar, pos + 124, 12).trim().isEmpty() ? "0" : field(tar, pos + 124, 12).trim(), 8); + char type = (char) tar[pos + 156]; + int data = pos + 512; + if (type == 'x' || type == 'L') { + String body = new String(tar, data, (int) size, StandardCharsets.UTF_8); + if (type == 'L') { + longName = body.trim(); + } else { + for (String rec : body.split("\n")) { + int sp = rec.indexOf(' '); + if (sp > 0 && rec.startsWith("path=", sp + 1)) { + longName = rec.substring(sp + 6); + } + } + } + } else if (type == '0' || type == '\0') { + String full = longName != null ? longName : prefix.isEmpty() ? name : prefix + "/" + name; + longName = null; + if (safe(full)) { + byte[] b = new byte[(int) size]; + System.arraycopy(tar, data, b, 0, (int) size); + out.put(full, b); + } + } else { + longName = null; + } + pos = data + (int) ((size + 511) / 512 * 512); + } + return out; + } + + private static String field(byte[] b, int off, int len) { + int end = off; + while (end < off + len && b[end] != 0) { + end++; + } + return new String(b, off, end - off, StandardCharsets.UTF_8); + } + + /// Rejects an entry that would land outside the target directory. + private static boolean safe(String path) { + return !path.startsWith("/") && !path.contains("\\") && !("/" + path + "/").contains("/../"); + } + + private static byte[] readAll(InputStream in) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + byte[] buf = new byte[16384]; + int n; + while ((n = in.read(buf)) > 0) { + out.write(buf, 0, n); + } + return out.toByteArray(); + } + + /// Makes `target` hold exactly `sources`, keeping the lock and index. + private static void writeTree(File target, Map sources) throws IOException { + if (target.isDirectory()) { + deletePy(target); + } + for (Map.Entry e : sources.entrySet()) { + File f = new File(target, e.getKey()); + Files.createDirectories(f.getParentFile().toPath()); + Files.write(f.toPath(), e.getValue().getBytes(StandardCharsets.UTF_8)); + } + } + + private static void deletePy(File dir) throws IOException { + File[] children = dir.listFiles(); + if (children == null) { + return; + } + for (File c : children) { + if (c.isDirectory() && !Files.isSymbolicLink(c.toPath())) { + deletePy(c); + String[] left = c.list(); + if (left != null && left.length == 0) { + Files.delete(c.toPath()); + } + } else if (c.getName().endsWith(".py")) { + Files.delete(c.toPath()); + } + } + } + + static Map readLock(File lock) throws BuildException { + Map out = new TreeMap(); + if (!lock.isFile()) { + return out; + } + try { + for (String line : Files.readAllLines(lock.toPath(), StandardCharsets.UTF_8)) { + String l = line.trim(); + if (l.isEmpty() || l.startsWith("#")) { + continue; + } + String[] f = l.split("\\s+"); + int eq = f[0].indexOf("=="); + if (f.length < 4 || eq < 0) { + throw new BuildException(lock + ": cannot read the line " + l); + } + Map kv = new TreeMap(); + for (int i = 1; i < f.length; i++) { + int e = f[i].indexOf('='); + if (e > 0) { + kv.put(f[i].substring(0, e), f[i].substring(e + 1)); + } + } + String name = f[0].substring(0, eq); + out.put(name, new Pinned(name, f[0].substring(eq + 2), kv.get("file"), kv.get("url"), kv.get("sha256"))); + } + } catch (IOException e) { + throw new BuildException("Cannot read " + lock + ": " + e.getMessage(), e); + } + return out; + } + + private static void writeLock(File lock, Iterable pins) throws IOException { + StringBuilder sb = new StringBuilder(); + sb.append("# Written by cn1:import-python-project. The packages in this directory are\n"); + sb.append("# exactly these; delete a line (or the file) to resolve that package again.\n"); + TreeSet lines = new TreeSet(); + for (Pinned p : pins) { + lines.add(p.name + "==" + p.version + " file=" + p.file + " sha256=" + p.sha256 + " url=" + p.url); + } + for (String l : lines) { + sb.append(l).append('\n'); + } + Files.createDirectories(lock.getParentFile().toPath()); + Files.write(lock.toPath(), sb.toString().getBytes(StandardCharsets.UTF_8)); + } + + private static byte[] download(String url) throws IOException { + HttpURLConnection c = (HttpURLConnection) new URL(url).openConnection(); + c.setConnectTimeout(30000); + c.setReadTimeout(120000); + c.setRequestProperty("Accept", "application/json, */*"); + c.setRequestProperty("User-Agent", "codenameone-python-import"); + int status = c.getResponseCode(); + if (status != 200) { + throw new IOException(url + " answered " + status); + } + try (InputStream in = c.getInputStream()) { + return readAll(in); + } + } +} diff --git a/maven/build-engine/src/main/java/com/codename1/maven/PythonProjectBuilder.java b/maven/build-engine/src/main/java/com/codename1/maven/PythonProjectBuilder.java index d457835a864..d1f684d7313 100644 --- a/maven/build-engine/src/main/java/com/codename1/maven/PythonProjectBuilder.java +++ b/maven/build-engine/src/main/java/com/codename1/maven/PythonProjectBuilder.java @@ -108,8 +108,19 @@ public boolean run() throws BuildException { return false; } Map sources = new TreeMap(); + Map nativeModules = new TreeMap(); try { collect(sourceDir, "", sources); + // PyPI packages import resolved beside the project; the + // project's own file wins where both have one. + File packages = new File(sourceDir.getParentFile(), PythonPackageResolver.PACKAGES_DIR); + if (packages.isDirectory()) { + Map vendored = new TreeMap(); + collect(packages, "", vendored); + vendored.putAll(sources); + sources = vendored; + nativeModules = PythonExtensionStubs.readIndex(new File(packages, PythonExtensionStubs.INDEX)); + } } catch (IOException e) { throw new BuildException("Could not read " + sourceDir + ": " + e.getMessage(), e); } @@ -129,7 +140,7 @@ public boolean run() throws BuildException { } Map out; try { - out = PythonTranspiler.transpile(sources, entry, classPath); + out = PythonTranspiler.transpile(sources, entry, classPath, nativeModules); } catch (PythonSyntaxError e) { throw new BuildException("src/main/python/" + e.file + ":" + e.line + ":" + e.col + ": " + e.reason, e); } catch (IllegalArgumentException e) { diff --git a/maven/build-engine/src/main/java/com/codename1/maven/PythonProjectImporter.java b/maven/build-engine/src/main/java/com/codename1/maven/PythonProjectImporter.java index 94edd3b59be..f657c0c0602 100644 --- a/maven/build-engine/src/main/java/com/codename1/maven/PythonProjectImporter.java +++ b/maven/build-engine/src/main/java/com/codename1/maven/PythonProjectImporter.java @@ -69,6 +69,8 @@ public static final class Result { /// Requirements the project declares, which have to be provided as /// source under `src/main/python` or by the runtime. public final List requirements = new ArrayList(); + /// The same requirements as written, versions and markers included. + public final List requirementSpecs = new ArrayList(); } public PythonProjectImporter(Log log) { @@ -108,7 +110,9 @@ public Result importProject(File source, File commonDir, String mainPackage, Str writeRecord(target, imported); String pyproject = read(new File(source, "pyproject.toml")); r.mainModule = mainModule(root, pyproject); - r.requirements.addAll(requirements(pyproject, read(new File(source, "requirements.txt")))); + String txt = read(new File(source, "requirements.txt")); + r.requirements.addAll(requirements(pyproject, txt)); + r.requirementSpecs.addAll(requirementSpecs(pyproject, txt)); if (r.mainModule != null) { updateSetting(new File(commonDir, "codenameone_settings.properties"), r.mainModule); if (mainPackage != null && mainClass != null && mainPackage.length() > 0 && mainClass.length() > 0) { @@ -207,6 +211,45 @@ static List requirements(String pyproject, String requirementsTxt) { return new ArrayList(names); } + /// The requirements of `[project] dependencies` and `requirements.txt` + /// as PEP 508 lines, comments and pip options left out. + static List requirementSpecs(String pyproject, String requirementsTxt) { + List out = new ArrayList(); + if (pyproject != null) { + String project = section(pyproject, "project"); + if (project != null) { + int at = project.indexOf("dependencies"); + int open = at < 0 ? -1 : project.indexOf('[', at); + // Quoted items up to the unquoted bracket that closes the list: + // an extra such as `requests[socks]` has brackets of its own. + for (int i = open + 1; open >= 0 && i < project.length(); i++) { + char c = project.charAt(i); + if (c == ']') { + break; + } + if (c == '"' || c == '\'') { + int end = project.indexOf(c, i + 1); + if (end < 0) { + break; + } + out.add(project.substring(i + 1, end).trim()); + i = end; + } + } + } + } + if (requirementsTxt != null) { + for (String line : requirementsTxt.split("\n")) { + int hash = line.indexOf(" #"); + String l = (hash >= 0 ? line.substring(0, hash) : line).trim(); + if (!l.isEmpty() && !l.startsWith("-") && !l.startsWith("#")) { + out.add(l); + } + } + } + return out; + } + private static void add(Set names, String spec) { int hash = spec.indexOf('#'); String s = hash >= 0 ? spec.substring(0, hash) : spec; diff --git a/maven/build-engine/src/main/java/com/codename1/maven/PythonRequirement.java b/maven/build-engine/src/main/java/com/codename1/maven/PythonRequirement.java new file mode 100644 index 00000000000..ef3c6c86d2a --- /dev/null +++ b/maven/build-engine/src/main/java/com/codename1/maven/PythonRequirement.java @@ -0,0 +1,515 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.maven; + +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; + +/// One requirement as `requirements.txt`, `pyproject.toml` and a wheel's +/// `Requires-Dist` write it (PEP 508): `name[extra] >=1.2,<2 ; marker`. +/// +/// Versions compare as PEP 440 orders them -- release segments, then +/// `a`/`b`/`rc` pre-releases, post- and dev-releases -- and specifiers +/// cover `==` (with `.*`), `!=`, `<`, `<=`, `>`, `>=`, `~=` and `===`. +/// Markers are evaluated for the environment a Codename One build targets: +/// CPython 3.13 on a POSIX system, so a dependency only Windows needs is +/// left out. +final class PythonRequirement { + /// The marker environment: what `python_version` and the rest answer. + static final String PYTHON_VERSION = "3.13"; + + final String name; + final List extras = new ArrayList(); + final List specifiers = new ArrayList(); + final String marker; + + private PythonRequirement(String name, String marker) { + this.name = name; + this.marker = marker; + } + + /// PEP 503's normal form, which PyPI's URLs and the lock file use. + static String normalize(String name) { + StringBuilder sb = new StringBuilder(); + boolean sep = false; + for (int i = 0; i < name.length(); i++) { + char c = name.charAt(i); + if (c == '-' || c == '_' || c == '.') { + sep = true; + } else { + if (sep && sb.length() > 0) { + sb.append('-'); + } + sep = false; + sb.append(Character.toLowerCase(c)); + } + } + return sb.toString(); + } + + /// Parses one requirement line, or answers null for one that is not a + /// named requirement (an option, a URL, a blank line). + static PythonRequirement parse(String line) { + String s = line; + int hash = s.indexOf(" #"); + if (hash >= 0) { + s = s.substring(0, hash); + } + if (s.startsWith("#")) { + return null; + } + s = s.trim(); + if (s.isEmpty() || s.startsWith("-") || s.contains("://")) { + return null; + } + String marker = null; + int semi = s.indexOf(';'); + if (semi >= 0) { + marker = s.substring(semi + 1).trim(); + s = s.substring(0, semi).trim(); + } + int end = 0; + while (end < s.length()) { + char c = s.charAt(end); + if (!(Character.isLetterOrDigit(c) || c == '-' || c == '_' || c == '.')) { + break; + } + end++; + } + if (end == 0) { + return null; + } + PythonRequirement r = new PythonRequirement(normalize(s.substring(0, end)), marker); + String rest = s.substring(end).trim(); + if (rest.startsWith("[")) { + int close = rest.indexOf(']'); + for (String e : rest.substring(1, close < 0 ? rest.length() : close).split(",")) { + if (!e.trim().isEmpty()) { + r.extras.add(normalize(e.trim())); + } + } + rest = close < 0 ? "" : rest.substring(close + 1).trim(); + } + if (rest.startsWith("(") && rest.endsWith(")")) { + rest = rest.substring(1, rest.length() - 1); + } + for (String spec : rest.split(",")) { + String t = spec.trim(); + if (t.isEmpty()) { + continue; + } + int i = 0; + while (i < t.length() && "=!<>~".indexOf(t.charAt(i)) >= 0) { + i++; + } + r.specifiers.add(new String[] {t.substring(0, i), t.substring(i).trim()}); + } + return r; + } + + /// Whether `version` satisfies every specifier. A pre-release is + /// accepted only when a specifier names one, as pip decides by default. + boolean accepts(String version) { + boolean pre = isPreRelease(version); + boolean prereleaseAsked = false; + for (String[] s : specifiers) { + if (isPreRelease(s[1].replace(".*", ""))) { + prereleaseAsked = true; + } + if (!matches(s[0], s[1], version)) { + return false; + } + } + return !pre || prereleaseAsked; + } + + /// Whether the marker holds for the build's environment, with `extras` + /// the extras requested of the distribution that declared it. + boolean applies(List requestedExtras) { + if (marker == null || marker.isEmpty()) { + return true; + } + return new MarkerParser(marker, requestedExtras).parse(); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(name); + for (int i = 0; i < specifiers.size(); i++) { + sb.append(i == 0 ? " " : ",").append(specifiers.get(i)[0]).append(specifiers.get(i)[1]); + } + return sb.toString(); + } + + static boolean isPreRelease(String version) { + Object[] v = parseVersion(version); + return v != null && (((int[]) v[1])[0] < 3 || ((int[]) v[3])[0] == 0); + } + + static boolean matches(String op, String spec, String version) { + if ("===".equals(op)) { + return spec.equals(version); + } + if (("==".equals(op) || "!=".equals(op)) && spec.endsWith(".*")) { + String prefix = spec.substring(0, spec.length() - 2); + int[] want = release(prefix); + int[] have = release(version); + boolean same = want != null && have != null; + for (int i = 0; same && i < want.length; i++) { + same = i < have.length ? have[i] == want[i] : want[i] == 0; + } + return "==".equals(op) == same; + } + int c = compare(version, spec); + if (c == Integer.MIN_VALUE) { + return false; + } + switch (op) { + case "==": + return c == 0; + case "!=": + return c != 0; + case "<": + return c < 0; + case "<=": + return c <= 0; + case ">": + return c > 0; + case ">=": + case "": + return c >= 0; + case "~=": { + int[] r = release(spec); + if (r == null || r.length < 2 || c < 0) { + return false; + } + StringBuilder prefix = new StringBuilder(); + for (int i = 0; i < r.length - 1; i++) { + prefix.append(i == 0 ? "" : ".").append(r[i]); + } + return matches("==", prefix + ".*", version); + } + default: + return false; + } + } + + /// Compares two versions; [Integer#MIN_VALUE] when either is not one. + static int compare(String a, String b) { + Object[] x = parseVersion(a); + Object[] y = parseVersion(b); + if (x == null || y == null) { + return Integer.MIN_VALUE; + } + int ex = ((int[]) x[4])[0]; + int ey = ((int[]) y[4])[0]; + if (ex != ey) { + return ex < ey ? -1 : 1; + } + int[] rx = (int[]) x[0]; + int[] ry = (int[]) y[0]; + for (int i = 0; i < Math.max(rx.length, ry.length); i++) { + int p = i < rx.length ? rx[i] : 0; + int q = i < ry.length ? ry[i] : 0; + if (p != q) { + return p < q ? -1 : 1; + } + } + for (int k = 1; k < 4; k++) { + int[] p = (int[]) x[k]; + int[] q = (int[]) y[k]; + for (int i = 0; i < 2; i++) { + if (p[i] != q[i]) { + return p[i] < q[i] ? -1 : 1; + } + } + } + return 0; + } + + private static int[] release(String v) { + Object[] p = parseVersion(v); + return p == null ? null : (int[]) p[0]; + } + + /// `{release, pre, post, dev, epoch}`, pre, post and dev each a pair whose + /// order sorts as PEP 440 does: a dev-release before its pre-release, + /// a pre-release before the final, the final before its post-release. + private static Object[] parseVersion(String version) { + String v = version.trim().toLowerCase(Locale.ROOT); + if (v.startsWith("v")) { + v = v.substring(1); + } + int bang = v.indexOf('!'); + int epoch = 0; + if (bang >= 0) { + try { + epoch = Integer.parseInt(v.substring(0, bang)); + } catch (NumberFormatException e) { + return null; + } + v = v.substring(bang + 1); + } + int plus = v.indexOf('+'); + if (plus >= 0) { + v = v.substring(0, plus); + } + int i = 0; + List rel = new ArrayList(); + while (i < v.length()) { + int j = i; + while (j < v.length() && Character.isDigit(v.charAt(j))) { + j++; + } + if (j == i) { + return null; + } + rel.add(Integer.valueOf(Integer.parseInt(v.substring(i, Math.min(j, i + 9))))); + i = j; + if (i < v.length() && v.charAt(i) == '.' && i + 1 < v.length() && Character.isDigit(v.charAt(i + 1))) { + i++; + } else { + break; + } + } + int[] release = new int[rel.size()]; + for (int k = 0; k < release.length; k++) { + release[k] = rel.get(k).intValue(); + } + int[] pre = {3, 0}; + int[] post = {0, 0}; + int[] dev = {1, 0}; + String rest = v.substring(i); + while (!rest.isEmpty()) { + rest = strip(rest); + String[] tags = {"alpha", "beta", "preview", "pre", "rc", "a", "b", "c", "post", "rev", "r", "dev"}; + String tag = null; + for (String t : tags) { + if (rest.startsWith(t)) { + tag = t; + break; + } + } + if (tag == null) { + if (!rest.isEmpty() && Character.isDigit(rest.charAt(0)) && post[0] == 0) { + tag = ""; + } else { + return rest.isEmpty() ? new Object[] {release, pre, post, dev} : null; + } + } + rest = strip(rest.substring(tag.length())); + int j = 0; + while (j < rest.length() && Character.isDigit(rest.charAt(j))) { + j++; + } + int n = j == 0 ? 0 : Integer.parseInt(rest.substring(0, Math.min(j, 9))); + rest = rest.substring(j); + if ("dev".equals(tag)) { + dev = new int[] {0, n}; + } else if ("post".equals(tag) || "rev".equals(tag) || "r".equals(tag) || tag.isEmpty()) { + post = new int[] {1, n}; + } else if (tag.startsWith("a")) { + pre = new int[] {0, n}; + } else if (tag.startsWith("b")) { + pre = new int[] {1, n}; + } else { + pre = new int[] {2, n}; + } + } + if (dev[0] == 0 && pre[0] == 3 && post[0] == 0) { + // 1.0.dev1 sorts before 1.0a1 + pre = new int[] {-1, 0}; + } + return new Object[] {release, pre, post, dev, new int[] {epoch}}; + } + + private static String strip(String s) { + int i = 0; + while (i < s.length() && (s.charAt(i) == '.' || s.charAt(i) == '-' || s.charAt(i) == '_')) { + i++; + } + return s.substring(i); + } + + /// PEP 508 markers: comparisons joined by `and`/`or`, in parentheses. + private static final class MarkerParser { + private final String s; + private final List extras; + private int pos; + + MarkerParser(String s, List extras) { + this.s = s; + this.extras = extras; + } + + boolean parse() { + boolean v = or(); + return v; + } + + private boolean or() { + boolean v = and(); + while (word("or")) { + boolean w = and(); + v = v || w; + } + return v; + } + + private boolean and() { + boolean v = atom(); + while (word("and")) { + boolean w = atom(); + v = v && w; + } + return v; + } + + private boolean atom() { + skip(); + if (pos < s.length() && s.charAt(pos) == '(') { + pos++; + boolean v = or(); + skip(); + if (pos < s.length() && s.charAt(pos) == ')') { + pos++; + } + return v; + } + String left = value(); + String op = operator(); + String right = value(); + return compareMarker(left, op, right); + } + + private boolean compareMarker(String left, String op, String right) { + boolean leftVar = left.startsWith("$"); + String var = leftVar ? left.substring(1) : right.startsWith("$") ? right.substring(1) : null; + if ("extra".equals(var)) { + String want = normalize(leftVar ? right : left); + boolean has = extras.contains(want); + return "==".equals(op) ? has : "!=".equals(op) ? !has : false; + } + String l = leftVar ? env(left.substring(1)) : left; + String r = right.startsWith("$") ? env(right.substring(1)) : right; + if ("in".equals(op)) { + return r.contains(l); + } + if ("not in".equals(op)) { + return !r.contains(l); + } + if (var != null && var.endsWith("version") && compare(l, r) != Integer.MIN_VALUE) { + return matches(op, leftVar ? r : l, leftVar ? l : r); + } + switch (op) { + case "==": + return l.equals(r); + case "!=": + return !l.equals(r); + default: + return false; + } + } + + private static String env(String name) { + switch (name) { + case "python_version": + return PYTHON_VERSION; + case "python_full_version": + return PYTHON_VERSION + ".0"; + case "os_name": + return "posix"; + case "sys_platform": + return "linux"; + case "platform_system": + return "Linux"; + case "platform_machine": + return "aarch64"; + case "platform_python_implementation": + return "CPython"; + case "implementation_name": + return "cpython"; + case "implementation_version": + return PYTHON_VERSION + ".0"; + default: + return ""; + } + } + + private String value() { + skip(); + if (pos >= s.length()) { + return ""; + } + char c = s.charAt(pos); + if (c == '"' || c == '\'') { + int end = s.indexOf(c, pos + 1); + if (end < 0) { + end = s.length(); + } + String v = s.substring(pos + 1, end); + pos = Math.min(s.length(), end + 1); + return v; + } + int start = pos; + while (pos < s.length() && (Character.isLetterOrDigit(s.charAt(pos)) || s.charAt(pos) == '_' + || s.charAt(pos) == '.')) { + pos++; + } + return "$" + s.substring(start, pos); + } + + private String operator() { + skip(); + if (word("not")) { + word("in"); + return "not in"; + } + if (word("in")) { + return "in"; + } + int start = pos; + while (pos < s.length() && "=!<>~".indexOf(s.charAt(pos)) >= 0) { + pos++; + } + return s.substring(start, pos); + } + + private boolean word(String w) { + skip(); + if (s.startsWith(w, pos) && (pos + w.length() == s.length() + || !Character.isLetterOrDigit(s.charAt(pos + w.length())))) { + pos += w.length(); + return true; + } + return false; + } + + private void skip() { + while (pos < s.length() && Character.isWhitespace(s.charAt(pos))) { + pos++; + } + } + } +} diff --git a/maven/build-engine/src/test/java/com/codename1/maven/PythonPackageResolverTest.java b/maven/build-engine/src/test/java/com/codename1/maven/PythonPackageResolverTest.java new file mode 100644 index 00000000000..3e5ed8ee1dd --- /dev/null +++ b/maven/build-engine/src/test/java/com/codename1/maven/PythonPackageResolverTest.java @@ -0,0 +1,223 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.maven; + +import com.codename1.build.SystemStreamLog; +import com.codename1.builders.BuildException; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/// [PythonRequirement], [PythonPackageResolver] and [PythonExtensionStubs] +/// against an index served from memory. +public class PythonPackageResolverTest { + @Rule + public TemporaryFolder tmp = new TemporaryFolder(); + + private final Map served = new HashMap(); + private final Map releases = new HashMap(); + + private void publish(String name, String version, String file, byte[] data) { + String url = "https://files.example/" + file; + served.put(url, data); + JSONObject rel = releases.get(name); + if (rel == null) { + rel = new JSONObject(); + releases.put(name, rel); + } + JSONArray files = rel.optJSONArray(version); + if (files == null) { + files = new JSONArray(); + rel.put(version, files); + } + files.put(new JSONObject().put("filename", file).put("url", url).put("size", data.length) + .put("digests", new JSONObject().put("sha256", PythonPackageResolver.sha256(data)))); + } + + private PythonPackageResolver resolver() { + return new PythonPackageResolver(new SystemStreamLog(), url -> { + String prefix = "https://pypi.org/pypi/"; + if (url.startsWith(prefix) && url.endsWith("/json")) { + String name = url.substring(prefix.length(), url.length() - 5); + JSONObject rel = releases.get(name); + if (rel == null) { + throw new IOException(url + " answered 404"); + } + return new JSONObject().put("releases", rel).toString().getBytes(StandardCharsets.UTF_8); + } + byte[] b = served.get(url); + if (b == null) { + throw new IOException(url + " answered 404"); + } + return b; + }); + } + + private static byte[] zip(Map entries) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + try (ZipOutputStream z = new ZipOutputStream(out)) { + for (Map.Entry e : entries.entrySet()) { + z.putNextEntry(new ZipEntry(e.getKey())); + z.write(e.getValue().getBytes(StandardCharsets.UTF_8)); + z.closeEntry(); + } + } + return out.toByteArray(); + } + + @Test + public void versionsOrderAsPep440() { + String[] ordered = {"1.0.dev1", "1.0a1", "1.0b2", "1.0rc1", "1.0", "1.0.post1", "1.1", "2!0.1"}; + for (int i = 1; i < ordered.length; i++) { + assertTrue(ordered[i - 1] + " < " + ordered[i], PythonRequirement.compare(ordered[i - 1], ordered[i]) < 0); + } + assertEquals(0, PythonRequirement.compare("1.0", "1.0.0")); + } + + @Test + public void specifiersAndMarkers() { + PythonRequirement r = PythonRequirement.parse("Foo_Bar[x] >=1.2, !=1.3.*, <2 ; python_version >= '3.8'"); + assertEquals("foo-bar", r.name); + assertEquals(Arrays.asList("x"), r.extras); + assertTrue(r.accepts("1.2")); + assertFalse(r.accepts("1.3.4")); + assertFalse(r.accepts("2.0")); + assertFalse("no pre-release unless asked", r.accepts("1.9b1")); + assertTrue(PythonRequirement.parse("x ~=1.4.2").accepts("1.4.9")); + assertFalse(PythonRequirement.parse("x ~=1.4.2").accepts("1.5")); + assertTrue(r.applies(Collections.emptyList())); + assertFalse(PythonRequirement.parse("pywin32 ; sys_platform == 'win32'").applies(Collections.emptyList())); + PythonRequirement extra = PythonRequirement.parse("socks ; extra == 'socks'"); + assertFalse(extra.applies(Collections.emptyList())); + assertTrue(extra.applies(Arrays.asList("socks"))); + } + + @Test + public void resolvesTransitivelyAndStubsCompiledModules() throws Exception { + Map app = new LinkedHashMap(); + app.put("app/__init__.py", "from fastlib import fast_sum\n"); + app.put("app-1.0.dist-info/METADATA", "Name: app\nVersion: 1.0\nRequires-Dist: fastlib>=2\n" + + "Requires-Dist: requests\nRequires-Dist: winonly; sys_platform == 'win32'\n\nbody\n"); + publish("app", "1.0", "app-1.0-py3-none-any.whl", zip(app)); + publish("app", "1.1b1", "app-1.1b1-py3-none-any.whl", zip(app)); + + Map fast = new LinkedHashMap(); + fast.put("fastlib/__init__.py", "from ._core import fast_sum, Vector\nfrom . import _io as io\n" + + "def total(xs):\n return io.read(xs) + io.SIZE\n" + + "try:\n from ._speedups import escape\nexcept ImportError:\n def escape(s):\n return s\n"); + fast.put("fastlib/_core.cpython-313-x86_64-linux-gnu.so", "ELF"); + fast.put("fastlib/_io.cpython-313-x86_64-linux-gnu.so", "ELF"); + fast.put("fastlib/_speedups.cpython-313-x86_64-linux-gnu.so", "ELF"); + fast.put("fastlib-2.1.dist-info/METADATA", "Name: fastlib\nVersion: 2.1\n"); + publish("fastlib", "1.9", "fastlib-1.9-py3-none-any.whl", zip(fast)); + publish("fastlib", "2.1", "fastlib-2.1-cp313-cp313-manylinux_2_17_x86_64.whl", zip(fast)); + + File common = tmp.newFolder("common"); + PythonPackageResolver.Result r = resolver().resolve(Arrays.asList("app"), common); + assertEquals(Arrays.asList("app==1.0", "fastlib==2.1"), r.installed); + assertEquals(Arrays.asList("requests"), r.provided); + assertEquals(Arrays.asList("fastlib._speedups"), r.optionalNativeModules); + assertEquals("com.codename1.python.ext.FastlibCore", r.nativeModules.get("fastlib._core")); + assertEquals("com.codename1.python.ext.FastlibIo", r.nativeModules.get("fastlib._io")); + + File pkgs = new File(common, "src/main/python-packages"); + assertTrue(new File(pkgs, "fastlib/__init__.py").isFile()); + assertFalse(new File(pkgs, "app-1.0.dist-info").exists()); + String lock = new String(Files.readAllBytes(new File(pkgs, "python.lock").toPath()), StandardCharsets.UTF_8); + assertTrue(lock, lock.contains("fastlib==2.1 file=fastlib-2.1-cp313-cp313-manylinux_2_17_x86_64.whl")); + + String core = new String(Files.readAllBytes(new File(common, + "src/main/java/com/codename1/python/ext/FastlibCore.java").toPath()), StandardCharsets.UTF_8); + assertTrue(core, core.contains("new NativeModule(\"fastlib._core\")")); + assertTrue(core, core.contains(".function(\"Vector\"") && core.contains(".function(\"fast_sum\"")); + String io = new String(Files.readAllBytes(new File(common, + "src/main/java/com/codename1/python/ext/FastlibIo.java").toPath()), StandardCharsets.UTF_8); + assertTrue(io, io.contains(".function(\"SIZE\"") && io.contains(".function(\"read\"")); + + // A filled-in stub survives a second import, which installs from the lock. + Files.write(new File(common, "src/main/java/com/codename1/python/ext/FastlibCore.java").toPath(), + "// mine\n".getBytes(StandardCharsets.UTF_8)); + publish("fastlib", "2.2", "fastlib-2.2-cp313-cp313-manylinux_2_17_x86_64.whl", zip(fast)); + PythonPackageResolver.Result again = resolver().resolve(Arrays.asList("app"), common); + assertTrue(again.installed.contains("fastlib==2.1")); + assertFalse(again.stubsWritten.contains("com.codename1.python.ext.FastlibCore")); + assertEquals("// mine\n", new String(Files.readAllBytes(new File(common, + "src/main/java/com/codename1/python/ext/FastlibCore.java").toPath()), StandardCharsets.UTF_8)); + assertEquals(PythonExtensionStubs.readIndex(new File(pkgs, PythonExtensionStubs.INDEX)), r.nativeModules); + } + + @Test + public void rejectsATamperedDownload() throws Exception { + Map m = new LinkedHashMap(); + m.put("lib/__init__.py", "x = 1\n"); + publish("lib", "1.0", "lib-1.0-py3-none-any.whl", zip(m)); + served.put("https://files.example/lib-1.0-py3-none-any.whl", "evil".getBytes(StandardCharsets.UTF_8)); + try { + resolver().resolve(Arrays.asList("lib"), tmp.newFolder("c")); + fail(); + } catch (BuildException e) { + assertTrue(e.getMessage(), e.getMessage().contains("SHA-256")); + } + } + + @Test + public void conflictingRequirementsFail() throws Exception { + Map m = new LinkedHashMap(); + m.put("lib/__init__.py", "x = 1\n"); + publish("lib", "1.0", "lib-1.0-py3-none-any.whl", zip(m)); + try { + resolver().resolve(Arrays.asList("lib>=2"), tmp.newFolder("c")); + fail(); + } catch (BuildException e) { + assertTrue(e.getMessage(), e.getMessage().contains("No release of lib")); + } + } + + @Test + public void requirementSpecsKeepVersions() { + assertEquals(Arrays.asList("requests[socks]>=2", "attrs", "numpy==2.1"), + PythonProjectImporter.requirementSpecs( + "[project]\nname = \"x\"\ndependencies = [\"requests[socks]>=2\", 'attrs']\n", + "# c\nnumpy==2.1 # pinned\n-r other.txt\n")); + } +} diff --git a/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/ImportPythonProjectMojo.java b/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/ImportPythonProjectMojo.java index ba63f51abd6..9018313a1c5 100644 --- a/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/ImportPythonProjectMojo.java +++ b/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/ImportPythonProjectMojo.java @@ -54,6 +54,14 @@ public class ImportPythonProjectMojo extends AbstractCN1Mojo { @Parameter(property = "cn1.python.import") private File pythonImport; + /// Leaves PyPI alone: the requirements are listed, not downloaded. + @Parameter(property = "cn1.python.offline", defaultValue = "false") + private boolean offline; + + /// An index that speaks PyPI's JSON API, in place of pypi.org. + @Parameter(property = "cn1.python.index") + private String index; + @Override protected void executeImpl() throws MojoExecutionException, MojoFailureException { File from = pythonImport != null ? pythonImport : source; @@ -81,10 +89,37 @@ protected void executeImpl() throws MojoExecutionException, MojoFailureException + (r.removedFiles == 0 ? "" : "; removed " + r.removedFiles + " the project no longer has")); getLog().info(r.mainModule == null ? "No entry module was found: the sources are imported as a library" : "Entry module: " + r.mainModule); - if (!r.requirements.isEmpty()) { - getLog().warn("The project depends on " + r.requirements + ". A pure-Python package builds when its " - + "sources are under src/main/python; see the developer guide for the packages the runtime " - + "provides."); + if (r.requirementSpecs.isEmpty()) { + return; + } + if (offline) { + getLog().warn("The project depends on " + r.requirements + "; they were not downloaded " + + "(-Dcn1.python.offline). Import again without it to vendor them."); + return; + } + PythonPackageResolver.Result p; + try { + PythonPackageResolver resolver = new PythonPackageResolver(MavenLog.of(getLog())); + if (index != null && !index.isEmpty()) { + resolver.index(index); + } + p = resolver.resolve(r.requirementSpecs, common); + } catch (com.codename1.builders.BuildException e) { + throw new MojoFailureException(e.getMessage(), e); + } + if (!p.provided.isEmpty()) { + getLog().info("Provided by the Codename One Python runtime: " + p.provided); + } + if (!p.installed.isEmpty()) { + getLog().info("Installed into src/main/" + PythonPackageResolver.PACKAGES_DIR + ": " + p.installed); + } + for (java.util.Map.Entry e : p.nativeModules.entrySet()) { + getLog().warn("C extension module " + e.getKey() + " is written in Java by " + e.getValue() + + (p.stubsWritten.contains(e.getValue()) ? " (stub generated: fill in its functions)" : "")); + } + if (!p.optionalNativeModules.isEmpty()) { + getLog().info("C accelerator modules with a pure-Python fallback, used instead: " + + p.optionalNativeModules); } } diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Builtins.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Builtins.java index fa427075b67..79a207a1b22 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Builtins.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Builtins.java @@ -833,7 +833,7 @@ static Object toFloat(Object x) { try { return toFloat(new String(cs)); } catch (PyException e) { - throw Py.valueError("could not convert string to float: " + Py.toRepr(x)); + throw Py.valueError("could not convert string to float: " + Py.toRepr(x)); //NOPMD PreserveStackTrace - CPython's error has no cause } } return Py.ofFloat(Py.asDouble(x)); diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Cn1Sql.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Cn1Sql.java index 46fca22e8c1..d14a7989a9b 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Cn1Sql.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Cn1Sql.java @@ -33,7 +33,8 @@ /// reports it ([Database#getColumnType(Row, int)]), and are inferred from /// their text where it does not. public final class Cn1Sql extends PySql { - private static int memoryCount; + private static final java.util.concurrent.atomic.AtomicInteger MEMORY_COUNT = + new java.util.concurrent.atomic.AtomicInteger(); /// Makes this the engine unless a host installed its own first. public static void installDefault() { @@ -45,7 +46,7 @@ public static void installDefault() { @Override public Connection open(String name) throws IOException { boolean memory = ":memory:".equals(name) || name.length() == 0; - String dbName = memory ? "cn1py-memory-" + System.currentTimeMillis() + "-" + (memoryCount++) + ".db" + String dbName = memory ? "cn1py-memory-" + System.currentTimeMillis() + "-" + MEMORY_COUNT.getAndIncrement() + ".db" : localName(name); if (memory && Database.exists(dbName)) { Database.delete(dbName); diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Gil.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Gil.java index a5e95f772e5..8ceb0cdb68b 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Gil.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Gil.java @@ -72,7 +72,8 @@ public static void exit() { if (owner != Thread.currentThread()) { //NOPMD CompareObjectsWithEquals - identity is meant return; } - if (--depth == 0) { + depth--; + if (depth == 0) { owner = null; MONITOR.notifyAll(); } diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/NativeModule.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/NativeModule.java new file mode 100644 index 00000000000..3592d5f79ab --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/NativeModule.java @@ -0,0 +1,169 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +/// A Python module written in Java: what takes the place of a C extension +/// module (`markupsafe._speedups`, `yaml._yaml`) in a Codename One build. +/// +/// A C extension is a module whose functions happen to be compiled from C. +/// The Python code above it imports and calls it like any other module, so +/// supporting the package means providing those functions in Java instead; +/// everything else in the package is transpiled as it is. The build +/// generates one class per extension module with a stub for every name the +/// package's Python code uses, and lists it in +/// `src/main/python-packages/cn1-native-modules.properties`; the generated +/// registry installs it before any module is imported. +/// +/// ```java +/// public static void install() { +/// new NativeModule("markupsafe._speedups") +/// .function("escape", args -> Escapes.html(String.valueOf(args[0]))) +/// .install(); +/// } +/// ``` +/// +/// A [Function] sees Java values -- a Python `str` as a `String`, an `int` +/// as an `Integer` or `Long`, a `float` as a `Double`, `bytes` as `byte[]`, +/// a list or tuple as a `java.util.List` and a dict as a `java.util.Map`, +/// `None` as null, a Java object as itself. Its result is converted back: +/// a `List` becomes a list, an `Object[]` a tuple, a `Map` a dict. A [PyBuiltin.Positional] given to [#raw] sees the +/// Python values untouched. +public final class NativeModule { + /// A module function over Java values. + public interface Function { + /// Runs the function with the call's positional arguments. + Object call(Object[] args); + } + + private final PyModule module; + + /// A module named `name`, the dotted name the Python code imports. + public NativeModule(String name) { + module = new PyModule(name, new String[0]); + } + + /// Adds the function `name` over Java values. + public NativeModule function(final String name, final Function f) { + module.setAttr(name, PyBuiltin.positional(name, a -> { + Object[] args = new Object[a.length]; + for (int i = 0; i < a.length; i++) { + args[i] = plain(a[i]); + } + return python(f.call(args)); + })); + return this; + } + + /// A Python value as plain Java: a list or tuple is a `java.util.List`, + /// a dict a `java.util.Map` (insertion ordered), each element converted + /// the same way; numbers, strings, bytes and `None` as `JavaBridge` + /// converts them. + static Object plain(Object v) { + if (v instanceof PyList) { + PyList l = (PyList) v; + java.util.ArrayList r = new java.util.ArrayList(l.size()); + for (int i = 0; i < l.size(); i++) { + r.add(plain(l.get(i))); + } + return r; + } + if (v instanceof PyTuple) { + PyTuple t = (PyTuple) v; + java.util.ArrayList r = new java.util.ArrayList(t.size()); + for (int i = 0; i < t.size(); i++) { + r.add(plain(t.get(i))); + } + return r; + } + if (v instanceof PyDict) { + PyDict d = (PyDict) v; + java.util.LinkedHashMap r = new java.util.LinkedHashMap(); + for (Object k : d.keyArray()) { + r.put(plain(k), plain(d.getItem(k))); + } + return r; + } + return JavaBridge.toJava(v); + } + + /// A Java result as Python: a `List` is a list, an `Object[]` a tuple, + /// a `Map` a dict and a `byte[]` bytes, converted all the way down. + static Object python(Object v) { + if (v instanceof java.util.List) { + java.util.List l = (java.util.List) v; + Object[] items = new Object[l.size()]; + for (int i = 0; i < items.length; i++) { + items[i] = python(l.get(i)); + } + return new PyList(items); + } + if (v instanceof Object[]) { + Object[] a = (Object[]) v; + Object[] items = new Object[a.length]; + for (int i = 0; i < items.length; i++) { + items[i] = python(a[i]); + } + return new PyTuple(items); + } + if (v instanceof java.util.Map) { + PyDict d = new PyDict(); + for (java.util.Map.Entry e : ((java.util.Map) v).entrySet()) { + d.setItem(python(e.getKey()), python(e.getValue())); + } + return d; + } + if (v instanceof byte[]) { + byte[] b = (byte[]) v; + byte[] copy = new byte[b.length]; + System.arraycopy(b, 0, copy, 0, b.length); + return new PyBytes(copy); + } + return Py.wrap(v); + } + + /// Adds the function `name` over Python values, for code that needs + /// Python's own containers or identity. + public NativeModule raw(String name, PyBuiltin.Positional f) { + module.setAttr(name, PyBuiltin.positional(name, f)); + return this; + } + + /// Adds the module attribute `name`, converted from Java. + public NativeModule constant(String name, Object value) { + module.setAttr(name, Py.wrap(value)); + return this; + } + + /// Makes the module importable, replacing any earlier one of that name. + public void install() { + PyModule.registerModule(module); + } + + /// The `NotImplementedError` a generated stub raises until it is filled + /// in, naming what is missing. + public static RuntimeException notImplemented(String what) { + return Py.error(Py.NOT_IMPLEMENTED_ERROR, what + " is a C extension function that has no Java implementation" + + " yet: implement it in the class the build generated for its module"); + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java index bcda3d3002d..e9be407ce2a 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java @@ -364,7 +364,7 @@ private static PyModule sql() { SQL.put(h, c); return PyInt.of(h); } catch (java.io.IOException e) { - throw Py.error(Py.OS_ERROR, String.valueOf(e.getMessage())); + throw Py.error(Py.OS_ERROR, String.valueOf(e.getMessage())); //NOPMD PreserveStackTrace - a Python exception has no Java cause } })); m.setAttr("run", PyBuiltin.positional("run", a -> { @@ -378,7 +378,7 @@ private static PyModule sql() { try { r = c.run(Py.toStr(a[1]), params, Py.isTrue(a[3])); } catch (java.io.IOException e) { - throw Py.error(Py.OS_ERROR, String.valueOf(e.getMessage())); + throw Py.error(Py.OS_ERROR, String.valueOf(e.getMessage())); //NOPMD PreserveStackTrace - a Python exception has no Java cause } Object[] cols = new Object[r.columns.length]; for (int i = 0; i < cols.length; i++) { @@ -418,7 +418,7 @@ private static void sqlControl(PyModule m, String name, int op) { c.close(); } } catch (java.io.IOException e) { - throw Py.error(Py.OS_ERROR, String.valueOf(e.getMessage())); + throw Py.error(Py.OS_ERROR, String.valueOf(e.getMessage())); //NOPMD PreserveStackTrace - a Python exception has no Java cause } return Py.NONE; })); diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java index 7ea6b3ed3b3..353e6e719a8 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java @@ -70,6 +70,19 @@ public final class PythonTranspiler { private PythonTranspiler() { } + /// Whether the runtime provides the top-level module `name` itself -- a + /// standard library module, or a package Codename One maps (`requests`, + /// `numpy`, `flask`) -- so a requirement on it needs nothing downloaded. + public static boolean provides(String name) { + for (String root : new String[] {STDLIB, SITE}) { + if (PythonTranspiler.class.getResource(root + name + ".py") != null + || PythonTranspiler.class.getResource(root + name + "/__init__.py") != null) { + return true; + } + } + return false; + } + /// @param sources relative `.py` path to source text /// @param mainModule the module run as `__main__`, or null for a library public static Map transpile(Map sources, String mainModule) { @@ -85,6 +98,19 @@ public static Map transpile(Map sources, String /// build error; otherwise `java.*` comes from the running JDK public static Map transpile(Map sources, String mainModule, List classPath) { + return transpile(sources, mainModule, classPath, java.util.Collections.emptyMap()); + } + + /// @param sources relative `.py` path to source text + /// @param mainModule the module run as `__main__`, or null for a library + /// @param classPath as for [#transpile(Map, String, List)] + /// @param nativeModules modules written in Java in place of a C + /// extension, by dotted module name: the fully qualified class + /// whose static `install()` registers it (a + /// `com.codename1.python.runtime.NativeModule`). The registry + /// installs each before any module runs + public static Map transpile(Map sources, String mainModule, + List classPath, Map nativeModules) { Map out = new LinkedHashMap(); Map classes = new LinkedHashMap(); Map trees = new LinkedHashMap(); @@ -160,7 +186,8 @@ public static Map transpile(Map sources, String if (server) { out.put(PACKAGE.replace('.', '/') + "/BackendServer.java", backendServer()); } - out.put(PACKAGE.replace('.', '/') + "/PythonModules.java", registry(classes, mainModule, interop != null, server)); + out.put(PACKAGE.replace('.', '/') + "/PythonModules.java", registry(classes, mainModule, interop != null, server, + nativeModules)); return out; } @@ -489,7 +516,8 @@ static String withLineMap(String java, String pyFile) { + java.substring(brace + 1); } - private static String registry(Map classes, String mainModule, boolean interop, boolean server) { + private static String registry(Map classes, String mainModule, boolean interop, boolean server, + Map nativeModules) { StringBuilder sb = new StringBuilder(); sb.append("// Generated by the Codename One Python transpiler.\n"); sb.append("package ").append(PACKAGE).append(";\n\n"); @@ -513,6 +541,9 @@ private static String registry(Map classes, String mainModule, b if (classes.containsKey("sqlite3")) { sb.append(" com.codename1.python.runtime.Cn1Sql.installDefault();\n"); } + for (String cls : new java.util.TreeSet(nativeModules.values())) { + sb.append(" ").append(cls).append(".install();\n"); + } for (String cls : classes.values()) { sb.append(" ").append(cls).append(".register();\n"); } diff --git a/maven/python-transpiler/src/test/java/com/codename1/python/transpiler/ConformanceTest.java b/maven/python-transpiler/src/test/java/com/codename1/python/transpiler/ConformanceTest.java index 6e7c3688255..53443efc681 100644 --- a/maven/python-transpiler/src/test/java/com/codename1/python/transpiler/ConformanceTest.java +++ b/maven/python-transpiler/src/test/java/com/codename1/python/transpiler/ConformanceTest.java @@ -85,9 +85,33 @@ private static void run(Path program) throws Exception { String name = program.getFileName().toString(); String stem = name.endsWith(".py") ? name.substring(0, name.length() - 3) : name; Map sources = new TreeMap(); + // A directory program may stand in for a C extension: natives.properties + // maps a module to the Java class (a .java file beside it) that + // replaces it. The module's .py there is what CPython ran for the + // expected output, and is left out here. + Map natives = new TreeMap(); + List javaSources = new ArrayList(); + Path nativesFile = program.resolve("natives.properties"); + if (Files.isDirectory(program) && Files.isRegularFile(nativesFile)) { + for (String line : Files.readAllLines(nativesFile, StandardCharsets.UTF_8)) { + int eq = line.indexOf('='); + if (eq > 0 && !line.startsWith("#")) { + natives.put(line.substring(0, eq).trim(), line.substring(eq + 1).trim()); + } + } + try (Stream s = Files.walk(program)) { + for (Path p : s.filter(x -> x.toString().endsWith(".java")).collect(Collectors.toList())) { + javaSources.add(p.toAbsolutePath().toString()); + } + } + } if (Files.isDirectory(program)) { try (Stream s = Files.walk(program)) { for (Path p : s.filter(x -> x.toString().endsWith(".py")).collect(Collectors.toList())) { + String rel = program.relativize(p).toString().replace(File.separatorChar, '/'); + if (natives.containsKey(rel.substring(0, rel.length() - 3).replace('/', '.'))) { + continue; + } sources.put(program.relativize(p).toString().replace(File.separatorChar, '/'), read(p)); } } @@ -101,7 +125,7 @@ private static void run(Path program) throws Exception { sources.put("_testbase.py", "BASE = \"http://127.0.0.1:" + server.port() + "\"\n"); } try { - run(stem, sources, server != null); + run(stem, sources, server != null, natives, javaSources); } finally { if (server != null) { server.stop(); @@ -109,7 +133,8 @@ private static void run(Path program) throws Exception { } } - private static void run(String stem, Map sources, boolean http) throws Exception { + private static void run(String stem, Map sources, boolean http, Map natives, + List javaSources) throws Exception { Path work = Files.createTempDirectory("py-conf-" + stem); Path gen = work.resolve("gen"); Path cls = work.resolve("classes"); @@ -119,12 +144,13 @@ private static void run(String stem, Map sources, boolean http) String core = location(com.codename1.io.Util.class); // Programs may import Java classes: the JDK's, and the core's. List classPath = Collections.singletonList(new File(core)); - for (Map.Entry e : PythonTranspiler.transpile(sources, "main", classPath).entrySet()) { + for (Map.Entry e : PythonTranspiler.transpile(sources, "main", classPath, natives).entrySet()) { Path f = gen.resolve(e.getKey()); Files.createDirectories(f.getParent()); Files.write(f, e.getValue().getBytes(StandardCharsets.US_ASCII)); args.add(f.toString()); } + args.addAll(javaSources); args.add(0, "-nowarn"); args.add(1, "--release"); args.add(2, "17"); diff --git a/maven/python-transpiler/src/test/resources/conformance/cext_.out b/maven/python-transpiler/src/test/resources/conformance/cext_.out new file mode 100644 index 00000000000..2e10b060a42 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/cext_.out @@ -0,0 +1,6 @@ +100 +[3.0, 5.0] +<a&b> +1.0 c-ext +[('k', 1)] +TypeError fast_sum() needs a list diff --git a/maven/python-transpiler/src/test/resources/conformance/cext_/FastlibCore.java b/maven/python-transpiler/src/test/resources/conformance/cext_/FastlibCore.java new file mode 100644 index 00000000000..0fc56e87f22 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/cext_/FastlibCore.java @@ -0,0 +1,46 @@ +package com.codename1.python.ext; + +import com.codename1.python.runtime.NativeModule; +import com.codename1.python.runtime.Py; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +public final class FastlibCore { + private FastlibCore() { + } + + public static void install() { + new NativeModule("fastlib._core") + .constant("VERSION", "1.0") + .function("fast_sum", args -> { + if (!(args[0] instanceof List)) { + throw Py.error(Py.TYPE_ERROR, "fast_sum() needs a list"); + } + long s = 0; + for (Object o : (List) args[0]) { + s += ((Number) o).longValue(); + } + return s; + }) + .function("scale", args -> { + double k = ((Number) args[1]).doubleValue(); + List out = new ArrayList(); + for (Object o : (List) args[0]) { + out.add(((Number) o).doubleValue() * k); + } + return out.toArray(); + }) + .function("join_words", args -> String.join("-", (List) args[0])) + .function("describe", args -> { + List out = new ArrayList(); + for (Map.Entry e : new TreeMap((Map) args[0]).entrySet()) { + out.add(new Object[] {e.getKey(), e.getValue()}); + } + return out; + }) + .install(); + } +} diff --git a/maven/python-transpiler/src/test/resources/conformance/cext_/fastlib/__init__.py b/maven/python-transpiler/src/test/resources/conformance/cext_/fastlib/__init__.py new file mode 100644 index 00000000000..306b4d7bc3c --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/cext_/fastlib/__init__.py @@ -0,0 +1,20 @@ +from ._core import fast_sum, scale, VERSION + +try: + from ._speedups import escape +except ImportError: + def escape(s): + return s.replace("&", "&").replace("<", "<").replace(">", ">") + + +def total(xs): + return fast_sum(xs) * 10 + + +def scaled(xs, k): + return list(scale(xs, k)) + + +def info(d): + from . import _core + return _core.describe(d) diff --git a/maven/python-transpiler/src/test/resources/conformance/cext_/fastlib/_core.py b/maven/python-transpiler/src/test/resources/conformance/cext_/fastlib/_core.py new file mode 100644 index 00000000000..5ce948cfb12 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/cext_/fastlib/_core.py @@ -0,0 +1,21 @@ +# What CPython runs for the expected output; the transpiled program +# runs Core.java in its place, as it would for a compiled module. +VERSION = "1.0" + + +def fast_sum(xs): + if not isinstance(xs, list): + raise TypeError("fast_sum() needs a list") + return sum(xs) + + +def scale(xs, k): + return tuple(x * k for x in xs) + + +def join_words(ws): + return "-".join(ws) + + +def describe(d): + return sorted(d.items()) diff --git a/maven/python-transpiler/src/test/resources/conformance/cext_/main.py b/maven/python-transpiler/src/test/resources/conformance/cext_/main.py new file mode 100644 index 00000000000..7d1ab7fcbbe --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/cext_/main.py @@ -0,0 +1,12 @@ +import fastlib +from fastlib import _core + +print(fastlib.total([1, 2, 3, 4])) +print(fastlib.scaled([1.5, 2.5], 2)) +print(fastlib.escape("")) +print(_core.VERSION, _core.join_words(["c", "ext"])) +print(fastlib.info({"k": 1})) +try: + _core.fast_sum("x") +except TypeError as e: + print("TypeError", e) diff --git a/maven/python-transpiler/src/test/resources/conformance/cext_/natives.properties b/maven/python-transpiler/src/test/resources/conformance/cext_/natives.properties new file mode 100644 index 00000000000..569ddcba310 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/cext_/natives.properties @@ -0,0 +1 @@ +fastlib._core=com.codename1.python.ext.FastlibCore From 447fe298fade9a4d95919a4f325802fec6861ea2 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 09:17:51 +0200 Subject: [PATCH 041/112] Python: split the static-field getter per class so importing FontImage compiles Importing a Java class with thousands of constants (FontImage's icon set) generated one getStatic method past the JVM's 64 KB code limit. Each class now gets its own getters of at most 256 fields, chained. Co-Authored-By: Claude Opus 5.5 --- .../codegen/JavaInteropGenerator.java | 29 +++++++++++++++---- .../conformance/java_interop_constants.out | 5 ++++ .../conformance/java_interop_constants.py | 13 +++++++++ 3 files changed, 41 insertions(+), 6 deletions(-) create mode 100644 maven/python-transpiler/src/test/resources/conformance/java_interop_constants.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/java_interop_constants.py diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/JavaInteropGenerator.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/JavaInteropGenerator.java index 65765001537..58f222d6c42 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/JavaInteropGenerator.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/JavaInteropGenerator.java @@ -60,6 +60,9 @@ public final class JavaInteropGenerator { private final ClassPathReader cp; /// Imported classes, by internal name, numbered in order. + /// Static fields per generated getter method. + private static final int STATIC_CHUNK = 256; + private final Map imported = new LinkedHashMap(); /// Python module (`java.com.codename1.ui`) to its imported names. private final Map> modules = new TreeMap>(); @@ -771,18 +774,32 @@ private String emit(String pkg) { } } } + // One method per class, and per STATIC_CHUNK fields of it, each + // falling through to the next: a class such as FontImage has + // thousands of constants, which in one method pass the JVM's 64 KB + // limit on a method's code. + StringBuilder fieldGetters = new StringBuilder(); body.append(" @Override\n public Object getStatic(int cls, String name) {\n"); body.append(" switch (cls) {\n"); for (Map.Entry> e : staticFields.entrySet()) { String c = javaType("L" + ids.get(e.getKey()) + ";"); - body.append(" case ").append(e.getKey()).append(":\n switch (name) {\n"); - for (Member f : e.getValue().values()) { - body.append(" case ").append(q(f.name)).append(": return ") - .append(toPy(f.desc, c + "." + f.name)).append(";\n"); + String fn = "getStatic" + e.getKey() + "_"; + body.append(" case ").append(e.getKey()).append(": return ").append(fn).append("0(name);\n"); + List fields = new ArrayList(e.getValue().values()); + for (int chunk = 0; chunk * STATIC_CHUNK < fields.size(); chunk++) { + fieldGetters.append(" private static Object ").append(fn).append(chunk).append("(String name) {\n"); + fieldGetters.append(" switch (name) {\n"); + int end = Math.min(fields.size(), (chunk + 1) * STATIC_CHUNK); + for (Member f : fields.subList(chunk * STATIC_CHUNK, end)) { + fieldGetters.append(" case ").append(q(f.name)).append(": return ") + .append(toPy(f.desc, c + "." + f.name)).append(";\n"); + } + fieldGetters.append(" default: return ") + .append(end < fields.size() ? fn + (chunk + 1) + "(name)" : "null").append(";\n"); + fieldGetters.append(" }\n }\n\n"); } - body.append(" default: return null;\n }\n"); } - body.append(" default: return null;\n }\n }\n\n"); + body.append(" default: return null;\n }\n }\n\n").append(fieldGetters); body.append(" @Override\n public boolean hasStatic(int cls, String name) {\n"); body.append(" switch (cls) {\n"); for (Map.Entry>> e : staticMethods.entrySet()) { diff --git a/maven/python-transpiler/src/test/resources/conformance/java_interop_constants.out b/maven/python-transpiler/src/test/resources/conformance/java_interop_constants.out new file mode 100644 index 00000000000..8ebdb619cf9 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/java_interop_constants.out @@ -0,0 +1,5 @@ +0xe951 +0xe145 +0xe5d5 +0xe7e2 +AttributeError diff --git a/maven/python-transpiler/src/test/resources/conformance/java_interop_constants.py b/maven/python-transpiler/src/test/resources/conformance/java_interop_constants.py new file mode 100644 index 00000000000..74f0bb769e5 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/java_interop_constants.py @@ -0,0 +1,13 @@ +# A Java class with thousands of constants (FontImage's icon set) imported +# whole. CPython cannot import java.*, so java_interop_constants.out is +# written by hand -- do not regenerate it. +from java.com.codename1.ui import FontImage + +print(hex(ord(FontImage.MATERIAL_10K))) +print(hex(ord(FontImage.MATERIAL_ADD))) +print(hex(ord(FontImage.MATERIAL_REFRESH))) +print(hex(ord(FontImage.MATERIAL_SWITCH_ACCESS_SHORTCUT_ADD))) +try: + FontImage.MATERIAL_NO_SUCH_ICON +except AttributeError: + print("AttributeError") From 766820cc6f66863fc3eb5344bd2c12578b67b772 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 09:17:51 +0200 Subject: [PATCH 042/112] Python guide: user interface examples, PyPI packages, C extensions, threads and sqlite3 Two complete applications show Python driving Codename One: a list loaded from the network on a worker thread, and a notes app with a dataclass model, sqlite3 storage and two forms. C extensions are documented as supported, by replacing each compiled module with generated Java. The chapter's snippets move into docs/demos (Java ones compile against the Python runtime there), as the guide's snippet gate requires. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/developer-guide-docs.yml | 7 +- docs/demos/common/pom.xml | 12 + .../python/CommonWorkloads.java | 39 +++ .../python/MarkupsafeSpeedups.java | 38 +++ .../python/PythonInteropSnippets.java | 40 +++ .../developer-guide/python-interop.py | 209 +++++++++++++++ .../developer-guide/python-interop.sh | 15 ++ docs/developer-guide/Python-Interop.asciidoc | 249 +++++++++++------- 8 files changed, 518 insertions(+), 91 deletions(-) create mode 100644 docs/demos/common/src/main/java/com/codenameone/developerguide/python/CommonWorkloads.java create mode 100644 docs/demos/common/src/main/java/com/codenameone/developerguide/python/MarkupsafeSpeedups.java create mode 100644 docs/demos/common/src/main/java/com/codenameone/developerguide/python/PythonInteropSnippets.java create mode 100644 docs/demos/common/src/main/snippets/developer-guide/python-interop.py create mode 100644 docs/demos/common/src/main/snippets/developer-guide/python-interop.sh diff --git a/.github/workflows/developer-guide-docs.yml b/.github/workflows/developer-guide-docs.yml index bd13b6d4b1a..ddde1e314e2 100644 --- a/.github/workflows/developer-guide-docs.yml +++ b/.github/workflows/developer-guide-docs.yml @@ -175,11 +175,12 @@ jobs: # Through retry.sh, transient failures only, like the demo install below: # this is the step that resolves the most from Central, and a single 502 # on a plugin dependency (maven-shared-utils) failed the whole guide build. - # dart-runtime and flutter-runtime: docs/demos/common depends on the Flutter - # runtime at this version, which nothing else selected here pulls in; they are + # dart-runtime, flutter-runtime and python-runtime: docs/demos/common depends on + # the Flutter and Python runtimes at this version, which nothing else selected + # here pulls in; they are # in the reactor because this job runs on JDK 17 (the flutter-modules profile). RETRY_ONLY_MATCHING=transient xvfb-run -a bash scripts/ci/retry.sh mvn -B -ntp -f maven/pom.xml \ - -pl core,javase,android,css-compiler,codenameone-maven-plugin,backend,backend-test,dart-runtime,flutter-runtime \ + -pl core,javase,android,css-compiler,codenameone-maven-plugin,backend,backend-test,dart-runtime,flutter-runtime,python-runtime \ -am install \ -Plocal-dev-javase \ -DskipTests \ diff --git a/docs/demos/common/pom.xml b/docs/demos/common/pom.xml index fb2175fc417..08462d272fc 100644 --- a/docs/demos/common/pom.xml +++ b/docs/demos/common/pom.xml @@ -33,6 +33,18 @@ provided + + + com.codenameone + codenameone-python-runtime + ${cn1.version} + provided + + diff --git a/docs/demos/common/src/main/java/com/codenameone/developerguide/python/CommonWorkloads.java b/docs/demos/common/src/main/java/com/codenameone/developerguide/python/CommonWorkloads.java new file mode 100644 index 00000000000..6760f7319e0 --- /dev/null +++ b/docs/demos/common/src/main/java/com/codenameone/developerguide/python/CommonWorkloads.java @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +// tag::python-interop-java-003[] +package com.codenameone.developerguide.python; + +public final class CommonWorkloads { + public static long intArithmetic() { + int a = 0x12345678; + int b = 0x9E3779B9; + long checksum = 0; + for (int i = 0; i < 40000000; i++) { + a = (a * 1103515245 + 12345) ^ (b >>> 3); + b = (b + a) * 5 - (a << 7); + checksum += (a ^ b) & 0xFFFF; + } + return checksum + a + b; + } +} +// end::python-interop-java-003[] diff --git a/docs/demos/common/src/main/java/com/codenameone/developerguide/python/MarkupsafeSpeedups.java b/docs/demos/common/src/main/java/com/codenameone/developerguide/python/MarkupsafeSpeedups.java new file mode 100644 index 00000000000..1f1bfb347b7 --- /dev/null +++ b/docs/demos/common/src/main/java/com/codenameone/developerguide/python/MarkupsafeSpeedups.java @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codenameone.developerguide.python; + +import com.codename1.python.runtime.NativeModule; + +/// The stub the Python import generates for a C extension module. +// tag::python-interop-java-002[] +public final class MarkupsafeSpeedups { + public static void install() { + new NativeModule("markupsafe._speedups") + .function("escape", args -> { + throw NativeModule.notImplemented("markupsafe._speedups.escape"); + }) + .install(); + } +} +// end::python-interop-java-002[] diff --git a/docs/demos/common/src/main/java/com/codenameone/developerguide/python/PythonInteropSnippets.java b/docs/demos/common/src/main/java/com/codenameone/developerguide/python/PythonInteropSnippets.java new file mode 100644 index 00000000000..26d0558bf01 --- /dev/null +++ b/docs/demos/common/src/main/java/com/codenameone/developerguide/python/PythonInteropSnippets.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codenameone.developerguide.python; + +import com.codename1.python.runtime.Py; +import com.codename1.python.runtime.PyModule; + +/// Java calling into translated Python, for the Python chapter. +public final class PythonInteropSnippets { + private PythonInteropSnippets() { + } + + static Object total(Object cart) { + // tag::python-interop-java-001[] + PyModule pricing = Py.importModule("pricing"); + Object total = Py.call(Py.getattr(pricing, "total"), cart); + // end::python-interop-java-001[] + return total; + } +} diff --git a/docs/demos/common/src/main/snippets/developer-guide/python-interop.py b/docs/demos/common/src/main/snippets/developer-guide/python-interop.py new file mode 100644 index 00000000000..f0a223a5947 --- /dev/null +++ b/docs/demos/common/src/main/snippets/developer-guide/python-interop.py @@ -0,0 +1,209 @@ +// Generated from docs/developer-guide source blocks. Edit the guide snippets here, not inline. + +// tag::python-interop-python-001[] +from java.com.codename1.ui import Form, Button, Label, TextField, Dialog +from java.com.codename1.ui.layouts import BoxLayout + + +def main(): + form = Form("Hello Python", BoxLayout.y()) + name = TextField("", "Your name") + out = Label("Hello, world!") + + def on_greet(event): + out.setText(f"Hello, {name.getText().strip()}!") + form.revalidate() + + button = Button("Greet") + button.addActionListener(on_greet) + form.add(name).add(button).add(out) + form.show() + + +main() +// end::python-interop-python-001[] + +// tag::python-interop-python-002[] +import threading + +import requests + +from java.com.codename1.components import InfiniteProgress, MultiButton +from java.com.codename1.ui import Display, FontImage, Form +from java.com.codename1.ui.layouts import BoxLayout + +API = "https://api.github.com/repos/codenameone/CodenameOne/releases" + + +def fetch_releases(): + response = requests.get(API, params={"per_page": 20}, timeout=15) + response.raise_for_status() + return [(r["name"] or r["tag_name"], r["published_at"][:10]) for r in response.json()] + + +def show_releases(form, releases): + form.removeAll() + for name, date in releases: + row = MultiButton(name) + row.setTextLine2(date) + row.addActionListener(lambda e, n=name: Display.getInstance().execute( + "https://github.com/codenameone/CodenameOne/releases/tag/" + n)) + form.add(row) + form.revalidate() + + +def load(form): + progress = InfiniteProgress().showInfiniteBlocking() + + def work(): + try: + releases = fetch_releases() + except requests.RequestException as e: + releases = [("Couldn't load releases", str(e))] + # Components belong to the EDT: hand the result back to it. + Display.getInstance().callSerially(lambda: (progress.dispose(), show_releases(form, releases))) + + threading.Thread(target=work, daemon=True).start() + + +def main(): + form = Form("Releases", BoxLayout.y()) + form.getToolbar().addMaterialCommandToRightBar("", FontImage.MATERIAL_REFRESH, lambda e: load(form)) + form.getContentPane().addPullToRefresh(lambda: load(form)) + form.show() + load(form) + + +main() +// end::python-interop-python-002[] + +// tag::python-interop-python-003[] +import sqlite3 +from dataclasses import dataclass +from datetime import date + +from java.com.codename1.components import MultiButton +from java.com.codename1.ui import Button, Dialog, FontImage, Form, TextArea, TextField +from java.com.codename1.ui.layouts import BorderLayout, BoxLayout + + +@dataclass +class Note: + id: int | None + title: str + body: str + day: str + + +class Notes: + def __init__(self, path: str = "notes.db"): + self.db = sqlite3.connect(path) + self.db.execute("CREATE TABLE IF NOT EXISTS note" + " (id INTEGER PRIMARY KEY, title TEXT, body TEXT, day TEXT)") + + def all(self) -> list[Note]: + rows = self.db.execute("SELECT id, title, body, day FROM note ORDER BY id DESC") + return [Note(*row) for row in rows] + + def save(self, note: Note) -> None: + with self.db: + if note.id is None: + self.db.execute("INSERT INTO note (title, body, day) VALUES (?, ?, ?)", + (note.title, note.body, note.day)) + else: + self.db.execute("UPDATE note SET title = ?, body = ? WHERE id = ?", + (note.title, note.body, note.id)) + + def delete(self, note: Note) -> None: + with self.db: + self.db.execute("DELETE FROM note WHERE id = ?", (note.id,)) + + +class NotesApp: + def __init__(self): + self.store = Notes() + self.home = Form("Notes", BoxLayout.y()) + self.home.getToolbar().addMaterialCommandToRightBar("", FontImage.MATERIAL_ADD, + lambda e: self.edit(Note(None, "", "", str(date.today())))) + self.refresh() + + def refresh(self): + self.home.removeAll() + for note in self.store.all(): + row = MultiButton(note.title or "Untitled") + row.setTextLine2(note.day) + row.addActionListener(lambda e, n=note: self.edit(n)) + self.home.add(row) + self.home.revalidate() + + def edit(self, note: Note): + form = Form("Edit", BorderLayout()) + title = TextField(note.title, "Title") + body = TextArea(note.body, 8, 40) + form.add(BorderLayout.NORTH, title).add(BorderLayout.CENTER, body) + + def back(e): + note.title, note.body = title.getText(), body.getText() + self.store.save(note) + self.refresh() + self.home.showBack() + + form.getToolbar().setBackCommand("", back) + if note.id is not None: + def delete(e): + if Dialog.show("Delete", f"Delete '{note.title}'?", "Delete", "Cancel"): + self.store.delete(note) + self.refresh() + self.home.showBack() + delete_button = Button("Delete") + delete_button.addActionListener(delete) + form.add(BorderLayout.SOUTH, delete_button) + form.show() + + def start(self): + self.home.show() + + +NotesApp().start() +// end::python-interop-python-003[] + +// tag::python-interop-python-004[] +from flask import Flask, jsonify + +app = Flask(__name__) + + +@app.get("/fib/") +def fib(n): + a, b = 0, 1 + for _ in range(n): + a, b = b, a + b + return jsonify({"n": n, "fib": str(a)}) + + +if __name__ == "__main__": + app.run(host="0.0.0.0", port=8080) +// end::python-interop-python-004[] + +// tag::python-interop-python-005[] +M32 = 0xFFFFFFFF + +def i32(v: int) -> int: + return ((v + 0x80000000) & M32) - 0x80000000 + +def int_arithmetic() -> int: + a: int = 0x12345678 + b: int = i32(0x9E3779B9) + checksum: int = 0 + for _ in range(40000000): + a = i32((a * 1103515245 + 12345) ^ ((b & M32) >> 3)) + b = i32((b + a) * 5 - (a << 7)) + checksum += (a ^ b) & 0xFFFF + return checksum + a + b +// end::python-interop-python-005[] + +// tag::python-interop-python-006[] +from java.com.codenameone.developerguide.python import CommonWorkloads + +checksum = CommonWorkloads.intArithmetic() +// end::python-interop-python-006[] diff --git a/docs/demos/common/src/main/snippets/developer-guide/python-interop.sh b/docs/demos/common/src/main/snippets/developer-guide/python-interop.sh new file mode 100644 index 00000000000..2a51a465558 --- /dev/null +++ b/docs/demos/common/src/main/snippets/developer-guide/python-interop.sh @@ -0,0 +1,15 @@ +// Generated from docs/developer-guide source blocks. Edit the guide snippets here, not inline. + +// tag::python-interop-bash-001[] +mvn cn1:import-python-project -Dsource=/path/to/my-python-app +// end::python-interop-bash-001[] + +// tag::python-interop-bash-002[] +jdb -sourcepath common/src/main/python -classpath ... MyAppStub +stop at com.codename1.generated.python.py_app$2eutil:6 +// end::python-interop-bash-002[] + +// tag::python-interop-bash-003[] +scripts/python-backend/build.sh scripts/python-backend/samples/flask-todo ./todo-server +./todo-server +// end::python-interop-bash-003[] diff --git a/docs/developer-guide/Python-Interop.asciidoc b/docs/developer-guide/Python-Interop.asciidoc index fd2b69eea73..23f4936c62c 100644 --- a/docs/developer-guide/Python-Interop.asciidoc +++ b/docs/developer-guide/Python-Interop.asciidoc @@ -15,11 +15,13 @@ Two things follow from compiling ahead of time rather than interpreting: * The program is known in full at build time. A module imported by a name computed while the program runs, `exec`, `eval` and `compile` of a - string, frame introspection and C extension modules are build errors that - name the file and the line. None of them reaches a device as a hidden failure. -* Packages that CPython implements in C -- `numpy`, `pydantic`'s core, - `yaml`'s scanner -- are provided as Codename One implementations of the same - public API. Pure Python packages are translated like your own code. + string, and frame introspection are build errors that name the file and the + line. None of them reaches a device as a hidden failure. +* Packages with C extensions are supported. The package's Python is + translated like your own code, and each compiled module is replaced by a + Java class with the same functions; see <>. + The most used ones -- `numpy`, `pydantic`'s core, `yaml`'s scanner -- are + already provided, as Codename One implementations of the same public API. === Adding Python to a project @@ -28,7 +30,7 @@ project into a Codename One application: [source,bash] ---- -mvn cn1:import-python-project -Dsource=/path/to/my-python-app +include::../demos/common/src/main/snippets/developer-guide/python-interop.sh[tag=python-interop-bash-001,indent=0] ---- The import copies the `.py` files, adds the Python runtime dependency and the @@ -69,8 +71,7 @@ With `jdb`: [source,bash] ---- -jdb -sourcepath common/src/main/python -classpath ... MyAppStub -stop at com.codename1.generated.python.py_app$2eutil:6 +include::../demos/common/src/main/snippets/developer-guide/python-interop.sh[tag=python-interop-bash-002,indent=0] ---- The mapping is applied to the compiled classes by the build's `remap-compat` @@ -84,26 +85,7 @@ Any class on the project's class path is importable under the `java.` prefix: [source,python] ---- -from java.com.codename1.ui import Form, Button, Label, TextField, Dialog -from java.com.codename1.ui.layouts import BoxLayout - - -def main(): - form = Form("Hello Python", BoxLayout.y()) - name = TextField("", "Your name") - out = Label("Hello, world!") - - def on_greet(event): - out.setText(f"Hello, {name.getText().strip()}!") - form.revalidate() - - button = Button("Greet") - button.addActionListener(on_greet) - form.add(name).add(button).add(out) - form.show() - - -main() +include::../demos/common/src/main/snippets/developer-guide/python-interop.py[tag=python-interop-python-001,indent=0] ---- A Python function or lambda passed where Java expects a listener or another @@ -131,22 +113,75 @@ Java exception arrives as a Python exception whose message names the Java type. To call into Python from Java, import the module through the runtime and call -its attributes: +its attributes. The build generates +`com.codename1.generated.python.PythonModules`, which registers every +translated module. When Python is the application's entry point, its main +class calls `PythonModules.registerAll()` at startup; a Java application +that uses Python as a library calls it once before importing anything: [source,java] ---- -PythonModules.registerAll(); -PyModule pricing = Py.importModule("pricing"); -Object total = Py.call(Py.getattr(pricing, "total"), cart); +include::../demos/common/src/main/java/com/codenameone/developerguide/python/PythonInteropSnippets.java[tag=python-interop-java-001,indent=0] ---- +=== Building the user interface in Python + +The user interface is Codename One's, called from Python, so everything in +the rest of this guide applies: layouts, themes and CSS, the component +inspector, and the same look on every platform. The examples below are +complete applications. Each is a `main.py` you can put in +`common/src/main/python`. + +==== A list loaded from the network + +The first example fetches JSON with `requests` on a worker thread, shows a +progress indicator while it waits and fills a list when the response +arrives. Pull the list down, or tap the toolbar button, to load it again. + +[source,python] +---- +include::../demos/common/src/main/snippets/developer-guide/python-interop.py[tag=python-interop-python-002,indent=0] +---- + +Three conventions carry over from Codename One's Java API: + +* Network calls run off the event dispatch thread. A Python thread is fine + for that, and `callSerially` hands the result back to the thread that owns + the components. +* A component added to a form that's already showing appears after + `revalidate()`. +* A Python function goes wherever an `ActionListener` or a `Runnable` is + expected. A default argument, as in `lambda e, n=name: ...`, captures the + loop's value for each row, as it would in any Python callback. + +==== A model, two forms and storage + +The second example keeps notes in SQLite through the standard `sqlite3` +module, models them as a dataclass and moves between a list and an editor +form. The editor saves when the user goes back, and asks before deleting. + +[source,python] +---- +include::../demos/common/src/main/snippets/developer-guide/python-interop.py[tag=python-interop-python-003,indent=0] +---- + +The structure is the one a Python developer would write for any user +interface toolkit: a storage class with no user interface code, and a class +that owns the forms and turns model objects into components. `show()` +moves forward to a form, `showBack()` returns with the reverse transition, +and the toolbar's back command is also what Android's back button runs. + === Libraries The standard library modules applications use most are included: `json`, `re`, `datetime`, `decimal`, `fractions`, `collections`, `itertools`, `functools`, `dataclasses`, `enum`, `typing`, `inspect`, `asyncio`, `logging`, `pathlib`, `os.path`, `csv`, `struct`, `hashlib`, `base64`, `uuid`, `random`, -`statistics`, `heapq`, `bisect`, `textwrap`, `string`, `tomllib` and others. +`statistics`, `heapq`, `bisect`, `textwrap`, `string`, `tomllib`, `sqlite3`, +`threading`, `queue`, `concurrent.futures`, `pickle`, `shelve`, +`urllib.request`, `http.client`, `email.message`, `zlib`, `gzip`, `zipfile`, +`zoneinfo`, `unittest` (with `unittest.mock`) and others. A small `pytest` is +included as well: `raises`, `approx`, marks, fixtures and parametrized tests. These third party packages are provided, each tested against the output of the real package: @@ -165,8 +200,88 @@ the real package: | `typing_extensions`, `six`, `tabulate`, `idna`, `packaging` | |=== -Import other pure Python packages by copying their sources into the source -directory; they translate like the application's own modules. +==== Packages from PyPI + +`cn1:import-python-project` installs the project's requirements, read from +`pyproject.toml` and `requirements.txt`, from PyPI. Each requirement and +everything it depends on is resolved to the newest release its version +specifiers accept, its download is checked against the SHA-256 PyPI +publishes, and its Python sources go into `common/src/main/python-packages`. +The build translates that directory together with `src/main/python`. +Requirements on packages the runtime already provides, such as `requests` +or `numpy`, aren't downloaded. + +The versions chosen are recorded in `python-packages/python.lock`. Importing +again installs the same versions, so a dependency changes only when you +delete its line from the lock, or the file itself, and import again. + +[cols="1,3",options="header"] +|=== +| Property | Meaning +| `cn1.python.offline` | `true` lists the requirements without downloading them. +| `cn1.python.index` | Another index that serves PyPI's JSON API, such as a company mirror. +|=== + +You can also copy a package's sources into the source directory by hand; +they translate like the application's own modules. + +[[python-c-extensions]] +==== C extensions + +A package with a C extension is supported. Its compiled modules are what +CPython loads from C; everything around them is ordinary Python. The build +translates that Python as usual, and each compiled module is replaced by a +Java class that registers the same functions. Most C extensions are small +accelerators or thin wrappers over a system facility, so the Java is usually +short, and often a call into a Codename One API that already does the job. + +The import finds the compiled modules (`.so` and `.pyd` files in a wheel, +`.pyx` sources in a source distribution) and the names the package's Python +uses from each one. For every module it generates a class in +`common/src/main/java/com/codename1/python/ext` with a stub per function, +and lists it in `python-packages/cn1-native-modules.properties`: + +[source,java] +---- +include::../demos/common/src/main/java/com/codenameone/developerguide/python/MarkupsafeSpeedups.java[tag=python-interop-java-002,indent=0] +---- + +Replace each stub's body with Java. The functions receive plain Java values +(a `Number` for an `int` or a `float`, `String`, `byte[]`, `java.util.List` +for a list or tuple, `java.util.Map` for a dict, `null` for `None`) and +return the same kinds; an `Object[]` comes back to Python as a tuple. Raise a +Python exception with `Py.error(Py.TYPE_ERROR, "...")` and its siblings, and +use `NativeModule.raw` for a function that needs the Python objects +themselves. Until a stub is filled in, calling it raises a +`NotImplementedError` that names it. The import never overwrites a class +that already exists, so importing again keeps your code. + +Many packages ship a C accelerator together with a pure Python version of +the same code, and import the compiled module inside +`try: ... except ImportError:`. When every import of a compiled module is +guarded that way, the import reports it as optional and generates nothing: +the pure Python version runs instead. + +==== Threads + +`threading`, `queue` and `concurrent.futures` run on Java threads. As in +CPython, one global interpreter lock serializes Python code: it's released +while a thread sleeps, waits on a lock, a queue or a future, or waits for a +network response, so threads help with blocking work and not with +computation. `ProcessPoolExecutor` is accepted and runs its tasks on threads. + +Codename One's user interface belongs to its event dispatch thread. Python +code started by the application runs on that thread, so update components +from a worker thread with `Display.getInstance().callSerially`, as Java code +does. + +==== `sqlite3` + +`sqlite3` runs on the platform's own SQLite through Codename One's +`Database`. On iOS, Android and the simulator a value comes back in the +storage class SQLite recorded for it. The Linux, Windows and JavaScript ports +don't report storage classes, so a value comes back as an `int` or a `float` +when its text is a number, and as a `str` otherwise. [[python-backends]] === Python backends @@ -176,21 +291,7 @@ The application starts the server the way it always does: [source,python] ---- -from flask import Flask, jsonify - -app = Flask(__name__) - - -@app.get("/fib/") -def fib(n): - a, b = 0, 1 - for _ in range(n): - a, b = b, a + b - return jsonify({"n": n, "fib": str(a)}) - - -if __name__ == "__main__": - app.run(host="0.0.0.0", port=8080) +include::../demos/common/src/main/snippets/developer-guide/python-interop.py[tag=python-interop-python-004,indent=0] ---- `app.run()` and `uvicorn.run(app)` are lowered onto the Codename One backend's @@ -198,8 +299,7 @@ HTTP server, so the same file still runs under CPython for development: [source,bash] ---- -scripts/python-backend/build.sh scripts/python-backend/samples/flask-todo ./todo-server -./todo-server +include::../demos/common/src/main/snippets/developer-guide/python-interop.sh[tag=python-interop-bash-003,indent=0] ---- Routing, converters, JSON bodies, `abort`, error handlers and the default @@ -276,9 +376,11 @@ more, using a full interpreter's memory per worker. The Codename One server's heap grows under load and stays at that size, which is the memory after load column; it doesn't grow further on longer runs. -Where an application needs CPython instead: packages with C extensions that -have no Codename One implementation (for example `pandas`, `scipy` or `torch`), -and code that builds source text and runs it with `exec`. +Where an application needs CPython instead: code that builds source text and +runs it with `exec`, and packages whose C code is the bulk of the package +rather than an accelerator under its Python, such as `scipy` or `torch`. +Replacing those means writing a numerical library in Java, not a few +functions. === Python against Java on Codename One @@ -317,7 +419,7 @@ image::img/python-perf-rss.svg["Peak memory of each CommonWorkloads benchmark on What the numbers say: * **Python reaches Java's memory.** On every workload the compiled Python uses - the memory the Java program uses, within a few percent and sometimes less: + about the memory the Java program uses, and sometimes less: ints, floats and int lists are held unboxed, a list of ints that fit 32 bits takes four bytes an item as a Java `int[]` does, and objects that don't escape a function are never allocated. @@ -345,50 +447,21 @@ The integer kernel in Python: [source,python] ---- -M32 = 0xFFFFFFFF - -def i32(v: int) -> int: - return ((v + 0x80000000) & M32) - 0x80000000 - -def int_arithmetic() -> int: - a: int = 0x12345678 - b: int = i32(0x9E3779B9) - checksum: int = 0 - for _ in range(40000000): - a = i32((a * 1103515245 + 12345) ^ ((b & M32) >> 3)) - b = i32((b + a) * 5 - (a << 7)) - checksum += (a ^ b) & 0xFFFF - return checksum + a + b +include::../demos/common/src/main/snippets/developer-guide/python-interop.py[tag=python-interop-python-005,indent=0] ---- The same function in Java, under `common/src/main/java`: [source,java] ---- -package com.bench; - -public final class CommonWorkloads { - public static long intArithmetic() { - int a = 0x12345678; - int b = 0x9E3779B9; - long checksum = 0; - for (int i = 0; i < 40000000; i++) { - a = (a * 1103515245 + 12345) ^ (b >>> 3); - b = (b + a) * 5 - (a << 7); - checksum += (a ^ b) & 0xFFFF; - } - return checksum + a + b; - } -} +include::../demos/common/src/main/java/com/codenameone/developerguide/python/CommonWorkloads.java[tag=python-interop-java-003,indent=0] ---- And the Python that calls it: [source,python] ---- -from java.com.bench import CommonWorkloads - -checksum = CommonWorkloads.intArithmetic() +include::../demos/common/src/main/snippets/developer-guide/python-interop.py[tag=python-interop-python-006,indent=0] ---- The Java `int` wraps by itself, so the masks disappear and the loop compiles From 723b12df323bd5ce1bbeae72ccc3b312409a022b Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 09:28:37 +0200 Subject: [PATCH 043/112] Python: range attributes and equality, builtins module names, __setattr__ before properties, namedtuple __module__, traceback module range has start/stop/step and compares and hashes as a sequence; builtins is in sys.modules once sys is imported; builtin functions report __module__ and __qualname__; a class's own __setattr__ runs before a property or data descriptor, and object.__setattr__ honours descriptors; namedtuple defaults __module__ to __main__ (the stdlib callers pass module=); a raised exception carries a frame-less __traceback__; new stdlib traceback module. Co-Authored-By: Claude Opus 5.5 --- .../codename1/python/runtime/Builtins.java | 2 +- .../java/com/codename1/python/runtime/Py.java | 2 + .../codename1/python/runtime/PyBuiltin.java | 7 +- .../codename1/python/runtime/PyException.java | 6 + .../codename1/python/runtime/PyInstance.java | 21 +- .../com/codename1/python/runtime/PyRange.java | 50 ++++ .../codename1/python/runtime/StdModules.java | 5 + .../python/stdlib/collections/__init__.py | 5 +- .../com/codename1/python/stdlib/statistics.py | 2 +- .../com/codename1/python/stdlib/traceback.py | 225 ++++++++++++++++++ .../com/codename1/python/stdlib/typing.py | 3 +- .../codename1/python/stdlib/urllib/parse.py | 6 +- .../conformance/builtins_module_names.out | 4 + .../conformance/builtins_module_names.py | 6 + .../conformance/namedtuple_module.out | 4 + .../conformance/namedtuple_module.py | 11 + .../conformance/range_attributes.out | 14 ++ .../resources/conformance/range_attributes.py | 15 ++ .../conformance/setattr_before_property.out | 7 + .../conformance/setattr_before_property.py | 60 +++++ .../conformance/traceback_module.out | 14 ++ .../resources/conformance/traceback_module.py | 49 ++++ 22 files changed, 502 insertions(+), 16 deletions(-) create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/traceback.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/builtins_module_names.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/builtins_module_names.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/namedtuple_module.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/namedtuple_module.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/range_attributes.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/range_attributes.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/setattr_before_property.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/setattr_before_property.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/traceback_module.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/traceback_module.py diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Builtins.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Builtins.java index 79a207a1b22..c52dc1da92d 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Builtins.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Builtins.java @@ -282,7 +282,7 @@ static void install() { o.define("__init__", OBJECT_INIT); o.define("__setattr__", PyBuiltin.positional("__setattr__", a -> { if (a[0] instanceof PyInstance) { - ((PyInstance) a[0]).setAttrDirect(Py.toStr(a[1]), a[2]); + ((PyInstance) a[0]).setAttrObject(Py.toStr(a[1]), a[2]); } else if (a[0] instanceof PyType) { // What a metaclass's super().__setattr__ reaches: store, // never back through the metaclass. diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Py.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Py.java index 584b3520543..cb834ec6ba9 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Py.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Py.java @@ -76,6 +76,8 @@ public final class Py { // ---- singletons ------------------------------------------------------ public static final Object NONE = new PyNone("None", null); + /// The type of an exception's `__traceback__`; its instances carry no frames. + public static final PyType TRACEBACK_TYPE = builtinType("traceback", null); public static final Object NOT_IMPLEMENTED = new PyNone("NotImplemented", NOT_IMPLEMENTED_TYPE); public static final Object ELLIPSIS = new PyNone("Ellipsis", ELLIPSIS_TYPE); public static final PyBool TRUE = new PyBool(true); diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyBuiltin.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyBuiltin.java index abd17fa08d2..a19388e196c 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyBuiltin.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyBuiltin.java @@ -134,9 +134,14 @@ public Object call(Object[] args, String[] kwNames) { @Override public Object getAttrOrNull(String attr) { - if ("__name__".equals(attr)) { + if ("__name__".equals(attr) || "__qualname__".equals(attr)) { return name; } + if ("__module__".equals(attr)) { + // A builtins function reports its module; any other built-in + // (a module's function, a bound helper) reports None here. + return Builtins.NAMES.get(String.valueOf(name)) == this ? Py.str("builtins") : Py.NONE; //NOPMD CompareObjectsWithEquals - identity is meant + } return super.getAttrOrNull(attr); } diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyException.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyException.java index 61d5f217ebc..2bb73a504ab 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyException.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyException.java @@ -39,6 +39,12 @@ public PyException(Object value) { if (h != Py.NONE && h != value && value instanceof PyInstance) { //NOPMD CompareObjectsWithEquals - identity is meant ((PyInstance) value).setAttrDirect("__context__", h); } + // A raised exception has a traceback. The translated program has no + // frame objects, so it is an empty `traceback` instance: what + // `traceback.format_exception` reads to print the header. + if (value instanceof PyInstance && ((PyInstance) value).getAttrOrNull("__traceback__") == Py.NONE) { //NOPMD CompareObjectsWithEquals - identity is meant + ((PyInstance) value).setAttrDirect("__traceback__", new PyInstance(Py.TRACEBACK_TYPE)); + } } /// `isinstance(value, type)`, where `type` may be a tuple of types. diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyInstance.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyInstance.java index 6eb1b5455d2..77dc23d1673 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyInstance.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyInstance.java @@ -215,6 +215,19 @@ public Object getAttrOrNull(String name) { @Override public void setAttr(String name, Object value) { + // A class's own __setattr__ runs first and decides, even for a name + // a property or another data descriptor owns, as in CPython. + Object sa = cls.lookup("__setattr__"); + if (sa != null && !(sa instanceof PyBuiltin)) { + Py.obj(sa).call(new Object[] {this, name, value}, Py.NO_KW); + return; + } + setAttrObject(name, value); + } + + /// `object.__setattr__(self, name, value)`: data descriptors, then the + /// plain store, never an overriding `__setattr__`. + public void setAttrObject(String name, Object value) { Object clsAttr = cls.lookup(name); if (clsAttr instanceof PyProperty) { ((PyProperty) clsAttr).set(this, value); @@ -227,16 +240,10 @@ public void setAttr(String name, Object value) { return; } } - Object sa = cls.lookup("__setattr__"); - if (sa != null && !(sa instanceof PyBuiltin)) { - Py.obj(sa).call(new Object[] {this, name, value}, Py.NO_KW); - return; - } store(name, value); } - /// `object.__setattr__(self, name, value)`: the plain store, bypassing an - /// overriding `__setattr__`. + /// The plain store, bypassing descriptors and an overriding `__setattr__`. public void setAttrDirect(String name, Object value) { store(name, value); } diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyRange.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyRange.java index 37c37402619..a2d9aafbe69 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyRange.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyRange.java @@ -43,6 +43,56 @@ public PyType type() { return Py.RANGE_TYPE; } + @Override + public Object getAttrOrNull(String name) { + if ("start".equals(name)) { + return PyInt.of(start); + } + if ("stop".equals(name)) { + return PyInt.of(stop); + } + if ("step".equals(name)) { + return PyInt.of(step); + } + return super.getAttrOrNull(name); + } + + /// Ranges compare as the sequences they produce: equal lengths, and when + /// not empty the same first element and (for more than one) the same step. + @Override + public Object eq(Object o) { + if (!(o instanceof PyRange)) { + return null; + } + PyRange r = (PyRange) o; + long n = length(); + if (n != r.length()) { + return Py.FALSE; + } + if (n == 0) { + return Py.TRUE; + } + if (start != r.start) { + return Py.FALSE; + } + return n == 1 || step == r.step ? Py.TRUE : Py.FALSE; + } + + /// Consistent with `eq`: hashes the length, and the first element and + /// step only where they decide equality. + @Override + public long hash() { + long n = length(); + long h = n * 1000003L; + if (n > 0) { + h = h * 31 + start; + if (n > 1) { + h = h * 31 + step; + } + } + return h == -1 ? -2 : h; + } + public long length() { if (step > 0) { return stop > start ? (stop - start + step - 1) / step : 0; diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java index e9be407ce2a..b5e0a91eeb6 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java @@ -767,6 +767,11 @@ private static PyModule sys() { m.setAttr("argv", new PyList(argv)); m.setAttr("path", new PyList()); m.setAttr("modules", PyModule.SYS_MODULES); + // CPython has builtins in sys.modules from start-up; nothing can + // look at sys.modules before importing sys, so this is the moment. + if (PyModule.find("builtins") == null) { + provide("builtins"); + } m.setAttr("float_info", PyTuple.of(Py.ofFloat(Double.MAX_VALUE), PyInt.of(1024), PyInt.of(308), Py.ofFloat(Double.MIN_NORMAL), PyInt.of(-1021), PyInt.of(-307), PyInt.of(15), PyInt.of(53), Py.ofFloat(Math.ulp(1.0)), PyInt.of(2), PyInt.of(1))); diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/collections/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/collections/__init__.py index d26033aa413..e6dc5937644 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/collections/__init__.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/collections/__init__.py @@ -524,8 +524,9 @@ def __getnewargs__(self): for index, name in enumerate(field_names): namespace[name] = property(_itemgetter(index), doc="Alias for field number " + str(index)) result = type(typename, (tuple,), namespace) - if module is not None: - result.__module__ = module + # CPython reads the caller's module from its frame; there are no frames + # here, so a caller outside __main__ passes module= (the stdlib does). + result.__module__ = "__main__" if module is None else module return result diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/statistics.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/statistics.py index 546f542ecd8..efeef56a423 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/statistics.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/statistics.py @@ -455,7 +455,7 @@ def correlation(x, y, /, *, method="linear"): raise StatisticsError("at least one of the inputs is constant") -LinearRegression = namedtuple("LinearRegression", ("slope", "intercept")) +LinearRegression = namedtuple("LinearRegression", ("slope", "intercept"), module="statistics") def linear_regression(x, y, /, *, proportional=False): diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/traceback.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/traceback.py new file mode 100644 index 00000000000..e29b895d6d1 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/traceback.py @@ -0,0 +1,225 @@ +"""Formatting exceptions the way CPython's `traceback` module does. + +The translated program has no frame objects, so the stack part of a +traceback is empty: `extract_tb` and `extract_stack` answer empty +`StackSummary` lists and a formatted traceback holds the header, the chained +exceptions and each exception's last line. Everything a program usually +reads -- `format_exc().splitlines()[-1]`, `format_exception_only` -- matches +CPython. +""" + +import sys + +__all__ = ["format_exc", "print_exc", "format_exception", "print_exception", + "format_exception_only", "extract_tb", "extract_stack", "format_tb", + "format_stack", "format_list", "print_tb", "print_stack", + "StackSummary", "FrameSummary", "TracebackException"] + +_CAUSE = "\nThe above exception was the direct cause of the following exception:\n\n" +_CONTEXT = "\nDuring handling of the above exception, another exception occurred:\n\n" + + +class FrameSummary: + def __init__(self, filename, lineno, name, *, lookup_line=True, locals=None, line=None): + self.filename = filename + self.lineno = lineno + self.name = name + self.line = line + self.locals = locals + + def __iter__(self): + return iter([self.filename, self.lineno, self.name, self.line]) + + def __repr__(self): + return "" % (self.filename, self.lineno, self.name) + + +class StackSummary(list): + @classmethod + def from_list(cls, a_list): + result = cls() + for f in a_list: + if isinstance(f, FrameSummary): + result.append(f) + else: + filename, lineno, name, line = f + result.append(FrameSummary(filename, lineno, name, line=line)) + return result + + def format(self): + out = [] + for f in self: + row = ' File "%s", line %s, in %s\n' % (f.filename, f.lineno, f.name) + if f.line: + row += " %s\n" % f.line.strip() + out.append(row) + return out + + +def extract_tb(tb, limit=None): + return StackSummary() + + +def extract_stack(f=None, limit=None): + return StackSummary() + + +def format_list(extracted_list): + return StackSummary.from_list(extracted_list).format() + + +def format_tb(tb, limit=None): + return [] + + +def format_stack(f=None, limit=None): + return [] + + +def print_tb(tb, limit=None, file=None): + pass + + +def print_stack(f=None, limit=None, file=None): + pass + + +def _type_name(etype): + name = getattr(etype, "__qualname__", None) or etype.__name__ + mod = getattr(etype, "__module__", "builtins") + if mod not in ("builtins", "__main__", None): + name = mod + "." + name + return name + + +def _only(exc): + if exc is None: + return ["NoneType: None\n"] + name = _type_name(type(exc)) + try: + msg = str(exc) + except Exception: + msg = "" + line = name + (": " + msg if msg else "") + "\n" + out = [line] + for note in getattr(exc, "__notes__", None) or (): + out.append(str(note) + "\n") + return out + + +class TracebackException: + def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None, + lookup_lines=True, capture_locals=False, compact=False, + max_group_width=15, max_group_depth=10, _seen=None): + if _seen is None: + _seen = set() + _seen.add(id(exc_value)) + self.exc_type = exc_type + self._value = exc_value + self._tb = exc_traceback + self.stack = StackSummary() + self.__cause__ = None + self.__context__ = None + self.__suppress_context__ = False + if exc_value is not None: + cause = getattr(exc_value, "__cause__", None) + context = getattr(exc_value, "__context__", None) + self.__suppress_context__ = bool(getattr(exc_value, "__suppress_context__", False)) + if cause is not None and id(cause) not in _seen: + self.__cause__ = TracebackException(type(cause), cause, getattr(cause, "__traceback__", None), _seen=_seen) + if context is not None and id(context) not in _seen: + self.__context__ = TracebackException(type(context), context, getattr(context, "__traceback__", None), _seen=_seen) + self.exc_type_str = _type_name(exc_type) if exc_type is not None else "NoneType" + + @classmethod + def from_exception(cls, exc, *args, **kwargs): + return cls(type(exc), exc, getattr(exc, "__traceback__", None), *args, **kwargs) + + def format_exception_only(self, *, show_group=False): + return iter(_only(self._value)) + + def format(self, *, chain=True): + out = [] + if chain: + if self.__cause__ is not None: + out.extend(self.__cause__.format(chain=chain)) + out.append(_CAUSE) + elif self.__context__ is not None and not self.__suppress_context__: + out.extend(self.__context__.format(chain=chain)) + out.append(_CONTEXT) + if self._tb is not None: + out.append("Traceback (most recent call last):\n") + out.extend(self.stack.format()) + out.extend(_only(self._value)) + return iter(out) + + def print(self, *, file=None, chain=True): + if file is None: + file = sys.stderr + for line in self.format(chain=chain): + print(line, file=file, end="") + + def __eq__(self, other): + if isinstance(other, TracebackException): + return self.exc_type is other.exc_type and self._value is other._value + return NotImplemented + + def __str__(self): + return str(self._value) + + +def _value_of(exc, value): + # format_exception(exc) and format_exception(type, value, tb) both work. + if value is _SENTINEL: + return exc + return value + + +_SENTINEL = object() + + +def format_exception(exc, value=_SENTINEL, tb=_SENTINEL, limit=None, chain=True): + v = _value_of(exc, value) + return list(TracebackException(type(v), v, getattr(v, "__traceback__", None)).format(chain=chain)) + + +def print_exception(exc, value=_SENTINEL, tb=_SENTINEL, limit=None, file=None, chain=True): + if file is None: + file = sys.stderr + for line in format_exception(exc, value, tb, limit, chain): + print(line, file=file, end="") + + +def format_exception_only(exc, value=_SENTINEL, *, show_group=False): + return _only(_value_of(exc, value)) + + +def format_exc(limit=None, chain=True): + e = sys.exception() + if e is None: + return "NoneType: None\n" + return "".join(format_exception(e, chain=chain)) + + +def print_exc(limit=None, file=None, chain=True): + if file is None: + file = sys.stderr + print(format_exc(limit, chain), file=file, end="") + + +def print_last(limit=None, file=None, chain=True): + exc = getattr(sys, "last_exc", None) + if exc is not None: + print_exception(exc, limit=limit, file=file, chain=chain) + + +def clear_frames(tb): + pass + + +def walk_tb(tb): + return iter(()) + + +def walk_stack(f): + return iter(()) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/typing.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/typing.py index a4097b279d2..b9f8f89021c 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/typing.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/typing.py @@ -680,7 +680,8 @@ def __new__(cls, typename, bases, ns): types = ns.get("__annotations__", {}) default_names = [n for n in types if n in ns] nm_tpl = collections.namedtuple(typename, list(types.keys()), - defaults=[ns[n] for n in default_names]) + defaults=[ns[n] for n in default_names], + module=ns.get("__module__", "__main__")) nm_tpl.__annotations__ = types for key, val in ns.items(): if key not in ("__annotations__", "__module__", "__qualname__") and key not in types and \ diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/urllib/parse.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/urllib/parse.py index 06bb94e6263..2f7a44baddd 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/urllib/parse.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/urllib/parse.py @@ -111,9 +111,9 @@ def _hostinfo(self): return hostname, port -_DefragResultBase = namedtuple("DefragResult", "url fragment") -_SplitResultBase = namedtuple("SplitResult", "scheme netloc path query fragment") -_ParseResultBase = namedtuple("ParseResult", "scheme netloc path params query fragment") +_DefragResultBase = namedtuple("DefragResult", "url fragment", module="urllib.parse") +_SplitResultBase = namedtuple("SplitResult", "scheme netloc path query fragment", module="urllib.parse") +_ParseResultBase = namedtuple("ParseResult", "scheme netloc path params query fragment", module="urllib.parse") class DefragResult(_DefragResultBase): diff --git a/maven/python-transpiler/src/test/resources/conformance/builtins_module_names.out b/maven/python-transpiler/src/test/resources/conformance/builtins_module_names.out new file mode 100644 index 00000000000..78cefa306f5 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/builtins_module_names.out @@ -0,0 +1,4 @@ +True +builtins len len +builtins isinstance +True diff --git a/maven/python-transpiler/src/test/resources/conformance/builtins_module_names.py b/maven/python-transpiler/src/test/resources/conformance/builtins_module_names.py new file mode 100644 index 00000000000..50e62bfb3cc --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/builtins_module_names.py @@ -0,0 +1,6 @@ +import sys +print("builtins" in sys.modules) +print(len.__module__, len.__qualname__, len.__name__) +print(print.__module__, isinstance.__qualname__) +import builtins +print(builtins.len is len) diff --git a/maven/python-transpiler/src/test/resources/conformance/namedtuple_module.out b/maven/python-transpiler/src/test/resources/conformance/namedtuple_module.out new file mode 100644 index 00000000000..3046ac9f848 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/namedtuple_module.out @@ -0,0 +1,4 @@ +__main__ P(x=1, y=2) +mine +__main__ N(a=3) +urllib.parse diff --git a/maven/python-transpiler/src/test/resources/conformance/namedtuple_module.py b/maven/python-transpiler/src/test/resources/conformance/namedtuple_module.py new file mode 100644 index 00000000000..b67a3795349 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/namedtuple_module.py @@ -0,0 +1,11 @@ +from collections import namedtuple +import typing +P = namedtuple("P", "x y") +print(P.__module__, P(1, 2)) +Q = namedtuple("Q", "a", module="mine") +print(Q.__module__) +class N(typing.NamedTuple): + a: int +print(N.__module__, N(3)) +from urllib.parse import urlsplit +print(type(urlsplit("http://a/b")).__module__) diff --git a/maven/python-transpiler/src/test/resources/conformance/range_attributes.out b/maven/python-transpiler/src/test/resources/conformance/range_attributes.out new file mode 100644 index 00000000000..d08e0a2f18d --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/range_attributes.out @@ -0,0 +1,14 @@ +2 11 3 +0 5 1 +True +True +True +True +False +False +False +True +True +True +2 +True diff --git a/maven/python-transpiler/src/test/resources/conformance/range_attributes.py b/maven/python-transpiler/src/test/resources/conformance/range_attributes.py new file mode 100644 index 00000000000..66ee0ad4b6f --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/range_attributes.py @@ -0,0 +1,15 @@ +r = range(2, 11, 3) +print(r.start, r.stop, r.step) +print(range(5).start, range(5).stop, range(5).step) +print(range(0, 3) == range(0, 3, 1)) +print(range(0) == range(5, 5)) +print(range(0, 3, 2) == range(0, 4, 2)) +print(range(1, 2, 5) == range(1, 3, 7)) +print(range(0, 3) == range(1, 4)) +print(range(0, 3) != range(0, 3)) +print(range(3) == [0, 1, 2]) +print(hash(range(0)) == hash(range(5, 5))) +print(hash(range(0, 3)) == hash(range(0, 3, 1))) +print(hash(range(1, 2, 5)) == hash(range(1, 3, 7))) +print(len({range(0), range(4, 4), range(3), range(0, 3, 1)})) +print(range(-5, 5, -1) == range(0)) diff --git a/maven/python-transpiler/src/test/resources/conformance/setattr_before_property.out b/maven/python-transpiler/src/test/resources/conformance/setattr_before_property.out new file mode 100644 index 00000000000..bd44370751c --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/setattr_before_property.out @@ -0,0 +1,7 @@ +setattr _v 1 +setattr value 4 +40 40 +read-only +still read-only +7 +[101] diff --git a/maven/python-transpiler/src/test/resources/conformance/setattr_before_property.py b/maven/python-transpiler/src/test/resources/conformance/setattr_before_property.py new file mode 100644 index 00000000000..b4e12d81bee --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/setattr_before_property.py @@ -0,0 +1,60 @@ +class Guarded: + def __init__(self): + self._v = 1 + + @property + def value(self): + return self._v + + def __setattr__(self, name, val): + print("setattr", name, val) + if name == "value": + object.__setattr__(self, "_v", val * 10) + return + object.__setattr__(self, name, val) + + +g = Guarded() +g.value = 4 +print(g.value, g._v) + + +class Plain: + @property + def ro(self): + return 7 + + +p = Plain() +try: + p.ro = 3 +except AttributeError: + print("read-only") +try: + object.__setattr__(p, "ro", 3) +except AttributeError: + print("still read-only") +print(p.ro) + + +class WithSetter: + def __init__(self): + self.log = [] + + @property + def x(self): + return 0 + + @x.setter + def x(self, v): + self.log.append(v) + + def __setattr__(self, name, val): + if name == "x": + val = val + 100 + object.__setattr__(self, name, val) + + +w = WithSetter() +w.x = 1 +print(w.log) diff --git a/maven/python-transpiler/src/test/resources/conformance/traceback_module.out b/maven/python-transpiler/src/test/resources/conformance/traceback_module.out new file mode 100644 index 00000000000..38d67b1fd63 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/traceback_module.out @@ -0,0 +1,14 @@ +Traceback (most recent call last): +ValueError: bad value +NoneType: None +["KeyError: 'k'\n"] +['ValueError: v\n'] +['RuntimeError\n'] +MyErr: custom +MyErr: custom +['Traceback (most recent call last):\n', 'ZeroDivisionError: division by zero\n', '\nThe above exception was the direct cause of the following exception:\n\n', 'Traceback (most recent call last):\n', "KeyError: 'x'\n"] +['Traceback (most recent call last):\n', "ValueError: invalid literal for int() with base 10: 'q'\n", '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', 'TypeError: t\n'] +z ['ValueError: z\n'] +ValueError +True +ValueError: to stdout diff --git a/maven/python-transpiler/src/test/resources/conformance/traceback_module.py b/maven/python-transpiler/src/test/resources/conformance/traceback_module.py new file mode 100644 index 00000000000..ca7e6518c3c --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/traceback_module.py @@ -0,0 +1,49 @@ +import traceback +import sys + +def fail(): + raise ValueError("bad value") + +try: + fail() +except ValueError: + text = traceback.format_exc() + lines = text.splitlines() + print(lines[0]) + print(lines[-1]) + +print(traceback.format_exc(), end="") +print(traceback.format_exception_only(KeyError("k"))) +print(traceback.format_exception_only(ValueError, ValueError("v"))) +print(traceback.format_exception_only(RuntimeError())) + +class MyErr(Exception): + pass + +e = MyErr("custom") +print(traceback.format_exception(e)[-1], end="") +print(traceback.format_exception(e)[0], end="") + +try: + try: + 1 / 0 + except ZeroDivisionError as z: + raise KeyError("x") from z +except KeyError as k: + out = traceback.format_exception(k) + print([l for l in out if not l.startswith(" ")]) + +try: + try: + int("q") + except ValueError: + raise TypeError("t") +except TypeError as t: + out = traceback.format_exception(t) + print([l for l in out if not l.startswith(" ")]) + +te = traceback.TracebackException.from_exception(ValueError("z")) +print(str(te), list(te.format_exception_only())) +print(te.exc_type_str) +print(isinstance(traceback.extract_stack(), list)) +traceback.print_exception(ValueError("to stdout"), file=sys.stdout) From c868ce51a9ab8f7f2f9feb38ad76017cb172d4ab Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 09:31:42 +0200 Subject: [PATCH 044/112] Python: Flask sessions, Jinja templates, multipart uploads and streamed responses Adds a clean-room jinja2 subset (interpreted, no eval), markupsafe and hmac, and wires flask.session (signed cookie), render_template(_string), flash, request.files/FileStorage and stream_with_context into the Flask mapping. Conformance: jinja_templates, flask_web_extras (outputs from CPython + Flask 3). Co-Authored-By: Claude Opus 5.5 --- .../codename1/python/site/flask/__init__.py | 492 ++++++- .../codename1/python/site/jinja2/__init__.py | 1139 +++++++++++++++++ .../python/site/markupsafe/__init__.py | 102 ++ .../com/codename1/python/stdlib/hmac.py | 87 ++ .../conformance/flask_web_extras.out | 15 + .../resources/conformance/flask_web_extras.py | 128 ++ .../resources/conformance/jinja_templates.out | 27 + .../resources/conformance/jinja_templates.py | 46 + 8 files changed, 2029 insertions(+), 7 deletions(-) create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/jinja2/__init__.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/markupsafe/__init__.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/hmac.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/flask_web_extras.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/flask_web_extras.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/jinja_templates.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/jinja_templates.py diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/flask/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/flask/__init__.py index 35763c8d869..9271afc9fc0 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/site/flask/__init__.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/flask/__init__.py @@ -7,7 +7,13 @@ ``(status, headers, body)``; the test client and the native server both go through it. Written from Flask's public documentation and behaviour. """ +import base64 as _base64 +import hashlib as _hashlib +import hmac as _hmac +import io as _io import json as _json +import os as _os +import sys as _sys import re as _re import uuid as _uuid from urllib.parse import parse_qsl, quote, quote_plus, unquote, urlencode @@ -17,7 +23,9 @@ default_exceptions) __all__ = ["Flask", "Blueprint", "Response", "request", "g", "jsonify", "make_response", - "abort", "redirect", "url_for", "current_app"] + "abort", "redirect", "url_for", "current_app", "session", "render_template", + "render_template_string", "stream_with_context", "flash", "get_flashed_messages", + "Markup", "escape"] _STATUS = { 100: "CONTINUE", 101: "SWITCHING PROTOCOLS", 200: "OK", 201: "CREATED", 202: "ACCEPTED", @@ -242,11 +250,16 @@ def __init__(self, response=None, status=None, headers=None, mimetype=None, cont else: self.status = status self._data = b"" + self.is_streamed = False if response is not None: if isinstance(response, (str, bytes, bytearray)): self.set_data(response) else: + # A generator body is drained here, while the request context is + # still pushed: the native server has no chunked-transfer API, so a + # streamed response is buffered and sent whole. self.set_data(b"".join([_b(c) for c in response])) + self.is_streamed = True self._cookies = [] @property @@ -356,6 +369,13 @@ def __init__(self, app, req): self.app = app self.request = req self.g = _AppGlobals() + self._session = None + + @property + def session(self): + if self._session is None: + self._session = self.app._open_session(self.request) + return self._session class _AppGlobals: @@ -391,6 +411,8 @@ def _get(self): return c.request if w == "g": return c.g + if w == "session": + return c.session return c.app def __getattr__(self, name): @@ -408,7 +430,24 @@ def __contains__(self, name): def __iter__(self): return iter(self._get()) + def __getitem__(self, key): + return self._get()[key] + + def __setitem__(self, key, value): + self._get()[key] = value + + def __delitem__(self, key): + del self._get()[key] + + def __len__(self): + return len(self._get()) + + def __eq__(self, other): + return self._get() == other + def __bool__(self): + if object.__getattribute__(self, "_what") == "session": + return bool(_stack) and bool(self._get()) return bool(_stack) def __repr__(self): @@ -448,11 +487,7 @@ def args(self): @property def form(self): - if self._form is None: - if self.mimetype == "application/x-www-form-urlencoded": - self._form = _parse_qs(self._body) - else: - self._form = MultiDict() + self._load_form() return self._form @property @@ -468,7 +503,19 @@ def values(self): @property def files(self): - return MultiDict() + self._load_form() + return self._files + + def _load_form(self): + if self._form is not None: + return + self._files = MultiDict() + if self.mimetype == "application/x-www-form-urlencoded": + self._form = _parse_qs(self._body) + elif self.mimetype == "multipart/form-data": + self._form, self._files = _parse_multipart(self.headers.get("Content-Type", ""), self._body) + else: + self._form = MultiDict() @property def cookies(self): @@ -555,6 +602,7 @@ def __repr__(self): request = _Proxy("request") g = _Proxy("g") current_app = _Proxy("app") +session = _Proxy("session") def _to_json(obj, compact=True, sort=True): @@ -817,6 +865,52 @@ def __init__(self, import_name=None, static_folder=None, template_folder=None, * self._bp_errors = {} self._sorted = None self.json = None + self.template_folder = "templates" if template_folder is None else template_folder + self._jinja_env = None + self._context_processors = [] + self.root_path = _root_path(import_name) + self.session_interface = SecureCookieSessionInterface() + + @property + def secret_key(self): + return self.config.get("SECRET_KEY") + + @secret_key.setter + def secret_key(self, value): + self.config["SECRET_KEY"] = value + + @property + def jinja_env(self): + if self._jinja_env is None: + from jinja2 import Environment, FileSystemLoader + folder = self.template_folder + if not _os.path.isabs(folder): + folder = _os.path.join(self.root_path, folder) + env = Environment(loader=FileSystemLoader(folder), autoescape=_select_autoescape) + env.globals["url_for"] = url_for + env.globals["get_flashed_messages"] = get_flashed_messages + env.globals["config"] = self.config + self._jinja_env = env + return self._jinja_env + + def context_processor(self, f): + self._context_processors.append(f) + return f + + def _template_context(self, ctx): + out = {} + if _stack: + c = _stack[-1] + out["request"] = c.request + out["session"] = c.session + out["g"] = c.g + for f in self._context_processors: + out.update(f()) + out.update(ctx) + return out + + def _open_session(self, req): + return self.session_interface.open_session(self, req) # -- registration def add_url_rule(self, rule, endpoint=None, view_func=None, **options): @@ -1052,6 +1146,8 @@ def dispatch(self, method, path, query, headers, body): if method == "HEAD": req.method = "HEAD" resp = self.full_dispatch(req) + if ctx._session is not None: + self.session_interface.save_session(self, ctx._session, resp) for f in self._teardown: try: f(None) @@ -1169,6 +1265,9 @@ def open(self, path="/", method="GET", json=None, data=None, query_string=None, for k, v in (headers.items() if hasattr(headers, "items") else headers): hdrs[k] = v body = b"" + if isinstance(data, dict) and any(_is_upload(v) for v in data.values()): + body, content_type = _encode_multipart(data) + data = None if json is not None: body = _to_json(json, True).encode("utf-8") content_type = content_type or "application/json" @@ -1218,3 +1317,382 @@ def __init__(self, body, status, headers): self.headers = Headers(headers) self._data = bytes(body) self.status_code = status + + +# ---- sessions -------------------------------------------------------------- + +class SecureCookieSession(dict): + """The session dict: tracks whether it was read and whether it changed.""" + + def __init__(self, initial=None): + dict.__init__(self, initial or {}) + self.modified = False + self.accessed = False + self.new = False + self.permanent = False + + def __getitem__(self, key): + self.accessed = True + return dict.__getitem__(self, key) + + def get(self, key, default=None): + self.accessed = True + return dict.get(self, key, default) + + def __contains__(self, key): + self.accessed = True + return dict.__contains__(self, key) + + def __setitem__(self, key, value): + self.accessed = True + self.modified = True + dict.__setitem__(self, key, value) + + def __delitem__(self, key): + self.accessed = True + self.modified = True + dict.__delitem__(self, key) + + def setdefault(self, key, default=None): + self.accessed = True + if not dict.__contains__(self, key): + self.modified = True + return dict.setdefault(self, key, default) + + def pop(self, key, *default): + self.accessed = True + self.modified = True + return dict.pop(self, key, *default) + + def popitem(self): + self.accessed = True + self.modified = True + return dict.popitem(self) + + def update(self, *a, **k): + self.accessed = True + self.modified = True + dict.update(self, *a, **k) + + def clear(self): + self.accessed = True + self.modified = True + dict.clear(self) + + +class NullSession(SecureCookieSession): + def _fail(self, *a, **k): + raise RuntimeError("The session is unavailable because no secret key was set. Set the" + " secret_key on the application to something unique and secret.") + + __setitem__ = _fail + __delitem__ = _fail + clear = _fail + pop = _fail + popitem = _fail + update = _fail + setdefault = _fail + + +def _b64e(b): + return _base64.urlsafe_b64encode(b).rstrip(b"=").decode("ascii") + + +def _b64d(s): + s = s.encode("ascii") + return _base64.urlsafe_b64decode(s + b"=" * (-len(s) % 4)) + + +class SecureCookieSessionInterface: + """Signed (not encrypted) cookie: ``base64url(json) "." base64url(HMAC-SHA1)``. + + The key is derived from the app's secret key and the salt + ``cookie-session``, but the cookie carries no timestamp, so it is not + byte-compatible with the cookie a CPython Flask server writes. + """ + salt = "cookie-session" + + def _key(self, app): + sk = app.secret_key + if isinstance(sk, str): + sk = sk.encode("utf-8") + return _hashlib.sha1(self.salt.encode("utf-8") + b"signer" + sk).digest() + + def _sign(self, app, payload): + return _b64e(_hmac.new(self._key(app), payload.encode("ascii"), _hashlib.sha1).digest()) + + def open_session(self, app, req): + if not app.secret_key: + return NullSession() + name = app.config.get("SESSION_COOKIE_NAME", "session") + raw = req.cookies.get(name) + if raw and "." in raw: + payload, sig = raw.rsplit(".", 1) + if _hmac.compare_digest(self._sign(app, payload), sig): + try: + data = _json.loads(_b64d(payload).decode("utf-8")) + if isinstance(data, dict): + return SecureCookieSession(data) + except ValueError: + pass + s = SecureCookieSession() + s.new = True + return s + + def save_session(self, app, sess, resp): + if isinstance(sess, NullSession): + return + name = app.config.get("SESSION_COOKIE_NAME", "session") + if sess.accessed: + vary = resp.headers.get("Vary") + if not vary: + resp.headers["Vary"] = "Cookie" + elif "cookie" not in vary.lower(): + resp.headers["Vary"] = vary + ", Cookie" + if not len(sess): + if sess.modified and not sess.new: + resp.delete_cookie(name) + return + if not sess.modified: + return + payload = _b64e(_json.dumps(dict(sess), separators=(",", ":"), sort_keys=True).encode("utf-8")) + resp.set_cookie(name, payload + "." + self._sign(app, payload), httponly=True) + + +def flash(message, category="message"): + flashes = session.get("_flashes", []) + flashes.append([category, message]) + session["_flashes"] = flashes + + +def get_flashed_messages(with_categories=False, category_filter=()): + c = _stack[-1] + cached = getattr(c, "_flashes", None) + if cached is None: + cached = c.session.pop("_flashes") if "_flashes" in c.session else [] + c._flashes = cached + out = [tuple(f) for f in cached] + if category_filter: + out = [f for f in out if f[0] in category_filter] + if not with_categories: + return [f[1] for f in out] + return out + + +# ---- templates ------------------------------------------------------------- + +def _root_path(import_name): + try: + mod = _sys.modules.get(import_name) + fn = getattr(mod, "__file__", None) + if fn: + return _os.path.dirname(_os.path.abspath(fn)) + except Exception: + pass + try: + return _os.getcwd() + except Exception: + return "." + + +def _select_autoescape(name): + if name is None: + return True + return name.lower().endswith((".html", ".htm", ".xml", ".xhtml", ".svg")) + + +def _current_app(): + if not _stack: + raise RuntimeError("Working outside of application context.") + return _stack[-1].app + + +def render_template(template_name_or_list, **context): + app = _current_app() + env = app.jinja_env + if isinstance(template_name_or_list, (list, tuple)): + t = env.select_template(template_name_or_list) + else: + t = env.get_template(template_name_or_list) + return t.render(app._template_context(context)) + + +def render_template_string(source, **context): + app = _current_app() + return app.jinja_env.from_string(source).render(app._template_context(context)) + + +def stream_with_context(generator_or_function): + """Keeps the request context around a generator body. + + Response drains a generator while the request context is still pushed + (the native server has no chunked API), so this only wraps a function. + """ + if hasattr(generator_or_function, "__next__") or not callable(generator_or_function): + return generator_or_function + + def wrapper(*a, **k): + return generator_or_function(*a, **k) + return wrapper + + +from markupsafe import Markup, escape # noqa: E402 + + +# ---- multipart ------------------------------------------------------------- + +class FileStorage: + def __init__(self, stream=None, filename=None, name=None, content_type=None, headers=None): + self.stream = stream if stream is not None else _io.BytesIO() + self.filename = filename + self.name = name + self.headers = Headers(headers or []) + if content_type is None: + content_type = self.headers.get("Content-Type") + self.content_type = content_type + + @property + def mimetype(self): + return (self.content_type or "").split(";")[0].strip().lower() + + def read(self, *a): + return self.stream.read(*a) + + def seek(self, *a): + return self.stream.seek(*a) + + def tell(self): + return self.stream.tell() + + def save(self, dst, buffer_size=16384): + if isinstance(dst, str) or hasattr(dst, "__fspath__"): + with open(str(dst), "wb") as f: + f.write(self.stream.read()) + else: + dst.write(self.stream.read()) + + def close(self): + self.stream.close() + + def __bool__(self): + return bool(self.filename) + + def __repr__(self): + return "" % (self.filename, self.content_type) + + +def _header_params(value): + """Splits ``form-data; name="a"; filename="b.txt"`` into its parameters.""" + out = {} + i = value.find(";") + if i < 0: + return out + rest = value[i + 1:] + n = len(rest) + j = 0 + while j < n: + while j < n and rest[j] in " ;\t": + j += 1 + k = rest.find("=", j) + if k < 0: + break + key = rest[j:k].strip().lower() + j = k + 1 + if j < n and rest[j] == '"': + j += 1 + buf = [] + while j < n and rest[j] != '"': + if rest[j] == "\\" and j + 1 < n: + j += 1 + buf.append(rest[j]) + j += 1 + j += 1 + out[key] = "".join(buf) + else: + e = rest.find(";", j) + if e < 0: + e = n + out[key] = rest[j:e].strip() + j = e + return out + + +def _parse_multipart(content_type, body): + form = MultiDict() + files = MultiDict() + boundary = _header_params(content_type).get("boundary") + if not boundary: + return form, files + delim = b"--" + boundary.encode("latin-1") + parts = body.split(delim) + for part in parts[1:]: + if part.startswith(b"--"): + break + if part.startswith(b"\r\n"): + part = part[2:] + if part.endswith(b"\r\n"): + part = part[:-2] + sep = part.find(b"\r\n\r\n") + if sep < 0: + continue + hdrs = Headers() + for line in part[:sep].decode("utf-8", "replace").split("\r\n"): + if ":" in line: + k, v = line.split(":", 1) + hdrs.add(k.strip(), v.strip()) + data = part[sep + 4:] + params = _header_params(hdrs.get("Content-Disposition", "")) + name = params.get("name") + if name is None: + continue + if "filename" in params: + files.add(name, FileStorage(_io.BytesIO(data), params["filename"], name, + hdrs.get("Content-Type", "application/octet-stream"), + hdrs.items())) + else: + form.add(name, data.decode("utf-8", "replace")) + return form, files + + +_GUESS = {"txt": "text/plain", "html": "text/html", "htm": "text/html", "css": "text/css", + "csv": "text/csv", "json": "application/json", "js": "text/javascript", + "png": "image/png", "jpg": "image/jpeg", "jpeg": "image/jpeg", "gif": "image/gif", + "pdf": "application/pdf", "xml": "application/xml", "zip": "application/zip"} + + +def _is_upload(v): + if isinstance(v, (list, tuple)) and v and hasattr(v[0], "read"): + return True + return hasattr(v, "read") + + +def _encode_multipart(data): + boundary = "----cn1FormBoundary7MA4YWxkTrZu0gW" + out = [] + for key, value in data.items(): + values = value if isinstance(value, list) else [value] + for v in values: + out.append(b"--" + boundary.encode("ascii") + b"\r\n") + if _is_upload(v): + if isinstance(v, tuple): + stream = v[0] + filename = v[1] if len(v) > 1 else None + ctype = v[2] if len(v) > 2 else None + else: + stream = v + filename = getattr(v, "name", None) + ctype = None + if filename is None: + filename = "" + if ctype is None: + ext = filename.rsplit(".", 1)[-1].lower() if "." in filename else "" + ctype = _GUESS.get(ext, "application/octet-stream") + out.append(('Content-Disposition: form-data; name="%s"; filename="%s"\r\n' + "Content-Type: %s\r\n\r\n" % (key, filename, ctype)).encode("utf-8")) + out.append(_b(stream.read())) + else: + out.append(('Content-Disposition: form-data; name="%s"\r\n\r\n' % key).encode("utf-8")) + out.append(_b(v)) + out.append(b"\r\n") + out.append(b"--" + boundary.encode("ascii") + b"--\r\n") + return b"".join(out), "multipart/form-data; boundary=" + boundary diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/jinja2/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/jinja2/__init__.py new file mode 100644 index 00000000000..6065bd242cb --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/jinja2/__init__.py @@ -0,0 +1,1139 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Clean-room subset of Jinja2, interpreted (no code generation, no eval). + +Supported: ``{{ expr }}`` with filters and tests, ``{% if/elif/else %}``, +``{% for %}`` (tuple targets, ``loop.*``, ``else``, inline ``if``), +``{% set %}``, ``{% extends %}``/``{% block %}`` with ``super()``, +``{% include %}``, ``{% raw %}``, ``{# comments #}`` and ``-`` whitespace +control. Written from Jinja's public template documentation. +""" +import os as _os + +from markupsafe import Markup, escape + +__all__ = ["Environment", "Template", "DictLoader", "FileSystemLoader", "BaseLoader", + "select_autoescape", "Markup", "escape", "Undefined", "TemplateNotFound", + "TemplateSyntaxError", "UndefinedError", "TemplateError"] + + +class TemplateError(Exception): + pass + + +class TemplateNotFound(TemplateError): + def __init__(self, name): + TemplateError.__init__(self, name) + self.name = name + + +class TemplateSyntaxError(TemplateError): + pass + + +class UndefinedError(TemplateError): + pass + + +class Undefined: + def __init__(self, hint=None, obj=None, name=None): + self._name = name + + def _fail(self): + raise UndefinedError("%r is undefined" % (self._name,)) + + def __getattr__(self, name): + if name.startswith("__"): + raise AttributeError(name) + self._fail() + + def __getitem__(self, key): + self._fail() + + def __call__(self, *a, **k): + self._fail() + + def __str__(self): + return "" + + def __html__(self): + return "" + + def __bool__(self): + return False + + def __iter__(self): + return iter(()) + + def __len__(self): + return 0 + + def __eq__(self, other): + return type(other) is type(self) + + def __ne__(self, other): + return not self.__eq__(other) + + def __hash__(self): + return id(type(self)) + + def __repr__(self): + return "Undefined" + + +def select_autoescape(enabled_extensions=("html", "htm", "xml"), disabled_extensions=(), + default_for_string=True, default=False): + en = tuple("." + e.lstrip(".") for e in enabled_extensions) + dis = tuple("." + e.lstrip(".") for e in disabled_extensions) + + def autoescape(name): + if name is None: + return default_for_string + low = name.lower() + if low.endswith(en): + return True + if low.endswith(dis): + return False + return default + return autoescape + + +class BaseLoader: + def get_source(self, environment, template): + raise TemplateNotFound(template) + + def load(self, environment, name, globals=None): + src, filename, _ = self.get_source(environment, name) + return environment._from_source(src, name, filename) + + +class DictLoader(BaseLoader): + def __init__(self, mapping): + self.mapping = mapping + + def get_source(self, environment, template): + if template in self.mapping: + return self.mapping[template], None, lambda: True + raise TemplateNotFound(template) + + +class FileSystemLoader(BaseLoader): + def __init__(self, searchpath, encoding="utf-8", followlinks=False): + if isinstance(searchpath, str): + searchpath = [searchpath] + self.searchpath = [str(p) for p in searchpath] + self.encoding = encoding + + def get_source(self, environment, template): + parts = [p for p in template.split("/") if p and p != "."] + if ".." in parts: + raise TemplateNotFound(template) + for base in self.searchpath: + fn = _os.path.join(base, *parts) + if _os.path.isfile(fn): + with open(fn, "r", encoding=self.encoding) as f: + return f.read(), fn, lambda: True + raise TemplateNotFound(template) + + +class ChoiceLoader(BaseLoader): + def __init__(self, loaders): + self.loaders = loaders + + def get_source(self, environment, template): + for l in self.loaders: + try: + return l.get_source(environment, template) + except TemplateNotFound: + pass + raise TemplateNotFound(template) + + +# ---- lexer --------------------------------------------------------------- + +def _lex(src): + """Splits a template into ("text", s), ("var", expr), ("block", stmt).""" + out = [] + i = 0 + n = len(src) + strip_next = False + while i < n: + j = -1 + k = i + while True: + k = src.find("{", k) + if k < 0 or k + 1 >= n: + k = -1 + break + if src[k + 1] in "{%#": + j = k + break + k += 1 + text = src[i:] if j < 0 else src[i:j] + if strip_next: + text = text.lstrip() + strip_next = False + if j < 0: + if text: + out.append(["text", text]) + break + kind = src[j + 1] + start = j + 2 + if start < n and src[start] == "-": + text = text.rstrip() + start += 1 + elif start < n and src[start] == "+": + start += 1 + if text: + out.append(["text", text]) + close = {"{": "}}", "%": "%}", "#": "#}"}[kind] + e = _find_close(src, start, close, kind == "#") + if e < 0: + raise TemplateSyntaxError("unexpected end of template") + inner = src[start:e] + if inner.endswith("-"): + inner = inner[:-1] + strip_next = True + elif inner.endswith("+"): + inner = inner[:-1] + i = e + 2 + if kind == "#": + continue + inner = inner.strip() + if kind == "%" and inner == "raw": + m = src.find("{%", i) + while m >= 0: + q = src.find("%}", m) + body = src[m + 2:q].strip(" -+\t\r\n") + if body == "endraw": + out.append(["text", src[i:m]]) + i = q + 2 + break + m = src.find("{%", m + 2) + continue + out.append(["var" if kind == "{" else "block", inner]) + return out + + +def _find_close(src, start, close, comment): + i = start + q = None + n = len(src) + while i < n: + c = src[i] + if not comment: + if q is not None: + if c == "\\": + i += 2 + continue + if c == q: + q = None + i += 1 + continue + if c in "'\"": + q = c + i += 1 + continue + if src.startswith(close, i): + return i + i += 1 + return -1 + + +# ---- expressions ----------------------------------------------------------- + +_OPS2 = ("==", "!=", "<=", ">=", "//", "**") + + +def _tokenize(s): + toks = [] + i = 0 + n = len(s) + while i < n: + c = s[i] + if c.isspace(): + i += 1 + elif c.isdigit(): + j = i + while j < n and (s[j].isdigit() or s[j] == "_"): + j += 1 + if j + 1 < n and s[j] == "." and s[j + 1].isdigit(): + j += 1 + while j < n and s[j].isdigit(): + j += 1 + toks.append(("num", float(s[i:j].replace("_", "")))) + else: + toks.append(("num", int(s[i:j].replace("_", "")))) + i = j + elif c.isalpha() or c == "_": + j = i + while j < n and (s[j].isalnum() or s[j] == "_"): + j += 1 + toks.append(("name", s[i:j])) + i = j + elif c in "'\"": + j = i + 1 + buf = [] + while j < n and s[j] != c: + if s[j] == "\\" and j + 1 < n: + j += 1 + buf.append({"n": "\n", "t": "\t", "r": "\r"}.get(s[j], s[j])) + else: + buf.append(s[j]) + j += 1 + toks.append(("str", "".join(buf))) + i = j + 1 + else: + if s[i:i + 2] in _OPS2: + toks.append(("op", s[i:i + 2])) + i += 2 + else: + toks.append(("op", c)) + i += 1 + toks.append(("end", None)) + return toks + + +class _P: + def __init__(self, s): + self.t = _tokenize(s) + self.i = 0 + + def peek(self, k=0): + return self.t[self.i + k] + + def next(self): + tok = self.t[self.i] + self.i += 1 + return tok + + def at(self, kind, val=None): + tok = self.t[self.i] + return tok[0] == kind and (val is None or tok[1] == val) + + def accept(self, kind, val=None): + if self.at(kind, val): + return self.next() + return None + + def expect(self, kind, val=None): + tok = self.accept(kind, val) + if tok is None: + raise TemplateSyntaxError("expected %s %s, got %r" % (kind, val or "", self.peek()[1])) + return tok + + def done(self): + return self.at("end") + + # grammar + def expr(self, with_cond=True): + e = self.or_() + if with_cond and self.accept("name", "if"): + c = self.or_() + other = None + if self.accept("name", "else"): + other = self.expr() + return ("cond", c, e, other) + return e + + def or_(self): + e = self.and_() + while self.accept("name", "or"): + e = ("or", e, self.and_()) + return e + + def and_(self): + e = self.not_() + while self.accept("name", "and"): + e = ("and", e, self.not_()) + return e + + def not_(self): + if self.accept("name", "not"): + return ("not", self.not_()) + return self.compare() + + def compare(self): + e = self.concat() + while True: + tok = self.peek() + if tok[0] == "op" and tok[1] in ("==", "!=", "<", ">", "<=", ">="): + self.next() + e = ("cmp", tok[1], e, self.concat()) + elif tok == ("name", "in"): + self.next() + e = ("cmp", "in", e, self.concat()) + elif tok == ("name", "not") and self.peek(1) == ("name", "in"): + self.next() + self.next() + e = ("cmp", "notin", e, self.concat()) + else: + return e + + def concat(self): + e = self.add() + while self.accept("op", "~"): + e = ("concat", e, self.add()) + return e + + def add(self): + e = self.mul() + while self.at("op", "+") or self.at("op", "-"): + op = self.next()[1] + e = ("bin", op, e, self.mul()) + return e + + def mul(self): + e = self.unary() + while self.peek()[0] == "op" and self.peek()[1] in ("*", "/", "//", "%"): + op = self.next()[1] + e = ("bin", op, e, self.unary()) + return e + + def unary(self): + if self.accept("op", "-"): + return ("neg", self.unary()) + if self.accept("op", "+"): + return self.unary() + e = self.postfix(self.primary()) + if self.accept("op", "**"): + e = ("bin", "**", e, self.unary()) + return e + + def args(self): + args = [] + kw = [] + while not self.accept("op", ")"): + if self.at("name") and self.peek(1) == ("op", "="): + k = self.next()[1] + self.next() + kw.append((k, self.expr())) + else: + args.append(self.expr()) + if not self.accept("op", ","): + self.expect("op", ")") + break + return args, kw + + def postfix(self, e): + while True: + if self.accept("op", "."): + tok = self.next() + e = ("getattr", e, str(tok[1])) + elif self.accept("op", "["): + k = self.expr() + self.expect("op", "]") + e = ("getitem", e, k) + elif self.accept("op", "("): + a, kw = self.args() + e = ("call", e, a, kw) + elif self.accept("op", "|"): + name = self.expect("name")[1] + a, kw = [], [] + if self.accept("op", "("): + a, kw = self.args() + e = ("filter", name, e, a, kw) + elif self.at("name", "is"): + self.next() + neg = bool(self.accept("name", "not")) + name = self.expect("name")[1] + a = [] + if self.accept("op", "("): + a, _ = self.args() + elif self.peek()[0] in ("num", "str") or (self.at("name") and self.peek()[1] not in + ("and", "or", "else", "if", "in", "not")): + a = [self.primary()] + e = ("test", name, e, a, neg) + else: + return e + + def primary(self): + tok = self.next() + k, v = tok + if k in ("num", "str"): + if k == "str": + while self.at("str"): + v += self.next()[1] + return ("const", v) + if k == "name": + if v in ("true", "True"): + return ("const", True) + if v in ("false", "False"): + return ("const", False) + if v in ("none", "None"): + return ("const", None) + return ("name", v) + if k == "op" and v == "(": + if self.accept("op", ")"): + return ("tuple", []) + e = self.expr() + if self.accept("op", ","): + items = [e] + while not self.at("op", ")"): + items.append(self.expr()) + if not self.accept("op", ","): + break + self.expect("op", ")") + return ("tuple", items) + self.expect("op", ")") + return e + if k == "op" and v == "[": + items = [] + while not self.accept("op", "]"): + items.append(self.expr()) + if not self.accept("op", ","): + self.expect("op", "]") + break + return ("list", items) + if k == "op" and v == "{": + items = [] + while not self.accept("op", "}"): + key = self.expr() + self.expect("op", ":") + items.append((key, self.expr())) + if not self.accept("op", ","): + self.expect("op", "}") + break + return ("dict", items) + raise TemplateSyntaxError("unexpected %r" % (v,)) + + def targets(self): + names = [self.expect("name")[1]] + while self.accept("op", ","): + names.append(self.expect("name")[1]) + return names + + +def _parse_expr(s): + p = _P(s) + e = p.expr() + if not p.done(): + raise TemplateSyntaxError("unexpected %r" % (p.peek()[1],)) + return e + + +# ---- statements ------------------------------------------------------------ + +def _parse(tokens): + pos = [0] + blocks = {} + meta = {"extends": None} + + def body(ends): + nodes = [] + while pos[0] < len(tokens): + kind, val = tokens[pos[0]] + pos[0] += 1 + if kind == "text": + nodes.append(("text", val)) + elif kind == "var": + nodes.append(("out", _parse_expr(val))) + else: + word = val.split(None, 1)[0] if val else "" + rest = val[len(word):].strip() + if word in ends: + return nodes, word, rest + nodes.append(stmt(word, rest)) + if ends: + raise TemplateSyntaxError("missing %s" % ends[-1]) + return nodes, None, None + + def stmt(word, rest): + if word == "if": + branches = [] + cond = _parse_expr(rest) + other = [] + while True: + b, end, r = body(("elif", "else", "endif")) + branches.append((cond, b)) + if end == "elif": + cond = _parse_expr(r) + continue + if end == "else": + other, _, _ = body(("endif",)) + break + return ("if", branches, other) + if word == "for": + p = _P(rest) + targets = p.targets() + p.expect("name", "in") + it = p.expr(False) + filt = None + if p.accept("name", "if"): + filt = p.expr(False) + b, end, _ = body(("else", "endfor")) + other = [] + if end == "else": + other, _, _ = body(("endfor",)) + return ("for", targets, it, filt, b, other) + if word == "set": + p = _P(rest) + targets = p.targets() + if p.accept("op", "="): + e = p.expr() + if p.accept("op", ","): + items = [e] + while not p.done(): + items.append(p.expr()) + if not p.accept("op", ","): + break + e = ("tuple", items) + return ("set", targets, e) + b, _, _ = body(("endset",)) + return ("setblock", targets[0], b) + if word == "block": + name = rest.split()[0] + b, _, _ = body(("endblock",)) + blocks[name] = b + return ("block", name) + if word == "extends": + meta["extends"] = _parse_expr(rest) + return ("text", "") + if word == "include": + ignore = False + if rest.endswith("ignore missing"): + ignore = True + rest = rest[:-len("ignore missing")] + for suffix in ("with context", "without context"): + if rest.endswith(suffix): + rest = rest[:-len(suffix)] + return ("include", _parse_expr(rest.strip()), ignore) + if word == "with": + p = _P(rest) + assigns = [] + while not p.done(): + n = p.expect("name")[1] + p.expect("op", "=") + assigns.append((n, p.expr())) + p.accept("op", ",") + b, _, _ = body(("endwith",)) + return ("with", assigns, b) + raise TemplateSyntaxError("Encountered unknown tag %r." % (word,)) + + nodes, _, _ = body(()) + return nodes, blocks, meta["extends"] + + +# ---- runtime --------------------------------------------------------------- + +class _Loop: + def __init__(self, items): + self._items = items + self.length = len(items) + self.index0 = 0 + + index = property(lambda self: self.index0 + 1) + revindex = property(lambda self: self.length - self.index0) + revindex0 = property(lambda self: self.length - self.index0 - 1) + first = property(lambda self: self.index0 == 0) + last = property(lambda self: self.index0 == self.length - 1) + previtem = property(lambda self: self._items[self.index0 - 1] if self.index0 > 0 else Undefined()) + nextitem = property(lambda self: self._items[self.index0 + 1] + if self.index0 + 1 < self.length else Undefined()) + + def cycle(self, *args): + return args[self.index0 % len(args)] + + +def _attr(obj, name): + if isinstance(obj, Undefined): + obj._fail() + try: + return getattr(obj, name) + except AttributeError: + pass + try: + return obj[name] + except (TypeError, LookupError, AttributeError): + return Undefined(name=name) + + +def _item(obj, key): + if isinstance(obj, Undefined): + obj._fail() + try: + return obj[key] + except (TypeError, LookupError, AttributeError): + pass + if isinstance(key, str): + try: + return getattr(obj, key) + except AttributeError: + pass + return Undefined(name=key) + + +def _do_default(v, default="", boolean=False): + if isinstance(v, Undefined) or (boolean and not v): + return default + return v + + +def _do_join(env_ae, v, d="", attribute=None): + items = list(v) + if attribute is not None: + items = [_attr(x, attribute) for x in items] + if env_ae: + return escape(d).join(items) + return str(d).join(str(x) for x in items) + + +def _do_title(s): + out = [] + up = True + for ch in str(s): + if ch.isalnum(): + out.append(ch.upper() if up else ch.lower()) + up = False + else: + out.append(ch) + up = ch in " \t\n\r-([{" + r = "".join(out) + return Markup(r) if isinstance(s, Markup) else r + + +def _keep(s, r): + return Markup(r) if isinstance(s, Markup) else r + + +_FILTERS = { + "safe": lambda v: Markup(str(v)), + "upper": lambda v: _keep(v, str(v).upper()), + "lower": lambda v: _keep(v, str(v).lower()), + "capitalize": lambda v: _keep(v, str(v).capitalize()), + "title": _do_title, + "trim": lambda v, chars=None: _keep(v, str(v).strip(chars)), + "length": len, + "count": len, + "default": _do_default, + "d": _do_default, + "first": lambda v: next(iter(v), Undefined()), + "last": lambda v: list(v)[-1] if len(list(v)) else Undefined(), + "reverse": lambda v: v[::-1] if isinstance(v, str) else list(reversed(list(v))), + "sort": lambda v, reverse=False: sorted(v, reverse=reverse), + "list": list, + "string": str, + "int": lambda v, default=0: _to_int(v, default), + "float": lambda v, default=0.0: _to_float(v, default), + "abs": abs, + "round": lambda v, precision=0: round(v, precision), + "replace": lambda v, old, new: _keep(v, str(v).replace(old, new)), + "sum": lambda v, start=0: sum(v, start), + "min": min, + "max": max, + "unique": lambda v: list(dict.fromkeys(v)), + "striptags": lambda v: Markup(escape(v) if not isinstance(v, Markup) else v).striptags(), + "items": lambda v: list(v.items()), + "dictsort": lambda v: sorted(v.items()), + "wordcount": lambda v: len(str(v).split()), + "center": lambda v, width=80: str(v).center(width), + "truncate": lambda v, length=255, killwords=False, end="...": _truncate(str(v), length, killwords, end), + "format": lambda v, *a, **k: str(v) % (a or k), + "tojson": lambda v: Markup(_tojson(v)), +} + + +def _to_int(v, default): + try: + return int(v) + except (TypeError, ValueError): + try: + return int(float(v)) + except (TypeError, ValueError): + return default + + +def _to_float(v, default): + try: + return float(v) + except (TypeError, ValueError): + return default + + +def _truncate(s, length, killwords, end): + if len(s) <= length + 5: + return s + if killwords: + return s[:length - len(end)] + end + r = s[:length - len(end)].rsplit(" ", 1)[0] + return r + end + + +def _tojson(v): + import json + s = json.dumps(v, sort_keys=True) + return (s.replace("<", "\\u003c").replace(">", "\\u003e").replace("&", "\\u0026") + .replace("'", "\\u0027")) + + +def _test(name, v, args): + if name == "defined": + return not isinstance(v, Undefined) + if name == "undefined": + return isinstance(v, Undefined) + if name == "none": + return v is None + if name in ("string",): + return isinstance(v, str) + if name == "number": + return isinstance(v, (int, float)) and not isinstance(v, bool) + if name in ("integer",): + return isinstance(v, int) and not isinstance(v, bool) + if name == "boolean": + return isinstance(v, bool) + if name == "even": + return v % 2 == 0 + if name == "odd": + return v % 2 == 1 + if name == "divisibleby": + return v % args[0] == 0 + if name == "mapping": + return isinstance(v, dict) + if name == "iterable": + try: + iter(v) + return True + except TypeError: + return False + if name in ("sameas",): + return v is args[0] + if name in ("eq", "equalto", "=="): + return v == args[0] + if name in ("ne", "!="): + return v != args[0] + if name in ("lower",): + return str(v).islower() + if name in ("upper",): + return str(v).isupper() + if name == "in": + return v in args[0] + if name == "true": + return v is True + if name == "false": + return v is False + raise TemplateSyntaxError("No test named %r." % (name,)) + + +class _Frame: + def __init__(self, tmpl, vars, blocks, out): + self.tmpl = tmpl + self.vars = vars + self.blocks = blocks + self.out = out + + +class Template: + def __init__(self, *args, **kwargs): + pass + + def __new__(cls, source=None, autoescape=False, **kw): + if source is None: + return object.__new__(cls) + env = Environment(autoescape=autoescape, **kw) + return env.from_string(source) + + def _setup(self, env, source, name, filename): + self.environment = env + self.name = name + self.filename = filename + if not env.keep_trailing_newline and source.endswith("\n"): + source = source[:-1] + if source.endswith("\r"): + source = source[:-1] + self._nodes, self._blocks, self._extends = _parse(_lex(source)) + ae = env.autoescape + self.autoescape = bool(ae(name)) if callable(ae) else bool(ae) + + def render(self, *args, **kwargs): + vars = dict(self.environment.globals) + if args: + vars.update(dict(args[0])) + vars.update(kwargs) + out = [] + self._render_root(vars, {}, out) + return "".join(out) + + def generate(self, *args, **kwargs): + yield self.render(*args, **kwargs) + + def stream(self, *args, **kwargs): + return self.generate(*args, **kwargs) + + def _render_root(self, vars, blocks, out): + merged = dict(blocks) + for k, b in self._blocks.items(): + merged.setdefault(k, []) + merged[k] = merged[k] + [(self, b)] + if self._extends is not None: + fr = _Frame(self, vars, merged, []) + for n in self._nodes: + if n[0] == "set": + self._exec(n, fr) + parent = self._eval(self._extends, fr) + if not isinstance(parent, Template): + parent = self.environment.get_template(parent) + parent._render_root(vars, merged, out) + return + self._exec_list(self._nodes, _Frame(self, vars, merged, out)) + + # -- execution + def _exec_list(self, nodes, fr): + for n in nodes: + self._exec(n, fr) + + def _exec(self, n, fr): + k = n[0] + if k == "text": + fr.out.append(n[1]) + elif k == "out": + fr.out.append(self._finalize(self._eval(n[1], fr))) + elif k == "if": + for cond, body in n[1]: + if self._eval(cond, fr): + self._exec_list(body, fr) + return + self._exec_list(n[2], fr) + elif k == "for": + self._for(n, fr) + elif k == "set": + v = self._eval(n[2], fr) + self._assign(fr.vars, n[1], v) + elif k == "setblock": + sub = _Frame(fr.tmpl, fr.vars, fr.blocks, []) + self._exec_list(n[2], sub) + s = "".join(sub.out) + fr.vars[n[1]] = Markup(s) if self.autoescape else s + elif k == "block": + self._render_block(n[1], 0, fr) + elif k == "include": + name = self._eval(n[1], fr) + try: + if isinstance(name, (list, tuple)): + t = self.environment.select_template(name) + else: + t = name if isinstance(name, Template) else self.environment.get_template(name) + except TemplateNotFound: + if n[2]: + return + raise + t._render_root(dict(fr.vars), {}, fr.out) + elif k == "with": + vars = dict(fr.vars) + for name, e in n[1]: + vars[name] = self._eval(e, fr) + self._exec_list(n[2], _Frame(fr.tmpl, vars, fr.blocks, fr.out)) + + def _render_block(self, name, level, fr): + chain = fr.blocks.get(name) + if not chain: + return + owner, body = chain[level] + vars = dict(fr.vars) + if level + 1 < len(chain): + def sup(): + sub = _Frame(fr.tmpl, fr.vars, fr.blocks, []) + self._render_block(name, level + 1, sub) + return Markup("".join(sub.out)) + vars["super"] = sup + owner._exec_list(body, _Frame(owner, vars, fr.blocks, fr.out)) + + def _assign(self, vars, targets, v): + if len(targets) == 1: + vars[targets[0]] = v + else: + vals = list(v) + if len(vals) != len(targets): + raise ValueError("not enough values to unpack") + for t, x in zip(targets, vals): + vars[t] = x + + def _for(self, n, fr): + _, targets, it, filt, body, other = n + seq = self._eval(it, fr) + if isinstance(seq, dict): + seq = list(seq.keys()) + items = list(seq) + if filt is not None: + kept = [] + for x in items: + tmp = dict(fr.vars) + self._assign(tmp, targets, x) + if self._eval(filt, _Frame(fr.tmpl, tmp, fr.blocks, fr.out)): + kept.append(x) + items = kept + if not items: + self._exec_list(other, fr) + return + loop = _Loop(items) + for idx, x in enumerate(items): + loop.index0 = idx + vars = dict(fr.vars) + vars["loop"] = loop + self._assign(vars, targets, x) + self._exec_list(body, _Frame(fr.tmpl, vars, fr.blocks, fr.out)) + + def _finalize(self, v): + if isinstance(v, Undefined): + return "" + if self.autoescape: + return str(escape(v)) + if hasattr(v, "__html__") and not isinstance(v, str): + return str(v.__html__()) + return str(v) + + # -- expressions + def _eval(self, e, fr): + k = e[0] + if k == "const": + return e[1] + if k == "name": + name = e[1] + if name in fr.vars: + return fr.vars[name] + return Undefined(name=name) + if k == "getattr": + return _attr(self._eval(e[1], fr), e[2]) + if k == "getitem": + return _item(self._eval(e[1], fr), self._eval(e[2], fr)) + if k == "call": + f = self._eval(e[1], fr) + args = [self._eval(a, fr) for a in e[2]] + kw = dict((kk, self._eval(v, fr)) for kk, v in e[3]) + return f(*args, **kw) + if k == "filter": + name = e[1] + v = self._eval(e[2], fr) + args = [self._eval(a, fr) for a in e[3]] + kw = dict((kk, self._eval(x, fr)) for kk, x in e[4]) + if name in self.environment.filters: + return self.environment.filters[name](v, *args, **kw) + if name in ("escape", "e", "forceescape"): + return escape(str(v)) if name == "forceescape" else escape(v) + if name == "join": + return _do_join(self.autoescape, v, *args, **kw) + f = _FILTERS.get(name) + if f is None: + raise TemplateSyntaxError("No filter named %r." % (name,)) + return f(v, *args, **kw) + if k == "test": + r = _test(e[1], self._eval(e[2], fr), [self._eval(a, fr) for a in e[3]]) + return (not r) if e[4] else r + if k == "not": + return not self._eval(e[1], fr) + if k == "and": + a = self._eval(e[1], fr) + return self._eval(e[2], fr) if a else a + if k == "or": + a = self._eval(e[1], fr) + return a if a else self._eval(e[2], fr) + if k == "cond": + if self._eval(e[1], fr): + return self._eval(e[2], fr) + return self._eval(e[3], fr) if e[3] is not None else Undefined() + if k == "concat": + a = self._eval(e[1], fr) + b = self._eval(e[2], fr) + if self.autoescape: + return Markup(str(escape(a)) + str(escape(b))) + return str(a) + str(b) + if k == "neg": + return -self._eval(e[1], fr) + if k == "bin": + a = self._eval(e[2], fr) + b = self._eval(e[3], fr) + op = e[1] + if op == "+": + return a + b + if op == "-": + return a - b + if op == "*": + return a * b + if op == "/": + return a / b + if op == "//": + return a // b + if op == "%": + return a % b + return a ** b + if k == "cmp": + a = self._eval(e[2], fr) + b = self._eval(e[3], fr) + op = e[1] + if op == "==": + return a == b + if op == "!=": + return a != b + if op == "<": + return a < b + if op == ">": + return a > b + if op == "<=": + return a <= b + if op == ">=": + return a >= b + if op == "in": + return a in b + return a not in b + if k == "list": + return [self._eval(x, fr) for x in e[1]] + if k == "tuple": + return tuple(self._eval(x, fr) for x in e[1]) + if k == "dict": + return dict((self._eval(a, fr), self._eval(b, fr)) for a, b in e[1]) + raise TemplateSyntaxError("bad node %r" % (k,)) + + +class Environment: + def __init__(self, loader=None, autoescape=False, keep_trailing_newline=False, + undefined=Undefined, **kw): + self.loader = loader + self.autoescape = autoescape + self.keep_trailing_newline = keep_trailing_newline + self.undefined = undefined + self.filters = {} + self.tests = {} + self.globals = {"range": range, "dict": dict, "lipsum": lambda *a, **k: "", + "cycler": None, "namespace": _Namespace} + self._cache = {} + + def from_string(self, source, globals=None, template_class=None): + t = self._from_source(source, None, None) + if globals: + t.globals = globals + return t + + def _from_source(self, source, name, filename): + t = Template.__new__(Template) + t._setup(self, source, name, filename) + return t + + def get_template(self, name, parent=None, globals=None): + if isinstance(name, Template): + return name + if name in self._cache: + return self._cache[name] + if self.loader is None: + raise TypeError("no loader for this environment specified") + t = self.loader.load(self, name, globals) + self._cache[name] = t + return t + + def select_template(self, names, parent=None, globals=None): + for n in names: + try: + return self.get_template(n) + except TemplateNotFound: + pass + raise TemplateNotFound(", ".join(str(n) for n in names)) + + get_or_select_template = select_template + + def list_templates(self): + m = getattr(self.loader, "mapping", None) + return sorted(m) if m is not None else [] + + +class _Namespace: + def __init__(self, **kw): + self.__dict__.update(kw) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/markupsafe/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/markupsafe/__init__.py new file mode 100644 index 00000000000..f96613cf3fa --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/markupsafe/__init__.py @@ -0,0 +1,102 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Clean-room subset of markupsafe: Markup and escape.""" + +__all__ = ["Markup", "escape", "escape_silent", "soft_str"] + + +def _esc(s): + return (s.replace("&", "&").replace(">", ">").replace("<", "<") + .replace("'", "'").replace('"', """)) + + +class Markup(str): + """A string that is already safe to insert into HTML.""" + + def __new__(cls, base="", encoding=None, errors="strict"): + if hasattr(base, "__html__"): + base = base.__html__() + if encoding is not None: + base = base.decode(encoding, errors) + return str.__new__(cls, base) + + def __html__(self): + return self + + def __add__(self, other): + if isinstance(other, str) or hasattr(other, "__html__"): + return Markup(str(self) + str(escape(other))) + return NotImplemented + + def __radd__(self, other): + if isinstance(other, str) or hasattr(other, "__html__"): + return Markup(str(escape(other)) + str(self)) + return NotImplemented + + def join(self, seq): + return Markup(str.join(self, [str(escape(s)) for s in seq])) + + def upper(self): + return Markup(str.upper(self)) + + def lower(self): + return Markup(str.lower(self)) + + def title(self): + return Markup(str.title(self)) + + def strip(self, chars=None): + return Markup(str.strip(self, chars)) + + def format(self, *args, **kwargs): + args = [escape(a) for a in args] + kwargs = dict((k, escape(v)) for k, v in kwargs.items()) + return Markup(str.format(self, *args, **kwargs)) + + def __mod__(self, arg): + if isinstance(arg, tuple): + arg = tuple(escape(a) for a in arg) + else: + arg = escape(arg) + return Markup(str.__mod__(self, arg)) + + def __repr__(self): + return "Markup(" + str.__repr__(self) + ")" + + def striptags(self): + out = [] + depth = 0 + for ch in str(self): + if ch == "<": + depth += 1 + elif ch == ">" and depth: + depth -= 1 + elif not depth: + out.append(ch) + return " ".join("".join(out).split()) + + def unescape(self): + s = str(self) + for a, b in (("<", "<"), (">", ">"), (""", '"'), ("'", "'"), + (""", '"'), ("&", "&")): + s = s.replace(a, b) + return s + + +def escape(s): + if hasattr(s, "__html__"): + return Markup(s.__html__()) + return Markup(_esc(str(s))) + + +def escape_silent(s): + if s is None: + return Markup() + return escape(s) + + +def soft_str(s): + if not isinstance(s, str): + return str(s) + return s diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/hmac.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/hmac.py new file mode 100644 index 00000000000..be79870d434 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/hmac.py @@ -0,0 +1,87 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""hmac (RFC 2104) over the hashlib in this runtime.""" +import hashlib as _hashlib + +__all__ = ["new", "compare_digest", "digest", "HMAC"] + +digest_size = None + + +def _ctor(digestmod): + if digestmod is None: + raise TypeError("Missing required argument 'digestmod'.") + if isinstance(digestmod, str): + name = digestmod + return lambda d=b"": _hashlib.new(name, d) + if hasattr(digestmod, "new"): + return digestmod.new + return digestmod + + +def _bytes(b): + if isinstance(b, str): + raise TypeError("Strings must be encoded before hashing") + return bytes(b) + + +class HMAC: + def __init__(self, key, msg=None, digestmod=None): + key = _bytes(key) + ctor = _ctor(digestmod) + self._ctor = ctor + probe = ctor() + bs = getattr(probe, "block_size", 64) + self.block_size = bs + self.digest_size = probe.digest_size + if len(key) > bs: + key = ctor(key).digest() + key = key + b"\x00" * (bs - len(key)) + self._inner = ctor(bytes(b ^ 0x36 for b in key)) + self._outer = ctor(bytes(b ^ 0x5C for b in key)) + self.name = "hmac-" + probe.name + if msg is not None: + self.update(msg) + + def update(self, msg): + self._inner.update(_bytes(msg)) + + def copy(self): + c = HMAC.__new__(HMAC) + c.__dict__.update(self.__dict__) + c._inner = self._inner.copy() + c._outer = self._outer.copy() + return c + + def digest(self): + o = self._outer.copy() + o.update(self._inner.copy().digest()) + return o.digest() + + def hexdigest(self): + return self.digest().hex() + + +def new(key, msg=None, digestmod=None): + return HMAC(key, msg, digestmod) + + +def digest(key, msg, digest): + return HMAC(key, msg, digest).digest() + + +def compare_digest(a, b): + if isinstance(a, str) and isinstance(b, str): + a = a.encode("ascii") + b = b.encode("ascii") + elif isinstance(a, str) or isinstance(b, str): + raise TypeError("unsupported operand types(s) or combination of types") + a = bytes(a) + b = bytes(b) + r = len(a) ^ len(b) + if len(a) != len(b): + b = a + for x, y in zip(a, b): + r |= x ^ y + return r == 0 diff --git a/maven/python-transpiler/src/test/resources/conformance/flask_web_extras.out b/maven/python-transpiler/src/test/resources/conformance/flask_web_extras.out new file mode 100644 index 00000000000..ab96f14739e --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/flask_web_extras.out @@ -0,0 +1,15 @@ +200 no session None False +200 ok Cookie True +ann visits=1 +ann visits=2 +user=ann msgs=['welcome ann'] +user=ann msgs=[] +bye ann True +None visits=1 +user=None msgs=[] +

LIST

  • 1. <a>
  • 2. b
+Hi & bye, Ann Lee! /inline None +200 notes.txt text/plain 12 b'hello\r\nworld' form={'title': 'T1'} extras=[('a.bin', b'x1'), ('b.bin', b'x2')] missing=False +200 text/plain chunk0;chunk1;chunk2;done +ab +error: The session is unavailable because no secret key was set. Set the secret_key on the application to something unique and secret. diff --git a/maven/python-transpiler/src/test/resources/conformance/flask_web_extras.py b/maven/python-transpiler/src/test/resources/conformance/flask_web_extras.py new file mode 100644 index 00000000000..abe603ba59a --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/flask_web_extras.py @@ -0,0 +1,128 @@ +import io + +from flask import (Flask, Response, request, session, render_template, render_template_string, + stream_with_context, flash, get_flashed_messages) +from jinja2 import DictLoader + +app = Flask(__name__) +app.secret_key = "conformance-secret" +app.jinja_env.loader = DictLoader({ + "layout.html": "

{% block title %}Site{% endblock %}

{% block content %}{% endblock %}", + "items.html": "{% extends 'layout.html' %}{% block title %}{{ title|upper }}{% endblock %}" + "{% block content %}
    {% for i in items %}
  • " + "{{ loop.index }}. {{ i }}
    • {% endfor %}
{% endblock %}", +}) + + +@app.route("/login", methods=["POST"]) +def login(): + session["user"] = request.form["user"] + session["visits"] = 0 + flash("welcome " + request.form["user"]) + return "ok" + + +@app.route("/visit") +def visit(): + session["visits"] = session.get("visits", 0) + 1 + return "%s visits=%d" % (session.get("user"), session["visits"]) + + +@app.route("/peek") +def peek(): + msgs = get_flashed_messages() + return "user=%s msgs=%s" % (session.get("user"), msgs) + + +@app.route("/logout") +def logout(): + who = session.pop("user", None) + session.clear() + return "bye " + str(who) + + +@app.route("/plain") +def plain(): + return "no session" + + +@app.route("/items") +def items(): + return render_template("items.html", title="list", items=["", "b"]) + + +@app.route("/inline") +def inline(): + return render_template_string("{{ greeting }}, {{ name|title }}! {{ request.path }} {{ session.get('user') }}", + greeting="Hi & bye", name="ann lee") + + +@app.route("/upload", methods=["POST"]) +def upload(): + f = request.files["doc"] + data = f.read() + others = request.files.getlist("extra") + return "%s %s %d %r form=%s extras=%s missing=%s" % ( + f.filename, f.content_type, len(data), data, request.form.to_dict(), + [(o.filename, o.read()) for o in others], "nope" in request.files) + + +@app.route("/stream") +def stream(): + def gen(): + for i in range(3): + yield "chunk%d;" % i + yield request.args.get("tail", "end") + return Response(stream_with_context(gen()), mimetype="text/plain") + + +@app.route("/gen") +def gen_view(): + def g(): + yield b"a" + yield "b" + return g() + + +c = app.test_client() +r = c.get("/plain") +print(r.status_code, r.get_data(as_text=True), r.headers.get("Vary"), "Set-Cookie" in r.headers) +r = c.post("/login", data={"user": "ann"}) +print(r.status_code, r.get_data(as_text=True), r.headers.get("Vary"), "Set-Cookie" in r.headers) +for _ in range(2): + print(c.get("/visit").get_data(as_text=True)) +print(c.get("/peek").get_data(as_text=True)) +print(c.get("/peek").get_data(as_text=True)) +r = c.get("/logout") +print(r.get_data(as_text=True), "Set-Cookie" in r.headers) +print(c.get("/visit").get_data(as_text=True)) + +c2 = app.test_client() +c2.set_cookie("session", "forged.value") +print(c2.get("/peek").get_data(as_text=True)) + +print(c.get("/items").get_data(as_text=True)) +print(c.get("/inline").get_data(as_text=True)) + +r = c.post("/upload", data={"doc": (io.BytesIO(b"hello\r\nworld"), "notes.txt"), "title": "T1", + "extra": [(io.BytesIO(b"x1"), "a.bin", "application/x-test"), + (io.BytesIO(b"x2"), "b.bin")]}) +print(r.status_code, r.get_data(as_text=True)) + +r = c.get("/stream?tail=done") +print(r.status_code, r.mimetype, r.get_data(as_text=True)) +print(c.get("/gen").get_data(as_text=True)) + +app2 = Flask("nokey") + + +@app2.route("/") +def nokey(): + try: + session["x"] = 1 + except RuntimeError as e: + return "error: " + str(e) + return "stored" + + +print(app2.test_client().get("/").get_data(as_text=True)) diff --git a/maven/python-transpiler/src/test/resources/conformance/jinja_templates.out b/maven/python-transpiler/src/test/resources/conformance/jinja_templates.out new file mode 100644 index 00000000000..d6f27b2ee74 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/jinja_templates.out @@ -0,0 +1,27 @@ +Hello World! +None|True|3.5|| +a1b 14 3 1 -4 +HELLO WORLD hello world Hello World 11 1, 2, 3 +none empty 0 +mid +yes+U +1/3:a first +2/3:b +3/3:c last + +empty + +a=1;b=2; +1a 3b 5a +4 3 +AB C D +Ann 30 6 ABC +no [1, 2, 3] (1, 2) +<b>&'"</b> &'" <b>&'"</b> ok +<a><br> + +<x>

<z>

+Child - Base +

part <me>

<me>
+ +TemplateNotFound nope.html diff --git a/maven/python-transpiler/src/test/resources/conformance/jinja_templates.py b/maven/python-transpiler/src/test/resources/conformance/jinja_templates.py new file mode 100644 index 00000000000..52fb3acfab2 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/jinja_templates.py @@ -0,0 +1,46 @@ +from jinja2 import Environment, DictLoader, Template, select_autoescape +from markupsafe import Markup, escape + +print(Template("Hello {{ name }}!").render(name="World")) +print(Template("{{ x }}|{{ y }}|{{ n }}|{{ missing }}|").render(x=None, y=True, n=3.5)) +print(Template("{{ 'a' ~ 1 ~ 'b' }} {{ 2 + 3 * 4 }} {{ 7 // 2 }} {{ 7 % 3 }} {{ -n }}").render(n=4)) +print(Template("{{ s|upper }} {{ s|lower }} {{ s|title }} {{ s|length }} {{ xs|join(', ') }}").render( + s="hello wORLD", xs=[1, 2, 3])) +print(Template("{{ m|default('none') }} {{ e|default('empty', true) }} {{ v|d('x') }}").render(e="", v=0)) +print(Template("{% if n > 5 %}big{% elif n > 2 %}mid{% else %}small{% endif %}").render(n=3)) +print(Template("{% if a and not b %}yes{% endif %}{% if 'x' in s %}+{% endif %}{% if k is defined %}D{% else %}U{% endif %}").render( + a=1, b=0, s="xyz")) + +loop_t = Template("""{% for x in xs -%} +{{ loop.index }}/{{ loop.length }}:{{ x }}{% if loop.first %} first{% endif %}{% if loop.last %} last{% endif %} +{% else %}empty +{% endfor %}""") +print(loop_t.render(xs=["a", "b", "c"])) +print(loop_t.render(xs=[])) +print(Template("{% for k, v in d.items() %}{{ k }}={{ v }};{% endfor %}").render(d={"a": 1, "b": 2})) +print(Template("{% for x in xs if x % 2 %}{{ x }}{{ loop.cycle('a', 'b') }} {% endfor %}").render(xs=range(7))) +print(Template("{% set total = xs|length * 2 %}{{ total }} {% set a, b = 1, 2 %}{{ a + b }}").render(xs=[1, 2])) +print(Template("A{# a comment #}B {%- if true %} C {%- endif %} D").render()) +print(Template("{{ user.name }} {{ user['age'] }} {{ items[1] }} {{ 'abc'.upper() }}").render( + user={"name": "Ann", "age": 30}, items=[5, 6])) +print(Template("{{ 'yes' if flag else 'no' }} {{ [1, 2] + [3] }} {{ (1, 2) }}").render(flag=False)) + +env = Environment(autoescape=True) +print(env.from_string("{{ v }} {{ v|safe }} {{ v|e }} {{ m }}").render(v="&'\"", m=Markup("ok"))) +print(env.from_string("{{ xs|join('
') }}").render(xs=["", Markup("")])) +print(Template("{{ v }}").render(v="")) +print(escape(""), escape(Markup("")), Markup("

%s

") % "") + +env = Environment(loader=DictLoader({ + "base.html": "{% block title %}Base{% endblock %}\n
{% block body %}{% endblock %}
\n", + "child.html": "{% extends 'base.html' %}{% block title %}Child - {{ super() }}{% endblock %}" + "{% block body %}{% include 'part.html' %} {{ who }}{% endblock %}", + "part.html": "

part {{ who }}

", + "plain.txt": "{{ who }}", +}), autoescape=select_autoescape(["html"])) +print(env.get_template("child.html").render(who="")) +print(env.get_template("plain.txt").render(who="")) +try: + env.get_template("nope.html") +except Exception as e: + print(type(e).__name__, e) From 57b8c4829f6899d16fa975c17b9dcdf380820066 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 09:32:07 +0200 Subject: [PATCH 045/112] Python guide: Flask sessions, templates, uploads and streaming Co-Authored-By: Claude Opus 5.5 --- docs/developer-guide/Python-Interop.asciidoc | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/developer-guide/Python-Interop.asciidoc b/docs/developer-guide/Python-Interop.asciidoc index 23f4936c62c..57a9199272c 100644 --- a/docs/developer-guide/Python-Interop.asciidoc +++ b/docs/developer-guide/Python-Interop.asciidoc @@ -304,9 +304,13 @@ include::../demos/common/src/main/snippets/developer-guide/python-interop.sh[tag Routing, converters, JSON bodies, `abort`, error handlers and the default error pages of Flask, and FastAPI's path, query, body and header parameters, -dependencies, `response_model` and OpenAPI generation are supported. Flask -sessions, templates, file uploads and streaming responses aren't, nor are -FastAPI WebSockets and middleware. +dependencies, `response_model` and OpenAPI generation are supported, and +Flask sessions (signed cookies), `flash`, Jinja templates (`extends`, `block`, +`include`, loops, conditions and the common filters, with autoescaping), +multipart file uploads and streamed responses work as well. A streamed +response is sent once its generator finishes. Jinja macros aren't supported yet, and a session cookie +written by CPython's Flask isn't readable here, or the reverse. FastAPI +WebSockets and middleware aren't supported yet. === Compared with CPython, PyPy and GraalPy From 758edcb6e8c3a4acf8d3ffd19bede84907937c0c Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 09:51:35 +0200 Subject: [PATCH 046/112] Python: Pillow subset (PIL.Image, ImageDraw, ImageOps, ImageFilter.BoxBlur) with pure PNG codec over zlib Pixels and loops live in the new _cn1image runtime module, following Pillow's fixed point code (luma, bilinear resample, affine nearest, mask blend, box blur) so output is byte identical. JPEG and other formats go through Codename One's image codecs when a display is running. Co-Authored-By: Claude Opus 5.5 --- .../com/codename1/python/runtime/PyImage.java | 650 ++++++++++++++++++ .../codename1/python/runtime/StdModules.java | 4 + .../com/codename1/python/site/PIL/Image.py | 475 +++++++++++++ .../codename1/python/site/PIL/ImageColor.py | 38 + .../codename1/python/site/PIL/ImageDraw.py | 177 +++++ .../codename1/python/site/PIL/ImageFilter.py | 26 + .../com/codename1/python/site/PIL/ImageOps.py | 31 + .../com/codename1/python/site/PIL/__init__.py | 16 + .../test/resources/conformance/pil_basic.out | 58 ++ .../test/resources/conformance/pil_basic.py | 94 +++ .../test/resources/conformance/pil_draw.out | 16 + .../test/resources/conformance/pil_draw.py | 57 ++ .../test/resources/conformance/pil_png.out | 12 + .../src/test/resources/conformance/pil_png.py | 86 +++ 14 files changed, 1740 insertions(+) create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/PyImage.java create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/Image.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/ImageColor.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/ImageDraw.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/ImageFilter.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/ImageOps.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/__init__.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/pil_basic.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/pil_basic.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/pil_draw.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/pil_draw.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/pil_png.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/pil_png.py diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyImage.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyImage.java new file mode 100644 index 00000000000..64de5863635 --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyImage.java @@ -0,0 +1,650 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +import java.io.ByteArrayOutputStream; + +/// The `_cn1image` module: pixel storage and the hot loops behind the +/// bundled `PIL` package. An image is a [Img] handle holding interleaved +/// 8-bit bands (1 for "L", 3 for "RGB", 4 for "RGBA"), exactly the layout of +/// Pillow's `tobytes()`. Every arithmetic here follows Pillow's own fixed +/// point code so results are byte identical with it. +final class PyImage { + /// Pillow's resampling precision, `32 - 8 - 2`. + private static final int PRECISION_BITS = 22; + + private PyImage() { + } + + /// One image: `w` by `h` pixels of `bands` interleaved bytes. + static final class Img { + final int w; + final int h; + final int bands; + final byte[] d; + + Img(int w, int h, int bands) { + this.w = w; + this.h = h; + this.bands = bands; + this.d = new byte[w * h * bands]; + } + } + + private static Img img(Object o) { + if (o instanceof PyJavaObject && ((PyJavaObject) o).value instanceof Img) { + return (Img) ((PyJavaObject) o).value; + } + throw Py.typeError("not an image"); + } + + private static Object wrap(Img i) { + return new PyJavaObject(i); + } + + private static int i(Object o) { + return (int) Py.asLong(o); + } + + private static byte[] bytes(Object o) { + byte[] b = PyBytes.like(o); + if (b == null) { + throw Py.typeError("a bytes-like object is required"); + } + return b; + } + + static PyModule module() { + PyModule m = new PyModule("_cn1image", new String[0]); + m.setAttr("new", PyBuiltin.positional("new", a -> { + Img r = new Img(i(a[0]), i(a[1]), i(a[2])); + fill(r, 0, 0, r.w, r.h, bytes(a[3]), null); + return wrap(r); + })); + m.setAttr("frombytes", PyBuiltin.positional("frombytes", a -> { + Img r = new Img(i(a[0]), i(a[1]), i(a[2])); + byte[] b = bytes(a[3]); + if (b.length < r.d.length) { + throw Py.error(Py.VALUE_ERROR, "not enough image data"); + } + System.arraycopy(b, 0, r.d, 0, r.d.length); + return wrap(r); + })); + m.setAttr("size", PyBuiltin.positional("size", a -> { + Img s = img(a[0]); + return new PyTuple(new Object[] {PyInt.of(s.w), PyInt.of(s.h)}); + })); + m.setAttr("tobytes", PyBuiltin.positional("tobytes", a -> { + Img s = img(a[0]); + byte[] out = new byte[s.d.length]; + System.arraycopy(s.d, 0, out, 0, out.length); + return new PyBytes(out); + })); + m.setAttr("getpixel", PyBuiltin.positional("getpixel", a -> { + Img s = img(a[0]); + int x = i(a[1]); + int y = i(a[2]); + if (x < 0 || y < 0 || x >= s.w || y >= s.h) { + throw Py.error(Py.INDEX_ERROR, "image index out of range"); + } + int p = (y * s.w + x) * s.bands; + Object[] t = new Object[s.bands]; + for (int b = 0; b < s.bands; b++) { + t[b] = PyInt.of(s.d[p + b] & 0xff); + } + return new PyTuple(t); + })); + m.setAttr("putpixel", PyBuiltin.positional("putpixel", a -> { + Img s = img(a[0]); + int x = i(a[1]); + int y = i(a[2]); + if (x < 0 || y < 0 || x >= s.w || y >= s.h) { + throw Py.error(Py.INDEX_ERROR, "image index out of range"); + } + System.arraycopy(bytes(a[3]), 0, s.d, (y * s.w + x) * s.bands, s.bands); + return Py.NONE; + })); + m.setAttr("crop", PyBuiltin.positional("crop", a -> wrap(crop(img(a[0]), i(a[1]), i(a[2]), i(a[3]), i(a[4]))))); + m.setAttr("fill", PyBuiltin.positional("fill", a -> { + Img s = img(a[0]); + fill(s, i(a[1]), i(a[2]), i(a[3]), i(a[4]), bytes(a[5]), a[6] == Py.NONE ? null : img(a[6])); //NOPMD CompareObjectsWithEquals - identity is meant + return Py.NONE; + })); + m.setAttr("paste", PyBuiltin.positional("paste", a -> { + paste(img(a[0]), img(a[1]), i(a[2]), i(a[3]), a[4] == Py.NONE ? null : img(a[4])); //NOPMD CompareObjectsWithEquals - identity is meant + return Py.NONE; + })); + m.setAttr("transpose", PyBuiltin.positional("transpose", a -> wrap(transpose(img(a[0]), i(a[1]))))); + m.setAttr("affine", PyBuiltin.positional("affine", a -> { + Object[] c = Py.toArray(a[3]); + double[] k = new double[6]; + for (int j = 0; j < 6; j++) { + k[j] = Py.asDouble(c[j]); + } + return wrap(affine(img(a[0]), i(a[1]), i(a[2]), k, bytes(a[4]))); + })); + m.setAttr("resample", PyBuiltin.positional("resample", a -> { + Object[] c = Py.toArray(a[3]); + double[] box = new double[4]; + for (int j = 0; j < 4; j++) { + box[j] = Py.asDouble(c[j]); + } + return wrap(resample(img(a[0]), i(a[1]), i(a[2]), box)); + })); + m.setAttr("convert", PyBuiltin.positional("convert", a -> wrap(convert(img(a[0]), i(a[1]))))); + m.setAttr("band", PyBuiltin.positional("band", a -> { + Img s = img(a[0]); + int b = i(a[1]); + Img r = new Img(s.w, s.h, 1); + for (int p = 0, n = s.w * s.h; p < n; p++) { + r.d[p] = s.d[p * s.bands + b]; + } + return wrap(r); + })); + m.setAttr("merge", PyBuiltin.positional("merge", a -> { + Object[] parts = Py.toArray(a[0]); + Img first = img(parts[0]); + Img r = new Img(first.w, first.h, parts.length); + for (int b = 0; b < parts.length; b++) { + Img s = img(parts[b]); + if (s.w != r.w || s.h != r.h || s.bands != 1) { + throw Py.error(Py.VALUE_ERROR, "images do not match"); + } + for (int p = 0, n = r.w * r.h; p < n; p++) { + r.d[p * r.bands + b] = s.d[p]; + } + } + return wrap(r); + })); + m.setAttr("getbbox", PyBuiltin.positional("getbbox", a -> getbbox(img(a[0]), Py.isTrue(a[1])))); + m.setAttr("point", PyBuiltin.positional("point", a -> { + Img s = img(a[0]); + byte[] lut = bytes(a[1]); + Img r = new Img(s.w, s.h, s.bands); + for (int p = 0; p < s.d.length; p += s.bands) { + for (int b = 0; b < s.bands; b++) { + r.d[p + b] = lut[(b << 8) + (s.d[p + b] & 0xff)]; + } + } + return wrap(r); + })); + m.setAttr("boxblur", PyBuiltin.positional("boxblur", a -> { + Img s = img(a[0]); + int r = i(a[1]); + Img t = blurRows(s, r); + Img u = blurRows(transpose(t, 5), r); + return wrap(transpose(u, 5)); + })); + m.setAttr("png_unfilter", PyBuiltin.positional("png_unfilter", a -> + new PyBytes(unfilter(bytes(a[0]), i(a[1]), i(a[2]), i(a[3]))))); + m.setAttr("png_expand", PyBuiltin.positional("png_expand", a -> + wrap(expand(bytes(a[0]), i(a[1]), i(a[2]), i(a[3]), i(a[4]), + PyBytes.like(a[5]), PyBytes.like(a[6]))))); + m.setAttr("png_rows", PyBuiltin.positional("png_rows", a -> { + Img s = img(a[0]); + int row = s.w * s.bands; + byte[] out = new byte[(row + 1) * s.h]; + for (int y = 0; y < s.h; y++) { + System.arraycopy(s.d, y * row, out, y * (row + 1) + 1, row); + } + return new PyBytes(out); + })); + m.setAttr("native_available", PyBuiltin.positional("native_available", a -> + com.codename1.ui.Display.isInitialized() ? Py.TRUE : Py.FALSE)); + m.setAttr("native_decode", PyBuiltin.positional("native_decode", a -> { + byte[] b = bytes(a[0]); + com.codename1.ui.Image ci = com.codename1.ui.Image.createImage(b, 0, b.length); + int w = ci.getWidth(); + int h = ci.getHeight(); + int[] argb = ci.getRGB(); + Img r = new Img(w, h, 4); + for (int p = 0; p < w * h; p++) { + int c = argb[p]; + r.d[p * 4] = (byte) (c >> 16); + r.d[p * 4 + 1] = (byte) (c >> 8); + r.d[p * 4 + 2] = (byte) c; + r.d[p * 4 + 3] = (byte) (c >>> 24); + } + return wrap(r); + })); + m.setAttr("native_encode", PyBuiltin.positional("native_encode", a -> { + Img s = convert(img(a[0]), 4); + int[] argb = new int[s.w * s.h]; + for (int p = 0; p < argb.length; p++) { + argb[p] = (s.d[p * 4 + 3] & 0xff) << 24 | (s.d[p * 4] & 0xff) << 16 + | (s.d[p * 4 + 1] & 0xff) << 8 | (s.d[p * 4 + 2] & 0xff); + } + com.codename1.ui.Image ci = com.codename1.ui.Image.createImage(argb, s.w, s.h); + com.codename1.ui.util.ImageIO io = com.codename1.ui.util.ImageIO.getImageIO(); + if (io == null) { + throw Py.error(Py.OS_ERROR, "this platform cannot encode " + Py.toStr(a[1])); + } + ByteArrayOutputStream bo = new ByteArrayOutputStream(); + try { + io.save(ci, bo, Py.toStr(a[1]), (float) Py.asDouble(a[2])); + } catch (java.io.IOException e) { + throw Py.error(Py.OS_ERROR, String.valueOf(e.getMessage())); + } + return new PyBytes(bo.toByteArray()); + })); + return m; + } + + private static Img crop(Img s, int x0, int y0, int x1, int y1) { + Img r = new Img(Math.max(0, x1 - x0), Math.max(0, y1 - y0), s.bands); + for (int y = 0; y < r.h; y++) { + int sy = y + y0; + if (sy < 0 || sy >= s.h) { + continue; + } + for (int x = 0; x < r.w; x++) { + int sx = x + x0; + if (sx >= 0 && sx < s.w) { + System.arraycopy(s.d, (sy * s.w + sx) * s.bands, r.d, (y * r.w + x) * s.bands, s.bands); + } + } + } + return r; + } + + /// Pillow's `DIV255(in1 * (255 - mask) + in2 * mask)`. + private static int blend(int out, int in, int mask) { + int t = out * (255 - mask) + in * mask + 128; + return ((t >> 8) + t) >> 8; + } + + private static int maskAt(Img mask, int x, int y) { + return mask.d[(y * mask.w + x) * mask.bands + mask.bands - 1] & 0xff; + } + + private static void fill(Img s, int x0, int y0, int x1, int y1, byte[] color, Img mask) { + int ax = Math.max(0, x0); + int ay = Math.max(0, y0); + int bx = Math.min(s.w, x1); + int by = Math.min(s.h, y1); + for (int y = ay; y < by; y++) { + for (int x = ax; x < bx; x++) { + int p = (y * s.w + x) * s.bands; + if (mask == null) { + System.arraycopy(color, 0, s.d, p, s.bands); + } else { + int k = maskAt(mask, x - x0, y - y0); + for (int b = 0; b < s.bands; b++) { + s.d[p + b] = (byte) blend(s.d[p + b] & 0xff, color[b] & 0xff, k); + } + } + } + } + } + + private static void paste(Img dst, Img src, int x0, int y0, Img mask) { + for (int y = 0; y < src.h; y++) { + int dy = y + y0; + if (dy < 0 || dy >= dst.h) { + continue; + } + for (int x = 0; x < src.w; x++) { + int dx = x + x0; + if (dx < 0 || dx >= dst.w) { + continue; + } + int p = (dy * dst.w + dx) * dst.bands; + int q = (y * src.w + x) * src.bands; + if (mask == null) { + System.arraycopy(src.d, q, dst.d, p, dst.bands); + } else { + int k = maskAt(mask, x, y); + for (int b = 0; b < dst.bands; b++) { + dst.d[p + b] = (byte) blend(dst.d[p + b] & 0xff, src.d[q + b] & 0xff, k); + } + } + } + } + } + + /// Pillow's `Transpose` values: 0 flip left-right, 1 flip top-bottom, + /// 2 rotate 90, 3 rotate 180, 4 rotate 270, 5 transpose, 6 transverse. + private static Img transpose(Img s, int op) { + boolean swap = op == 2 || op == 4 || op == 5 || op == 6; + Img r = swap ? new Img(s.h, s.w, s.bands) : new Img(s.w, s.h, s.bands); + for (int y = 0; y < r.h; y++) { + for (int x = 0; x < r.w; x++) { + int sx; + int sy; + switch (op) { + case 0: sx = s.w - 1 - x; sy = y; break; + case 1: sx = x; sy = s.h - 1 - y; break; + case 2: sx = s.w - 1 - y; sy = x; break; + case 3: sx = s.w - 1 - x; sy = s.h - 1 - y; break; + case 4: sx = y; sy = s.h - 1 - x; break; + case 5: sx = y; sy = x; break; + case 6: sx = s.w - 1 - y; sy = s.h - 1 - x; break; + default: throw Py.error(Py.VALUE_ERROR, "unknown transpose method"); + } + System.arraycopy(s.d, (sy * s.w + sx) * s.bands, r.d, (y * r.w + x) * s.bands, s.bands); + } + } + return r; + } + + /// Pillow's `COORD`: truncation, with every negative coordinate outside. + private static int coord(double v) { + return v < 0.0 ? -1 : (int) v; + } + + /// Nearest neighbour affine transform, accumulating the coordinates the + /// way Pillow's `affine_transform` does so the rounding is the same. + private static Img affine(Img s, int w, int h, double[] a, byte[] fill) { + Img r = new Img(w, h, s.bands); + double xo = a[2] + a[1] * 0.5 + a[0] * 0.5; + double yo = a[5] + a[4] * 0.5 + a[3] * 0.5; + for (int y = 0; y < h; y++) { + double xx = xo; + double yy = yo; + for (int x = 0; x < w; x++) { + int xin = coord(xx); + int yin = coord(yy); + int p = (y * w + x) * s.bands; + if (xin >= 0 && yin >= 0 && xin < s.w && yin < s.h) { + System.arraycopy(s.d, (yin * s.w + xin) * s.bands, r.d, p, s.bands); + } else { + System.arraycopy(fill, 0, r.d, p, s.bands); + } + xx += a[0]; + yy += a[3]; + } + xo += a[1]; + yo += a[4]; + } + return r; + } + + /// Bilinear coefficients for one axis, as Pillow's `precompute_coeffs` + /// and `normalize_coeffs_8bpc` make them. Returns `ksize`; `bounds` + /// receives `xmin, count` per output position. + private static int coeffs(int inSize, double in0, double in1, int outSize, int[] bounds, int[][] out) { + double scale = (in1 - in0) / outSize; + double filterscale = scale < 1.0 ? 1.0 : scale; + double support = filterscale; + int ksize = (int) Math.ceil(support) * 2 + 1; + int[] kk = new int[outSize * ksize]; + double[] k = new double[ksize]; + for (int xx = 0; xx < outSize; xx++) { + double center = in0 + (xx + 0.5) * scale; + double ww = 0.0; + double ss = 1.0 / filterscale; + int xmin = (int) (center - support + 0.5); + if (xmin < 0) { + xmin = 0; + } + int xmax = (int) (center + support + 0.5); + if (xmax > inSize) { + xmax = inSize; + } + xmax -= xmin; + for (int x = 0; x < xmax; x++) { + double t = (x + xmin - center + 0.5) * ss; + if (t < 0) { + t = -t; + } + double w = t < 1.0 ? 1.0 - t : 0.0; + k[x] = w; + ww += w; + } + for (int x = 0; x < xmax; x++) { + double v = ww != 0.0 ? k[x] / ww : k[x]; + kk[xx * ksize + x] = v < 0 ? (int) (-0.5 + v * (1 << PRECISION_BITS)) + : (int) (0.5 + v * (1 << PRECISION_BITS)); + } + bounds[xx * 2] = xmin; + bounds[xx * 2 + 1] = xmax; + } + out[0] = kk; + return ksize; + } + + private static byte clip8(int v) { + int s = v >> PRECISION_BITS; + return (byte) (s < 0 ? 0 : s > 255 ? 255 : s); + } + + private static Img resample(Img s, int w, int h, double[] box) { + Img cur = s; + boolean horizontal = w != s.w || box[0] != 0 || box[2] != w; + boolean vertical = h != s.h || box[1] != 0 || box[3] != h; + if (horizontal) { + int[] bounds = new int[w * 2]; + int[][] kh = new int[1][]; + int ks = coeffs(cur.w, box[0], box[2], w, bounds, kh); + Img r = new Img(w, cur.h, cur.bands); + for (int y = 0; y < cur.h; y++) { + for (int x = 0; x < w; x++) { + int xmin = bounds[x * 2]; + int n = bounds[x * 2 + 1]; + for (int b = 0; b < cur.bands; b++) { + int ss = 1 << (PRECISION_BITS - 1); + for (int j = 0; j < n; j++) { + ss += (cur.d[(y * cur.w + xmin + j) * cur.bands + b] & 0xff) * kh[0][x * ks + j]; + } + r.d[(y * w + x) * cur.bands + b] = clip8(ss); + } + } + } + cur = r; + } + if (vertical) { + int[] bounds = new int[h * 2]; + int[][] kv = new int[1][]; + int ks = coeffs(cur.h, box[1], box[3], h, bounds, kv); + Img r = new Img(cur.w, h, cur.bands); + for (int y = 0; y < h; y++) { + int ymin = bounds[y * 2]; + int n = bounds[y * 2 + 1]; + for (int x = 0; x < cur.w; x++) { + for (int b = 0; b < cur.bands; b++) { + int ss = 1 << (PRECISION_BITS - 1); + for (int j = 0; j < n; j++) { + ss += (cur.d[((ymin + j) * cur.w + x) * cur.bands + b] & 0xff) * kv[0][y * ks + j]; + } + r.d[(y * cur.w + x) * cur.bands + b] = clip8(ss); + } + } + } + cur = r; + } + if (cur == s) { //NOPMD CompareObjectsWithEquals - identity is meant + cur = crop(s, 0, 0, s.w, s.h); + } + return cur; + } + + private static Img convert(Img s, int bands) { + Img r = new Img(s.w, s.h, bands); + for (int p = 0, n = s.w * s.h; p < n; p++) { + int q = p * s.bands; + int cr = s.d[q] & 0xff; + int cg = s.bands == 1 ? cr : s.d[q + 1] & 0xff; + int cb = s.bands == 1 ? cr : s.d[q + 2] & 0xff; + int ca = s.bands == 4 ? s.d[q + 3] & 0xff : 255; + int o = p * bands; + if (bands == 1) { + // Pillow's L24: ITU-R 601-2 luma in 16-bit fixed point + r.d[o] = (byte) ((cr * 19595 + cg * 38470 + cb * 7471 + 0x8000) >> 16); + } else { + r.d[o] = (byte) cr; + r.d[o + 1] = (byte) cg; + r.d[o + 2] = (byte) cb; + if (bands == 4) { + r.d[o + 3] = (byte) ca; + } + } + } + return r; + } + + private static Object getbbox(Img s, boolean alphaOnly) { + int x0 = s.w; + int y0 = s.h; + int x1 = -1; + int y1 = -1; + int from = alphaOnly && s.bands == 4 ? 3 : 0; + for (int y = 0; y < s.h; y++) { + for (int x = 0; x < s.w; x++) { + int p = (y * s.w + x) * s.bands; + boolean set = false; + for (int b = from; b < s.bands; b++) { + if (s.d[p + b] != 0) { + set = true; + break; + } + } + if (set) { + x0 = Math.min(x0, x); + y0 = Math.min(y0, y); + x1 = Math.max(x1, x); + y1 = Math.max(y1, y); + } + } + } + if (x1 < 0) { + return Py.NONE; + } + return new PyTuple(new Object[] {PyInt.of(x0), PyInt.of(y0), PyInt.of(x1 + 1), PyInt.of(y1 + 1)}); + } + + /// One horizontal pass of Pillow's box blur for an integer radius. + private static Img blurRows(Img s, int r) { + Img o = new Img(s.w, s.h, s.bands); + int ww = (1 << 24) / (r * 2 + 1); + int fw = ((1 << 24) - (r * 2 + 1) * ww) / 2; + int last = s.w - 1; + for (int y = 0; y < s.h; y++) { + for (int b = 0; b < s.bands; b++) { + int base = y * s.w * s.bands + b; + for (int x = 0; x < s.w; x++) { + long acc = 0; + for (int k = x - r; k <= x + r; k++) { + acc += s.d[base + Math.max(0, Math.min(last, k)) * s.bands] & 0xff; + } + long edge = (s.d[base + Math.max(0, Math.min(last, x - r - 1)) * s.bands] & 0xff) + + (s.d[base + Math.max(0, Math.min(last, x + r + 1)) * s.bands] & 0xff); + o.d[base + x * s.bands] = (byte) ((acc * ww + edge * fw + (1 << 23)) >> 24); + } + } + } + return o; + } + + private static int paeth(int a, int b, int c) { + int p = a + b - c; + int pa = Math.abs(p - a); + int pb = Math.abs(p - b); + int pc = Math.abs(p - c); + if (pa <= pb && pa <= pc) { + return a; + } + return pb <= pc ? b : c; + } + + /// Reverses the five PNG row filters; `data` holds `h` rows of a filter + /// byte followed by `row` bytes, and `bpp` is the filter's byte distance. + private static byte[] unfilter(byte[] data, int row, int h, int bpp) { + if (data.length < (row + 1) * h) { + throw Py.error(Py.OS_ERROR, "image file is truncated"); + } + byte[] out = new byte[row * h]; + for (int y = 0; y < h; y++) { + int ft = data[y * (row + 1)] & 0xff; + int in = y * (row + 1) + 1; + int o = y * row; + for (int x = 0; x < row; x++) { + int raw = data[in + x] & 0xff; + int a = x >= bpp ? out[o + x - bpp] & 0xff : 0; + int b = y > 0 ? out[o - row + x] & 0xff : 0; + int c = x >= bpp && y > 0 ? out[o - row + x - bpp] & 0xff : 0; + int v; + switch (ft) { + case 0: v = raw; break; + case 1: v = raw + a; break; + case 2: v = raw + b; break; + case 3: v = raw + ((a + b) >> 1); break; + case 4: v = raw + paeth(a, b, c); break; + default: throw Py.error(Py.OS_ERROR, "unknown PNG filter type " + ft); + } + out[o + x] = (byte) v; + } + } + return out; + } + + private static int sample(byte[] raw, int rowStart, int index, int depth) { + if (depth == 8) { + return raw[rowStart + index] & 0xff; + } + if (depth == 16) { + return raw[rowStart + index * 2] & 0xff; + } + int bit = index * depth; + int v = (raw[rowStart + (bit >> 3)] & 0xff) >> (8 - depth - (bit & 7)) & ((1 << depth) - 1); + return v; + } + + /// Unpacks unfiltered PNG rows into an image: gray to 1 band, RGB to 3, + /// gray+alpha, RGBA and palette (to 3, or 4 with a transparency chunk). + private static Img expand(byte[] raw, int w, int h, int ctype, int depth, byte[] plte, byte[] trns) { + int channels = ctype == 0 || ctype == 3 ? 1 : ctype == 2 ? 3 : ctype == 4 ? 2 : 4; + int row = (w * channels * depth + 7) / 8; + int bands = ctype == 0 ? 1 : ctype == 2 ? 3 : ctype == 3 && trns == null ? 3 : 4; + Img r = new Img(w, h, bands); + for (int y = 0; y < h; y++) { + int rs = y * row; + for (int x = 0; x < w; x++) { + int o = (y * w + x) * bands; + if (ctype == 3) { + int idx = sample(raw, rs, x, depth); + for (int b = 0; b < 3; b++) { + r.d[o + b] = plte != null && idx * 3 + b < plte.length ? plte[idx * 3 + b] : 0; + } + if (bands == 4) { + r.d[o + 3] = idx < trns.length ? trns[idx] : (byte) 255; + } + continue; + } + int scale = depth < 8 ? 255 / ((1 << depth) - 1) : 1; + if (ctype == 4) { + int g = sample(raw, rs, x * 2, depth); + r.d[o] = (byte) g; + r.d[o + 1] = (byte) g; + r.d[o + 2] = (byte) g; + r.d[o + 3] = (byte) sample(raw, rs, x * 2 + 1, depth); + continue; + } + for (int b = 0; b < channels; b++) { + r.d[o + b] = (byte) (sample(raw, rs, x * channels + b, depth) * scale); + } + } + } + return r; + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java index b5e0a91eeb6..37aa74940ef 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java @@ -82,6 +82,10 @@ static boolean provide(String name) { PyModule.registerModule(PyZlib.module()); return true; } + if ("_cn1image".equals(name)) { + PyModule.registerModule(PyImage.module()); + return true; + } if ("_cn1tz".equals(name)) { PyModule.registerModule(tz()); return true; diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/Image.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/Image.py new file mode 100644 index 00000000000..9b75c7e8e8f --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/Image.py @@ -0,0 +1,475 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Pillow's Image module for "L", "RGB" and "RGBA" images. + +The pixel loops follow Pillow's own fixed point code (luma conversion, +bilinear resampling, nearest neighbour affine transforms, mask blending), so +pixels come out byte identical. Bicubic, Lanczos, box and Hamming resampling +are done bilinearly. PNG decoding returns "P" and "LA" files as "RGB"/"RGBA". +""" + +import builtins +import math +import os +import struct +import zlib + +import _cn1image +from PIL import ImageColor, UnidentifiedImageError + + +class Resampling: + NEAREST = 0 + LANCZOS = 1 + BILINEAR = 2 + BICUBIC = 3 + BOX = 4 + HAMMING = 5 + + +class Transpose: + FLIP_LEFT_RIGHT = 0 + FLIP_TOP_BOTTOM = 1 + ROTATE_90 = 2 + ROTATE_180 = 3 + ROTATE_270 = 4 + TRANSPOSE = 5 + TRANSVERSE = 6 + + +NEAREST = Resampling.NEAREST +LANCZOS = ANTIALIAS = Resampling.LANCZOS +BILINEAR = Resampling.BILINEAR +BICUBIC = Resampling.BICUBIC +BOX = Resampling.BOX +HAMMING = Resampling.HAMMING +FLIP_LEFT_RIGHT = Transpose.FLIP_LEFT_RIGHT +FLIP_TOP_BOTTOM = Transpose.FLIP_TOP_BOTTOM +ROTATE_90 = Transpose.ROTATE_90 +ROTATE_180 = Transpose.ROTATE_180 +ROTATE_270 = Transpose.ROTATE_270 +TRANSPOSE = Transpose.TRANSPOSE +TRANSVERSE = Transpose.TRANSVERSE + +_BANDS = {"L": 1, "RGB": 3, "RGBA": 4} +_BAND_NAMES = {"L": ("L",), "RGB": ("R", "G", "B"), "RGBA": ("R", "G", "B", "A")} +_PNG_SIG = b"\x89PNG\r\n\x1a\n" +_EXT = {".png": "PNG", ".jpg": "JPEG", ".jpeg": "JPEG", ".gif": "GIF", ".bmp": "BMP", ".webp": "WEBP"} + + +def _bands(mode): + if mode not in _BANDS: + raise ValueError("unrecognized image mode: %s (only L, RGB and RGBA are supported)" % mode) + return _BANDS[mode] + + +def _color(mode, color): + """The pixel bytes of `color` in `mode`.""" + n = _bands(mode) + if color is None: + return bytes(n) + if isinstance(color, str): + color = ImageColor.getcolor(color, mode) + if isinstance(color, float): + color = int(color) + if isinstance(color, int): + if n == 1: + return bytes([color & 255]) + return bytes([(color >> (8 * i)) & 255 for i in range(n)]) + c = [int(v) for v in color] + if n == 1: + return bytes([c[0] & 255]) + if len(c) == 3 and n == 4: + c.append(255) + return bytes([v & 255 for v in c[:n]]) + + +def _wrap(mode, size, handle): + im = Image() + im.mode = mode + im._size = (int(size[0]), int(size[1])) + im._im = handle + return im + + +class Image: + format = None + + def __init__(self): + self.mode = "" + self._size = (0, 0) + self._im = None + self.info = {} + + @property + def size(self): + return self._size + + @property + def width(self): + return self._size[0] + + @property + def height(self): + return self._size[1] + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + + def close(self): + pass + + def load(self): + return None + + def getbands(self): + return _BAND_NAMES[self.mode] + + def _new(self, handle, size=None, mode=None): + return _wrap(mode or self.mode, size or self._size, handle) + + def copy(self): + return self._new(_cn1image.crop(self._im, 0, 0, self.width, self.height)) + + def getpixel(self, xy): + x, y = int(xy[0]), int(xy[1]) + if x < 0: + x += self.width + if y < 0: + y += self.height + t = _cn1image.getpixel(self._im, x, y) + return t[0] if self.mode == "L" else t + + def putpixel(self, xy, value): + x, y = int(xy[0]), int(xy[1]) + if x < 0: + x += self.width + if y < 0: + y += self.height + _cn1image.putpixel(self._im, x, y, _color(self.mode, value)) + + def getdata(self, band=None): + raw = self.tobytes() + n = _bands(self.mode) + if n == 1: + return list(raw) + if band is not None: + return list(raw[band::n]) + return [tuple(raw[i:i + n]) for i in range(0, len(raw), n)] + + def histogram(self): + raw = self.tobytes() + n = _bands(self.mode) + h = [0] * (256 * n) + for i, v in enumerate(raw): + h[(i % n) * 256 + v] += 1 + return h + + def getextrema(self): + n = _bands(self.mode) + raw = self.tobytes() + ext = tuple((min(raw[b::n]), max(raw[b::n])) for b in range(n)) + return ext[0] if n == 1 else ext + + def tobytes(self, encoder_name="raw", *args): + return _cn1image.tobytes(self._im) + + def frombytes(self, data): + self._im = _cn1image.frombytes(self.width, self.height, _bands(self.mode), data) + + def crop(self, box=None): + if box is None: + return self.copy() + x0, y0, x1, y1 = (int(round(v)) for v in box) + if x1 < x0: + raise ValueError("Coordinate 'right' is less than 'left'") + if y1 < y0: + raise ValueError("Coordinate 'lower' is less than 'upper'") + return self._new(_cn1image.crop(self._im, x0, y0, x1, y1), (x1 - x0, y1 - y0)) + + def resize(self, size, resample=None, box=None, reducing_gap=None): + w, h = int(size[0]), int(size[1]) + if box is None: + box = (0, 0) + self._size + if resample is None: + resample = BICUBIC + if (w, h) == self._size and tuple(box) == (0, 0) + self._size: + return self.copy() + if resample == NEAREST: + a = [(box[2] - box[0]) / w, 0, box[0], 0, (box[3] - box[1]) / h, box[1]] + return self._new(_cn1image.affine(self._im, w, h, a, bytes(4)), (w, h)) + return self._new(_cn1image.resample(self._im, w, h, tuple(float(v) for v in box)), (w, h)) + + def thumbnail(self, size, resample=BICUBIC, reducing_gap=2.0): + x, y = math.floor(size[0]), math.floor(size[1]) + if x >= self.width and y >= self.height: + return + aspect = self.width / self.height + + def round_aspect(number, key): + return max(min(math.floor(number), math.ceil(number), key=key), 1) + + if x / y >= aspect: + x = round_aspect(y * aspect, key=lambda n: abs(aspect - n / y)) + else: + y = round_aspect(x / aspect, key=lambda n: 0 if n == 0 else abs(aspect - x / n)) + im = self.resize((x, y), resample) + self._im = im._im + self._size = im._size + + def transpose(self, method): + h = _cn1image.transpose(self._im, int(method)) + w, ht = self._size + size = (ht, w) if method in (ROTATE_90, ROTATE_270, TRANSPOSE, TRANSVERSE) else (w, ht) + return self._new(h, size) + + def transform(self, size, method, data=None, resample=NEAREST, fill=1, fillcolor=None): + if method != 0: + raise ValueError("only AFFINE transforms are supported") + w, h = int(size[0]), int(size[1]) + return self._new(_cn1image.affine(self._im, w, h, [float(v) for v in data], + _color(self.mode, fillcolor)), (w, h)) + + def rotate(self, angle, resample=NEAREST, expand=0, center=None, translate=None, fillcolor=None): + angle = angle % 360.0 + if not (center or translate): + if angle == 0: + return self.copy() + if angle == 180: + return self.transpose(ROTATE_180) + if angle in (90, 270) and (expand or self.width == self.height): + return self.transpose(ROTATE_90 if angle == 90 else ROTATE_270) + w, h = self.size + post_trans = (0, 0) if translate is None else translate + if center is None: + center = (w / 2, h / 2) + angle = -math.radians(angle) + matrix = [round(math.cos(angle), 15), round(math.sin(angle), 15), 0.0, + round(-math.sin(angle), 15), round(math.cos(angle), 15), 0.0] + + def transform(x, y, m): + a, b, c, d, e, f = m + return a * x + b * y + c, d * x + e * y + f + + matrix[2], matrix[5] = transform(-center[0] - post_trans[0], -center[1] - post_trans[1], matrix) + matrix[2] += center[0] + matrix[5] += center[1] + if expand: + xx = [] + yy = [] + for x, y in ((0, 0), (w, 0), (w, h), (0, h)): + tx, ty = transform(x, y, matrix) + xx.append(tx) + yy.append(ty) + nw = math.ceil(max(xx)) - math.floor(min(xx)) + nh = math.ceil(max(yy)) - math.floor(min(yy)) + matrix[2], matrix[5] = transform(-(nw - w) / 2.0, -(nh - h) / 2.0, matrix) + w, h = nw, nh + return self.transform((w, h), 0, matrix, resample, fillcolor=fillcolor) + + def convert(self, mode=None, matrix=None, dither=None, palette=None, colors=256): + if mode is None or mode == self.mode: + return self.copy() + return self._new(_cn1image.convert(self._im, _bands(mode)), mode=mode) + + def paste(self, im, box=None, mask=None): + if box is not None and len(box) == 2 and isinstance(im, Image): + box = tuple(box) + (box[0] + im.width, box[1] + im.height) + if box is None: + box = (0, 0) + (im.size if isinstance(im, Image) else self._size) + x0, y0 = int(box[0]), int(box[1]) + m = None if mask is None else mask._im + if isinstance(im, Image): + if len(box) == 4 and (box[2] - box[0], box[3] - box[1]) != im.size: + raise ValueError("images do not match") + if im.mode != self.mode: + im = im.convert(self.mode) + _cn1image.paste(self._im, im._im, x0, y0, m) + else: + x1, y1 = (int(box[2]), int(box[3])) if len(box) == 4 else (mask.width + x0, mask.height + y0) + _cn1image.fill(self._im, x0, y0, x1, y1, _color(self.mode, im), m) + + def alpha_composite(self, im, dest=(0, 0), source=(0, 0)): + raise NotImplementedError("alpha_composite is not supported") + + def split(self): + if self.mode == "L": + return (self.copy(),) + return tuple(self._new(_cn1image.band(self._im, b), mode="L") for b in range(_bands(self.mode))) + + def getchannel(self, channel): + if isinstance(channel, str): + channel = self.getbands().index(channel) + return self._new(_cn1image.band(self._im, channel), mode="L") + + def putalpha(self, alpha): + if self.mode != "RGBA": + rgba = self.convert("RGBA") + self.mode, self._im = "RGBA", rgba._im + if not isinstance(alpha, Image): + alpha = new("L", self._size, alpha) + r, g, b, _ = self.split() + self._im = merge("RGBA", (r, g, b, alpha))._im + + def getbbox(self, alpha_only=True): + return _cn1image.getbbox(self._im, alpha_only) + + def point(self, lut, mode=None): + n = _bands(self.mode) + if callable(lut): + lut = [lut(i) for i in range(256)] * n + else: + lut = list(lut) + if len(lut) == 256: + lut = lut * n + if len(lut) != 256 * n: + raise ValueError("wrong number of lut entries") + table = bytes(max(0, min(255, int(v))) for v in lut) + return self._new(_cn1image.point(self._im, table)) + + def filter(self, f): + if isinstance(f, type): + f = f() + return f.filter(self) + + def save(self, fp, format=None, **params): + filename = None + if isinstance(fp, (str, os.PathLike)): + filename = os.fspath(fp) + if format is None: + ext = os.path.splitext(filename or "")[1].lower() + if ext not in _EXT: + raise ValueError("unknown file extension: " + ext) + format = _EXT[ext] + format = format.upper() + if format == "PNG": + data = _png_encode(self) + else: + data = _native_encode(self, format, params.get("quality", 75)) + if filename is not None: + with builtins.open(filename, "wb") as f: + f.write(data) + else: + fp.write(data) + + def show(self, title=None): + pass + + def __eq__(self, other): + return (isinstance(other, Image) and self.mode == other.mode + and self.size == other.size and self.tobytes() == other.tobytes()) + + def __repr__(self): + return "" % (self.mode, self.width, self.height) + + +def new(mode, size, color=0): + w, h = int(size[0]), int(size[1]) + return _wrap(mode, (w, h), _cn1image.new(w, h, _bands(mode), _color(mode, color))) + + +def frombytes(mode, size, data, decoder_name="raw", *args): + w, h = int(size[0]), int(size[1]) + return _wrap(mode, (w, h), _cn1image.frombytes(w, h, _bands(mode), data)) + + +def merge(mode, bands): + if len(bands) != _bands(mode): + raise ValueError("wrong number of bands") + for b in bands: + if b.mode != "L": + raise ValueError("mode mismatch") + if mode == "L": + return bands[0].copy() + return _wrap(mode, bands[0].size, _cn1image.merge([b._im for b in bands])) + + +def open(fp, mode="r", formats=None): + if isinstance(fp, (str, os.PathLike)): + with builtins.open(os.fspath(fp), "rb") as f: + data = f.read() + else: + data = fp.read() + if data[:8] == _PNG_SIG: + im = _png_decode(data) + im.format = "PNG" + return im + kind = None + if data[:3] == b"\xff\xd8\xff": + kind = "JPEG" + elif data[:4] == b"GIF8": + kind = "GIF" + elif data[:2] == b"BM": + kind = "BMP" + elif data[:4] == b"RIFF" and data[8:12] == b"WEBP": + kind = "WEBP" + if kind is None: + raise UnidentifiedImageError("cannot identify image file %r" % (fp if isinstance(fp, str) else fp,)) + if not _cn1image.native_available(): + raise OSError(kind + " needs a running Codename One display to decode; only PNG is built in") + handle = _cn1image.native_decode(data) + im = _wrap("RGBA", _cn1image.size(handle), handle) + im.format = kind + if kind in ("JPEG", "BMP"): + im = im.convert("RGB") + im.format = kind + return im + + +def _native_encode(im, format, quality): + if not _cn1image.native_available(): + raise OSError(format + " needs a running Codename One display to encode; only PNG is built in") + if format == "JPEG" and im.mode == "RGBA": + raise OSError("cannot write mode RGBA as JPEG") + return _cn1image.native_encode(im._im, format.lower(), quality / 100.0) + + +def _chunk(kind, body): + return struct.pack(">I", len(body)) + kind + body + struct.pack(">I", zlib.crc32(kind + body) & 0xffffffff) + + +def _png_encode(im): + ctype = {"L": 0, "RGB": 2, "RGBA": 6}[im.mode] + head = struct.pack(">IIBBBBB", im.width, im.height, 8, ctype, 0, 0, 0) + body = zlib.compress(_cn1image.png_rows(im._im), 6) + return _PNG_SIG + _chunk(b"IHDR", head) + _chunk(b"IDAT", body) + _chunk(b"IEND", b"") + + +def _png_decode(data): + pos = 8 + idat = [] + plte = None + trns = None + head = None + while pos + 8 <= len(data): + length, kind = struct.unpack(">I4s", data[pos:pos + 8]) + body = data[pos + 8:pos + 8 + length] + pos += 12 + length + if kind == b"IHDR": + head = struct.unpack(">IIBBBBB", body) + elif kind == b"PLTE": + plte = bytes(body) + elif kind == b"tRNS": + trns = bytes(body) + elif kind == b"IDAT": + idat.append(body) + elif kind == b"IEND": + break + if head is None: + raise UnidentifiedImageError("cannot identify image file") + w, h, depth, ctype, _, _, interlace = head + if interlace: + raise OSError("interlaced PNG files are not supported") + channels = {0: 1, 2: 3, 3: 1, 4: 2, 6: 4}[ctype] + row = (w * channels * depth + 7) // 8 + bpp = max(1, channels * depth // 8) + raw = _cn1image.png_unfilter(zlib.decompress(b"".join(idat)), row, h, bpp) + if ctype != 3: + trns = None + handle = _cn1image.png_expand(raw, w, h, ctype, depth, plte, trns) + mode = "L" if ctype == 0 else "RGB" if ctype == 2 or (ctype == 3 and trns is None) else "RGBA" + return _wrap(mode, (w, h), handle) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/ImageColor.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/ImageColor.py new file mode 100644 index 00000000000..73644f71b93 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/ImageColor.py @@ -0,0 +1,38 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Color names and `#rgb` strings, as Pillow's ImageColor reads them.""" + +colormap = { + "black": (0, 0, 0), "white": (255, 255, 255), "red": (255, 0, 0), + "lime": (0, 255, 0), "green": (0, 128, 0), "blue": (0, 0, 255), + "yellow": (255, 255, 0), "cyan": (0, 255, 255), "aqua": (0, 255, 255), + "magenta": (255, 0, 255), "fuchsia": (255, 0, 255), "gray": (128, 128, 128), + "grey": (128, 128, 128), "silver": (192, 192, 192), "maroon": (128, 0, 0), + "olive": (128, 128, 0), "navy": (0, 0, 128), "purple": (128, 0, 128), + "teal": (0, 128, 128), "orange": (255, 165, 0), "pink": (255, 192, 203), + "brown": (165, 42, 42), "gold": (255, 215, 0), "transparent": (0, 0, 0, 0), +} + + +def getrgb(color): + c = color.lower() + if c in colormap: + return colormap[c] + if c.startswith("#"): + h = c[1:] + if len(h) in (3, 4): + return tuple(int(ch * 2, 16) for ch in h) + if len(h) in (6, 8): + return tuple(int(h[i:i + 2], 16) for i in range(0, len(h), 2)) + raise ValueError("unknown color specifier: %r" % color) + + +def getcolor(color, mode): + rgb = getrgb(color) + alpha = rgb[3] if len(rgb) == 4 else 255 + if mode == "L": + return (rgb[0] * 299 + rgb[1] * 587 + rgb[2] * 114) // 1000 + if mode == "RGBA": + return rgb[:3] + (alpha,) + return rgb[:3] diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/ImageDraw.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/ImageDraw.py new file mode 100644 index 00000000000..3551bf52c08 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/ImageDraw.py @@ -0,0 +1,177 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Pillow's ImageDraw: points, lines, rectangles, ellipses and polygons. + +Points, rectangles and horizontal or vertical lines match Pillow pixel for +pixel. Diagonal lines are Bresenham's, and ellipses and polygons are filled +by pixel centre, so their edges can differ from Pillow's by a pixel. Nothing +is antialiased and there is no text. +""" + +import _cn1image +from PIL import Image as _Image + + +def _points(xy): + xy = list(xy) + if xy and not isinstance(xy[0], (tuple, list)): + return [(xy[i], xy[i + 1]) for i in range(0, len(xy), 2)] + return [tuple(p) for p in xy] + + +def _box(xy): + p = _points(xy) + (x0, y0), (x1, y1) = p[0], p[1] + if x1 < x0: + raise ValueError("x1 must be greater than or equal to x0") + if y1 < y0: + raise ValueError("y1 must be greater than or equal to y0") + return int(x0), int(y0), int(x1), int(y1) + + +class ImageDraw: + def __init__(self, im, mode=None): + self.im = im + self.mode = im.mode + self.ink = _Image._color(im.mode, -1) + self.fill = False + + def _ink(self, color): + return None if color is None else _Image._color(self.mode, color) + + def _rect(self, x0, y0, x1, y1, ink): + # inclusive corners + _cn1image.fill(self.im._im, x0, y0, x1 + 1, y1 + 1, ink, None) + + def _inks(self, outline, fill): + ink = self._ink(outline) + fill_ink = self._ink(fill) + if ink is None and fill_ink is None: + if self.fill: + fill_ink = self.ink + else: + ink = self.ink + return ink, fill_ink + + def point(self, xy, fill=None): + ink = self._ink(fill) if fill is not None else self.ink + for x, y in _points(xy): + x, y = int(x), int(y) + self._rect(x, y, x, y, ink) + + def _segment(self, x0, y0, x1, y1, ink, width): + if y0 == y1 or x0 == x1: + if x1 < x0: + x0, x1 = x1, x0 + if y1 < y0: + y0, y1 = y1, y0 + lo = (width - 1) // 2 + hi = width - 1 - lo + if y0 == y1: + self._rect(x0, y0 - lo, x1, y0 + hi, ink) + else: + self._rect(x0 - lo, y0, x0 + hi, y1, ink) + return + dx = abs(x1 - x0) + dy = -abs(y1 - y0) + sx = 1 if x0 < x1 else -1 + sy = 1 if y0 < y1 else -1 + err = dx + dy + while True: + self._rect(x0, y0, x0, y0, ink) + if x0 == x1 and y0 == y1: + return + e2 = 2 * err + if e2 >= dy: + err += dy + x0 += sx + if e2 <= dx: + err += dx + y0 += sy + + def line(self, xy, fill=None, width=0, joint=None): + ink = self._ink(fill) if fill is not None else self.ink + p = [(int(x), int(y)) for x, y in _points(xy)] + for i in range(len(p) - 1): + self._segment(p[i][0], p[i][1], p[i + 1][0], p[i + 1][1], ink, max(1, width)) + + def rectangle(self, xy, fill=None, outline=None, width=1): + x0, y0, x1, y1 = _box(xy) + ink, fill_ink = self._inks(outline, fill) + if fill_ink is not None: + self._rect(x0, y0, x1, y1, fill_ink) + if ink is not None and ink != fill_ink and width != 0: + for i in range(width): + if x0 + i > x1 - i or y0 + i > y1 - i: + break + self._rect(x0 + i, y0 + i, x1 - i, y0 + i, ink) + self._rect(x0 + i, y1 - i, x1 - i, y1 - i, ink) + self._rect(x0 + i, y0 + i, x0 + i, y1 - i, ink) + self._rect(x1 - i, y0 + i, x1 - i, y1 - i, ink) + + def _spans(self, x0, y0, x1, y1): + cx = (x0 + x1 + 1) / 2.0 + cy = (y0 + y1 + 1) / 2.0 + rx = (x1 - x0 + 1) / 2.0 + ry = (y1 - y0 + 1) / 2.0 + for y in range(y0, y1 + 1): + t = (y + 0.5 - cy) / ry + if t * t > 1: + continue + half = rx * (1 - t * t) ** 0.5 + a = int(cx - half + 0.5) + b = int(cx + half - 0.5) + if b >= a: + yield y, a, b + + def ellipse(self, xy, fill=None, outline=None, width=1): + x0, y0, x1, y1 = _box(xy) + ink, fill_ink = self._inks(outline, fill) + spans = list(self._spans(x0, y0, x1, y1)) + if fill_ink is not None: + for y, a, b in spans: + self._rect(a, y, b, y, fill_ink) + if ink is not None and ink != fill_ink and width != 0: + inner = {} + if x1 - x0 >= 2 * width and y1 - y0 >= 2 * width: + for y, a, b in self._spans(x0 + width, y0 + width, x1 - width, y1 - width): + inner[y] = (a, b) + for y, a, b in spans: + if y in inner: + ia, ib = inner[y] + self._rect(a, y, ia - 1, y, ink) + self._rect(ib + 1, y, b, y, ink) + else: + self._rect(a, y, b, y, ink) + + def polygon(self, xy, fill=None, outline=None, width=1): + p = [(float(x), float(y)) for x, y in _points(xy)] + ink, fill_ink = self._inks(outline, fill) + if fill_ink is not None and len(p) > 2: + ys = [q[1] for q in p] + for y in range(int(min(ys)), int(max(ys)) + 1): + yc = y + 0.5 + xs = [] + for i in range(len(p)): + (ax, ay), (bx, by) = p[i], p[(i + 1) % len(p)] + if (ay <= yc < by) or (by <= yc < ay): + xs.append(ax + (yc - ay) * (bx - ax) / (by - ay)) + xs.sort() + for i in range(0, len(xs) - 1, 2): + a = int(xs[i] + 0.5) + b = int(xs[i + 1] - 0.5) + if b >= a: + self._rect(a, y, b, y, fill_ink) + if ink is not None and ink != fill_ink: + q = [(int(x), int(y)) for x, y in p] + for i in range(len(q)): + a, b = q[i], q[(i + 1) % len(q)] + self._segment(a[0], a[1], b[0], b[1], ink, max(1, width)) + + def text(self, xy, text, fill=None, font=None, *args, **kwargs): + raise NotImplementedError("ImageDraw.text is not supported") + + +def Draw(im, mode=None): + return ImageDraw(im, mode) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/ImageFilter.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/ImageFilter.py new file mode 100644 index 00000000000..8a166630da5 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/ImageFilter.py @@ -0,0 +1,26 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Pillow's ImageFilter: a box blur of a whole-number radius.""" + +import _cn1image + + +class Filter: + pass + + +class BoxBlur(Filter): + name = "BoxBlur" + + def __init__(self, radius): + if radius < 0: + raise ValueError("radius must be >= 0") + if radius != int(radius): + raise ValueError("only whole-number BoxBlur radii are supported") + self.radius = int(radius) + + def filter(self, image): + if self.radius == 0: + return image.copy() + return image._new(_cn1image.boxblur(image._im, self.radius)) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/ImageOps.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/ImageOps.py new file mode 100644 index 00000000000..dedacf6ffed --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/ImageOps.py @@ -0,0 +1,31 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""A few of Pillow's ImageOps helpers.""" + +from PIL import Image + + +def grayscale(image): + return image.convert("L") + + +def flip(image): + return image.transpose(Image.FLIP_TOP_BOTTOM) + + +def mirror(image): + return image.transpose(Image.FLIP_LEFT_RIGHT) + + +def invert(image): + return image.point(lambda v: 255 - v) + + +def expand(image, border=0, fill=0): + if isinstance(border, int): + border = (border,) * 4 + left, top, right, bottom = border + out = Image.new(image.mode, (image.width + left + right, image.height + top + bottom), fill) + out.paste(image, (left, top)) + return out diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/__init__.py new file mode 100644 index 00000000000..9e652241f9f --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/__init__.py @@ -0,0 +1,16 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""A Pillow subset: images in "L", "RGB" and "RGBA" with PNG built in. + +Pixels live in the runtime's `_cn1image` module, which also does every loop +over them. PNG is read and written here over zlib, so it works on servers as +well as devices; JPEG and the other formats go through Codename One's own +image codecs and need a running display. +""" + +__version__ = "11.0.0" + + +class UnidentifiedImageError(OSError): + pass diff --git a/maven/python-transpiler/src/test/resources/conformance/pil_basic.out b/maven/python-transpiler/src/test/resources/conformance/pil_basic.out new file mode 100644 index 00000000000..61c4d9dcd56 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/pil_basic.out @@ -0,0 +1,58 @@ +RGB (3, 2) 3 2 RGB (10, 20, 30) (10, 20, 30) +RGBA (3, 2) 3 2 RGBA (1, 2, 3, 4) (1, 2, 3, 4) +L (3, 2) 3 2 L 77 77 +RGB (3, 2) 3 2 RGB (255, 0, 0) (255, 0, 0) +L (3, 2) 3 2 L 173 173 +RGBA (3, 2) 3 2 RGBA (51, 102, 153, 255) (51, 102, 153, 255) +RGB (3, 2) 3 2 RGB (86, 52, 18) (86, 52, 18) +RGBA (3, 2) 3 2 RGBA (5, 6, 7, 255) (5, 6, 7, 255) +conv 30aeec5a8ff988b7 df8161af64f2aea4 ccb7940aec92d446 0c26e17e3685eeca ea98f2a3c2caa88a 4d0a15fb0e6b0f8c +px (155, 21, 67) 66 (180, 118, 51, 48) +bands ('R', 'G', 'B', 'A') ('R', 'G', 'B') ('L',) +crop (6, 4) aa74708a3c99af88 42c80fa961fbf570 +copy True +transpose 0 (11, 7) c97c423bedca7d8c +transpose 1 (11, 7) f49bf5f4057170fa +transpose 2 (7, 11) 21649e90f9cec3f8 +transpose 3 (11, 7) 6a18e2be07eb1c0a +transpose 4 (7, 11) 5389077858487ad2 +transpose 5 (7, 11) ef05d48ef38bc74f +transpose 6 (7, 11) d65d64a690273e9b +rotate 90 True (7, 11) 43ebef4f15b0c518 +rotate 180 False (11, 7) 2513ca399c3f379e +rotate 270 True (7, 11) 1dd1408b2b7e9867 +rotate 90 False (11, 7) aa08f0060bcf06d9 +rotate 30 False (11, 7) ac0c042f4cc55c71 +rotate 45 True (13, 13) a2129e8ffea62f24 +rotate -20 True (13, 11) 049f5525869c996f +rotate fill 5a2fe40f1e402296 +nearest (5, 3) edf6a00a5c80dfbe df605ceb850541b7 +bilinear (5, 3) 439f9218ec55a260 64fd8205dc6758e1 +nearest (22, 14) 7edd865aae132fc1 ebc29fe3f059de5b +bilinear (22, 14) 1583b294fd3d94bf bc1441eeb131db28 +nearest (11, 3) 283648da60b87f61 58443460c79c397c +bilinear (11, 3) 83433bab339353e5 ac14d803e1b24cde +nearest (4, 7) 5db553a6f9ae0108 a7f16bb4c185f97f +bilinear (4, 7) 3b0ac914ab6e9b62 d2a35ccddcc17e03 +nearest (17, 7) f5df00970f2b7c23 b3fb35a4457f7cd3 +bilinear (17, 7) d2660f07c74dcf75 59d723c0357512ff +box 5257779b9803d7df +thumb (6, 4) +thumb (128, 85) +paste c8a7a72c2b4812ca +paste color 3cecb217b58f4978 +paste mask 281a6dc7bf86231a +paste gray 0927c59cf4f36055 (38, 38, 38) +paste color mask 0738976c4ee66c00 +split L 099561e6763f8640 be7149bedee57af9 599b3d2d957d174c acd65f39e46d11a5 +merge RGB 3fbf4493276025dc (27, 10, 96) +channel acd65f39e46d11a5 be7149bedee57af9 +bytes 231 df8161af64f2aea4 +bbox empty None +bbox (3, 2, 8, 5) +bbox alpha (6, 6, 7, 7) (0, 0, 10, 10) +point 94223524082f80ea 8e172aff6e7a9022 +point lut 909f035184e2ada3 +data [1, 12, 25, 39, 54] (111, 9, 7) (1, 137) ((0, 255), (0, 118), (1, 247)) +hist 77 1 +eq True False diff --git a/maven/python-transpiler/src/test/resources/conformance/pil_basic.py b/maven/python-transpiler/src/test/resources/conformance/pil_basic.py new file mode 100644 index 00000000000..9fc4a6d2b94 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/pil_basic.py @@ -0,0 +1,94 @@ +# Pillow Image basics: modes, pixels, conversion, geometry, paste, bands. +import hashlib + +from PIL import Image + + +def h(im): + return hashlib.sha256(im.tobytes()).hexdigest()[:16] + + +def pattern(w, h_): + im = Image.new("RGBA", (w, h_)) + for y in range(h_): + for x in range(w): + im.putpixel((x, y), ((x * 37 + y * 11) % 256, (x * x + y * 3) % 256, + (x * y * 5 + 7) % 256, (x * 13 + y * 29) % 256)) + return im + + +for mode, color in (("RGB", (10, 20, 30)), ("RGBA", (1, 2, 3, 4)), ("L", 77), ("RGB", "red"), + ("L", "orange"), ("RGBA", "#336699"), ("RGB", 0x123456), ("RGBA", (5, 6, 7))): + im = Image.new(mode, (3, 2), color) + print(mode, im.size, im.width, im.height, im.mode, im.getpixel((1, 1)), im.getpixel((-1, -1))) + +src = pattern(11, 7) +rgb = src.convert("RGB") +gray = src.convert("L") +print("conv", h(src), h(rgb), h(gray), h(gray.convert("RGB")), h(gray.convert("RGBA")), h(rgb.convert("RGBA"))) +print("px", rgb.getpixel((3, 4)), gray.getpixel((3, 4)), src.getpixel((10, 6))) +print("bands", src.getbands(), rgb.getbands(), gray.getbands()) + +c = rgb.crop((2, 1, 8, 5)) +print("crop", c.size, h(c), h(rgb.crop((-2, -1, 4, 3)))) +print("copy", rgb.copy().tobytes() == rgb.tobytes()) + +for t in (Image.FLIP_LEFT_RIGHT, Image.FLIP_TOP_BOTTOM, Image.ROTATE_90, Image.ROTATE_180, + Image.ROTATE_270, Image.TRANSPOSE, Image.TRANSVERSE): + r = src.transpose(t) + print("transpose", t, r.size, h(r)) + +for angle, expand in ((90, True), (180, False), (270, True), (90, False), (30, False), (45, True), (-20, True)): + r = rgb.rotate(angle, expand=expand) + print("rotate", angle, expand, r.size, h(r)) +print("rotate fill", h(gray.rotate(33, fillcolor=200))) + +for size in ((5, 3), (22, 14), (11, 3), (4, 7), (17, 7)): + print("nearest", size, h(rgb.resize(size, Image.NEAREST)), h(gray.resize(size, Image.Resampling.NEAREST))) + print("bilinear", size, h(rgb.resize(size, Image.BILINEAR)), h(gray.resize(size, Image.BILINEAR))) +print("box", h(rgb.resize((4, 4), Image.BILINEAR, box=(1, 1, 9, 6)))) + +t = rgb.copy() +t.thumbnail((6, 6)) +print("thumb", t.size) +t = Image.new("L", (300, 200)) +t.thumbnail((128, 128)) +print("thumb", t.size) + +dst = Image.new("RGB", (8, 8), (9, 9, 9)) +dst.paste(rgb.crop((0, 0, 4, 4)), (2, 3)) +print("paste", h(dst)) +dst.paste((200, 100, 50), (0, 0, 3, 2)) +print("paste color", h(dst)) +dst.paste(src.crop((0, 0, 5, 5)), (4, 4), src.crop((0, 0, 5, 5))) +print("paste mask", h(dst)) +dst.paste(gray.crop((1, 1, 4, 4)), (-1, -1)) +print("paste gray", h(dst), dst.getpixel((0, 0))) +mask = gray.crop((0, 0, 8, 8)) +dst.paste((255, 255, 255), (0, 0, 8, 8), mask) +print("paste color mask", h(dst)) + +r, g, b, a = src.split() +print("split", r.mode, h(r), h(g), h(b), h(a)) +m = Image.merge("RGB", (b, g, r)) +print("merge", m.mode, h(m), m.getpixel((2, 2))) +print("channel", h(src.getchannel("A")), h(src.getchannel(1))) + +raw = rgb.tobytes() +print("bytes", len(raw), h(Image.frombytes("RGB", rgb.size, raw))) + +z = Image.new("RGB", (10, 10)) +print("bbox empty", z.getbbox()) +z.putpixel((3, 4), (0, 0, 1)) +z.putpixel((7, 2), (5, 0, 0)) +print("bbox", z.getbbox()) +za = Image.new("RGBA", (10, 10), (50, 50, 50, 0)) +za.putpixel((6, 6), (0, 0, 0, 9)) +print("bbox alpha", za.getbbox(), za.getbbox(alpha_only=False)) + +print("point", h(gray.point(lambda v: 255 - v)), h(rgb.point(lambda v: v // 2))) +lut = [255 - i for i in range(256)] + list(range(256)) + [i // 3 for i in range(256)] +print("point lut", h(rgb.point(lut))) +print("data", list(gray.getdata())[:5], rgb.getdata()[3], gray.getextrema(), rgb.getextrema()) +print("hist", sum(gray.histogram()), gray.histogram()[gray.getpixel((0, 0))]) +print("eq", rgb == rgb.copy(), rgb == gray) diff --git a/maven/python-transpiler/src/test/resources/conformance/pil_draw.out b/maven/python-transpiler/src/test/resources/conformance/pil_draw.out new file mode 100644 index 00000000000..847a654234f --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/pil_draw.out @@ -0,0 +1,16 @@ +rect fill 80c8423f670f5f1a (255, 0, 0) (255, 0, 0) (255, 255, 255) +rect outline 2f106f3826e270d8 (0, 0, 255) (255, 255, 255) +rect both 5635def20aa833c4 +point 15c7687d9235a7e1 (1, 2, 3) (0, 0, 0) +line 23cc8175c644bd71 +gray 3413754910daba3f +rgba e83c78a7f4e0ddae (10, 20, 30, 128) (255, 255, 255, 255) +shapes 255 0 100 0 +ops 61b880fec894e58f L +flip af03c525972014e1 710110561d00ba8a +invert 234db1cdf3ca7cf5 +expand (13, 11) f5456b313691b49a +blur 1 fe8c55a2001f568e aab93c3fdadaf042 +blur 2 c88414354f4ff247 626ee244a4d13126 +blur 3 751694fb485447e3 3e6c1d44e1f1b76d +blur0 True diff --git a/maven/python-transpiler/src/test/resources/conformance/pil_draw.py b/maven/python-transpiler/src/test/resources/conformance/pil_draw.py new file mode 100644 index 00000000000..a729dd8df5e --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/pil_draw.py @@ -0,0 +1,57 @@ +# Pillow ImageDraw, ImageOps and ImageFilter.BoxBlur. +import hashlib + +from PIL import Image, ImageDraw, ImageFilter, ImageOps + + +def h(im): + return hashlib.sha256(im.tobytes()).hexdigest()[:16] + + +im = Image.new("RGB", (20, 16), "white") +d = ImageDraw.Draw(im) +d.rectangle([2, 3, 9, 7], fill=(255, 0, 0)) +print("rect fill", h(im), im.getpixel((2, 3)), im.getpixel((9, 7)), im.getpixel((10, 7))) +d.rectangle([(11, 1), (18, 12)], outline="blue") +print("rect outline", h(im), im.getpixel((11, 5)), im.getpixel((12, 5))) +d.rectangle((1, 9, 8, 14), fill=(0, 255, 0), outline=(0, 0, 0), width=2) +print("rect both", h(im)) +d.point([(0, 0), (19, 15)], fill=(1, 2, 3)) +d.point((5, 0, 6, 0), fill="black") +print("point", h(im), im.getpixel((19, 15)), im.getpixel((6, 0))) +d.line([(0, 15), (19, 15)], fill=(9, 9, 9)) +d.line((10, 0, 10, 15), fill=(8, 8, 8)) +d.line([(0, 8), (5, 8), (5, 2)], fill=(7, 7, 7)) +print("line", h(im)) + +g = Image.new("L", (12, 12), 0) +dg = ImageDraw.Draw(g) +dg.rectangle([0, 0, 11, 11], outline=200) +dg.rectangle([3, 3, 4, 8], fill=90) +dg.line([(6, 2), (6, 9)], fill=255) +print("gray", h(g)) + +a = Image.new("RGBA", (10, 10), (0, 0, 0, 0)) +da = ImageDraw.Draw(a) +da.rectangle([2, 2, 5, 5], fill=(10, 20, 30, 128)) +da.rectangle([4, 4, 8, 8], fill=(255, 255, 255)) +print("rgba", h(a), a.getpixel((3, 3)), a.getpixel((5, 5))) + +e = Image.new("L", (21, 15), 0) +de = ImageDraw.Draw(e) +de.ellipse([2, 2, 18, 12], fill=255) +dp = ImageDraw.Draw(e) +dp.polygon([(0, 0), (6, 0), (0, 6)], fill=100) +print("shapes", e.getpixel((10, 7)), e.getpixel((0, 14)), e.getpixel((1, 1)), e.getpixel((20, 0))) + +src = Image.new("RGB", (9, 7)) +for y in range(7): + for x in range(9): + src.putpixel((x, y), (x * 28, y * 36, (x * y * 7) % 256)) +print("ops", h(ImageOps.grayscale(src)), ImageOps.grayscale(src).mode) +print("flip", h(ImageOps.flip(src)), h(ImageOps.mirror(src))) +print("invert", h(ImageOps.invert(src))) +print("expand", ImageOps.expand(src, 2, (1, 2, 3)).size, h(ImageOps.expand(src, 2, (1, 2, 3)))) +for r in (1, 2, 3): + print("blur", r, h(src.filter(ImageFilter.BoxBlur(r))), h(src.convert("L").filter(ImageFilter.BoxBlur(r)))) +print("blur0", src.filter(ImageFilter.BoxBlur(0)) == src) diff --git a/maven/python-transpiler/src/test/resources/conformance/pil_png.out b/maven/python-transpiler/src/test/resources/conformance/pil_png.out new file mode 100644 index 00000000000..f57fb73393c --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/pil_png.out @@ -0,0 +1,12 @@ +bit (9, 4) PNG 218be6c5703006d7 (0, 0, 0, 255) (0, 0, 0, 255) +l (13, 9) PNG c66c5aa77eb7683c (18, 18, 18, 255) (131, 131, 131, 255) +la (13, 9) PNG 70bd7ea122b88380 (18, 18, 18, 42) (131, 131, 131, 132) +p (7, 5) PNG 64baabd8ac9cc55c (59, 7, 17, 255) (29, 23, 47, 255) +pt (7, 5) PNG 7a9001b5a8c45b0a (59, 7, 17, 255) (29, 23, 47, 255) +rgb (13, 9) PNG 401d364a6bc512f4 (48, 4, 12, 255) (20, 168, 231, 255) +rgba (13, 9) PNG 8ce2da94f384f776 (48, 4, 12, 42) (20, 168, 231, 132) +L L (13, 9) True 90adee0b26352f08 +RGB RGB (13, 9) True f8ecc7d7089db771 +RGBA RGBA (13, 9) True 8ce2da94f384f776 +path RGBA (13, 9) True +UnidentifiedImageError True diff --git a/maven/python-transpiler/src/test/resources/conformance/pil_png.py b/maven/python-transpiler/src/test/resources/conformance/pil_png.py new file mode 100644 index 00000000000..d3ce0a2b2e1 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/pil_png.py @@ -0,0 +1,86 @@ +# Pillow PNG: decoding files Pillow wrote (every filter, gray, gray+alpha, +# RGB, RGBA, palette with and without transparency, 1-bit) and round trips +# through BytesIO and a file path. +import hashlib +import io +import os + +from PIL import Image + +PNGS = { + 'rgb': bytes.fromhex( + '89504e470d0a1a0a0000000d494844520000000d0000000908020000006618313d000000' + '6649444154789c85ccc10a80201084e11115a925242aa24b1719e8d2a5f77fb82831a4ac' + 'e03b0ceccf2ac051811ab4a0032b50c006f4600b76e0008e30a221ca8afe6152577f3359' + 'e73fdcba3e621a9767371515bb39b3c551e8781cc2694d23bcfd0ba297dc0ee85a0f3336' + '0aa31f0000000049454e44ae426082' + ), + 'rgba': bytes.fromhex( + '89504e470d0a1a0a0000000d494844520000000d000000090806000000e97aa66a000000' + '7949444154789c8dccc10ac2301084e1096b29ba0ca5a88887f55206bc78f1fd1faea442' + '08d6480e1fd985fd9380114aa00cd4006a047504e5a0086a0235833a83ba82ba81073784' + 'a7816efdaae8d4ed2b9abafc882e85aab9d688ee2d91df3fd1a3166eefb237236d074bb8' + '2d747be5b96845f1f9351f3d77561def15d7edd13cae0000000049454e44ae426082' + ), + 'l': bytes.fromhex( + '89504e470d0a1a0a0000000d494844520000000d000000090800000000cc11f9b6000000' + '5a49444154789c25cbcb0980301444510ce67d66a22228c1fedb08b8b31aed415eb23ccc' + 'dc69e2b26efbfe1c67ad57ca226ae60e824c5954d50c004b49b9d3636489a7983547688e' + '6e849f623c3b7a1768e62fc83b36510d80e5071a4f0ab3e941397c0000000049454e44ae' + '426082' + ), + 'la': bytes.fromhex( + '89504e470d0a1a0a0000000d494844520000000d00000009080400000043736ee1000000' + '7749444154789c4d8e5d0a833010062bd4c4fc4c4550d4c2deff1285bef534f60eb29b44' + 'e4639f8699a4eb1e0978313231f16366616565e7d98bc3e11918080422c92e15e46d0aa3' + '2d91c98a7ab3746a154f610d3a733e2cd5baa1162cd61f2ff11ebc80bcdb5b0d682ec8c1' + '6ec16fb30c4be060ab9fcf9c31cc11aa5b8c049e0000000049454e44ae426082' + ), + 'p': bytes.fromhex( + '89504e470d0a1a0a0000000d49484452000000070000000504030000007bb4ebeb000000' + '30504c5445f3295ada2252e42616c61d3bb91907aa162f981325850f25700d1e5b09164b' + '09133b07111d172f23050a18020700000770e3c8270000002449444154789c011900e6ff' + '00fda8642002fef0ff0002f0fff0e002efefde000200efdfc0b15d0e284316c091000000' + '0049454e44ae426082' + ), + 'pt': bytes.fromhex( + '89504e470d0a1a0a0000000d49484452000000070000000504030000007bb4ebeb000000' + '30504c5445f3295ada2252e42616c61d3bb91907aa162f981325850f25700d1e5b09164b' + '09133b07111d172f23050a18020700000770e3c8270000000474524e53ffffff00402aa9' + 'f40000002449444154789c011900e6ff00fda8642002fef0ff0002f0fff0e002efefde00' + '0200efdfc0b15d0e284316c0910000000049454e44ae426082' + ), + 'bit': bytes.fromhex( + '89504e470d0a1a0a0000000d494844520000000900000004010000000074742863000000' + '0b49444154789c6360400000000c00017c4c44c20000000049454e44ae426082' + ), +} + + +def h(im): + return hashlib.sha256(im.tobytes()).hexdigest()[:16] + + +for k in sorted(PNGS): + im = Image.open(io.BytesIO(PNGS[k])) + rgba = im.convert("RGBA") + print(k, im.size, im.format, h(rgba), rgba.getpixel((1, 1)), rgba.getpixel((im.width - 1, im.height - 1))) + +src = Image.open(io.BytesIO(PNGS["rgba"])) +for mode in ("L", "RGB", "RGBA"): + im = src.convert(mode) + buf = io.BytesIO() + im.save(buf, "PNG") + back = Image.open(io.BytesIO(buf.getvalue())) + print(mode, back.mode, back.size, back.tobytes() == im.tobytes(), h(back)) + +path = "pil_png_conformance_tmp.png" +src.save(path) +with Image.open(path) as again: + print("path", again.mode, again.size, again.tobytes() == src.tobytes()) +os.remove(path) + +try: + Image.open(io.BytesIO(b"not an image at all")) +except Exception as e: + print(type(e).__name__, isinstance(e, OSError)) From 026029d67d26a3fe32c2f6f306e94b62e9e52df7 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 09:54:53 +0200 Subject: [PATCH 047/112] Python guide: Pillow subset Co-Authored-By: Claude Opus 5.5 --- docs/developer-guide/Python-Interop.asciidoc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/developer-guide/Python-Interop.asciidoc b/docs/developer-guide/Python-Interop.asciidoc index 57a9199272c..8c7bd9eac00 100644 --- a/docs/developer-guide/Python-Interop.asciidoc +++ b/docs/developer-guide/Python-Interop.asciidoc @@ -196,6 +196,13 @@ the real package: views, fancy indexing, ufuncs, reductions, sorting and `numpy.linalg`. The kernels are Java loops over primitive arrays. `numpy.random` draws from Python's generator, so its streams differ from NumPy's. +| `Pillow` | `PIL.Image` in the `RGB`, `RGBA` and `L` modes: conversion, + cropping, resizing, rotation, pasting, channels and point operations, + computed as Pillow computes them. PNG is read and written on every platform; + JPEG, GIF, BMP and WebP go through Codename One's image support and need the + application's display, so a server reads and writes PNG only. `ImageDraw` + draws points, lines, rectangles, ellipses and polygons, but not text. + Resizing filters other than `NEAREST` and `BILINEAR` resize bilinearly. | `flask`, `fastapi`, `uvicorn` | See <>. | `typing_extensions`, `six`, `tabulate`, `idna`, `packaging` | |=== From e92e3ce40e49757dda7198f83d5d6a7754e0fc49 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 09:56:50 +0200 Subject: [PATCH 048/112] Python: pandas subset (Series, DataFrame, groupby, merge, concat, CSV/JSON I/O) Pure-Python pandas over the numpy subset: inferred int64/float64/bool/str/object columns with NaN promotion, index alignment, loc/iloc, describe, groupby on one key, inner/left merge, and pandas 3's default repr. Conformance programs pandas_series, pandas_frame and pandas_io hold it to real pandas 3.1 output. Co-Authored-By: Claude Opus 5.5 --- .../codename1/python/site/pandas/__init__.py | 1840 +++++++++++++++++ .../resources/conformance/pandas_frame.out | 155 ++ .../resources/conformance/pandas_frame.py | 56 + .../test/resources/conformance/pandas_io.out | 40 + .../test/resources/conformance/pandas_io.py | 24 + .../resources/conformance/pandas_series.out | 156 ++ .../resources/conformance/pandas_series.py | 53 + 7 files changed, 2324 insertions(+) create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/pandas/__init__.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/pandas_frame.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/pandas_frame.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/pandas_io.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/pandas_io.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/pandas_series.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/pandas_series.py diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/pandas/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/pandas/__init__.py new file mode 100644 index 00000000000..a395be6aa00 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/pandas/__init__.py @@ -0,0 +1,1840 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""A pandas subset: Series and DataFrame over plain Python lists. + +A column is a list of Python values plus a dtype name -- int64, float64, +bool, str or object -- inferred the way pandas infers it: a missing value in +a numeric column promotes it to float64 and is stored as float('nan'). +Conversion to numpy happens only at the edges (`values`, `to_numpy`). + +Formatting follows pandas' default repr for frames that fit the screen. +Not provided: datetime indexes, MultiIndex, pivot, categoricals, rolling +windows, and the truncated repr of long frames. +""" + +import math as _math +import re as _re +import json as _json +import csv as _csv +import io as _io +import collections as _collections +import builtins as _b +import numpy as _np + +__version__ = "3.1.0" + +nan = float("nan") +NA = nan + + +# ---- values and dtypes ---------------------------------------------------- + +def _isna(v): + return v is None or (isinstance(v, float) and v != v) + + +def isna(v): + if isinstance(v, (Series, DataFrame)): + return v.isna() + return _isna(v) + + +isnull = isna + + +def notna(v): + if isinstance(v, (Series, DataFrame)): + return v.notna() + return not _isna(v) + + +notnull = notna + + +class _Dtype: + def __init__(self, name): + self.name = name + + def __str__(self): + return self.name + + def __repr__(self): + if self.name in ("str", "object"): + return self.name + return "dtype('" + self.name + "')" + + def __eq__(self, other): + if isinstance(other, _Dtype): + return other.name == self.name + return _dtype_name(other) == self.name + + def __ne__(self, other): + return not self.__eq__(other) + + def __hash__(self): + return hash(self.name) + + @property + def kind(self): + return {"int64": "i", "float64": "f", "bool": "b"}.get(self.name, "O") + + +def _dtype_name(t): + if isinstance(t, _Dtype): + return t.name + if t is int or t == "int" or t == "int64" or t == "int32": + return "int64" + if t is float or t == "float" or t == "float64" or t == "float32": + return "float64" + if t is bool or t == "bool": + return "bool" + if t is str or t == "str" or t == "string": + return "str" + return "object" + + +def _is_num(v): + return isinstance(v, (int, float)) and not isinstance(v, bool) + + +def _infer(values): + """Returns (dtype name, values converted to that dtype's storage).""" + if not values: + return "object", [] + kinds = set() + for v in values: + if v is None or (isinstance(v, float) and v != v): + kinds.add("na") + elif isinstance(v, bool): + kinds.add("bool") + elif isinstance(v, int): + kinds.add("int") + elif isinstance(v, float): + kinds.add("float") + elif isinstance(v, str): + kinds.add("str") + else: + kinds.add("obj") + if kinds == {"bool"}: + return "bool", list(values) + if kinds == {"int"}: + return "int64", list(values) + if kinds <= {"int", "float", "na"}: + return "float64", [nan if _isna(v) else float(v) for v in values] + if kinds <= {"str", "na"} and "str" in kinds: + return "str", [nan if _isna(v) else v for v in values] + return "object", list(values) + + +def _cast(values, dt): + if dt == "float64": + return [nan if _isna(v) else float(v) for v in values] + if dt == "int64": + out = [] + for v in values: + if _isna(v): + raise ValueError("cannot convert NA to integer") + out.append(int(v)) + return out + if dt == "bool": + return [bool(v) for v in values] + if dt == "str": + return [v if _isna(v) else str(v) for v in values] + return list(values) + + +def _numeric(dt): + return dt in ("int64", "float64", "bool") + + +# ---- formatting ----------------------------------------------------------- + +def _fmt_float_list(vals): + finite = [abs(v) for v in vals if not _isna(v) and v not in (float("inf"), float("-inf"))] + sci = any(v > 1e6 for v in finite) or any(0 < v < 1e-6 for v in finite) + out = [] + for v in vals: + if _isna(v): + out.append("NaN") + elif v == float("inf"): + out.append(" inf") + elif v == float("-inf"): + out.append("-inf") + elif sci: + out.append("%.6e" % v if v < 0 else " %.6e" % v) + else: + out.append("%.6f" % v if v < 0 else " %.6f" % v) + if sci: + return out + + def num(x): + return "." in x and x.strip() not in ("NaN", "inf", "-inf") + while True: + nums = [x for x in out if num(x)] + if not nums or not all(x.endswith("0") for x in nums): + break + out = [x[:-1] if num(x) else x for x in out] + return [x + "0" if num(x) and x.endswith(".") else x for x in out] + + +def _fmt_obj(v): + if v is None: + return "None" + if isinstance(v, float) and v != v: + return "NaN" + return str(v) + + +def _fmt_values(values, dt): + if dt == "float64": + return _fmt_float_list(values) + if dt == "int64": + return [str(v) if v < 0 else " " + str(v) for v in values] + return [" " + _fmt_obj(v) for v in values] + + +def _rjust_all(strs): + w = max([len(x) for x in strs] + [0]) + return [x.rjust(w) for x in strs] + + +def _label(v): + return _fmt_obj(v) + + +# ---- Index ---------------------------------------------------------------- + +class Index: + def __init__(self, data=None, name=None, dtype=None): + if isinstance(data, Index): + data = data._v + self._v = list(data) if data is not None else [] + self.name = name + self._range = False + self._pos = None + + @classmethod + def _rng(cls, n): + ix = cls(range(n)) + ix._range = True + return ix + + def _loc(self): + if self._pos is None: + d = {} + for i, v in enumerate(self._v): + if v not in d: + d[v] = i + self._pos = d + return self._pos + + def get_loc(self, key): + d = self._loc() + if key in d: + return d[key] + raise KeyError(key) + + def __len__(self): + return len(self._v) + + def __iter__(self): + return iter(self._v) + + def __contains__(self, key): + return key in self._loc() + + def __getitem__(self, i): + if isinstance(i, slice): + return Index(self._v[i], self.name) + return self._v[i] + + def tolist(self): + return list(self._v) + + to_list = tolist + + @property + def values(self): + return _to_np(self._v, self.dtype.name) + + @property + def dtype(self): + return _Dtype(_infer(self._v)[0] if self._v else "object") + + def equals(self, other): + return list(other) == self._v + + def __eq__(self, other): + if isinstance(other, (list, Index)): + return [a == b for a, b in zip(self._v, other)] + return [a == other for a in self._v] + + def __hash__(self): + return id(self) + + def __repr__(self): + if self._range: + return "RangeIndex(start=0, stop=%d, step=1)" % len(self._v) + items = ", ".join(repr(v) for v in self._v) + s = "Index([" + items + "], dtype='" + self.dtype.name + "'" + if self.name is not None: + s += ", name=" + repr(self.name) + return s + ")" + + __str__ = __repr__ + + +RangeIndex = Index + + +def _to_np(values, dt): + if dt in ("int64", "float64", "bool"): + return _np.array(values) + return list(values) + + +# ---- Series --------------------------------------------------------------- + +def _same_index(a, b): + return a._v == b._v + + +def _union(a, b): + seen = set(a._v) + out = list(a._v) + for v in b._v: + if v not in seen: + seen.add(v) + out.append(v) + try: + out.sort() + except TypeError: + pass + return out + + +class Series: + def __init__(self, data=None, index=None, dtype=None, name=None, copy=False): + if isinstance(data, Series): + if name is None: + name = data.name + if index is None: + index = data._index + data = data._v + if isinstance(data, dict): + if index is None: + index = list(data.keys()) + data = list(data.values()) + elif data is None: + data = [] + elif isinstance(data, _np.ndarray): + data = data.tolist() + elif isinstance(data, (str, int, float, bool)): + n = len(index) if index is not None else 1 + data = [data] * n + else: + data = list(data) + if dtype is not None: + self._dt = _dtype_name(dtype) + self._v = _cast(data, self._dt) + else: + self._dt, self._v = _infer(data) + if index is None: + self._index = Index._rng(len(self._v)) + elif isinstance(index, Index): + self._index = index + else: + self._index = Index(index) + self.name = name + + @classmethod + def _make(cls, values, index, name=None, dt=None): + s = cls.__new__(cls) + if dt is None: + s._dt, s._v = _infer(values) + else: + s._dt, s._v = dt, values + s._index = index if isinstance(index, Index) else Index(index) + s.name = name + return s + + # attributes + @property + def index(self): + return self._index + + @index.setter + def index(self, value): + self._index = value if isinstance(value, Index) else Index(value) + + @property + def dtype(self): + return _Dtype(self._dt) + + dtypes = dtype + + @property + def values(self): + return _to_np(self._v, self._dt) + + def to_numpy(self): + return _to_np(self._v, self._dt) + + def tolist(self): + return list(self._v) + + to_list = tolist + + @property + def shape(self): + return (len(self._v),) + + @property + def size(self): + return len(self._v) + + @property + def empty(self): + return len(self._v) == 0 + + @property + def loc(self): + return _SLoc(self) + + @property + def iloc(self): + return _SILoc(self) + + @property + def str(self): + return _StrAccessor(self) + + def __len__(self): + return len(self._v) + + def __iter__(self): + return iter(self._v) + + def __contains__(self, key): + return key in self._index + + def keys(self): + return self._index + + def items(self): + return zip(self._index._v, self._v) + + def copy(self): + return Series._make(list(self._v), Index(self._index._v, self._index.name), self.name, self._dt) + + def _take(self, positions): + ix = Index([self._index._v[i] for i in positions], self._index.name) + return Series._make([self._v[i] for i in positions], ix, self.name, self._dt) + + def _mask_positions(self, mask): + m = mask._v if isinstance(mask, Series) else list(mask) + return [i for i, b in enumerate(m) if b is True or (b is not False and not _isna(b) and b)] + + def __getitem__(self, key): + if isinstance(key, Series) and key._dt == "bool": + return self._take(self._mask_positions(key)) + if isinstance(key, slice): + return self._take(range(len(self._v))[key]) + if isinstance(key, list): + if key and isinstance(key[0], bool): + return self._take(self._mask_positions(key)) + return self._take([self._index.get_loc(k) for k in key]) + if key in self._index: + return self._v[self._index.get_loc(key)] + if isinstance(key, int) and not isinstance(key, bool): + return self._v[key] + raise KeyError(key) + + def __setitem__(self, key, value): + if isinstance(key, Series) and key._dt == "bool": + pos = self._mask_positions(key) + elif key in self._index: + pos = [self._index.get_loc(key)] + else: + self._index = Index(self._index._v + [key], self._index.name) + self._v.append(value) + self._dt, self._v = _infer(self._v) + return + vals = list(self._v) + for p in pos: + vals[p] = value + self._dt, self._v = _infer(vals) + + def get(self, key, default=None): + if key in self._index: + return self._v[self._index.get_loc(key)] + return default + + # arithmetic + def _binop(self, other, op, name=None): + if isinstance(other, DataFrame): + return NotImplemented + if isinstance(other, Series): + if _same_index(self._index, other._index): + a, b, ix = self._v, other._v, self._index + else: + labels = _union(self._index, other._index) + a = [self._v[self._index.get_loc(k)] if k in self._index else nan for k in labels] + b = [other._v[other._index.get_loc(k)] if k in other._index else nan for k in labels] + ix = Index(labels) + nm = self.name if self.name == other.name else None + out = [_apply2(op, x, y) for x, y in zip(a, b)] + return Series._make(out, ix, nm) + if isinstance(other, (list, tuple, _np.ndarray)): + o = list(other) + return Series._make([_apply2(op, x, y) for x, y in zip(self._v, o)], self._index, self.name) + return Series._make([_apply2(op, x, other) for x in self._v], self._index, self.name) + + def __add__(self, o): return self._binop(o, "add") + def __radd__(self, o): return self._binop(o, "radd") + def __sub__(self, o): return self._binop(o, "sub") + def __rsub__(self, o): return self._binop(o, "rsub") + def __mul__(self, o): return self._binop(o, "mul") + def __rmul__(self, o): return self._binop(o, "mul") + def __truediv__(self, o): return self._binop(o, "div") + def __rtruediv__(self, o): return self._binop(o, "rdiv") + def __floordiv__(self, o): return self._binop(o, "fdiv") + def __mod__(self, o): return self._binop(o, "mod") + def __pow__(self, o): return self._binop(o, "pow") + def __eq__(self, o): return self._binop(o, "eq") + def __ne__(self, o): return self._binop(o, "ne") + def __lt__(self, o): return self._binop(o, "lt") + def __le__(self, o): return self._binop(o, "le") + def __gt__(self, o): return self._binop(o, "gt") + def __ge__(self, o): return self._binop(o, "ge") + def __and__(self, o): return self._binop(o, "and") + def __or__(self, o): return self._binop(o, "or") + def __xor__(self, o): return self._binop(o, "xor") + + def __hash__(self): + return id(self) + + add = __add__ + sub = __sub__ + mul = __mul__ + div = truediv = __truediv__ + + def __neg__(self): + return Series._make([-v for v in self._v], self._index, self.name) + + def __abs__(self): + return Series._make([abs(v) for v in self._v], self._index, self.name) + + def abs(self): + return self.__abs__() + + def __invert__(self): + return Series._make([not v for v in self._v], self._index, self.name, "bool") + + def __bool__(self): + raise ValueError("The truth value of a Series is ambiguous. " + "Use a.empty, a.bool(), a.item(), a.any() or a.all().") + + def round(self, decimals=0): + if self._dt != "float64": + return self.copy() + return Series._make([v if _isna(v) else round(v, decimals) for v in self._v], + self._index, self.name, "float64") + + def between(self, left, right): + return (self >= left) & (self <= right) + + def isin(self, values): + vs = list(values) + return Series._make([v in vs for v in self._v], self._index, self.name, "bool") + + # missing data + def isna(self): + return Series._make([_isna(v) for v in self._v], self._index, self.name, "bool") + + isnull = isna + + def notna(self): + return Series._make([not _isna(v) for v in self._v], self._index, self.name, "bool") + + notnull = notna + + def fillna(self, value): + return Series._make([value if _isna(v) else v for v in self._v], self._index, self.name) + + def dropna(self): + return self._take([i for i, v in enumerate(self._v) if not _isna(v)]) + + # reductions + def _valid(self): + return [v for v in self._v if not _isna(v)] + + def sum(self): + vs = self._valid() + if self._dt == "str": + return "".join(vs) + total = 0 + for v in vs: + total = total + v + if self._dt == "bool": + return int(total) + return total + + def prod(self): + p = 1 + for v in self._valid(): + p = p * v + return p + + def count(self): + return len(self._valid()) + + def mean(self): + vs = self._valid() + if not vs: + return nan + return float(sum(vs)) / len(vs) + + def var(self, ddof=1): + vs = self._valid() + if len(vs) <= ddof: + return nan + m = float(sum(vs)) / len(vs) + return sum((v - m) * (v - m) for v in vs) / (len(vs) - ddof) + + def std(self, ddof=1): + v = self.var(ddof) + return v if v != v else _math.sqrt(v) + + def min(self): + vs = self._valid() + return min(vs) if vs else nan + + def max(self): + vs = self._valid() + return max(vs) if vs else nan + + def idxmax(self): + best = None + for k, v in zip(self._index._v, self._v): + if not _isna(v) and (best is None or v > best[1]): + best = (k, v) + return best[0] + + def idxmin(self): + best = None + for k, v in zip(self._index._v, self._v): + if not _isna(v) and (best is None or v < best[1]): + best = (k, v) + return best[0] + + def quantile(self, q=0.5): + return _quantile(sorted(self._valid()), q) + + def median(self): + return self.quantile(0.5) + + def any(self): + return any(self._valid()) + + def all(self): + return all(self._valid()) + + def nunique(self): + return len(set(self._valid())) + + def unique(self): + seen = [] + s = set() + for v in self._v: + key = "__nan__" if _isna(v) else v + if key not in s: + s.add(key) + seen.append(v) + return _to_np(seen, self._dt) + + def value_counts(self, ascending=False): + counts = {} + order = [] + for v in self._valid(): + if v not in counts: + counts[v] = 0 + order.append(v) + counts[v] += 1 + order.sort(key=lambda k: counts[k], reverse=not ascending) + return Series._make([counts[k] for k in order], Index(order, self.name), "count", "int64") + + def describe(self): + vals = [float(self.count()), self.mean(), self.std(), float(self.min()), + self.quantile(0.25), self.quantile(0.5), self.quantile(0.75), float(self.max())] + return Series._make(vals, Index(_DESCRIBE), self.name, "float64") + + def cumsum(self): + out = [] + total = 0 + for v in self._v: + if _isna(v): + out.append(nan) + else: + total = total + v + out.append(total) + return Series._make(out, self._index, self.name) + + def cumprod(self): + out = [] + total = 1 + for v in self._v: + if _isna(v): + out.append(nan) + else: + total = total * v + out.append(total) + return Series._make(out, self._index, self.name) + + def diff(self): + out = [nan] + for i in range(1, len(self._v)): + out.append(self._v[i] - self._v[i - 1]) + return Series._make(out, self._index, self.name, "float64" if self._v else None) + + # transforms + def apply(self, func): + return Series._make([func(v) for v in self._v], self._index, self.name) + + def map(self, arg): + if isinstance(arg, dict): + return Series._make([arg.get(v, nan) for v in self._v], self._index, self.name) + if isinstance(arg, Series): + return Series._make([arg.get(v, nan) for v in self._v], self._index, self.name) + return Series._make([v if _isna(v) else arg(v) for v in self._v], self._index, self.name) + + def astype(self, dtype): + dt = _dtype_name(dtype) + return Series._make(_cast(self._v, dt), self._index, self.name, dt) + + def rename(self, name): + s = self.copy() + s.name = name + return s + + def reset_index(self, drop=False): + if drop: + return Series._make(list(self._v), Index._rng(len(self._v)), self.name, self._dt) + return DataFrame({self._index.name or "index": self._index._v, + (self.name if self.name is not None else 0): self._v}) + + def to_frame(self, name=None): + n = name if name is not None else (self.name if self.name is not None else 0) + return DataFrame._from_series([Series._make(self._v, self._index, n, self._dt)], self._index) + + def to_dict(self): + return dict(zip(self._index._v, self._v)) + + def head(self, n=5): + return self._take(range(min(n, len(self._v)))) + + def tail(self, n=5): + return self._take(range(max(0, len(self._v) - n), len(self._v))) + + def sort_values(self, ascending=True): + return self._take(_order(self._v, ascending)) + + def sort_index(self, ascending=True): + return self._take(_order(self._index._v, ascending)) + + def equals(self, other): + return isinstance(other, Series) and _eq_lists(self._v, other._v) and self._dt == other._dt + + # repr + def __repr__(self): + if not self._v: + return "Series([], " + ("Name: %s, " % self.name if self.name is not None else "") + \ + "dtype: " + self._dt + ")" + labels = [_label(v) for v in self._index._v] + w = max(len(x) for x in labels) + vals = _rjust_all(_fmt_values(self._v, self._dt)) + lines = [] + if self._index.name is not None: + lines.append(str(self._index.name)) + for lab, v in zip(labels, vals): + lines.append(lab.ljust(w) + " " + v) + foot = [] + if self.name is not None: + foot.append("Name: " + str(self.name)) + foot.append("dtype: " + self._dt) + lines.append(", ".join(foot)) + return "\n".join(lines) + + __str__ = __repr__ + + +_DESCRIBE = ["count", "mean", "std", "min", "25%", "50%", "75%", "max"] + + +def _eq_lists(a, b): + if len(a) != len(b): + return False + for x, y in zip(a, b): + if _isna(x) and _isna(y): + continue + if x != y: + return False + return True + + +def _quantile(vs, q): + if not vs: + return nan + pos = (len(vs) - 1) * q + lo = int(_math.floor(pos)) + hi = min(lo + 1, len(vs) - 1) + frac = pos - lo + return float(vs[lo]) + (float(vs[hi]) - float(vs[lo])) * frac + + +def _order(values, ascending=True): + good = [i for i, v in enumerate(values) if not _isna(v)] + bad = [i for i, v in enumerate(values) if _isna(v)] + good.sort(key=lambda i: values[i], reverse=not ascending) + return good + bad + + +def _apply2(op, x, y): + if op == "eq": + return x == y + if op == "ne": + return x != y + if _isna(x) or _isna(y): + if op in ("lt", "le", "gt", "ge"): + return False + if op in ("and", "or", "xor"): + return False + return nan + if op == "add": + return x + y + if op == "radd": + return y + x + if op == "sub": + return x - y + if op == "rsub": + return y - x + if op == "mul": + return x * y + if op == "div": + if y == 0: + return nan if x == 0 else (float("inf") if x > 0 else float("-inf")) + return x / y + if op == "rdiv": + if x == 0: + return nan if y == 0 else (float("inf") if y > 0 else float("-inf")) + return y / x + if op == "fdiv": + return x // y + if op == "mod": + return x % y + if op == "pow": + return x ** y + if op == "lt": + return x < y + if op == "le": + return x <= y + if op == "gt": + return x > y + if op == "ge": + return x >= y + if op == "and": + return bool(x and y) + if op == "or": + return bool(x or y) + if op == "xor": + return bool(x) != bool(y) + raise ValueError(op) + + +class _StrAccessor: + def __init__(self, s): + self._s = s + + def _map(self, f, dt=None): + s = self._s + return Series._make([nan if _isna(v) else f(v) for v in s._v], s._index, s.name, dt) + + def lower(self): return self._map(lambda v: v.lower()) + def upper(self): return self._map(lambda v: v.upper()) + def title(self): return self._map(lambda v: v.title()) + def capitalize(self): return self._map(lambda v: v.capitalize()) + def strip(self, chars=None): return self._map(lambda v: v.strip(chars)) + def lstrip(self, chars=None): return self._map(lambda v: v.lstrip(chars)) + def rstrip(self, chars=None): return self._map(lambda v: v.rstrip(chars)) + def len(self): return self._map(lambda v: len(v)) + def startswith(self, p): return self._map(lambda v: v.startswith(p)) + def endswith(self, p): return self._map(lambda v: v.endswith(p)) + def split(self, sep=None): return self._map(lambda v: v.split(sep), "object") + + def replace(self, pat, repl, regex=False): + if regex: + return self._map(lambda v: _re.sub(pat, repl, v)) + return self._map(lambda v: v.replace(pat, repl)) + + def contains(self, pat, case=True, regex=True): + if not regex: + if case: + return self._map(lambda v: pat in v) + return self._map(lambda v: pat.lower() in v.lower()) + flags = 0 if case else _re.IGNORECASE + return self._map(lambda v: _re.search(pat, v, flags) is not None) + + def __getitem__(self, i): + return self._map(lambda v: v[i]) + + +class _SLoc: + def __init__(self, s): + self._s = s + + def __getitem__(self, key): + s = self._s + if isinstance(key, Series) or (isinstance(key, list) and key and isinstance(key[0], bool)): + return s._take(s._mask_positions(key)) + if isinstance(key, list): + return s._take([s._index.get_loc(k) for k in key]) + if isinstance(key, slice): + a = 0 if key.start is None else s._index.get_loc(key.start) + b = len(s._v) if key.stop is None else s._index.get_loc(key.stop) + 1 + return s._take(range(a, b)) + return s._v[s._index.get_loc(key)] + + def __setitem__(self, key, value): + self._s[key] = value + + +class _SILoc: + def __init__(self, s): + self._s = s + + def __getitem__(self, key): + s = self._s + if isinstance(key, slice): + return s._take(range(len(s._v))[key]) + if isinstance(key, list): + return s._take(key) + return s._v[key] + + +# ---- DataFrame ------------------------------------------------------------ + +class DataFrame: + def __init__(self, data=None, index=None, columns=None, dtype=None): + cols = [] + if data is None: + data = {} + if isinstance(data, DataFrame): + cols = [c.copy() for c in data._cols] + index = data._index + elif isinstance(data, dict): + n = None + for v in data.values(): + if isinstance(v, Series): + if index is None: + index = v._index + n = len(v) + elif isinstance(v, (list, tuple, _np.ndarray)): + n = len(v) + if n is None: + n = len(index) if index is not None else 1 + for k, v in data.items(): + if columns is not None and k not in columns: + continue + if isinstance(v, Series): + vals = v._v + elif isinstance(v, (list, tuple)): + vals = list(v) + elif isinstance(v, _np.ndarray): + vals = v.tolist() + else: + vals = [v] * n + cols.append(Series(vals, dtype=dtype, name=k)) + elif isinstance(data, (list, tuple)): + rows = list(data) + if rows and isinstance(rows[0], dict): + names = [] + for r in rows: + for k in r: + if k not in names: + names.append(k) + if columns is not None: + names = list(columns) + for k in names: + cols.append(Series([r.get(k) for r in rows], dtype=dtype, name=k)) + elif rows and isinstance(rows[0], Series): + names = list(rows[0]._index._v) + for j, k in enumerate(names): + cols.append(Series([r._v[j] for r in rows], dtype=dtype, name=k)) + if index is None: + index = [r.name for r in rows] + else: + width = len(rows[0]) if rows else 0 + names = list(columns) if columns is not None else list(range(width)) + for j, k in enumerate(names): + cols.append(Series([r[j] for r in rows], dtype=dtype, name=k)) + elif isinstance(data, _np.ndarray): + DataFrame.__init__(self, data.tolist(), index, columns, dtype) + return + n = len(cols[0]) if cols else (len(index) if index is not None else 0) + if index is None: + ix = Index._rng(n) + elif isinstance(index, Index): + ix = index + else: + ix = Index(index) + for c in cols: + c._index = ix + self._cols = cols + self._index = ix + + @classmethod + def _from_series(cls, cols, index): + df = cls.__new__(cls) + for c in cols: + c._index = index + df._cols = cols + df._index = index + return df + + def _col(self, name): + for c in self._cols: + if c.name == name: + return c + raise KeyError(name) + + def _names(self): + return [c.name for c in self._cols] + + # attributes + @property + def columns(self): + return Index(self._names()) + + @columns.setter + def columns(self, names): + for c, n in zip(self._cols, list(names)): + c.name = n + + @property + def index(self): + return self._index + + @index.setter + def index(self, value): + ix = value if isinstance(value, Index) else Index(value) + self._index = ix + for c in self._cols: + c._index = ix + + @property + def shape(self): + return (len(self._index), len(self._cols)) + + @property + def size(self): + return len(self._index) * len(self._cols) + + @property + def empty(self): + return len(self._index) == 0 or len(self._cols) == 0 + + @property + def dtypes(self): + return Series._make([_Dtype(c._dt) for c in self._cols], Index(self._names()), None, "object") + + @property + def values(self): + return self.to_numpy() + + def to_numpy(self): + rows = [[c._v[i] for c in self._cols] for i in range(len(self._index))] + if all(_numeric(c._dt) for c in self._cols): + return _np.array(rows) + return rows + + @property + def loc(self): + return _DLoc(self, False) + + @property + def iloc(self): + return _DLoc(self, True) + + @property + def T(self): + rows = [self._row(i) for i in range(len(self._index))] + return DataFrame({lab: r._v for lab, r in zip(self._index._v, rows)}, index=self._names()) + + def __len__(self): + return len(self._index) + + def __iter__(self): + return iter(self._names()) + + def __contains__(self, name): + return name in self._names() + + def keys(self): + return self.columns + + def items(self): + return [(c.name, c.copy()) for c in self._cols] + + def copy(self): + ix = Index(self._index._v, self._index.name) + ix._range = self._index._range + return DataFrame._from_series([c.copy() for c in self._cols], ix) + + def _take(self, positions, names=None): + positions = list(positions) + ix = Index([self._index._v[i] for i in positions], self._index.name) + if self._index._range and positions == list(range(positions[0] if positions else 0, + (positions[0] if positions else 0) + len(positions))) \ + and (not positions or positions[0] == 0): + ix._range = True + cols = self._cols if names is None else [self._col(n) for n in names] + out = [Series._make([c._v[i] for i in positions], ix, c.name, c._dt) for c in cols] + return DataFrame._from_series(out, ix) + + def _row(self, i): + vals = [c._v[i] for c in self._cols] + dts = set(c._dt for c in self._cols) + if len(dts) == 1: + dt = dts.pop() + elif dts <= {"int64", "float64"}: + dt = "float64" + vals = [float(v) for v in vals] + else: + dt = "object" + return Series._make(vals, Index(self._names()), self._index._v[i], dt) + + def _mask(self, key): + return self._cols[0]._mask_positions(key) if self._cols else [] + + def __getitem__(self, key): + if isinstance(key, Series) and key._dt == "bool": + return self._take(self._mask(key)) + if isinstance(key, slice): + return self._take(range(len(self._index))[key]) + if isinstance(key, (list, Index)): + key = list(key) + if key and isinstance(key[0], bool): + return self._take(self._mask(key)) + return self._take(range(len(self._index)), key) + c = self._col(key) + return Series._make(c._v, self._index, c.name, c._dt) + + def __getattr__(self, name): + if name.startswith("_"): + raise AttributeError(name) + for c in self.__dict__.get("_cols", []): + if c.name == name: + return self[name] + raise AttributeError("'DataFrame' object has no attribute '" + name + "'") + + def _as_column(self, value, name): + n = len(self._index) + if isinstance(value, Series): + if _same_index(value._index, self._index): + vals = list(value._v) + else: + vals = [value._v[value._index.get_loc(k)] if k in value._index else nan + for k in self._index._v] + elif isinstance(value, _np.ndarray): + vals = value.tolist() + elif isinstance(value, (list, tuple)): + vals = list(value) + else: + vals = [value] * n + return Series(vals, index=self._index, name=name) + + def __setitem__(self, key, value): + if isinstance(key, list): + for k, v in zip(key, value._cols if isinstance(value, DataFrame) else value): + self[k] = v + return + col = self._as_column(value, key) + for i, c in enumerate(self._cols): + if c.name == key: + self._cols[i] = col + return + if not self._cols and len(self._index) == 0: + self._index = Index._rng(len(col)) + col._index = self._index + self._cols.append(col) + + def __delitem__(self, key): + self._cols = [c for c in self._cols if c.name != key] + + def insert(self, loc, column, value): + self._cols.insert(loc, self._as_column(value, column)) + + def pop(self, key): + s = self[key] + del self[key] + return s + + def get(self, key, default=None): + return self[key] if key in self._names() else default + + def head(self, n=5): + return self._take(range(min(n, len(self._index)))) + + def tail(self, n=5): + return self._take(range(max(0, len(self._index) - n), len(self._index))) + + # missing data + def isna(self): + return DataFrame._from_series([c.isna() for c in self._cols], self._index) + + isnull = isna + + def notna(self): + return DataFrame._from_series([c.notna() for c in self._cols], self._index) + + notnull = notna + + def fillna(self, value): + out = [] + for c in self._cols: + v = value.get(c.name, nan) if isinstance(value, dict) else value + out.append(c.fillna(v)) + return DataFrame._from_series(out, self._index) + + def dropna(self, how="any", subset=None): + cols = self._cols if subset is None else [self._col(n) for n in subset] + keep = [] + for i in range(len(self._index)): + miss = [_isna(c._v[i]) for c in cols] + if (how == "any" and not any(miss)) or (how == "all" and not all(miss)): + keep.append(i) + return self._take(keep) + + # reductions over columns + def _reduce(self, name, numeric_only=False): + names, vals = [], [] + for c in self._cols: + if numeric_only and not _numeric(c._dt): + continue + names.append(c.name) + vals.append(getattr(c, name)()) + return Series._make(vals, Index(names)) + + def sum(self, numeric_only=False): return self._reduce("sum", numeric_only) + def count(self): return self._reduce("count") + def mean(self, numeric_only=False): return self._reduce("mean", numeric_only) + def median(self, numeric_only=False): return self._reduce("median", numeric_only) + def std(self, numeric_only=False): return self._reduce("std", numeric_only) + def min(self, numeric_only=False): return self._reduce("min", numeric_only) + def max(self, numeric_only=False): return self._reduce("max", numeric_only) + def nunique(self): return self._reduce("nunique") + + def describe(self): + num = [c for c in self._cols if c._dt in ("int64", "float64")] + out = [c.describe() for c in num] + ix = Index(_DESCRIBE) + return DataFrame._from_series([Series._make(s._v, ix, s.name, "float64") for s in out], ix) + + # reshaping + def sort_values(self, by, ascending=True): + keys = by if isinstance(by, list) else [by] + asc = ascending if isinstance(ascending, list) else [ascending] * len(keys) + pos = list(range(len(self._index))) + for k, a in reversed(list(zip(keys, asc))): + vals = self._col(k)._v + good = [i for i in pos if not _isna(vals[i])] + bad = [i for i in pos if _isna(vals[i])] + good.sort(key=lambda i: vals[i], reverse=not a) + pos = good + bad + return self._take(pos) + + def sort_index(self, ascending=True): + return self._take(_order(self._index._v, ascending)) + + def reset_index(self, drop=False): + df = self.copy() + df.index = Index._rng(len(self._index)) + if not drop: + name = self._index.name if self._index.name is not None else "index" + df._cols.insert(0, Series(list(self._index._v), index=df._index, name=name)) + return df + + def set_index(self, key): + c = self._col(key) + df = self.drop(columns=[key]) + df.index = Index(list(c._v), key) + return df + + def groupby(self, by): + return GroupBy(self, by) + + def merge(self, right, on=None, how="inner", left_on=None, right_on=None, suffixes=("_x", "_y")): + return merge(self, right, on=on, how=how, left_on=left_on, right_on=right_on, suffixes=suffixes) + + def assign(self, **kwargs): + df = self.copy() + for k, v in kwargs.items(): + df[k] = v(df) if callable(v) else v + return df + + def rename(self, columns=None, index=None): + df = self.copy() + if columns: + for c in df._cols: + if c.name in columns: + c.name = columns[c.name] + if index: + df.index = Index([index.get(v, v) for v in df._index._v], df._index.name) + return df + + def drop(self, labels=None, axis=0, index=None, columns=None): + if labels is not None: + if axis in (1, "columns"): + columns = labels + else: + index = labels + df = self + if columns is not None: + gone = columns if isinstance(columns, list) else [columns] + df = df._take(range(len(df._index)), [n for n in df._names() if n not in gone]) + if index is not None: + gone = index if isinstance(index, list) else [index] + df = df._take([i for i, v in enumerate(df._index._v) if v not in gone]) + return df + + def drop_duplicates(self, subset=None): + names = subset or self._names() + seen = set() + keep = [] + for i in range(len(self._index)): + key = tuple(self._col(n)._v[i] for n in names) + if key not in seen: + seen.add(key) + keep.append(i) + return self._take(keep) + + def apply(self, func, axis=0): + if axis in (1, "columns"): + res = [func(self._row(i)) for i in range(len(self._index))] + return Series._make(res, self._index) + res = [func(self[c.name]) for c in self._cols] + if res and isinstance(res[0], Series): + return DataFrame._from_series([Series._make(r._v, r._index, c.name, r._dt) + for r, c in zip(res, self._cols)], res[0]._index) + return Series._make(res, Index(self._names())) + + def applymap(self, func): + return DataFrame._from_series([c.apply(func) for c in self._cols], self._index) + + map = applymap + + def equals(self, other): + if not isinstance(other, DataFrame) or self._names() != other._names(): + return False + return all(a.equals(b) for a, b in zip(self._cols, other._cols)) + + # iteration + def iterrows(self): + for i in range(len(self._index)): + yield self._index._v[i], self._row(i) + + def itertuples(self, index=True, name="Pandas"): + fields = (["Index"] if index else []) + [str(n) for n in self._names()] + T = _collections.namedtuple(name, fields, rename=True) + for i in range(len(self._index)): + vals = ([self._index._v[i]] if index else []) + [c._v[i] for c in self._cols] + yield T(*vals) + + # conversion + def to_dict(self, orient="dict"): + if orient == "records": + return [{c.name: c._v[i] for c in self._cols} for i in range(len(self._index))] + if orient == "list": + return {c.name: list(c._v) for c in self._cols} + if orient == "index": + return {self._index._v[i]: {c.name: c._v[i] for c in self._cols} + for i in range(len(self._index))} + return {c.name: dict(zip(self._index._v, c._v)) for c in self._cols} + + def to_csv(self, path_or_buf=None, index=True, sep=",", header=True): + out = [] + + def cell(v): + if _isna(v): + return "" + s = str(v) + if sep in s or '"' in s or "\n" in s: + s = '"' + s.replace('"', '""') + '"' + return s + if header: + hdr = ([cell(self._index.name) if self._index.name is not None else ""] if index else []) + out.append(sep.join(hdr + [cell(n) for n in self._names()])) + for i in range(len(self._index)): + row = ([cell(self._index._v[i])] if index else []) + [cell(c._v[i]) for c in self._cols] + out.append(sep.join(row)) + text = "\n".join(out) + "\n" + if path_or_buf is None: + return text + if isinstance(path_or_buf, str): + with open(path_or_buf, "w") as f: + f.write(text) + else: + path_or_buf.write(text) + return None + + def to_json(self, path_or_buf=None, orient="columns"): + def clean(v): + return None if _isna(v) else v + if orient == "records": + obj = [{str(c.name): clean(c._v[i]) for c in self._cols} for i in range(len(self._index))] + elif orient == "list" or orient == "values": + obj = [[clean(c._v[i]) for c in self._cols] for i in range(len(self._index))] + elif orient == "index": + obj = {str(self._index._v[i]): {str(c.name): clean(c._v[i]) for c in self._cols} + for i in range(len(self._index))} + else: + obj = {str(c.name): {str(k): clean(v) for k, v in zip(self._index._v, c._v)} + for c in self._cols} + text = _json.dumps(obj, separators=(",", ":")) + if path_or_buf is None: + return text + if isinstance(path_or_buf, str): + with open(path_or_buf, "w") as f: + f.write(text) + else: + path_or_buf.write(text) + return None + + # repr + def __repr__(self): + names = self._names() + if not self._cols or not len(self._index): + return "Empty DataFrame\nColumns: [" + ", ".join(_label(n) for n in names) + \ + "]\nIndex: [" + ", ".join(_label(v) for v in self._index._v) + "]" + named = self._index.name is not None + idx = [""] + ([str(self._index.name)] if named else []) + [_label(v) for v in self._index._v] + iw = max(len(x) for x in idx) + lines = [x.ljust(iw) for x in idx] + for c in self._cols: + head = (" " if _numeric(c._dt) else "") + _label(c.name) + cells = _fmt_values(c._v, c._dt) + w = max([len(head)] + [len(x) for x in cells]) + colstrs = [head.rjust(w)] + ([" " * w] if named else []) + [x.rjust(w) for x in cells] + lines = [a + " " + b for a, b in zip(lines, colstrs)] + return "\n".join(lines) + + __str__ = __repr__ + + +class _DLoc: + def __init__(self, df, positional): + self._df = df + self._pos = positional + + def _rows(self, key): + df = self._df + n = len(df._index) + if isinstance(key, Series) and key._dt == "bool": + return df._mask(key), False + if isinstance(key, slice): + if self._pos or (key.start is None and key.stop is None): + return list(range(n)[key]), False + a = 0 if key.start is None else df._index.get_loc(key.start) + b = n if key.stop is None else df._index.get_loc(key.stop) + 1 + return list(range(a, b)), False + if isinstance(key, list): + if key and isinstance(key[0], bool): + return df._mask(key), False + return (list(key) if self._pos else [df._index.get_loc(k) for k in key]), False + return [key if self._pos else df._index.get_loc(key)], True + + def _colnames(self, key): + names = self._df._names() + if isinstance(key, slice): + if self._pos: + return names[key], False + a = 0 if key.start is None else names.index(key.start) + b = len(names) if key.stop is None else names.index(key.stop) + 1 + return names[a:b], False + if isinstance(key, list): + return ([names[k] for k in key] if self._pos else list(key)), False + return [names[key] if self._pos else key], True + + def __getitem__(self, key): + df = self._df + if isinstance(key, tuple): + rk, ck = key + else: + rk, ck = key, slice(None) + rows, one_row = self._rows(rk) + cols, one_col = self._colnames(ck) + if one_row and one_col: + return df._col(cols[0])._v[rows[0]] + if one_col: + c = df._col(cols[0]) + return c._take(rows) + sub = df._take(rows, cols) + if one_row: + return sub._row(0) + return sub + + def __setitem__(self, key, value): + df = self._df + if isinstance(key, tuple): + rk, ck = key + else: + rk, ck = key, slice(None) + cols, _ = self._colnames(ck) + if not self._pos and not isinstance(rk, (Series, slice, list)) and rk not in df._index: + df._index = Index(df._index._v + [rk], df._index.name) + for c in df._cols: + c._v = c._v + [nan] + c._index = df._index + rows, _ = self._rows(rk) + for name in cols: + if name not in df._names(): + df[name] = nan + c = df._col(name) + vals = list(c._v) + for r in rows: + vals[r] = value + dt, vals = _infer(vals) + c._dt, c._v = dt, vals + + +# ---- groupby -------------------------------------------------------------- + +def _agg_one(series, how): + if callable(how): + return how(series) + if how == "size": + return len(series) + if how == "first": + return series._v[0] + if how == "last": + return series._v[-1] + return getattr(series, how)() + + +class GroupBy: + def __init__(self, df, by, cols=None): + self._df = df + self._by = by + keys = df._col(by)._v + groups = {} + for i, k in enumerate(keys): + if _isna(k): + continue + if k not in groups: + groups[k] = [] + groups[k].append(i) + self._keys = sorted(groups.keys()) + self._groups = groups + self._cols = cols + + def _value_cols(self): + if self._cols is not None: + return self._cols + return [n for n in self._df._names() if n != self._by] + + def _index(self): + return Index(list(self._keys), self._by) + + def __getitem__(self, key): + if isinstance(key, list): + return GroupBy(self._df, self._by, key) + return SeriesGroupBy(self, key) + + def __iter__(self): + for k in self._keys: + yield k, self._df._take(self._groups[k]) + + @property + def groups(self): + return {k: self._df._index[0:0].__class__([self._df._index._v[i] for i in self._groups[k]]) + for k in self._keys} + + def get_group(self, key): + return self._df._take(self._groups[key]) + + def _agg(self, spec, numeric_only=False): + ix = self._index() + out = [] + for name in self._value_cols(): + how = spec.get(name) if isinstance(spec, dict) else spec + if how is None: + continue + col = self._df._col(name) + if numeric_only and not _numeric(col._dt): + continue + hows = how if isinstance(how, list) else [how] + for h in hows: + vals = [_agg_one(col._take(self._groups[k]), h) for k in self._keys] + out.append(Series._make(vals, ix, name)) + return DataFrame._from_series(out, ix) + + def agg(self, spec): + return self._agg(spec) + + aggregate = agg + + def sum(self): return self._agg("sum") + def mean(self): return self._agg("mean", True) + def median(self): return self._agg("median", True) + def std(self): return self._agg("std", True) + def min(self): return self._agg("min") + def max(self): return self._agg("max") + def first(self): return self._agg("first") + def last(self): return self._agg("last") + def nunique(self): return self._agg("nunique") + + def count(self): + df = self._agg("count") + for c in df._cols: + c._dt = "int64" + return df + + def size(self): + return Series._make([len(self._groups[k]) for k in self._keys], self._index(), None, "int64") + + def apply(self, func): + return Series._make([func(self._df._take(self._groups[k])) for k in self._keys], self._index()) + + +class SeriesGroupBy: + def __init__(self, gb, name): + self._gb = gb + self._name = name + + def agg(self, how): + gb = self._gb + col = gb._df._col(self._name) + if isinstance(how, list): + return DataFrame._from_series( + [Series._make([_agg_one(col._take(gb._groups[k]), h) for k in gb._keys], gb._index(), + h if isinstance(h, str) else h.__name__) for h in how], gb._index()) + vals = [_agg_one(col._take(gb._groups[k]), how) for k in gb._keys] + return Series._make(vals, gb._index(), self._name) + + aggregate = agg + apply = agg + + def sum(self): return self.agg("sum") + def mean(self): return self.agg("mean") + def median(self): return self.agg("median") + def std(self): return self.agg("std") + def min(self): return self.agg("min") + def max(self): return self.agg("max") + def count(self): return self.agg("count") + def nunique(self): return self.agg("nunique") + def first(self): return self.agg("first") + def last(self): return self.agg("last") + + def size(self): + return self._gb.size() + + +# ---- combining ------------------------------------------------------------ + +def merge(left, right, on=None, how="inner", left_on=None, right_on=None, suffixes=("_x", "_y")): + lk = left_on or on + rk = right_on or on + lkeys = left._col(lk)._v + rkeys = right._col(rk)._v + rpos = {} + for j, k in enumerate(rkeys): + rpos.setdefault(k, []).append(j) + pairs = [] + for i, k in enumerate(lkeys): + if k in rpos: + for j in rpos[k]: + pairs.append((i, j)) + elif how in ("left", "outer"): + pairs.append((i, None)) + if how == "outer": + matched = set(j for _, j in pairs if j is not None) + for j in range(len(rkeys)): + if j not in matched: + pairs.append((None, j)) + lnames = left._names() + rnames = [n for n in right._names() if not (n == rk and lk == rk)] + data = {} + order = [] + for n in lnames: + name = n + suffixes[0] if (n in rnames and n != lk) else n + col = left._col(n)._v + if n == lk: + data[name] = [col[i] if i is not None else rkeys[j] for i, j in pairs] + else: + data[name] = [col[i] if i is not None else None for i, j in pairs] + order.append(name) + for n in rnames: + name = n + suffixes[1] if n in lnames else n + col = right._col(n)._v + data[name] = [col[j] if j is not None else None for i, j in pairs] + order.append(name) + return DataFrame({k: data[k] for k in order}) + + +def concat(objs, axis=0, ignore_index=False): + objs = list(objs) + if objs and isinstance(objs[0], Series) and axis in (1, "columns"): + return DataFrame({s.name: s for s in objs}) + if objs and isinstance(objs[0], Series): + vals, labels = [], [] + for s in objs: + vals.extend(s._v) + labels.extend(s._index._v) + ix = Index._rng(len(vals)) if ignore_index else Index(labels) + return Series._make(vals, ix, objs[0].name) + if axis in (1, "columns"): + cols = [] + for d in objs: + cols.extend(c.copy() for c in d._cols) + return DataFrame._from_series(cols, objs[0]._index) + names = [] + for d in objs: + for n in d._names(): + if n not in names: + names.append(n) + labels = [] + for d in objs: + labels.extend(d._index._v) + ix = Index._rng(len(labels)) if ignore_index else Index(labels) + cols = [] + for n in names: + vals = [] + for d in objs: + if n in d._names(): + vals.extend(d._col(n)._v) + else: + vals.extend([None] * len(d._index)) + cols.append(Series._make(vals, ix, n)) + return DataFrame._from_series(cols, ix) + + +# ---- I/O ------------------------------------------------------------------ + +_NA_STRINGS = ("", "NaN", "nan", "NA", "N/A", "null", "NULL", "None", "n/a", "") + + +def _parse_column(cells): + vals = [] + kind = "int" + for s in cells: + if s in _NA_STRINGS: + vals.append(None) + continue + vals.append(s) + present = [v for v in vals if v is not None] + if present and all(v in ("True", "False", "TRUE", "FALSE", "true", "false") for v in present) \ + and len(present) == len(vals): + return [v.lower() == "true" for v in vals] + for v in present: + try: + int(v) + except ValueError: + kind = "float" + break + if kind == "float": + for v in present: + try: + float(v) + except ValueError: + kind = "str" + break + if kind == "int": + return [int(v) if v is not None else None for v in vals] + if kind == "float": + return [float(v) if v is not None else None for v in vals] + return vals + + +def read_csv(filepath_or_buffer, sep=",", header=0, names=None, index_col=None, usecols=None): + if isinstance(filepath_or_buffer, str): + with open(filepath_or_buffer) as f: + text = f.read() + else: + text = filepath_or_buffer.read() + rows = [r for r in _csv.reader(_io.StringIO(text), delimiter=sep) if r] + if names is not None: + cols = list(names) + body = rows[1:] if header == 0 and header is not None and names is None else rows + if header == 0 and names is not None and header is not None and header != "infer": + body = rows + elif header is None: + cols = list(range(len(rows[0]))) if rows else [] + body = rows + else: + cols = rows[0] if rows else [] + body = rows[1:] + data = {} + for j, n in enumerate(cols): + if usecols is not None and n not in usecols: + continue + data[n] = _parse_column([r[j] if j < len(r) else "" for r in body]) + df = DataFrame(data) + if index_col is not None: + key = cols[index_col] if isinstance(index_col, int) else index_col + df = df.set_index(key) + return df + + +def read_json(path_or_buf, orient=None): + if isinstance(path_or_buf, str): + if path_or_buf.lstrip()[:1] in ("[", "{"): + text = path_or_buf + else: + with open(path_or_buf) as f: + text = f.read() + else: + text = path_or_buf.read() + obj = _json.loads(text) + if isinstance(obj, list): + if obj and isinstance(obj[0], list): + return DataFrame(obj) + return DataFrame(obj) + if orient == "index": + labels = list(obj.keys()) + return DataFrame([obj[k] for k in labels], index=labels) + labels = [] + for col in obj.values(): + for k in col: + if k not in labels: + labels.append(k) + data = {c: [obj[c].get(k) for k in labels] for c in obj} + try: + labels = [int(k) for k in labels] + except ValueError: + pass + return DataFrame(data, index=labels) + + +def to_numeric(arg, errors="raise"): + def conv(v): + if _is_num(v) or _isna(v): + return v + try: + return int(v) + except (ValueError, TypeError): + try: + return float(v) + except (ValueError, TypeError): + if errors == "coerce": + return nan + raise + if isinstance(arg, Series): + return Series._make([conv(v) for v in arg._v], arg._index, arg.name) + if isinstance(arg, list): + return _np.array([conv(v) for v in arg]) + return conv(arg) + + +def unique(values): + return Series(values).unique() diff --git a/maven/python-transpiler/src/test/resources/conformance/pandas_frame.out b/maven/python-transpiler/src/test/resources/conformance/pandas_frame.out new file mode 100644 index 00000000000..5b1491728df --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/pandas_frame.out @@ -0,0 +1,155 @@ + name age score +0 Ann 28 88.50 +1 Bob 35 92.00 +2 Cy 42 79.25 +3 Dee 19 95.00 +(4, 3) ['name', 'age', 'score'] 4 +Index(['name', 'age', 'score'], dtype='str') +RangeIndex(start=0, stop=4, step=1) +name str +age int64 +score float64 +dtype: object +0 28 +1 35 +2 42 +3 19 +Name: age, dtype: int64 + name score +0 Ann 88.50 +1 Bob 92.00 +2 Cy 79.25 +3 Dee 95.00 + name age score +0 Ann 28 88.50 +1 Bob 35 92.00 +2 Cy 42 79.25 + name age score +0 Ann 28 88.5 +1 Bob 35 92.0 + name age score +3 Dee 19 95.0 +name Bob +age 35 +score 92.0 +Name: 1, dtype: object +Cy 28 95.0 + name age score +1 Bob 35 92.00 +2 Cy 42 79.25 + name score +1 Bob 92.0 +3 Dee 95.0 + name age score passed double +0 Ann 28 88.50 True 56 +1 Bob 35 92.00 True 70 +2 Cy 42 79.25 False 84 +3 Dee 19 95.00 True 38 + age score +count 4.000000 4.000000 +mean 31.000000 88.687500 +std 9.831921 6.829394 +min 19.000000 79.250000 +25% 25.750000 86.187500 +50% 31.500000 90.250000 +75% 36.750000 92.750000 +max 42.000000 95.000000 + name age score passed +3 Dee 19 95.00 True +0 Ann 28 88.50 True +1 Bob 35 92.00 True +2 Cy 42 79.25 False + name age score passed +3 Dee 19 95.00 True +1 Bob 35 92.00 True +0 Ann 28 88.50 True +2 Cy 42 79.25 False +31.0 95.0 + pts ast +team +a 9 4.0 +b 6 2.0 + pts ast +team +a 3.0 1.333333 +b 3.0 1.000000 + pts ast +team +a 3 3 +b 2 2 +team +a 3 +b 2 +dtype: int64 +team +a 9 +b 6 +Name: pts, dtype: int64 + pts ast +team +a 9 2.0 +b 6 1.5 + k a b +0 1 x 10 +1 3 z 30 + k a b +0 1 x 10.0 +1 2 y NaN +2 3 z 30.0 + k a +0 1 x +1 2 y +2 3 z +0 1 x +1 2 y +2 3 z + k a b +0 1 x NaN +1 2 y NaN +2 3 z NaN +3 1 NaN 10.0 +4 3 NaN 30.0 +5 4 NaN 40.0 + k a c +0 1 x 3 +1 2 y 6 +2 3 z 9 + k letter +0 1 x +1 2 y +2 3 z + k +0 1 +1 2 +2 3 + k a +1 2 y +pts 5.0 +ast 2.0 +dtype: float64 +0 1.0 +1 1.0 +2 6.0 +3 6.0 +4 5.0 +dtype: float64 +[{'team': 'a', 'pts': 1, 'ast': 1.0}, {'team': 'b', 'pts': 2, 'ast': 0.5}, {'team': 'a', 'pts': 3, 'ast': 2.0}, {'team': 'b', 'pts': 4, 'ast': 1.5}, {'team': 'a', 'pts': 5, 'ast': 1.0}] +{'team': ['a', 'b', 'a', 'b', 'a'], 'pts': [1, 2, 3, 4, 5], 'ast': [1.0, 0.5, 2.0, 1.5, 1.0]} +{'k': {0: 1, 1: 2, 2: 3}, 'a': {0: 'x', 1: 'y', 2: 'z'}} +0 1 x +1 2 y +2 3 z +0 1 x +1 2 y +2 3 z + a b +0 1 2 +1 3 4 + x y +0 1 2.5 +1 3 4.0 + a b +0 1 NaN +1 2 5.0 +[28 35 42 19] +False 15 diff --git a/maven/python-transpiler/src/test/resources/conformance/pandas_frame.py b/maven/python-transpiler/src/test/resources/conformance/pandas_frame.py new file mode 100644 index 00000000000..31a96b4d0a5 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/pandas_frame.py @@ -0,0 +1,56 @@ +import pandas as pd + +df = pd.DataFrame({"name": ["Ann", "Bob", "Cy", "Dee"], "age": [28, 35, 42, 19], "score": [88.5, 92.0, 79.25, 95.0]}) +print(df) +print(df.shape, list(df.columns), len(df)) +print(df.columns) +print(df.index) +print(df.dtypes) +print(df["age"]) +print(df[["name", "score"]]) +print(df[df["age"] > 25]) +print(df.head(2)) +print(df.tail(1)) +print(df.loc[1]) +print(df.loc[2, "name"], df.iloc[0, 1], df.iloc[3]["score"]) +print(df.iloc[1:3]) +print(df.loc[df["score"] > 90, ["name", "score"]]) +df["passed"] = df["score"] >= 80 +df["double"] = df["age"] * 2 +print(df) +del df["double"] +print(df.describe()) +print(df.sort_values("age")) +print(df.sort_values(by="score", ascending=False)) +print(df["age"].mean(), df["score"].max()) +g = pd.DataFrame({"team": ["a", "b", "a", "b", "a"], "pts": [1, 2, 3, 4, 5], "ast": [1.0, 0.5, 2.0, 1.5, 1.0]}) +print(g.groupby("team").sum()) +print(g.groupby("team").mean()) +print(g.groupby("team").count()) +print(g.groupby("team").size()) +print(g.groupby("team")["pts"].sum()) +print(g.groupby("team").agg({"pts": "sum", "ast": "max"})) +left = pd.DataFrame({"k": [1, 2, 3], "a": ["x", "y", "z"]}) +right = pd.DataFrame({"k": [1, 3, 4], "b": [10, 30, 40]}) +print(pd.merge(left, right, on="k")) +print(left.merge(right, on="k", how="left")) +print(pd.concat([left, left])) +print(pd.concat([left, right], ignore_index=True)) +print(left.assign(c=lambda d: d["k"] * 3)) +print(left.rename(columns={"a": "letter"})) +print(left.drop(columns=["a"])) +print(left.drop([0, 2])) +print(g[["pts", "ast"]].apply(lambda col: col.max())) +print(g.apply(lambda r: r["pts"] * r["ast"], axis=1)) +print(g.to_dict(orient="records")) +print(g.to_dict(orient="list")) +print(left.to_dict()) +for row in left.itertuples(): + print(row.Index, row.k, row.a) +for i, row in left.iterrows(): + print(i, row["k"], row["a"]) +print(pd.DataFrame([{"a": 1, "b": 2}, {"a": 3, "b": 4}])) +print(pd.DataFrame([[1, 2.5], [3, 4.0]], columns=["x", "y"])) +print(pd.DataFrame([{"a": 1}, {"a": 2, "b": 5}])) +print(df["age"].values) +print(df.empty, g.size) diff --git a/maven/python-transpiler/src/test/resources/conformance/pandas_io.out b/maven/python-transpiler/src/test/resources/conformance/pandas_io.out new file mode 100644 index 00000000000..b07df6092a3 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/pandas_io.out @@ -0,0 +1,40 @@ + id name price qty +0 1 pen 1.50 10.0 +1 2 book 12.00 NaN +2 3 lamp 30.25 2.0 +id int64 +name str +price float64 +qty float64 +dtype: object +[False, True, False] + id name price qty +0 1 pen 1.50 10.0 +1 2 book 12.00 0.0 +2 3 lamp 30.25 2.0 + id name price qty +0 1 pen 1.50 10.0 +2 3 lamp 30.25 2.0 +id,name,price,qty +1,pen,1.5,10.0 +2,book,12.0, +3,lamp,30.25,2.0 + +,id,name,price,qty +0,1,pen,1.5,10.0 +1,2,book,12.0, +2,3,lamp,30.25,2.0 + +[{"id":1,"name":"pen","price":1.5,"qty":10.0},{"id":2,"name":"book","price":12.0,"qty":null},{"id":3,"name":"lamp","price":30.25,"qty":2.0}] + a b +0 1 x +1 2 y +a int64 +b str +dtype: object +True (3, 4) +0 15.0 +1 NaN +2 60.5 +dtype: float64 +75.5 diff --git a/maven/python-transpiler/src/test/resources/conformance/pandas_io.py b/maven/python-transpiler/src/test/resources/conformance/pandas_io.py new file mode 100644 index 00000000000..5ae9d262abb --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/pandas_io.py @@ -0,0 +1,24 @@ +import io +import os +import pandas as pd + +text = "id,name,price,qty\n1,pen,1.5,10\n2,book,12,\n3,lamp,30.25,2\n" +df = pd.read_csv(io.StringIO(text)) +print(df) +print(df.dtypes) +print(df["qty"].isna().tolist()) +print(df.fillna(0)) +print(df.dropna()) +print(df.to_csv(index=False)) +print(df.to_csv()) +print(df.to_json(orient="records")) +back = pd.read_json(io.StringIO('[{"a": 1, "b": "x"}, {"a": 2, "b": "y"}]')) +print(back) +print(back.dtypes) +path = "cn1_pandas_io_test.csv" +df.to_csv(path, index=False) +again = pd.read_csv(path) +print(again.equals(df), again.shape) +os.remove(path) +print(df["price"] * df["qty"]) +print((df["price"] * df["qty"]).sum()) diff --git a/maven/python-transpiler/src/test/resources/conformance/pandas_series.out b/maven/python-transpiler/src/test/resources/conformance/pandas_series.out new file mode 100644 index 00000000000..2540757d6e9 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/pandas_series.out @@ -0,0 +1,156 @@ +0 1 +1 2 +2 3 +3 4 +dtype: int64 +int64 4 10 2.5 4 1 +a 1.50 +b 2.25 +c 3.00 +Name: price, dtype: float64 +2.25 3.0 1.5 +['a', 'b', 'c'] price +a 3.0 +b 4.5 +c 6.0 +Name: price, dtype: float64 +a 4.50 +b 4.25 +c 4.00 +dtype: float64 +0 False +1 False +2 True +3 True +dtype: bool +2 3 +3 4 +dtype: int64 +0 1 +1 2 +dtype: int64 +2 3 +3 4 +dtype: int64 +1.290994 4 2.5 +0 a +1 b +2 a +3 c +4 a +Name: letter, dtype: str +letter +a 3 +b 1 +c 1 +Name: count, dtype: int64 +['a', 'b', 'c'] +0 1.0 +1 NaN +2 3.0 +dtype: float64 +0 False +1 True +2 False +dtype: bool +0 1.0 +1 0.0 +2 3.0 +dtype: float64 +0 1.0 +2 3.0 +dtype: float64 +4.0 2 2.0 +0 10 +1 20 +2 30 +3 40 +dtype: int64 +0 1 +1 2 +2 1 +3 3 +4 1 +Name: letter, dtype: int64 +0 1.0 +1 2.0 +2 3.0 +3 4.0 +dtype: float64 +a 1 +b 2 +c 3 +Name: price, dtype: int64 +1 1 +2 2 +0 3 +dtype: int64 +x 1 +y 2 +z 3 +dtype: int64 +0 3 +2 2 +1 1 +dtype: int64 +0 1 +1 3 +2 6 +3 10 +dtype: int64 +0 Apple +1 banana +2 Cherry +dtype: str +0 apple +1 banana +2 cherry +dtype: str +0 APPLE +1 BANANA +2 CHERRY +dtype: str +0 7 +1 6 +2 6 +dtype: int64 +0 False +1 True +2 False +dtype: bool +[1, 2, 3, 4] [1 2 3 4] +[1 2 3 4] +0 True +1 False +2 True +dtype: bool +0 1.0 +1 NaN +2 3.0 +dtype: float64 +1 2 +2 3 +dtype: int64 +0 -1 +1 -2 +2 -3 +3 -4 +dtype: int64 +0 False +1 True +2 False +3 False +dtype: bool +2 +x 10 +y 20 +dtype: int64 +0 0.123457 +1 12.500000 +dtype: float64 +RangeIndex(start=0, stop=4, step=1) +Index(['a', 'b', 'c'], dtype='str') +0 11.0 +1 22.0 +2 NaN +dtype: float64 diff --git a/maven/python-transpiler/src/test/resources/conformance/pandas_series.py b/maven/python-transpiler/src/test/resources/conformance/pandas_series.py new file mode 100644 index 00000000000..d1636d70ea6 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/pandas_series.py @@ -0,0 +1,53 @@ +import pandas as pd + +s = pd.Series([1, 2, 3, 4]) +print(s) +print(s.dtype, len(s), s.sum(), s.mean(), s.max(), s.min()) +f = pd.Series([1.5, 2.25, 3.0], index=["a", "b", "c"], name="price") +print(f) +print(f["b"], f.loc["c"], f.iloc[0]) +print(list(f.index), f.name) +print(f * 2) +print(f + pd.Series([1.0, 2.0, 3.0], index=["c", "b", "a"])) +print(s > 2) +print(s[s > 2]) +print(s.head(2)) +print(s.tail(2)) +print(round(s.std(), 6), s.count(), s.median()) +w = pd.Series(["a", "b", "a", "c", "a"], name="letter") +print(w) +print(w.value_counts()) +print(list(w.unique())) +n = pd.Series([1.0, float("nan"), 3.0]) +print(n) +print(n.isna()) +print(n.fillna(0)) +print(n.dropna()) +print(n.sum(), n.count(), n.mean()) +print(s.apply(lambda x: x * 10)) +print(w.map({"a": 1, "b": 2, "c": 3})) +print(s.astype(float)) +print(f.astype(int)) +print(pd.Series([3, 1, 2]).sort_values()) +print(pd.Series([3, 1, 2], index=["z", "x", "y"]).sort_index()) +print(pd.Series([3, 1, 2]).sort_values(ascending=False)) +print(s.cumsum()) +t = pd.Series([" Apple ", "banana", "Cherry"]) +print(t.str.strip()) +print(t.str.lower()) +print(t.str.upper()) +print(t.str.len()) +print(t.str.contains("an")) +print(s.tolist(), s.to_numpy()) +print(s.values) +print(pd.Series([True, False, True])) +print(pd.Series([1, None, 3])) +print(s.iloc[1:3]) +print(-s) +print(s == 2) +print((s % 2 == 0).sum()) +print(pd.Series({"x": 10, "y": 20})) +print(pd.Series([0.123456789, 12.5])) +print(s.index) +print(f.index) +print(pd.Series([1, 2]) + pd.Series([10, 20, 30])) From b5d6be2b29adb4186ed266f7ad60829eeb798686 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 09:57:09 +0200 Subject: [PATCH 049/112] Python guide: pandas subset Co-Authored-By: Claude Opus 5.5 --- docs/developer-guide/Python-Interop.asciidoc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/developer-guide/Python-Interop.asciidoc b/docs/developer-guide/Python-Interop.asciidoc index 8c7bd9eac00..1b0df468d2a 100644 --- a/docs/developer-guide/Python-Interop.asciidoc +++ b/docs/developer-guide/Python-Interop.asciidoc @@ -196,6 +196,12 @@ the real package: views, fancy indexing, ufuncs, reductions, sorting and `numpy.linalg`. The kernels are Java loops over primitive arrays. `numpy.random` draws from Python's generator, so its streams differ from NumPy's. +| `pandas` | `Series` and `DataFrame` over the `numpy` subset: index + alignment, selection with `loc`, `iloc` and masks, missing values, + statistics and `describe`, `groupby` on one key, `merge`, `concat`, and + CSV and JSON reading and writing. Printing matches pandas. Long frames + print in full, and date indexes, `MultiIndex`, pivot tables, categoricals + and rolling windows aren't supported. | `Pillow` | `PIL.Image` in the `RGB`, `RGBA` and `L` modes: conversion, cropping, resizing, rotation, pasting, channels and point operations, computed as Pillow computes them. PNG is read and written on every platform; From 6a6ad35b6d907e6fd473fa807b1c166fade028dc Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 10:14:37 +0200 Subject: [PATCH 050/112] Python: FastAPI middleware (http, CORS, GZip, TrustedHost, BaseHTTPMiddleware) and WebSockets The dispatch core is async end to end so a middleware's call_next runs inside the same event loop; the stack is built with the last added outermost, as Starlette does. Websocket endpoints are driven step by step by a session that both TestClient.websocket_connect and the platform server feed; uvicorn.run wires them to the backend HttpServer's websocket fallback through PyServer. Co-Authored-By: Claude Opus 5.5 --- .../codename1/python/runtime/PyServer.java | 82 +++++++ .../codename1/python/runtime/StdModules.java | 7 +- .../transpiler/api/PythonTranspiler.java | 91 +++++++- .../codename1/python/site/fastapi/__init__.py | 6 +- .../python/site/fastapi/applications.py | 95 +++++++- .../python/site/fastapi/exceptions.py | 17 ++ .../site/fastapi/middleware/__init__.py | 5 + .../python/site/fastapi/middleware/cors.py | 5 + .../python/site/fastapi/middleware/gzip.py | 5 + .../site/fastapi/middleware/trustedhost.py | 5 + .../codename1/python/site/fastapi/routing.py | 91 ++++++-- .../python/site/fastapi/websockets.py | 5 + .../python/site/starlette/datastructures.py | 6 + .../site/starlette/middleware/__init__.py | 24 ++ .../python/site/starlette/middleware/base.py | 21 ++ .../python/site/starlette/middleware/cors.py | 96 ++++++++ .../python/site/starlette/middleware/gzip.py | 34 +++ .../site/starlette/middleware/trustedhost.py | 41 ++++ .../python/site/starlette/testclient.py | 107 +++++++++ .../python/site/starlette/websockets.py | 184 ++++++++++++++++ .../codename1/python/site/uvicorn/__init__.py | 53 ++++- .../conformance/fastapi_middleware_ws.out | 33 +++ .../conformance/fastapi_middleware_ws.py | 206 ++++++++++++++++++ 23 files changed, 1179 insertions(+), 40 deletions(-) create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/middleware/__init__.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/middleware/cors.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/middleware/gzip.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/middleware/trustedhost.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/websockets.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/middleware/__init__.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/middleware/base.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/middleware/cors.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/middleware/gzip.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/middleware/trustedhost.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/websockets.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/fastapi_middleware_ws.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/fastapi_middleware_ws.py diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyServer.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyServer.java index 5e92a3fe08a..f98d41958bc 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyServer.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyServer.java @@ -48,6 +48,88 @@ public static void install(PyServer server) { /// Serves until the server stops. public abstract void serve(Object dispatch, String host, int port); + /// Serves `dispatch`, and websocket connections through `webSockets` + /// when it is not null. `webSockets` is the callable `uvicorn.run` + /// passes: `(path, query, headers, send)` answering a connection object + /// with `text(str)`, `binary(bytes)` and `closed(code, reason)`. A server + /// that cannot upgrade a connection refuses to start rather than serving + /// an application whose websocket routes nothing could reach. + public void serve(Object dispatch, String host, int port, Object webSockets) { + if (webSockets != null) { + throw Py.error(Py.OS_ERROR, "this HTTP server does not support websockets"); + } + serve(dispatch, host, port); + } + + /// What a server does with the messages a Python websocket endpoint sends. + public interface WebSocketSink { + /// The endpoint accepted; the handshake has already completed. + void accept(String subprotocol); + + void text(String value); + + void binary(byte[] value); + + void close(int code, String reason); + } + + /// Opens one websocket connection in Python; answers the connection + /// object [#webSocketText], [#webSocketBinary] and [#webSocketClosed] + /// take. Messages the endpoint sends, including any it sends while + /// opening, reach `sink`. + public static Object openWebSocket(Object webSockets, String target, String[] headers, + final WebSocketSink sink) { + Object[] req = request("GET", target, headers, null); + PyBuiltin send = PyBuiltin.positional("send", a -> { + String kind = Py.toStr(a[0]); + if ("accept".equals(kind)) { + sink.accept(a[1] == Py.NONE ? null : Py.toStr(a[1])); //NOPMD CompareObjectsWithEquals - identity is meant + } else if ("text".equals(kind)) { + sink.text(Py.toStr(a[1])); + } else if ("bytes".equals(kind)) { + sink.binary(PyBytes.like(a[1])); + } else { + sink.close((int) Py.asLong(a[1]), a[2] == Py.NONE ? "" : Py.toStr(a[2])); //NOPMD CompareObjectsWithEquals - identity is meant + } + return Py.NONE; + }); + Gil.enter(); + try { + return Py.obj(webSockets).call(new Object[] {req[1], req[2], req[3], send}, Py.NO_KW); + } finally { + Gil.exit(); + } + } + + /// Delivers a text message to a connection [#openWebSocket] opened. + public static void webSocketText(Object connection, String value) { + event(connection, "text", value); + } + + /// Delivers a binary message to a connection [#openWebSocket] opened. + public static void webSocketBinary(Object connection, byte[] value) { + event(connection, "binary", new PyBytes(value)); + } + + /// Tells a connection [#openWebSocket] opened that the peer has gone. + public static void webSocketClosed(Object connection, int code, String reason) { + Gil.enter(); + try { + Py.callMethod(connection, "closed", Py.of(code), reason == null ? "" : reason); + } finally { + Gil.exit(); + } + } + + private static void event(Object connection, String name, Object value) { + Gil.enter(); + try { + Py.callMethod(connection, name, value); + } finally { + Gil.exit(); + } + } + /// The request as `dispatch` takes it. public static Object[] request(String method, String target, String[] headers, byte[] body) { int q = target.indexOf('?'); diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java index 37aa74940ef..a341fda878e 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java @@ -64,11 +64,12 @@ static boolean provide(String name) { if ("_cn1server".equals(name)) { PyModule m = new PyModule("_cn1server", new String[0]); m.setAttr("serve", PyBuiltin.positional("serve", a -> { - if (a.length != 3) { - throw Py.typeError("serve(dispatch, host, port)"); + if (a.length != 3 && a.length != 4) { + throw Py.typeError("serve(dispatch, host, port[, websockets])"); } + Object ws = a.length == 4 && a[3] != Py.NONE ? a[3] : null; //NOPMD CompareObjectsWithEquals - identity is meant PyServer.get().serve(a[0], a[1] == Py.NONE ? null : Py.toStr(a[1]), //NOPMD CompareObjectsWithEquals - identity is meant - (int) Py.asLong(a[2])); + (int) Py.asLong(a[2]), ws); return Py.NONE; })); PyModule.registerModule(m); diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java index 353e6e719a8..b205e516d1f 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java @@ -411,6 +411,8 @@ private static String backendServer() { return "// Generated by the Codename One Python transpiler.\n" + "package " + PACKAGE + ";\n\n" + "import com.codename1.backend.HttpServer;\n" + + "import com.codename1.backend.WebSocket;\n" + + "import com.codename1.backend.WebSocketSession;\n" + "import com.codename1.python.runtime.PyServer;\n" + "import java.util.Iterator;\n" + "import java.util.LinkedHashMap;\n" @@ -421,20 +423,89 @@ private static String backendServer() { + " }\n\n" + " @Override\n" + " public void serve(final Object dispatch, String host, int port) {\n" + + " serve(dispatch, host, port, null);\n" + + " }\n\n" + + " static String[] headers(HttpServer.Request r) {\n" + + " Map in = r.getHeaders();\n" + + " String[] headers = new String[in.size() * 2];\n" + + " int at = 0;\n" + + " for (Iterator it = in.entrySet().iterator(); it.hasNext();) {\n" + + " Map.Entry e = (Map.Entry) it.next();\n" + + " headers[at++] = String.valueOf(e.getKey());\n" + + " headers[at++] = String.valueOf(e.getValue());\n" + + " }\n" + + " return headers;\n" + + " }\n\n" + + " /** One websocket connection driven by the Python endpoint. */\n" + + " static final class Socket implements WebSocket, PyServer.WebSocketSink {\n" + + " private final Object open;\n" + + " private final String target;\n" + + " private final String[] headers;\n" + + " private WebSocketSession session;\n" + + " private Object connection;\n\n" + + " Socket(Object open, String target, String[] headers) {\n" + + " this.open = open;\n" + + " this.target = target;\n" + + " this.headers = headers;\n" + + " }\n\n" + + " public void onOpen(WebSocketSession s) {\n" + + " session = s;\n" + + " connection = PyServer.openWebSocket(open, target, headers, this);\n" + + " }\n\n" + + " public void onText(WebSocketSession s, String message) {\n" + + " if (connection != null) {\n" + + " PyServer.webSocketText(connection, message);\n" + + " }\n" + + " }\n\n" + + " public void onBinary(WebSocketSession s, byte[] message, int offset, int length) {\n" + + " if (connection != null) {\n" + + " byte[] copy = new byte[length];\n" + + " System.arraycopy(message, offset, copy, 0, length);\n" + + " PyServer.webSocketBinary(connection, copy);\n" + + " }\n" + + " }\n\n" + + " public void onClose(WebSocketSession s, int code, String reason) {\n" + + " if (connection != null) {\n" + + " PyServer.webSocketClosed(connection, code, reason);\n" + + " }\n" + + " }\n\n" + + " public void accept(String subprotocol) {\n" + + " }\n\n" + + " public void text(String value) {\n" + + " try {\n" + + " session.sendText(value);\n" + + " } catch (java.io.IOException e) {\n" + + " session.abort();\n" + + " }\n" + + " }\n\n" + + " public void binary(byte[] value) {\n" + + " try {\n" + + " session.sendBinary(value);\n" + + " } catch (java.io.IOException e) {\n" + + " session.abort();\n" + + " }\n" + + " }\n\n" + + " public void close(int code, String reason) {\n" + + " session.close(code, reason);\n" + + " }\n" + + " }\n\n" + + " @Override\n" + + " public void serve(final Object dispatch, String host, int port, final Object webSockets) {\n" + + " HttpServer.WebSocketRoutes routes = webSockets == null ? null : new HttpServer.WebSocketRoutes() {\n" + + " public void register(HttpServer.WebSocketRegistry registry) {\n" + + " registry.fallback(new HttpServer.WebSocketHandler() {\n" + + " public WebSocket open(HttpServer.Request r) {\n" + + " return new Socket(webSockets, r.getTarget(), headers(r));\n" + + " }\n" + + " });\n" + + " }\n" + + " };\n" + " HttpServer server;\n" + " try {\n" + " server = HttpServer.start(host, port, 512, 1, new HttpServer.Handler() {\n" + " public HttpServer.Response handle(HttpServer.Request r) {\n" - + " Map in = r.getHeaders();\n" - + " String[] headers = new String[in.size() * 2];\n" - + " int at = 0;\n" - + " for (Iterator it = in.entrySet().iterator(); it.hasNext();) {\n" - + " Map.Entry e = (Map.Entry) it.next();\n" - + " headers[at++] = String.valueOf(e.getKey());\n" - + " headers[at++] = String.valueOf(e.getValue());\n" - + " }\n" + " PyServer.Response res = PyServer.call(dispatch,\n" - + " PyServer.request(r.getMethod(), r.getTarget(), headers, r.getBodyBytes()));\n" + + " PyServer.request(r.getMethod(), r.getTarget(), headers(r), r.getBodyBytes()));\n" + " Map extra = new LinkedHashMap();\n" + " String type = null;\n" + " for (int i = 0; i + 1 < res.headers.length; i += 2) {\n" @@ -446,7 +517,7 @@ private static String backendServer() { + " }\n" + " return new HttpServer.Response(res.status, type, res.body, extra);\n" + " }\n" - + " });\n" + + " }, (com.codename1.backend.Tls) null, routes);\n" + " } catch (Exception e) {\n" + " throw new RuntimeException(\"cannot serve on port \" + port + \": \" + e.getMessage());\n" + " }\n" diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/__init__.py index d12d4eed26c..c97adc695ba 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/__init__.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/__init__.py @@ -10,12 +10,14 @@ from starlette.background import BackgroundTasks from starlette.requests import Request from starlette.responses import Response +from starlette.websockets import WebSocket, WebSocketDisconnect from fastapi import status from fastapi.applications import FastAPI -from fastapi.exceptions import HTTPException +from fastapi.exceptions import HTTPException, WebSocketException from fastapi.params import Body, Cookie, Depends, File, Form, Header, Path, Query, Security from fastapi.routing import APIRouter __all__ = ["FastAPI", "APIRouter", "Depends", "Query", "Path", "Body", "Header", "Cookie", "Form", - "File", "Security", "HTTPException", "Request", "Response", "BackgroundTasks", "status"] + "File", "Security", "HTTPException", "Request", "Response", "BackgroundTasks", "status", + "WebSocket", "WebSocketDisconnect", "WebSocketException"] diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/applications.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/applications.py index ab5abfee69f..a086bab2191 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/applications.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/applications.py @@ -15,7 +15,10 @@ from fastapi.exceptions import RequestValidationError, ResponseValidationError from fastapi.params import Undefined -from fastapi.routing import APIRouter, APIRoute, run_coro, run_route, _all_params, _all_body, _analyze +from fastapi.routing import (APIRouter, APIRoute, APIWebSocketRoute, run_coro, run_route, run_websocket, + _all_params, _all_body, _analyze) +from starlette.middleware.base import BaseHTTPMiddleware +from starlette.websockets import WebSocket, WebSocketDisconnect, _Session def _http_exception_handler(request, exc): @@ -33,7 +36,7 @@ class FastAPI(APIRouter): def __init__(self, *, debug=False, title="FastAPI", summary=None, description="", version="0.1.0", openapi_url="/openapi.json", docs_url="/docs", redoc_url="/redoc", default_response_class=None, dependencies=None, exception_handlers=None, - lifespan=None, tags=None, root_path="", **kw): + lifespan=None, tags=None, root_path="", middleware=None, **kw): APIRouter.__init__(self, dependencies=dependencies, default_response_class=default_response_class) self.debug = debug self.title = title @@ -59,8 +62,37 @@ def __init__(self, *, debug=False, title="FastAPI", summary=None, description="" self._lifespan = lifespan self._lifespan_cm = None self._openapi = None + self.user_middleware = [] + self._stack = None + for m in middleware or []: + self.user_middleware.append(m) # -- registration + def add_middleware(self, middleware_class, *args, **kwargs): + if self._stack is not None: + raise RuntimeError("Cannot add middleware after an application has started") + from starlette.middleware import Middleware + self.user_middleware.insert(0, Middleware(middleware_class, *args, **kwargs)) + + def middleware(self, middleware_type): + if middleware_type != "http": + raise ValueError("Currently only middleware(\"http\") is supported.") + + def deco(f): + self.add_middleware(BaseHTTPMiddleware, dispatch=f) + return f + return deco + + def _build_stack(self): + if self._stack is None: + app = self._core + for m in reversed(self.user_middleware): + cls, args, kwargs = m + app = cls(app, *args, **kwargs) + self._stack = app + return self._stack + + # -- registration (handlers) def exception_handler(self, exc_class_or_status): def deco(f): self.exception_handlers[exc_class_or_status] = f @@ -121,13 +153,13 @@ def _find_handler(self, exc): return self.exception_handlers[cls] return None - def _handle_exception(self, req, exc): + async def _handle_exception(self, req, exc): h = self._find_handler(exc) if h is None: raise exc r = h(req, exc) if inspect.iscoroutine(r): - r = run_coro(r) + r = await r return r def _route(self, req): @@ -135,6 +167,8 @@ def _route(self, req): method = req.method partial = None for r in self.routes: + if isinstance(r, APIWebSocketRoute): + continue vals = r.match(path) if vals is None: continue @@ -151,7 +185,7 @@ def _route(self, req): else: alt = path + "/" for r in self.routes: - if r.match(alt) is not None: + if not isinstance(r, APIWebSocketRoute) and r.match(alt) is not None: host = req.headers.get("host", "testserver") loc = "http://" + host + quote(alt) if req._query: @@ -159,9 +193,7 @@ def _route(self, req): return RedirectResponse(loc, status_code=307) raise StarletteHTTPException(404, "Not Found") - def _handle(self, method, path, query, headers, body): - req = Request(method.upper(), unquote(path), query, headers, body, self) - req.app = self + async def _core(self, req): try: if self.openapi_url and req._path == self.openapi_url and req.method == "GET": resp = JSONResponse(self.openapi()) @@ -170,9 +202,15 @@ def _handle(self, method, path, query, headers, body): if isinstance(route, Response): resp = route else: - resp = run_route(route, req, self.default_response_class or JSONResponse) + resp = await run_route(route, req, self.default_response_class or JSONResponse) except Exception as exc: - resp = self._handle_exception(req, exc) + resp = await self._handle_exception(req, exc) + return resp + + def _handle(self, method, path, query, headers, body): + req = Request(method.upper(), unquote(path), query, headers, body, self) + req.app = self + resp = run_coro(self._build_stack()(req)) if resp.background is not None: resp.background() return resp @@ -190,6 +228,41 @@ def _dispatch_for_client(self, method, path, query, headers, body, raise_excepti def dispatch(self, method, path, query, headers, body): return self._dispatch_for_client(method, path, query, headers, body, False) + # -- websockets + def websocket_open(self, path, query, headers, sink): + """Starts a websocket connection; returns its session, or None with no route. + + ``sink(message)`` receives the ASGI-shaped messages the endpoint sends + (``websocket.accept``, ``websocket.send``, ``websocket.close``). The + caller feeds client messages with ``session.deliver``. + """ + path = unquote(path) + route = None + vals = None + for r in self.routes: + if isinstance(r, APIWebSocketRoute): + vals = r.match(path) + if vals is not None: + route = r + break + session = _Session(sink) + ws = WebSocket(path, query, headers, session, self) + ws.path_params = vals or {} + session.start(self._run_ws(route, ws)) + return session + + async def _run_ws(self, route, ws): + from fastapi.exceptions import WebSocketException, WebSocketRequestValidationError + if route is None: + await ws.close(1000) + return + try: + await run_websocket(route, ws) + except WebSocketRequestValidationError: + await ws.close(1008) + except WebSocketException as e: + await ws.close(e.code, e.reason) + # -- OpenAPI def openapi(self): if self._openapi is None: @@ -206,7 +279,7 @@ def _build_openapi(self): schemas = {} need_validation = False for r in self.routes: - if not r.include_in_schema: + if isinstance(r, APIWebSocketRoute) or not r.include_in_schema: continue r._prepare() item = paths.setdefault(_re.sub(":[a-z]+\\}", "}", r.path), {}) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/exceptions.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/exceptions.py index ad04b36b976..b69a2bea72b 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/exceptions.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/exceptions.py @@ -35,3 +35,20 @@ def __str__(self): class FastAPIError(RuntimeError): pass + + +class WebSocketRequestValidationError(ValidationException): + pass + + +class WebSocketException(Exception): + def __init__(self, code, reason=None): + Exception.__init__(self) + self.code = code + self.reason = reason or "" + + def __str__(self): + return "%d: %s" % (self.code, self.reason) + + def __repr__(self): + return "WebSocketException(code=%d, reason=%r)" % (self.code, self.reason) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/middleware/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/middleware/__init__.py new file mode 100644 index 00000000000..7a12c74e88c --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/middleware/__init__.py @@ -0,0 +1,5 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Middleware (re-exported from starlette).""" +from starlette.middleware import Middleware # noqa diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/middleware/cors.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/middleware/cors.py new file mode 100644 index 00000000000..d8da3795433 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/middleware/cors.py @@ -0,0 +1,5 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""CORSMiddleware (re-exported from starlette).""" +from starlette.middleware.cors import CORSMiddleware # noqa diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/middleware/gzip.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/middleware/gzip.py new file mode 100644 index 00000000000..9ed0ca32cd7 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/middleware/gzip.py @@ -0,0 +1,5 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""GZipMiddleware (re-exported from starlette).""" +from starlette.middleware.gzip import GZipMiddleware # noqa diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/middleware/trustedhost.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/middleware/trustedhost.py new file mode 100644 index 00000000000..a65161cb1df --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/middleware/trustedhost.py @@ -0,0 +1,5 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""TrustedHostMiddleware (re-exported from starlette).""" +from starlette.middleware.trustedhost import TrustedHostMiddleware # noqa diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/routing.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/routing.py index e9f7cdf1270..92f39e519b3 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/routing.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/routing.py @@ -18,6 +18,7 @@ from starlette.exceptions import HTTPException as StarletteHTTPException from starlette.requests import Request from starlette.responses import JSONResponse, RedirectResponse, Response +from starlette.websockets import WebSocket from fastapi.encoders import jsonable_encoder from fastapi.exceptions import (HTTPException, RequestValidationError, ResponseValidationError) @@ -146,6 +147,9 @@ def _analyze(call, path_names): elif isinstance(default, Param): field = default default = field.default + if isinstance(tp, type) and issubclass(tp, WebSocket): + dep.specials[name] = "websocket" + continue if isinstance(tp, type) and issubclass(tp, Request): dep.specials[name] = "request" continue @@ -272,10 +276,10 @@ def match(self, path): return vals -def _call(call, values): +async def _call(call, values): r = call(**values) if inspect.iscoroutine(r): - r = run_coro(r) + r = await r return r @@ -328,18 +332,18 @@ def _missing(ctx, loc): ctx.errors.append({"type": "missing", "loc": tuple(loc), "msg": "Field required", "input": None}) -def _run_dep(sub, subvals, ctx): +async def _run_dep(sub, subvals, ctx): r = sub.call(**subvals) if sub.is_gen: r2 = next(r) ctx.cleanups.append(r) return r2 if inspect.iscoroutine(r): - r = run_coro(r) + r = await r return r -def _solve(dep, ctx): +async def _solve(dep, ctx): req = ctx.req values = {} for name, sub, use_cache in dep.deps: @@ -348,15 +352,15 @@ def _solve(dep, ctx): values[name] = ctx.cache[key] continue before = len(ctx.errors) - subvals = _solve(sub, ctx) + subvals = await _solve(sub, ctx) if len(ctx.errors) > before: continue - r = _run_dep(sub, subvals, ctx) + r = await _run_dep(sub, subvals, ctx) if use_cache: ctx.cache[key] = r values[name] = r for name, kind in dep.specials.items(): - if kind == "request": + if kind == "request" or kind == "websocket": values[name] = req elif kind == "response": values[name] = ctx.sub @@ -511,7 +515,7 @@ def _serialize(route, raw): return jsonable_encoder(raw) -def run_route(route, req, default_class): +async def run_route(route, req, default_class): route._prepare() body, form, err = _read_body(route, req) if err is not None: @@ -520,15 +524,15 @@ def run_route(route, req, default_class): ctx.form = form try: for pre in route._pre: - pv = _solve(pre, ctx) + pv = await _solve(pre, ctx) if not ctx.errors: - _run_dep(pre, pv, ctx) - values = _solve(route._dep, ctx) + await _run_dep(pre, pv, ctx) + values = await _solve(route._dep, ctx) if ctx.errors: raise RequestValidationError(list(ctx.errors), body=body if body is not Undefined else None) raw = route.endpoint(**values) if inspect.iscoroutine(raw): - raw = run_coro(raw) + raw = await raw if isinstance(raw, Response): resp = raw else: @@ -557,6 +561,50 @@ def run_route(route, req, default_class): return resp +class APIWebSocketRoute: + """A websocket endpoint: path parameters and dependencies resolve as for HTTP.""" + + def __init__(self, path, endpoint, name=None, dependencies=None): + self.path = path + self.endpoint = endpoint + self.name = name or endpoint.__name__ + self.regex, self.path_names = _compile(path) + self.dependencies = list(dependencies or []) + self.methods = set() + self.include_in_schema = False + self.body_params = [] + self._dep = None + self._pre = None + + def _prepare(self): + if self._dep is None: + self._dep = _analyze(self.endpoint, self.path_names) + self._pre = [_analyze(d.dependency, self.path_names) for d in self.dependencies] + + def match(self, path): + return APIRoute.match(self, path) + + +async def run_websocket(route, ws): + route._prepare() + ctx = _Ctx(route, ws, Undefined) + try: + for pre in route._pre: + pv = await _solve(pre, ctx) + if not ctx.errors: + await _run_dep(pre, pv, ctx) + values = await _solve(route._dep, ctx) + if ctx.errors: + from fastapi.exceptions import WebSocketRequestValidationError + raise WebSocketRequestValidationError(list(ctx.errors)) + await _call(route.endpoint, values) + _finish_cleanups(ctx, None) + except Exception as exc: + if ctx.cleanups: + _finish_cleanups(ctx, exc) + raise + + def _verb_decorator(self, methods, path, kw): def deco(f): self.add_api_route(path, f, methods=methods, **kw) @@ -585,6 +633,19 @@ def add_api_route(self, path, endpoint, *, methods=None, **kw): kw["deprecated"] = self.deprecated self.routes.append(APIRoute(self.prefix + path, endpoint, methods or ["GET"], **kw)) + def add_api_websocket_route(self, path, endpoint, name=None, dependencies=None): + self.routes.append(APIWebSocketRoute(self.prefix + path, endpoint, name=name, + dependencies=list(self.dependencies) + list(dependencies or []))) + + def websocket(self, path, name=None, dependencies=None): + def deco(f): + self.add_api_websocket_route(path, f, name=name, dependencies=dependencies) + return f + return deco + + def websocket_route(self, path, name=None): + return self.websocket(path, name=name) + def api_route(self, path, *, methods=None, **kw): return _verb_decorator(self, methods or ["GET"], path, kw) @@ -613,6 +674,10 @@ def include_router(self, router, *, prefix="", tags=None, dependencies=None, default_response_class=None, responses=None, deprecated=None, include_in_schema=True, **kw): for r in router.routes: + if isinstance(r, APIWebSocketRoute): + self.routes.append(APIWebSocketRoute(prefix + r.path, r.endpoint, name=r.name, + dependencies=list(dependencies or []) + list(r.dependencies))) + continue n = APIRoute(prefix + r.path, r.endpoint, r.methods, name=r.name, status_code=r.status_code, response_model=r.response_model, response_class=r.response_class or default_response_class or self.default_response_class, diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/websockets.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/websockets.py new file mode 100644 index 00000000000..ab3b31c1f2b --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/websockets.py @@ -0,0 +1,5 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Websockets (re-exported from starlette).""" +from starlette.websockets import WebSocket, WebSocketDisconnect, WebSocketState # noqa diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/datastructures.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/datastructures.py index f87ee22bae7..4c2fd96b569 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/datastructures.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/datastructures.py @@ -106,6 +106,12 @@ def update(self, other): for k, v in items: self[k] = v + def add_vary_header(self, vary): + existing = self.get("vary") + if existing is not None: + vary = ", ".join([existing, vary]) + self["vary"] = vary + class QueryParams: """Immutable multi-dict of decoded query parameters; the last value wins on get.""" diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/middleware/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/middleware/__init__.py new file mode 100644 index 00000000000..0a7c2d2efa8 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/middleware/__init__.py @@ -0,0 +1,24 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Middleware over the pure dispatch core. + +There is no ASGI here: an "app" a middleware wraps is an async callable +taking a request and answering a response. Every middleware class takes +that app first and implements ``async __call__(request)``; the application +builds the stack with the last one added outermost, as Starlette does. +Websocket connections do not pass through it. +""" + + +class Middleware: + def __init__(self, cls, *args, **kwargs): + self.cls = cls + self.args = args + self.kwargs = kwargs + + def __iter__(self): + return iter((self.cls, self.args, self.kwargs)) + + def __repr__(self): + return "Middleware(%s)" % getattr(self.cls, "__name__", repr(self.cls)) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/middleware/base.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/middleware/base.py new file mode 100644 index 00000000000..3d74299d581 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/middleware/base.py @@ -0,0 +1,21 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""BaseHTTPMiddleware: ``dispatch(request, call_next)``.""" + + +class BaseHTTPMiddleware: + def __init__(self, app, dispatch=None): + self.app = app + self.dispatch_func = self.dispatch if dispatch is None else dispatch + + async def call_next(self, request): + resp = await self.app(request) + resp._streamed = True + return resp + + async def __call__(self, request): + return await self.dispatch_func(request, self.call_next) + + async def dispatch(self, request, call_next): + raise NotImplementedError() diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/middleware/cors.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/middleware/cors.py new file mode 100644 index 00000000000..a2ec9c8d4b0 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/middleware/cors.py @@ -0,0 +1,96 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""CORSMiddleware, written from Starlette's documented behaviour.""" +import re + +from starlette.responses import PlainTextResponse + +ALL_METHODS = ("DELETE", "GET", "HEAD", "OPTIONS", "PATCH", "POST", "PUT", "QUERY") +SAFELISTED_HEADERS = {"Accept", "Accept-Language", "Content-Language", "Content-Type"} + + +class CORSMiddleware: + def __init__(self, app, allow_origins=(), allow_methods=("GET",), allow_headers=(), + allow_credentials=False, allow_origin_regex=None, expose_headers=(), max_age=600, + allow_private_network=False): + if "*" in allow_methods: + allow_methods = ALL_METHODS + self.app = app + self.allow_origins = allow_origins + self.allow_methods = allow_methods + self.allow_all_origins = "*" in allow_origins + self.allow_all_headers = "*" in allow_headers + self.allow_origin_regex = re.compile(allow_origin_regex) if allow_origin_regex is not None else None + self.allow_credentials = allow_credentials + self.preflight_explicit_allow_origin = not self.allow_all_origins or allow_credentials + simple = {} + if self.allow_all_origins: + simple["Access-Control-Allow-Origin"] = "*" + if allow_credentials: + simple["Access-Control-Allow-Credentials"] = "true" + if expose_headers: + simple["Access-Control-Expose-Headers"] = ", ".join(expose_headers) + pre = {"Vary": "Origin, Access-Control-Request-Method, Access-Control-Request-Headers, " + "Access-Control-Request-Private-Network"} + if not self.preflight_explicit_allow_origin: + pre["Access-Control-Allow-Origin"] = "*" + pre["Access-Control-Allow-Methods"] = ", ".join(allow_methods) + pre["Access-Control-Max-Age"] = str(max_age) + allowed = sorted(SAFELISTED_HEADERS | set(allow_headers)) + if allowed and not self.allow_all_headers: + pre["Access-Control-Allow-Headers"] = ", ".join(allowed) + if allow_credentials: + pre["Access-Control-Allow-Credentials"] = "true" + self.simple_headers = simple + self.preflight_headers = pre + self.allow_headers = [h.lower() for h in allowed] + + def is_allowed_origin(self, origin): + if self.allow_all_origins: + return True + if self.allow_origin_regex is not None and self.allow_origin_regex.fullmatch(origin): + return True + return origin in self.allow_origins + + async def __call__(self, request): + headers = request.headers + origin = headers.get("origin") + if origin is None: + resp = await self.app(request) + resp.headers.add_vary_header("Origin") + return resp + if request.method == "OPTIONS" and "access-control-request-method" in headers: + return self.preflight_response(headers) + resp = await self.app(request) + resp.headers.update(self.simple_headers) + if self.allow_all_origins and self.allow_credentials: + resp.headers["Access-Control-Allow-Origin"] = origin + elif not self.allow_all_origins and self.is_allowed_origin(origin): + resp.headers["Access-Control-Allow-Origin"] = origin + resp.headers.add_vary_header("Origin") + return resp + + def preflight_response(self, request_headers): + origin = request_headers["origin"] + method = request_headers["access-control-request-method"] + requested = request_headers.get("access-control-request-headers") + headers = dict(self.preflight_headers) + failures = [] + if self.is_allowed_origin(origin): + if self.preflight_explicit_allow_origin: + headers["Access-Control-Allow-Origin"] = origin + else: + failures.append("origin") + if method not in self.allow_methods: + failures.append("method") + if self.allow_all_headers and requested is not None: + headers["Access-Control-Allow-Headers"] = requested + elif requested is not None: + for h in [h.lower() for h in requested.split(",")]: + if h.strip() not in self.allow_headers: + failures.append("headers") + break + if failures: + return PlainTextResponse("Disallowed CORS " + ", ".join(failures), status_code=400, headers=headers) + return PlainTextResponse("OK", status_code=200, headers=headers) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/middleware/gzip.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/middleware/gzip.py new file mode 100644 index 00000000000..5add9710c2c --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/middleware/gzip.py @@ -0,0 +1,34 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""GZipMiddleware: compresses a response body of at least ``minimum_size`` bytes.""" +import gzip + + +class GZipMiddleware: + def __init__(self, app, minimum_size=500, compresslevel=9): + self.app = app + self.minimum_size = minimum_size + self.compresslevel = compresslevel + + async def __call__(self, request): + resp = await self.app(request) + if "content-encoding" in resp.headers: + return resp + ct = resp.headers.get("content-type", "") + if ct.startswith("text/event-stream"): + return resp + body = resp.body + # A response that came through BaseHTTPMiddleware is streamed under + # Starlette, and a streamed body is compressed whatever its size. + if len(body) < self.minimum_size and not getattr(resp, "_streamed", False): + return resp + # Vary is set on every response that would be compressed for a client + # that asked, so that a cache keeps the two encodings apart. + resp.headers.add_vary_header("Accept-Encoding") + if "gzip" not in request.headers.get("accept-encoding", ""): + return resp + resp.body = gzip.compress(body, compresslevel=self.compresslevel) + resp.headers["content-encoding"] = "gzip" + resp.headers["content-length"] = str(len(resp.body)) + return resp diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/middleware/trustedhost.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/middleware/trustedhost.py new file mode 100644 index 00000000000..44f33b46f73 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/middleware/trustedhost.py @@ -0,0 +1,41 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""TrustedHostMiddleware: refuses a request whose Host header is not allowed.""" +from starlette.responses import PlainTextResponse, RedirectResponse + +ENFORCE_DOMAIN_WILDCARD = "Domain wildcard patterns must be like '*.example.com'." + + +class TrustedHostMiddleware: + def __init__(self, app, allowed_hosts=None, www_redirect=True): + if allowed_hosts is None: + allowed_hosts = ["*"] + for pattern in allowed_hosts: + assert "*" not in pattern[1:], ENFORCE_DOMAIN_WILDCARD + if pattern.startswith("*") and pattern != "*": + assert pattern.startswith("*."), ENFORCE_DOMAIN_WILDCARD + self.app = app + self.allowed_hosts = list(allowed_hosts) + self.allow_any = "*" in allowed_hosts + self.www_redirect = www_redirect + + async def __call__(self, request): + if self.allow_any: + return await self.app(request) + host = request.headers.get("host", "").split(":")[0] + valid = False + www = False + for pattern in self.allowed_hosts: + if host == pattern or (pattern.startswith("*") and host.endswith(pattern[1:])): + valid = True + break + elif "www." + host == pattern: + www = True + if valid: + return await self.app(request) + if www and self.www_redirect: + netloc = request.headers.get("host", "") + url = "http://www." + netloc + request._path + ("?" + request._query if request._query else "") + return RedirectResponse(url) + return PlainTextResponse("Invalid host header", status_code=400) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/testclient.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/testclient.py index 1e874af6f1b..e4abb0fd486 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/testclient.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/testclient.py @@ -2,6 +2,7 @@ # Licensed under the GNU General Public License version 2 with the Classpath # exception, as the rest of the Codename One Python runtime. """A test client that calls the app's pure dispatch core.""" +import gzip as _gzip import json as _json from urllib.parse import urlencode, urljoin @@ -79,6 +80,19 @@ def __exit__(self, *a): self.app._shutdown() return False + def websocket_connect(self, url, subprotocols=None, headers=None, **kw): + if "://" not in url: + url = self.base_url.rstrip("/") + "/" + url.lstrip("/") + u = URL(url) + hdrs = MutableHeaders([("host", u.netloc), ("accept", "*/*"), ("accept-encoding", "gzip, deflate"), + ("user-agent", "testclient"), ("connection", "upgrade"), + ("sec-websocket-key", "testserver=="), ("sec-websocket-version", "13")]) + if subprotocols: + hdrs["sec-websocket-protocol"] = ", ".join(subprotocols) + if headers: + hdrs.update(headers) + return WebSocketTestSession(self.app, u.path, u.query, hdrs.items()) + def get(self, url, **kw): return self.request("GET", url, **kw) @@ -155,6 +169,9 @@ def _send(self, method, url, params, headers, json, data, content, cookies): self.raise_server_exceptions) except Exception: raise + for k, v in rh: + if k.lower() == "content-encoding" and v.strip().lower() == "gzip": + rb = _gzip.decompress(rb) resp = TestResponse(status, rh, rb, url if not query else url.split("?")[0] + "?" + query) for k, v in resp.headers.multi_items(): if k == "set-cookie": @@ -166,3 +183,93 @@ def _send(self, method, url, params, headers, json, data, content, cookies): else: self.cookies[ck] = cv return resp + + +class WebSocketTestSession: + __test__ = False + + def __init__(self, app, path, query, headers): + self.app = app + self.path = path + self.query = query + self.headers = headers + self.accepted_subprotocol = None + self.extra_headers = None + self._out = [] + self._session = None + + def _sink(self, message): + self._out.append(message) + + def _raise_error(self): + s = self._session + if s is not None and s.error is not None: + e = s.error + s.error = None + raise e + + def __enter__(self): + self._session = self.app.websocket_open(self.path, self.query, self.headers, self._sink) + self._raise_error() + m = self.receive() + if m["type"] == "websocket.close": + from starlette.websockets import WebSocketDisconnect + raise WebSocketDisconnect(m.get("code", 1000), m.get("reason", "")) + self.accepted_subprotocol = m.get("subprotocol") + self.extra_headers = m.get("headers") + return self + + def __exit__(self, *a): + s = self._session + if not s.done: + s.deliver({"type": "websocket.disconnect", "code": 1000}) + self._raise_error() + return False + + def send(self, message): + self._session.deliver(message) + self._raise_error() + + def send_text(self, data): + self.send({"type": "websocket.receive", "text": data}) + + def send_bytes(self, data): + self.send({"type": "websocket.receive", "bytes": bytes(data)}) + + def send_json(self, data, mode="text"): + text = _json.dumps(data, separators=(",", ":"), ensure_ascii=False) + if mode == "text": + self.send({"type": "websocket.receive", "text": text}) + else: + self.send({"type": "websocket.receive", "bytes": text.encode("utf-8")}) + + def close(self, code=1000, reason=None): + self.send({"type": "websocket.disconnect", "code": code, "reason": reason}) + + def receive(self): + self._raise_error() + if not self._out: + raise RuntimeError("the websocket endpoint is waiting for a message and sent nothing") + return self._out.pop(0) + + def _raise_on_close(self, m): + if m["type"] == "websocket.close": + from starlette.websockets import WebSocketDisconnect + raise WebSocketDisconnect(m.get("code", 1000), m.get("reason", "")) + + def receive_text(self): + m = self.receive() + self._raise_on_close(m) + return m["text"] + + def receive_bytes(self): + m = self.receive() + self._raise_on_close(m) + return m["bytes"] + + def receive_json(self, mode="text"): + m = self.receive() + self._raise_on_close(m) + if mode == "text": + return _json.loads(m["text"]) + return _json.loads(m["bytes"].decode("utf-8")) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/websockets.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/websockets.py new file mode 100644 index 00000000000..9a82534dfa6 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/websockets.py @@ -0,0 +1,184 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Server-side websockets without ASGI. + +The endpoint coroutine is driven by ``_Session`` one step at a time rather +than by an event loop: whenever it waits for a message that has not arrived, +it yields a ``_Suspend`` and the session returns to its caller (the test +client, or the platform server's frame callback). A message delivered later +resumes it. An endpoint that awaits anything else that really suspends -- a +timer, another task -- is not supported by this driver. +""" +import enum +import json as _json + +from starlette.datastructures import URL +from starlette.requests import HTTPConnection + + +class WebSocketState(enum.Enum): + CONNECTING = 0 + CONNECTED = 1 + DISCONNECTED = 2 + RESPONSE = 3 + + +class WebSocketDisconnect(Exception): + def __init__(self, code=1000, reason=None): + Exception.__init__(self) + self.code = code + self.reason = reason or "" + + +class _Suspend: + def __await__(self): + yield self + + +class WebSocket(HTTPConnection): + def __init__(self, path, query, headers, session, app=None): + HTTPConnection.__init__(self, "GET", path, query, headers, b"", app) + self.scope = {"type": "websocket", "path": path} + self._session = session + self.client_state = WebSocketState.CONNECTING + self.application_state = WebSocketState.CONNECTING + + @property + def url(self): + host = self.headers.get("host", "testserver") + return URL("ws://" + host + self._path + ("?" + self._query if self._query else "")) + + async def receive(self): + if self.client_state == WebSocketState.DISCONNECTED: + raise RuntimeError('Cannot call "receive" once a disconnect message has been received.') + inbox = self._session.inbox + while not inbox: + await _Suspend() + m = inbox.pop(0) + if m["type"] == "websocket.disconnect": + self.client_state = WebSocketState.DISCONNECTED + return m + + async def send(self, message): + t = message["type"] + if self.application_state == WebSocketState.DISCONNECTED: + raise RuntimeError('Cannot call "send" once a close message has been sent.') + if t == "websocket.accept": + self.application_state = WebSocketState.CONNECTED + self.client_state = WebSocketState.CONNECTED + elif t == "websocket.close": + self.application_state = WebSocketState.DISCONNECTED + elif self.application_state != WebSocketState.CONNECTED: + raise RuntimeError('WebSocket is not connected. Need to call "accept" first.') + self._session.outgoing(message) + + async def accept(self, subprotocol=None, headers=None): + if self.client_state == WebSocketState.CONNECTING: + await self.receive() + await self.send({"type": "websocket.accept", "subprotocol": subprotocol, "headers": headers or []}) + + def _raise_on_disconnect(self, m): + if m["type"] == "websocket.disconnect": + raise WebSocketDisconnect(m.get("code", 1000), m.get("reason")) + + def _check_connected(self): + if self.application_state != WebSocketState.CONNECTED: + raise RuntimeError('WebSocket is not connected. Need to call "accept" first.') + + async def receive_text(self): + self._check_connected() + m = await self.receive() + self._raise_on_disconnect(m) + return m["text"] + + async def receive_bytes(self): + self._check_connected() + m = await self.receive() + self._raise_on_disconnect(m) + return m["bytes"] + + async def receive_json(self, mode="text"): + self._check_connected() + m = await self.receive() + self._raise_on_disconnect(m) + if mode == "text": + return _json.loads(m["text"]) + return _json.loads(m["bytes"].decode("utf-8")) + + async def iter_text(self): + try: + while True: + yield await self.receive_text() + except WebSocketDisconnect: + pass + + async def iter_bytes(self): + try: + while True: + yield await self.receive_bytes() + except WebSocketDisconnect: + pass + + async def iter_json(self): + try: + while True: + yield await self.receive_json() + except WebSocketDisconnect: + pass + + async def send_text(self, data): + await self.send({"type": "websocket.send", "text": data}) + + async def send_bytes(self, data): + await self.send({"type": "websocket.send", "bytes": bytes(data)}) + + async def send_json(self, data, mode="text"): + text = _json.dumps(data, separators=(",", ":"), ensure_ascii=False) + if mode == "text": + await self.send({"type": "websocket.send", "text": text}) + else: + await self.send({"type": "websocket.send", "bytes": text.encode("utf-8")}) + + async def close(self, code=1000, reason=None): + await self.send({"type": "websocket.close", "code": code, "reason": reason or ""}) + + +class _Session: + """One connection: the inbox the endpoint reads and the coroutine it runs in. + + ``sink(message)`` receives every message the endpoint sends. ``deliver`` + queues a client message and runs the endpoint until it waits again. + """ + + def __init__(self, sink): + self.inbox = [{"type": "websocket.connect"}] + self.sink = sink + self.coro = None + self.done = False + self.error = None + + def outgoing(self, message): + self.sink(message) + + def start(self, coro): + self.coro = coro + self.pump() + + def deliver(self, message): + self.inbox.append(message) + self.pump() + + def pump(self): + while not self.done: + try: + y = self.coro.send(None) + except StopIteration: + self.done = True + return + except BaseException as e: # noqa + self.done = True + self.error = e + return + if isinstance(y, _Suspend) and not self.inbox: + return diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/uvicorn/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/uvicorn/__init__.py index da44a319336..2e45a28a21e 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/site/uvicorn/__init__.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/uvicorn/__init__.py @@ -4,6 +4,57 @@ """Clean-room subset of uvicorn: run an app through the platform server.""" +class _Connection: + """One upgraded connection, as the platform server drives it.""" + + def __init__(self, session): + self.session = session + + def _check(self): + s = self.session + if s.error is not None: + e = s.error + s.error = None + raise e + + def text(self, value): + if not self.session.done: + self.session.deliver({"type": "websocket.receive", "text": value}) + self._check() + + def binary(self, value): + if not self.session.done: + self.session.deliver({"type": "websocket.receive", "bytes": bytes(value)}) + self._check() + + def closed(self, code, reason): + if not self.session.done: + self.session.deliver({"type": "websocket.disconnect", "code": code, "reason": reason}) + self._check() + + +def _websockets(app): + def open_(path, query, headers, send): + def sink(m): + t = m["type"] + if t == "websocket.accept": + send("accept", m.get("subprotocol"), None) + elif t == "websocket.send": + if m.get("text") is not None: + send("text", m["text"], None) + else: + send("bytes", m["bytes"], None) + elif t == "websocket.close": + send("close", m.get("code", 1000), m.get("reason") or "") + conn = _Connection(app.websocket_open(path, query, headers, sink)) + conn._check() + return conn + return open_ + + def run(app, host="127.0.0.1", port=8000, **kw): import _cn1server - _cn1server.serve(app.dispatch, host, port) + if hasattr(app, "websocket_open"): + _cn1server.serve(app.dispatch, host, port, _websockets(app)) + else: + _cn1server.serve(app.dispatch, host, port) diff --git a/maven/python-transpiler/src/test/resources/conformance/fastapi_middleware_ws.out b/maven/python-transpiler/src/test/resources/conformance/fastapi_middleware_ws.out new file mode 100644 index 00000000000..5ec6798a2bd --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/fastapi_middleware_ws.out @@ -0,0 +1,33 @@ +200 {'msg': 'hi', 'tag': 't1'} 1 t1 +['tagger-in', 'timing-in /hello', 'timing-out 200', 'tagger-out'] +cors None +404 {'detail': 'nope'} t1 ['tagger-in', 'timing-in /missing', 'timing-out 404', 'tagger-out'] +403 blocked by t1 None ['tagger-in'] +200 1000 gzip Origin, Accept-Encoding +200 1000 None 1000 +200 gzip +200 https://good.example true X-Tag Origin, Accept-Encoding +200 None true +200 OK https://good.example GET, POST Accept, Accept-Language, Content-Language, Content-Type, X-Custom 600 Origin, Access-Control-Request-Method, Access-Control-Request-Headers, Access-Control-Request-Private-Network +400 Disallowed CORS origin, method, headers +['tagger-in', 'timing-in /hello', 'timing-out 200', 'tagger-out', 'tagger-in', 'timing-in /hello', 'timing-out 200', 'tagger-out'] +400 Invalid host header +ok +ok +307 http://www.site.com/ +* Origin +200 * DELETE, GET, HEAD, OPTIONS, PATCH, POST, PUT, QUERY X-A, X-B 60 +raised boom +500 +echo a +echo b +['disconnect 1000'] +echo c +['disconnect 1000', 'disconnect 4000'] +{'room': 'lobby', 'limit': 1, 'path': '/rooms/lobby'} +{'got': {'n': 1}, 'prefix': '>'} +b'cba' +closed 1001 +/reject refused 1008 +/nowhere refused 1000 +[] diff --git a/maven/python-transpiler/src/test/resources/conformance/fastapi_middleware_ws.py b/maven/python-transpiler/src/test/resources/conformance/fastapi_middleware_ws.py new file mode 100644 index 00000000000..e0284f59e2f --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/fastapi_middleware_ws.py @@ -0,0 +1,206 @@ +from fastapi import Depends, FastAPI, HTTPException, Request, WebSocket, WebSocketDisconnect +from fastapi.middleware.cors import CORSMiddleware +from fastapi.middleware.gzip import GZipMiddleware +from fastapi.middleware.trustedhost import TrustedHostMiddleware +from fastapi.responses import PlainTextResponse +from fastapi.testclient import TestClient +from starlette.middleware.base import BaseHTTPMiddleware + +app = FastAPI() +order = [] + + +@app.middleware("http") +async def timing(request: Request, call_next): + order.append("timing-in " + request.url.path) + response = await call_next(request) + response.headers["X-Timing"] = "1" + order.append("timing-out %d" % response.status_code) + return response + + +class Tagger(BaseHTTPMiddleware): + def __init__(self, app, tag): + super().__init__(app) + self.tag = tag + + async def dispatch(self, request, call_next): + order.append("tagger-in") + if request.url.path == "/blocked": + return PlainTextResponse("blocked by " + self.tag, status_code=403) + request.state.tag = self.tag + response = await call_next(request) + response.headers["X-Tag"] = self.tag + order.append("tagger-out") + return response + + +app.add_middleware(Tagger, tag="t1") +app.add_middleware(CORSMiddleware, allow_origins=["https://good.example"], allow_methods=["GET", "POST"], + allow_headers=["X-Custom"], allow_credentials=True, expose_headers=["X-Tag"]) +app.add_middleware(GZipMiddleware, minimum_size=100) + + +@app.get("/hello") +def hello(request: Request): + return {"msg": "hi", "tag": request.state.tag} + + +@app.get("/big") +def big(): + return PlainTextResponse("x" * 1000) + + +@app.get("/missing") +def missing(): + raise HTTPException(404, "nope") + + +@app.get("/blocked") +def blocked(): + return {"never": True} + + +client = TestClient(app) + +r = client.get("/hello") +print(r.status_code, r.json(), r.headers["x-timing"], r.headers["x-tag"]) +print(order) +del order[:] +print("cors" , r.headers.get("access-control-allow-origin")) + +r = client.get("/missing") +print(r.status_code, r.json(), r.headers.get("x-tag"), order) +del order[:] + +r = client.get("/blocked") +print(r.status_code, r.text, r.headers.get("x-timing"), order) +del order[:] + +r = client.get("/big") +print(r.status_code, len(r.text), r.headers.get("content-encoding"), r.headers.get("vary")) +r = client.get("/big", headers={"accept-encoding": "identity"}) +print(r.status_code, len(r.text), r.headers.get("content-encoding"), r.headers.get("content-length")) +r = client.get("/hello", headers={"accept-encoding": "gzip"}) +print(r.status_code, r.headers.get("content-encoding")) +del order[:] + +# CORS simple and preflight requests +r = client.get("/hello", headers={"Origin": "https://good.example"}) +print(r.status_code, r.headers.get("access-control-allow-origin"), r.headers.get("access-control-allow-credentials"), + r.headers.get("access-control-expose-headers"), r.headers.get("vary")) +r = client.get("/hello", headers={"Origin": "https://evil.example"}) +print(r.status_code, r.headers.get("access-control-allow-origin"), r.headers.get("access-control-allow-credentials")) +r = client.options("/hello", headers={"Origin": "https://good.example", "Access-Control-Request-Method": "POST", + "Access-Control-Request-Headers": "x-custom"}) +print(r.status_code, r.text, r.headers.get("access-control-allow-origin"), r.headers.get("access-control-allow-methods"), + r.headers.get("access-control-allow-headers"), r.headers.get("access-control-max-age"), r.headers.get("vary")) +r = client.options("/hello", headers={"Origin": "https://evil.example", "Access-Control-Request-Method": "DELETE", + "Access-Control-Request-Headers": "x-other"}) +print(r.status_code, r.text) +print(order) +del order[:] + +# A wildcard CORS app and trusted hosts +app2 = FastAPI() +app2.add_middleware(CORSMiddleware, allow_origins=["*"], allow_methods=["*"], allow_headers=["*"], max_age=60) +app2.add_middleware(TrustedHostMiddleware, allowed_hosts=["example.com", "*.example.org", "www.site.com"]) + + +@app2.get("/") +def root(): + return "ok" + + +c2 = TestClient(app2) +print(c2.get("/").status_code, c2.get("/").text) +print(c2.get("/", headers={"host": "example.com"}).json()) +print(c2.get("/", headers={"host": "api.example.org"}).json()) +r = c2.get("/", headers={"host": "site.com"}, follow_redirects=False) +print(r.status_code, r.headers.get("location")) +r = c2.get("/", headers={"host": "example.com", "Origin": "https://any.where"}) +print(r.headers.get("access-control-allow-origin"), r.headers.get("vary")) +r = c2.options("/", headers={"host": "example.com", "Origin": "https://any.where", + "Access-Control-Request-Method": "PATCH", "Access-Control-Request-Headers": "X-A, X-B"}) +print(r.status_code, r.headers.get("access-control-allow-origin"), r.headers.get("access-control-allow-methods"), + r.headers.get("access-control-allow-headers"), r.headers.get("access-control-max-age")) + + +@app2.get("/boom") +def boom(): + raise ValueError("boom") + + +try: + c2.get("/boom", headers={"host": "example.com"}) +except ValueError as e: + print("raised", e) +print(TestClient(app2, raise_server_exceptions=False).get("/boom", headers={"host": "example.com"}).status_code) + +# WebSockets +log = [] + + +def get_prefix(): + return ">" + + +@app.websocket("/ws") +async def echo(websocket: WebSocket): + await websocket.accept() + try: + while True: + text = await websocket.receive_text() + await websocket.send_text("echo " + text) + except WebSocketDisconnect as e: + log.append("disconnect %d" % e.code) + + +@app.websocket("/rooms/{room}") +async def room(websocket: WebSocket, room: str, prefix: str = Depends(get_prefix), limit: int = 2): + await websocket.accept() + await websocket.send_json({"room": room, "limit": limit, "path": websocket.url.path}) + for _ in range(limit): + data = await websocket.receive_json() + await websocket.send_json({"got": data, "prefix": prefix}) + b = await websocket.receive_bytes() + await websocket.send_bytes(b[::-1]) + await websocket.close(code=1001) + + +@app.websocket("/reject") +async def reject(websocket: WebSocket): + await websocket.close(code=1008) + + +with client.websocket_connect("/ws") as ws: + ws.send_text("a") + print(ws.receive_text()) + ws.send_text("b") + print(ws.receive_text()) +print(log) + +with client.websocket_connect("/ws") as ws: + ws.send_text("c") + print(ws.receive_text()) + ws.close(code=4000) +print(log) + +with client.websocket_connect("/rooms/lobby?limit=1") as ws: + print(ws.receive_json()) + ws.send_json({"n": 1}) + print(ws.receive_json()) + ws.send_bytes(b"abc") + print(ws.receive_bytes()) + try: + ws.receive_text() + except WebSocketDisconnect as e: + print("closed", e.code) + +for path in ("/reject", "/nowhere"): + try: + with client.websocket_connect(path) as ws: + print("connected?") + except WebSocketDisconnect as e: + print(path, "refused", e.code) +print(order) From f9233fa9cd5fe26ed5df028e65ff419224894279 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 10:15:07 +0200 Subject: [PATCH 051/112] Python guide: FastAPI middleware and WebSockets Co-Authored-By: Claude Opus 5.5 --- docs/developer-guide/Python-Interop.asciidoc | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/developer-guide/Python-Interop.asciidoc b/docs/developer-guide/Python-Interop.asciidoc index 1b0df468d2a..b2382cfef50 100644 --- a/docs/developer-guide/Python-Interop.asciidoc +++ b/docs/developer-guide/Python-Interop.asciidoc @@ -322,8 +322,14 @@ Flask sessions (signed cookies), `flash`, Jinja templates (`extends`, `block`, `include`, loops, conditions and the common filters, with autoescaping), multipart file uploads and streamed responses work as well. A streamed response is sent once its generator finishes. Jinja macros aren't supported yet, and a session cookie -written by CPython's Flask isn't readable here, or the reverse. FastAPI -WebSockets and middleware aren't supported yet. +written by CPython's Flask isn't readable here, or the reverse. + +FastAPI middleware works as in Starlette: `@app.middleware("http")`, +`BaseHTTPMiddleware` subclasses, `CORSMiddleware`, `GZipMiddleware` and +`TrustedHostMiddleware`. WebSocket endpoints are supported, including +`TestClient.websocket_connect`. A WebSocket endpoint can't yet wait on +something that suspends, such as `asyncio.sleep`, while its socket is open, +and middleware doesn't apply to WebSocket connections. === Compared with CPython, PyPy and GraalPy From 3dbe16fa0fe787b1b8f2eda79a2622bc9f293795 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 10:22:38 +0200 Subject: [PATCH 052/112] Python: SQLAlchemy 2.x subset (Core + declarative ORM over sqlite3) Core: create_engine (sqlite only), connect/begin, text() with :name params, Result/Row/scalars/mappings, MetaData/Table/Column with common types, create_all/drop_all, select/insert/update/delete with where, joins, order_by, limit/offset, group_by, func, and_/or_/not_, returning. ORM: DeclarativeBase, Mapped, mapped_column, relationship with back_populates (one-to-many / many-to-one, lazy select), Session with identity map, flush ordering, expire_on_commit, refresh and rollback. Runtime fix: obj.__dict__.pop/popitem/clear left the attribute set on the instance; InstanceDict now removes it (conformance test instance_dict_pop). Co-Authored-By: Claude Opus 5.5 --- .../codename1/python/runtime/PyInstance.java | 32 +- .../python/site/sqlalchemy/__init__.py | 1497 +++++++++++++++++ .../codename1/python/site/sqlalchemy/exc.py | 84 + .../python/site/sqlalchemy/orm/__init__.py | 875 ++++++++++ .../conformance/instance_dict_pop.out | 6 + .../conformance/instance_dict_pop.py | 21 + .../resources/conformance/sqlalchemy_core.out | 34 + .../resources/conformance/sqlalchemy_core.py | 104 ++ .../resources/conformance/sqlalchemy_orm.out | 22 + .../resources/conformance/sqlalchemy_orm.py | 94 ++ 10 files changed, 2768 insertions(+), 1 deletion(-) create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/__init__.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/exc.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/orm/__init__.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/instance_dict_pop.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/instance_dict_pop.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/sqlalchemy_core.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/sqlalchemy_core.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/sqlalchemy_orm.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/sqlalchemy_orm.py diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyInstance.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyInstance.java index 77dc23d1673..081bc69a2da 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyInstance.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyInstance.java @@ -642,9 +642,39 @@ void setLong(long k, Object value) { @Override public void delItem(Object key) { - super.delItem(key); + // Not super.delItem: it removes through remove(), which is overridden below and + // would delete the attribute a second time. + if (super.remove(key) == null) { + throw Py.keyError(key); + } self.delAttrDirect(Py.toStr(key)); } + + // pop() and popitem() remove through here, clear() through clear(): without these + // overrides `obj.__dict__.pop(name)` answered the value and left the attribute set. + @Override + public Object remove(Object key) { + Object v = super.remove(key); + if (v != null) { + self.delAttrDirect(Py.toStr(key)); + } + return v; + } + + @Override + public void clear() { + Object[] snapshot = new Object[used]; + int n = 0; + for (int i = 0; i < used; i++) { + if (keys[i] != null) { + snapshot[n++] = keys[i]; + } + } + super.clear(); + for (int i = 0; i < n; i++) { + self.delAttrDirect(Py.toStr(snapshot[i])); + } + } } } diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/__init__.py new file mode 100644 index 00000000000..890b8dad28a --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/__init__.py @@ -0,0 +1,1497 @@ +"""A SQLAlchemy 2.x subset for Codename One: Core over the DB-API ``sqlite3`` module. + +Only the SQLite dialect exists. Statements compile to SQL with ``?`` placeholders and run +through ``sqlite3``; every result is fetched eagerly. The ORM lives in ``sqlalchemy.orm`` +and plugs in through two hooks: a mapped class carries ``__sa_mapper__`` (selecting it +yields objects) and a relationship attribute answers ``_join_info()`` (joining on it). + +Unsupported: other dialects, connection pooling options, reflection, events, ``echo`` +output, subqueries/CTEs/unions, ``Enum``/``JSON``/``Numeric`` decimal handling. +""" + +import datetime as _dt +import sqlite3 as _sqlite3 + +from . import exc + +__version__ = "2.0.0" + + +# ----------------------------------------------------------------------------- types + +class TypeEngine: + _ddl = "NULL" + + def __init__(self, *args, **kwargs): + pass + + def ddl(self): + return self._ddl + + def bind(self, value): + return value + + def result(self, value): + return value + + +class NullType(TypeEngine): + _ddl = "NULL" + + +class Integer(TypeEngine): + _ddl = "INTEGER" + + +class BigInteger(Integer): + _ddl = "BIGINT" + + +class SmallInteger(Integer): + _ddl = "SMALLINT" + + +class String(TypeEngine): + _ddl = "VARCHAR" + + def __init__(self, length=None, *args, **kwargs): + self.length = length + + def ddl(self): + if self.length: + return "%s(%d)" % (self._ddl, self.length) + return self._ddl + + +class Unicode(String): + pass + + +class Text(String): + _ddl = "TEXT" + + +class UnicodeText(Text): + pass + + +class Float(TypeEngine): + _ddl = "FLOAT" + + def result(self, value): + return None if value is None else float(value) + + +class Numeric(Float): + _ddl = "NUMERIC" + + +class Boolean(TypeEngine): + _ddl = "BOOLEAN" + + def bind(self, value): + if value is None: + return None + return 1 if value else 0 + + def result(self, value): + return None if value is None else bool(value) + + +def _num(s, a, b): + return int(s[a:b]) + + +class DateTime(TypeEngine): + _ddl = "DATETIME" + + def bind(self, value): + if value is None or isinstance(value, str): + return value + return "%04d-%02d-%02d %02d:%02d:%02d.%06d" % ( + value.year, value.month, value.day, value.hour, value.minute, value.second, value.microsecond) + + def result(self, value): + if not isinstance(value, str): + return value + s = value.replace("T", " ") + micro = 0 + if len(s) > 20 and s[19] == ".": + frac = (s[20:26] + "000000")[:6] + micro = int(frac) + hour = _num(s, 11, 13) if len(s) >= 13 else 0 + minute = _num(s, 14, 16) if len(s) >= 16 else 0 + second = _num(s, 17, 19) if len(s) >= 19 else 0 + return _dt.datetime(_num(s, 0, 4), _num(s, 5, 7), _num(s, 8, 10), hour, minute, second, micro) + + +class Date(TypeEngine): + _ddl = "DATE" + + def bind(self, value): + if value is None or isinstance(value, str): + return value + return "%04d-%02d-%02d" % (value.year, value.month, value.day) + + def result(self, value): + if not isinstance(value, str): + return value + return _dt.date(_num(value, 0, 4), _num(value, 5, 7), _num(value, 8, 10)) + + +class LargeBinary(TypeEngine): + _ddl = "BLOB" + + +def _to_type(t): + if t is None: + return NullType() + if isinstance(t, type): + return t() + return t + + +def _is_type(a): + if isinstance(a, TypeEngine): + return True + return isinstance(a, type) and issubclass(a, TypeEngine) + + +# ----------------------------------------------------------------------------- compiler + +class _Ctx: + def __init__(self, qualify=True): + self.params = [] + self.qualify = qualify + self.columns_clause = False + + +class ClauseElement: + type = NullType() + + def _compile(self, ctx): + raise NotImplementedError(type(self).__name__) + + def _tables(self, acc): + pass + + def __str__(self): + return self._compile(_Ctx()) + + +def _lit(value, typ=None): + if isinstance(value, ClauseElement): + return value + return BindParameter(value, typ) + + +class ColumnElement(ClauseElement): + key = None + name = None + + def __hash__(self): + return id(self) + + def _op(self, op, other): + return BinaryExpression(self, op, _lit(other, self.type)) + + def __eq__(self, other): + if other is None: + return BinaryExpression(self, "IS", Null()) + return self._op("=", other) + + def __ne__(self, other): + if other is None: + return BinaryExpression(self, "IS NOT", Null()) + return self._op("!=", other) + + def __lt__(self, other): + return self._op("<", other) + + def __le__(self, other): + return self._op("<=", other) + + def __gt__(self, other): + return self._op(">", other) + + def __ge__(self, other): + return self._op(">=", other) + + def __add__(self, other): + return Grouping(self._op("+", other), self.type) + + def __sub__(self, other): + return Grouping(self._op("-", other), self.type) + + def __mul__(self, other): + return Grouping(self._op("*", other), self.type) + + def __truediv__(self, other): + return Grouping(self._op("/", other), self.type) + + def __mod__(self, other): + return Grouping(self._op("%", other), self.type) + + def __neg__(self): + return UnaryExpression(self, "-", None, self.type) + + def __invert__(self): + return not_(self) + + def in_(self, values): + return InExpression(self, [_lit(v, self.type) for v in values], False) + + def not_in(self, values): + return InExpression(self, [_lit(v, self.type) for v in values], True) + + notin_ = not_in + + def like(self, pattern, escape=None): + return self._op("LIKE", pattern) + + def not_like(self, pattern, escape=None): + return self._op("NOT LIKE", pattern) + + def ilike(self, pattern, escape=None): + return BinaryExpression(Function("lower", [self]), "LIKE", Function("lower", [_lit(pattern, self.type)])) + + def startswith(self, s): + return self._op("LIKE", s + "%") + + def endswith(self, s): + return self._op("LIKE", "%" + s) + + def contains(self, s): + return self._op("LIKE", "%" + s + "%") + + def between(self, lo, hi): + return and_(self >= lo, self <= hi) + + def is_(self, other): + return BinaryExpression(self, "IS", _lit(other) if other is not None else Null()) + + def is_not(self, other): + return BinaryExpression(self, "IS NOT", _lit(other) if other is not None else Null()) + + isnot = is_not + + def desc(self): + return UnaryExpression(self, None, "DESC", self.type) + + def asc(self): + return UnaryExpression(self, None, "ASC", self.type) + + def label(self, name): + return Label(name, self) + + +class Null(ColumnElement): + def _compile(self, ctx): + return "NULL" + + +def null(): + return Null() + + +class BindParameter(ColumnElement): + def __init__(self, value, typ=None): + self.value = value + self.type = _to_type(typ) + + def _compile(self, ctx): + ctx.params.append(self.type.bind(self.value)) + return "?" + + +def literal(value, type_=None): + return BindParameter(value, type_) + + +class BinaryExpression(ColumnElement): + def __init__(self, left, op, right): + self.left = left + self.op = op + self.right = right + self.type = Boolean() if op not in ("+", "-", "*", "/", "%") else left.type + + def _compile(self, ctx): + return "%s %s %s" % (self.left._compile(ctx), self.op, self.right._compile(ctx)) + + def _tables(self, acc): + self.left._tables(acc) + self.right._tables(acc) + + def __bool__(self): + if self.op == "=": + return self.left is self.right + if self.op == "!=": + return self.left is not self.right + raise TypeError("Boolean value of this clause is not defined") + + +class Grouping(ColumnElement): + def __init__(self, elem, typ): + self.elem = elem + self.type = typ + + def _compile(self, ctx): + return "(" + self.elem._compile(ctx) + ")" + + def _tables(self, acc): + self.elem._tables(acc) + + +class InExpression(ColumnElement): + def __init__(self, left, items, negate): + self.left = left + self.items = items + self.negate = negate + self.type = Boolean() + + def _compile(self, ctx): + inner = ", ".join(i._compile(ctx) for i in self.items) + return "%s %sIN (%s)" % (self.left._compile(ctx), "NOT " if self.negate else "", inner) + + def _tables(self, acc): + self.left._tables(acc) + + +class UnaryExpression(ColumnElement): + def __init__(self, elem, prefix, suffix, typ=None): + self.elem = elem + self.prefix = prefix + self.suffix = suffix + self.type = typ if typ is not None else NullType() + + def _compile(self, ctx): + s = self.elem._compile(ctx) + if self.prefix: + s = self.prefix + s if self.prefix == "-" else "%s (%s)" % (self.prefix, s) + if self.suffix: + s = s + " " + self.suffix + return s + + def _tables(self, acc): + self.elem._tables(acc) + + +def desc(col): + return col.desc() + + +def asc(col): + return col.asc() + + +class BooleanClauseList(ColumnElement): + def __init__(self, op, items): + self.op = op + self.items = list(items) + self.type = Boolean() + + def _compile(self, ctx): + if len(self.items) == 1: + return self.items[0]._compile(ctx) + return "(" + (" %s " % self.op).join(i._compile(ctx) for i in self.items) + ")" + + def _tables(self, acc): + for i in self.items: + i._tables(acc) + + +def and_(*clauses): + return BooleanClauseList("AND", clauses) + + +def or_(*clauses): + return BooleanClauseList("OR", clauses) + + +def not_(clause): + return UnaryExpression(clause, "NOT", None, Boolean()) + + +def true(): + return TextClause("1") + + +def false(): + return TextClause("0") + + +class Label(ColumnElement): + def __init__(self, name, elem): + self.name = self.key = name + self.elem = elem + self.type = elem.type + + def _compile(self, ctx): + if ctx.columns_clause: + ctx.columns_clause = False + s = self.elem._compile(ctx) + ctx.columns_clause = True + return "%s AS %s" % (s, self.name) + return self.name + + def _tables(self, acc): + self.elem._tables(acc) + + +class Function(ColumnElement): + def __init__(self, name, args): + self.name = self.key = name + self.args = [_lit(a) for a in args] + if name == "count": + self.type = Integer() + elif self.args and name in ("sum", "max", "min", "abs", "coalesce", "lower", "upper"): + self.type = self.args[0].type + elif name == "avg": + self.type = Float() + else: + self.type = NullType() + + def _compile(self, ctx): + if not self.args and self.name == "count": + return "count(*)" + cc = ctx.columns_clause + ctx.columns_clause = False + s = "%s(%s)" % (self.name, ", ".join(a._compile(ctx) for a in self.args)) + ctx.columns_clause = cc + return s + + def _tables(self, acc): + for a in self.args: + a._tables(acc) + + +class _FuncNamespace: + def __getattr__(self, name): + if name.startswith("__"): + raise AttributeError(name) + + def make(*args): + return Function(name, args) + return make + + +func = _FuncNamespace() + + +class TextClause(ColumnElement): + def __init__(self, text_): + self.text = text_ + + def _compile(self, ctx): + return self.text + + def _execute_on(self, conn, params, session=None): + if isinstance(params, list): + cur = conn._run(self.text, params, many=True) + else: + cur = conn._run(self.text, params if params is not None else {}) + return CursorResult._from_cursor(cur, None) + + +def text(sql): + return TextClause(sql) + + +# ----------------------------------------------------------------------------- schema + +class ForeignKey: + def __init__(self, column, ondelete=None, onupdate=None, **kw): + self._spec = column + self.parent = None + self.ondelete = ondelete + + def _target(self): + if isinstance(self._spec, Column): + return self._spec.table.name, self._spec.name + dot = self._spec.rfind(".") + return self._spec[:dot], self._spec[dot + 1:] + + @property + def column(self): + tname, cname = self._target() + return self.parent.table.metadata.tables[tname].c[cname] + + def references(self, table): + return self._target()[0] == table.name + + +class Column(ColumnElement): + def __init__(self, *args, **kw): + self.name = None + self.type = None + self.foreign_keys = [] + for a in args: + if isinstance(a, str): + self.name = a + elif isinstance(a, ForeignKey): + self.foreign_keys.append(a) + a.parent = self + elif _is_type(a): + self.type = _to_type(a) + if self.type is None: + self.type = Integer() if self.foreign_keys else NullType() + self.key = kw.get("key", self.name) + self.primary_key = kw.get("primary_key", False) + self.nullable = kw.get("nullable", not self.primary_key) + self.default = kw.get("default") + self.unique = kw.get("unique", False) + self.index = kw.get("index", False) + self.autoincrement = kw.get("autoincrement", "auto") + self.table = None + + def _compile(self, ctx): + if ctx.qualify and self.table is not None: + return "%s.%s" % (self.table.name, self.name) + return self.name + + def _tables(self, acc): + if self.table is not None: + _add_unique(acc, self.table) + + def _default_value(self): + d = self.default + if callable(d): + try: + return d() + except TypeError: + return d(None) + return d + + +def _add_unique(acc, item): + for x in acc: + if x is item: + return + acc.append(item) + + +class ColumnCollection: + def __init__(self): + self._cols = [] + self._by_key = {} + + def _add(self, col): + self._cols.append(col) + self._by_key[col.key] = col + + def __getattr__(self, name): + if name.startswith("_"): + raise AttributeError(name) + try: + return self._by_key[name] + except KeyError: + raise AttributeError(name) + + def __getitem__(self, key): + if isinstance(key, int): + return self._cols[key] + return self._by_key[key] + + def __iter__(self): + return iter(list(self._cols)) + + def __len__(self): + return len(self._cols) + + def __contains__(self, key): + return key in self._by_key + + def keys(self): + return [c.key for c in self._cols] + + def values(self): + return list(self._cols) + + +class FromClause(ClauseElement): + def join(self, right, onclause=None, isouter=False, full=False): + return Join(self, _from_obj(right), onclause, isouter) + + def outerjoin(self, right, onclause=None, full=False): + return Join(self, _from_obj(right), onclause, True) + + +class Table(FromClause): + def __init__(self, name, metadata, *args, **kw): + self.name = name + self.metadata = metadata + self.c = self.columns = ColumnCollection() + self.constraints = [] + for a in args: + if isinstance(a, Column): + a.table = self + self.c._add(a) + metadata.tables[name] = self + metadata._order.append(self) + + @property + def primary_key(self): + return [c for c in self.c if c.primary_key] + + @property + def foreign_keys(self): + out = [] + for c in self.c: + out.extend(c.foreign_keys) + return out + + def _compile(self, ctx): + return self.name + + def _tables(self, acc): + _add_unique(acc, self) + + def _all_tables(self): + return [self] + + def _create_sql(self): + parts = [] + for c in self.c: + s = "%s %s" % (c.name, c.type.ddl()) + if not c.nullable: + s += " NOT NULL" + parts.append(s) + pk = self.primary_key + if pk: + parts.append("PRIMARY KEY (%s)" % ", ".join(c.name for c in pk)) + for c in self.c: + if c.unique: + parts.append("UNIQUE (%s)" % c.name) + for c in self.c: + for fk in c.foreign_keys: + tname, cname = fk._target() + s = "FOREIGN KEY(%s) REFERENCES %s (%s)" % (c.name, tname, cname) + if fk.ondelete: + s += " ON DELETE " + fk.ondelete + parts.append(s) + return "CREATE TABLE IF NOT EXISTS %s (\n\t%s\n)" % (self.name, ", \n\t".join(parts)) + + def create(self, bind, checkfirst=True): + with bind.begin() as conn: + conn._run(self._create_sql(), ()) + + def drop(self, bind, checkfirst=True): + with bind.begin() as conn: + conn._run("DROP TABLE IF EXISTS " + self.name, ()) + + def insert(self): + return insert(self) + + def select(self): + return select(self) + + def update(self): + return update(self) + + def delete(self): + return delete(self) + + +class Join(FromClause): + def __init__(self, left, right, onclause, isouter): + self.left = left + self.right = right + self.isouter = isouter + if onclause is None: + onclause = _join_condition(left._all_tables(), right) + self.onclause = onclause + + def _all_tables(self): + return self.left._all_tables() + self.right._all_tables() + + def _tables(self, acc): + for t in self._all_tables(): + _add_unique(acc, t) + + def _compile(self, ctx): + kw = "LEFT OUTER JOIN" if self.isouter else "JOIN" + return "%s %s %s ON %s" % (self.left._compile(ctx), kw, self.right._compile(ctx), + self.onclause._compile(ctx)) + + +def _join_condition(left_tables, right): + for lt in left_tables: + for c in right.c: + for fk in c.foreign_keys: + if fk.references(lt): + return lt.c[fk._target()[1]] == c + for c in lt.c: + for fk in c.foreign_keys: + if fk.references(right): + return c == right.c[fk._target()[1]] + raise exc.ArgumentError("Can't find any foreign key relationships between the tables") + + +def _from_obj(x): + m = getattr(x, "__sa_mapper__", None) + if m is not None: + return m.table + return x + + +class MetaData: + def __init__(self, schema=None): + self.tables = {} + self._order = [] + + @property + def sorted_tables(self): + done = [] + pending = list(self._order) + while pending: + progressed = False + for t in list(pending): + ok = True + for fk in t.foreign_keys: + tn = fk._target()[0] + if tn != t.name and tn in self.tables and not any(d.name == tn for d in done): + ok = False + if ok: + done.append(t) + pending.remove(t) + progressed = True + if not progressed: + done.extend(pending) + break + return done + + def create_all(self, bind, tables=None, checkfirst=True): + with bind.begin() as conn: + for t in self.sorted_tables: + if tables is None or t in tables: + conn._run(t._create_sql(), ()) + + def drop_all(self, bind, tables=None, checkfirst=True): + with bind.begin() as conn: + for t in reversed(self.sorted_tables): + if tables is None or t in tables: + conn._run("DROP TABLE IF EXISTS " + t.name, ()) + + +# ----------------------------------------------------------------------------- statements + +class _Slot: + """One entry of a select's columns clause: a column expression, or a mapped entity.""" + + def __init__(self, elem=None, mapper=None): + self.elem = elem + self.mapper = mapper + if mapper is not None: + self.key = mapper.class_.__name__ + self.width = len(mapper.columns) + else: + self.key = elem.key if elem.key is not None else elem.name + self.width = 1 + + +def _slots(ent): + m = getattr(ent, "__sa_mapper__", None) + if m is not None: + return [_Slot(mapper=m)] + if isinstance(ent, Table): + return [_Slot(c) for c in ent.c] + return [_Slot(_lit(ent))] + + +class Select(ClauseElement): + def __init__(self, *entities): + if len(entities) == 1 and isinstance(entities[0], (list, tuple)): + entities = entities[0] + self._slots = [] + for e in entities: + self._slots.extend(_slots(e)) + self._where = [] + self._order = [] + self._group = [] + self._having = [] + self._joins = [] + self._from = [] + self._limit = None + self._offset = None + self._distinct = False + + def _clone(self): + s = Select() + s._slots = list(self._slots) + s._where = list(self._where) + s._order = list(self._order) + s._group = list(self._group) + s._having = list(self._having) + s._joins = list(self._joins) + s._from = list(self._from) + s._limit = self._limit + s._offset = self._offset + s._distinct = self._distinct + return s + + def where(self, *clauses): + s = self._clone() + s._where.extend(clauses) + return s + + filter = where + + def filter_by(self, **kw): + s = self._clone() + t = self._first_table() + for k in kw: + s._where.append(t.c[k] == kw[k]) + return s + + def order_by(self, *clauses): + s = self._clone() + for c in clauses: + if c is not None: + s._order.append(c) + return s + + def group_by(self, *clauses): + s = self._clone() + s._group.extend(clauses) + return s + + def having(self, *clauses): + s = self._clone() + s._having.extend(clauses) + return s + + def limit(self, n): + s = self._clone() + s._limit = n + return s + + def offset(self, n): + s = self._clone() + s._offset = n + return s + + def distinct(self): + s = self._clone() + s._distinct = True + return s + + def select_from(self, *froms): + s = self._clone() + s._from.extend(_from_obj(f) for f in froms) + return s + + def join(self, target, onclause=None, isouter=False, full=False): + s = self._clone() + info = getattr(target, "_join_info", None) + if info is not None and onclause is None: + target, onclause = info() + s._joins.append((_from_obj(target), onclause, isouter)) + return s + + def outerjoin(self, target, onclause=None, full=False): + return self.join(target, onclause, True) + + join_from = None + + def add_columns(self, *cols): + s = self._clone() + for c in cols: + s._slots.extend(_slots(c)) + return s + + def _first_table(self): + acc = [] + self._column_tables(acc) + return acc[0] + + def _column_tables(self, acc): + for sl in self._slots: + if sl.mapper is not None: + _add_unique(acc, sl.mapper.table) + else: + sl.elem._tables(acc) + + def _froms(self): + if self._from: + base = list(self._from) + else: + base = [] + self._column_tables(base) + if not base: + for w in self._where: + w._tables(base) + joined = [] + for target, _, _ in self._joins: + joined.append(target) + covered = [] + for b in base: + if isinstance(b, Join): + for t in b._all_tables(): + covered.append(t) + out = [] + for b in base: + if any(b is j for j in joined): + continue + if not isinstance(b, Join) and any(b is c for c in covered): + continue + out.append(b) + return out + + def _compile(self, ctx): + ctx.columns_clause = True + cols = [] + for sl in self._slots: + if sl.mapper is not None: + for c in sl.mapper.columns: + cols.append(c._compile(ctx)) + else: + cols.append(sl.elem._compile(ctx)) + ctx.columns_clause = False + sql = "SELECT " + ("DISTINCT " if self._distinct else "") + ", ".join(cols) + froms = self._froms() + if froms or self._joins: + if not froms: + froms = [self._joins[0][0]] + parts = [f._compile(ctx) for f in froms] + seen = [] + for f in froms: + seen.extend(f._all_tables()) + jsql = "" + for target, onclause, isouter in self._joins: + if onclause is None: + onclause = _join_condition(seen, target) + jsql += " %s %s ON %s" % ("LEFT OUTER JOIN" if isouter else "JOIN", target._compile(ctx), + onclause._compile(ctx)) + seen.append(target) + parts[0] = parts[0] + jsql + sql += " \nFROM " + ", ".join(parts) + if self._where: + sql += " \nWHERE " + " AND ".join(w._compile(ctx) for w in self._where) + if self._group: + sql += " GROUP BY " + ", ".join(g._compile(ctx) for g in self._group) + if self._having: + sql += " \nHAVING " + " AND ".join(h._compile(ctx) for h in self._having) + if self._order: + sql += " ORDER BY " + ", ".join(o._compile(ctx) for o in self._order) + if self._limit is not None or self._offset is not None: + sql += " \nLIMIT ?" + ctx.params.append(self._limit if self._limit is not None else -1) + if self._offset is not None: + sql += " OFFSET ?" + ctx.params.append(self._offset) + return sql + + def _execute_on(self, conn, params, session=None): + ctx = _Ctx() + sql = self._compile(ctx) + cur = conn._run(sql, ctx.params) + return CursorResult._from_cursor(cur, self._slots, session) + + def subquery(self, name=None): + raise NotImplementedError("subqueries are not supported") + + def scalar_subquery(self): + raise NotImplementedError("subqueries are not supported") + + +def select(*entities): + return Select(*entities) + + +def _values_from(args, kw): + d = {} + if args: + a = args[0] + if isinstance(a, dict): + d.update(a) + d.update(kw) + return d + + +def _key_of(table, k): + if isinstance(k, Column): + return k.key + return k + + +class _DML(ClauseElement): + def __init__(self, table): + self.table = _from_obj(table) + self._values = None + self._where = [] + self._returning = [] + + def _clone(self): + s = type(self)(self.table) + s._values = None if self._values is None else dict(self._values) + s._where = list(self._where) + s._returning = list(self._returning) + return s + + def values(self, *args, **kw): + s = self._clone() + d = {} if s._values is None else s._values + for k, v in _values_from(args, kw).items(): + d[_key_of(self.table, k)] = v + s._values = d + return s + + def where(self, *clauses): + s = self._clone() + s._where.extend(clauses) + return s + + filter = where + + def returning(self, *cols): + s = self._clone() + for c in cols: + if isinstance(c, Table): + s._returning.extend(c.c) + else: + s._returning.append(c) + return s + + def _returning_sql(self, ctx): + if not self._returning: + return "" + q = ctx.qualify + ctx.qualify = False + s = " RETURNING " + ", ".join(c._compile(ctx) for c in self._returning) + ctx.qualify = q + return s + + def _where_sql(self, ctx): + if not self._where: + return "" + return " WHERE " + " AND ".join(w._compile(ctx) for w in self._where) + + def _result(self, cur): + slots = [_Slot(_lit(c)) for c in self._returning] if self._returning else None + return CursorResult._from_cursor(cur, slots) + + +class Insert(_DML): + def _row(self, params): + row = {} + if self._values: + row.update(self._values) + if params: + row.update(params) + for c in self.table.c: + if c.key not in row and c.default is not None: + row[c.key] = c._default_value() + return row + + def _sql(self, row, ctx): + cols = [c for c in self.table.c if c.key in row] + if not cols: + return "INSERT INTO %s DEFAULT VALUES" % self.table.name + self._returning_sql(ctx) + vals = [] + for c in cols: + vals.append(_lit(row[c.key], c.type)._compile(ctx)) + return "INSERT INTO %s (%s) VALUES (%s)" % ( + self.table.name, ", ".join(c.name for c in cols), ", ".join(vals)) + self._returning_sql(ctx) + + def _execute_on(self, conn, params, session=None): + if isinstance(params, list) and len(params) > 1: + rows = [self._row(p) for p in params] + ctx = _Ctx() + sql = self._sql(rows[0], ctx) + seq = [] + for r in rows: + c2 = _Ctx() + self._sql(r, c2) + seq.append(c2.params) + cur = conn._run(sql, seq, many=True) + return self._result(cur) + if isinstance(params, list): + params = params[0] if params else None + row = self._row(params) + ctx = _Ctx() + cur = conn._run(self._sql(row, ctx), ctx.params) + res = self._result(cur) + pk = [] + for c in self.table.primary_key: + pk.append(row[c.key] if row.get(c.key) is not None else cur.lastrowid) + res.inserted_primary_key = Row(pk, dict((c.key, i) for i, c in enumerate(self.table.primary_key))) + return res + + +class Update(_DML): + def _execute_on(self, conn, params, session=None): + vals = dict(self._values or {}) + if isinstance(params, dict): + vals.update(params) + ctx = _Ctx() + sets = [] + for c in self.table.c: + if c.key in vals: + v = vals[c.key] + sets.append("%s=%s" % (c.name, _lit(v, c.type)._compile(ctx))) + sql = "UPDATE %s SET %s" % (self.table.name, ", ".join(sets)) + self._where_sql(ctx) + self._returning_sql(ctx) + return self._result(conn._run(sql, ctx.params)) + + +class Delete(_DML): + def _execute_on(self, conn, params, session=None): + ctx = _Ctx() + sql = "DELETE FROM " + self.table.name + self._where_sql(ctx) + self._returning_sql(ctx) + return self._result(conn._run(sql, ctx.params)) + + +def insert(table): + return Insert(table) + + +def update(table): + return Update(table) + + +def delete(table): + return Delete(table) + + +# ----------------------------------------------------------------------------- results + +class Row: + def __init__(self, values, keymap): + self.__dict__["_values"] = tuple(values) + self.__dict__["_keymap"] = keymap + + def __getattr__(self, name): + km = self.__dict__.get("_keymap") + if km is not None and name in km: + return self.__dict__["_values"][km[name]] + raise AttributeError("Could not locate column in row for column '%s'" % name) + + def __getitem__(self, i): + return self._values[i] + + def __iter__(self): + return iter(self._values) + + def __len__(self): + return len(self._values) + + def __eq__(self, other): + if isinstance(other, Row): + return self._values == other._values + if isinstance(other, tuple): + return self._values == other + return False + + def __ne__(self, other): + return not self.__eq__(other) + + def __hash__(self): + return hash(self._values) + + def __repr__(self): + return repr(self._values) + + def _asdict(self): + return dict((k, self._values[i]) for k, i in self._keymap.items()) + + @property + def _mapping(self): + return RowMapping(self._asdict()) + + @property + def _fields(self): + return tuple(self._keymap.keys()) + + def _tuple(self): + return self._values + + @property + def t(self): + return self._values + + def tuple(self): + return self._values + + +class RowMapping(dict): + pass + + +def _only(rows, allow_none): + if not rows: + if allow_none: + return None + raise exc.NoResultFound("No row was found when one was required") + if len(rows) > 1: + raise exc.MultipleResultsFound("Multiple rows were found when exactly one was required") + return rows[0] + + +class _Rows: + def __init__(self, rows): + self._rows = rows + self._pos = 0 + + def _take(self): + r = self._rows[self._pos:] + self._pos = len(self._rows) + return r + + def __iter__(self): + return iter(self._take()) + + def all(self): + return self._take() + + fetchall = all + + def fetchone(self): + if self._pos < len(self._rows): + self._pos += 1 + return self._rows[self._pos - 1] + return None + + def fetchmany(self, size=1): + r = self._rows[self._pos:self._pos + size] + self._pos += len(r) + return r + + def first(self): + r = self._take() + return r[0] if r else None + + def one(self): + return _only(self._take(), False) + + def one_or_none(self): + return _only(self._take(), True) + + def unique(self): + seen = [] + out = [] + for r in self._rows[self._pos:]: + if not any(r is s or r == s for s in seen): + seen.append(r) + out.append(r) + self._rows = out + self._pos = 0 + return self + + +class ScalarResult(_Rows): + pass + + +class MappingResult(_Rows): + pass + + +class CursorResult(_Rows): + def __init__(self, rows, keys, rowcount=-1, lastrowid=None): + _Rows.__init__(self, rows) + self._keys = keys + self.rowcount = rowcount + self.lastrowid = lastrowid + self.inserted_primary_key = None + + @staticmethod + def _from_cursor(cur, slots, session=None): + desc = cur.description + if desc is None: + return CursorResult([], [], cur.rowcount, cur.lastrowid) + raw = cur.fetchall() + if slots is None: + keys = [d[0] for d in desc] + km = dict((k, i) for i, k in enumerate(keys)) + return CursorResult([Row(r, km) for r in raw], keys, cur.rowcount, cur.lastrowid) + keys = [s.key for s in slots] + km = {} + for i, k in enumerate(keys): + if k is not None and k not in km: + km[k] = i + rows = [] + for r in raw: + vals = [] + pos = 0 + for s in slots: + if s.mapper is not None: + vals.append(s.mapper._load_row(session, r[pos:pos + s.width])) + else: + vals.append(s.elem.type.result(r[pos])) + pos += s.width + rows.append(Row(vals, km)) + return CursorResult(rows, keys, cur.rowcount, cur.lastrowid) + + def keys(self): + return list(self._keys) + + def scalars(self, index=0): + return ScalarResult([r[index] for r in self._take()]) + + def mappings(self): + return MappingResult([r._mapping for r in self._take()]) + + def scalar(self): + r = self._take() + return r[0][0] if r else None + + def scalar_one(self): + return _only(self._take(), False)[0] + + def scalar_one_or_none(self): + r = _only(self._take(), True) + return None if r is None else r[0] + + def tuples(self): + return self + + def close(self): + pass + + +Result = CursorResult + + +# ----------------------------------------------------------------------------- engine + +class Connection: + def __init__(self, engine): + self.engine = engine + self._dbapi = engine._raw_connection() + self.closed = False + + def _run(self, sql, params, many=False): + cur = self._dbapi.cursor() + try: + if many: + cur.executemany(sql, params) + else: + cur.execute(sql, params) + except _sqlite3.Error as e: + raise exc._wrap(e, sql, params) + return cur + + def execute(self, statement, parameters=None, execution_options=None): + if isinstance(statement, str): + raise exc.ArgumentError("Textual SQL expression should be explicitly declared as text()") + return statement._execute_on(self, parameters) + + def exec_driver_sql(self, sql, parameters=()): + return CursorResult._from_cursor(self._run(sql, parameters), None) + + def scalar(self, statement, parameters=None): + return self.execute(statement, parameters).scalar() + + def scalars(self, statement, parameters=None): + return self.execute(statement, parameters).scalars() + + def commit(self): + self._dbapi.commit() + + def rollback(self): + self._dbapi.rollback() + + def begin(self): + return _Tx(self, False) + + def in_transaction(self): + return self._dbapi.in_transaction + + def close(self): + if not self.closed: + self._dbapi.rollback() + self.closed = True + + def __enter__(self): + return self + + def __exit__(self, t, v, tb): + self.close() + return False + + +class _Tx: + def __init__(self, conn, close_after): + self.conn = conn + self.close_after = close_after + + def __enter__(self): + return self.conn + + def __exit__(self, t, v, tb): + if t is None: + self.conn.commit() + else: + self.conn.rollback() + if self.close_after: + self.conn.close() + return False + + def commit(self): + self.conn.commit() + + def rollback(self): + self.conn.rollback() + + +class _Dialect: + name = "sqlite" + driver = "pysqlite" + + +class URL: + def __init__(self, s, database): + self._s = s + self.drivername = s.split(":", 1)[0] + self.database = database + + def __str__(self): + return self._s + + +def make_url(s): + if not s.startswith("sqlite"): + raise exc.ArgumentError("Only sqlite:// URLs are supported, got %r" % s) + rest = s.split("://", 1)[1] if "://" in s else "" + db = rest[1:] if rest.startswith("/") else rest + q = db.find("?") + if q >= 0: + db = db[:q] + return URL(s, db if db else None) + + +class Engine: + def __init__(self, url, echo=False): + self.url = url + self.echo = echo + self.dialect = _Dialect() + self._dbapi = None + + def _raw_connection(self): + # One DB-API connection for the engine (SQLAlchemy's SingletonThreadPool for SQLite). + if self._dbapi is None: + self._dbapi = _sqlite3.connect(self.url.database or ":memory:") + return self._dbapi + + def raw_connection(self): + return self._raw_connection() + + def connect(self): + return Connection(self) + + def begin(self): + return _Tx(Connection(self), True) + + def dispose(self): + if self._dbapi is not None: + self._dbapi.close() + self._dbapi = None + + +def create_engine(url, echo=False, **kwargs): + if not isinstance(url, URL): + url = make_url(url) + return Engine(url, echo) + + +def inspect(obj): + return obj diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/exc.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/exc.py new file mode 100644 index 00000000000..e50208ad47e --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/exc.py @@ -0,0 +1,84 @@ +"""SQLAlchemy exception hierarchy (the subset the Codename One SQLAlchemy provides).""" + + +class SQLAlchemyError(Exception): + pass + + +class ArgumentError(SQLAlchemyError): + pass + + +class InvalidRequestError(SQLAlchemyError): + pass + + +class NoResultFound(InvalidRequestError): + pass + + +class MultipleResultsFound(InvalidRequestError): + pass + + +class NoSuchTableError(InvalidRequestError): + pass + + +class StatementError(SQLAlchemyError): + def __init__(self, message, statement=None, params=None, orig=None): + SQLAlchemyError.__init__(self, message) + self.statement = statement + self.params = params + self.orig = orig + + +class DBAPIError(StatementError): + pass + + +class InterfaceError(DBAPIError): + pass + + +class DatabaseError(DBAPIError): + pass + + +class DataError(DatabaseError): + pass + + +class OperationalError(DatabaseError): + pass + + +class IntegrityError(DatabaseError): + pass + + +class InternalError(DatabaseError): + pass + + +class ProgrammingError(DatabaseError): + pass + + +class NotSupportedError(DatabaseError): + pass + + +_BY_NAME = { + "InterfaceError": InterfaceError, "DatabaseError": DatabaseError, "DataError": DataError, + "OperationalError": OperationalError, "IntegrityError": IntegrityError, + "InternalError": InternalError, "ProgrammingError": ProgrammingError, + "NotSupportedError": NotSupportedError, +} + + +def _wrap(e, statement, params): + """Wraps a DB-API exception the way SQLAlchemy does, keeping the original as ``orig``.""" + cls = _BY_NAME.get(type(e).__name__, DBAPIError) + msg = "(sqlite3.%s) %s\n[SQL: %s]" % (type(e).__name__, e, statement) + return cls(msg, statement, params, e) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/orm/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/orm/__init__.py new file mode 100644 index 00000000000..792550028d2 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/orm/__init__.py @@ -0,0 +1,875 @@ +"""SQLAlchemy 2.0 declarative ORM subset: DeclarativeBase, Mapped, mapped_column, +relationship (one-to-many / many-to-one, lazy "select" loading) and Session. + +Mapping is done by a metaclass reading the class body's ``__annotations__``; nothing is +evaluated with ``exec``. Instance state lives in ``obj.__dict__``: a column attribute that +is absent from it is *expired* and is reloaded from the database on access, which is how +``expire_on_commit``, ``refresh`` and ``rollback`` work. Changes are found at flush time by +comparing the attribute values with a snapshot taken at load, so no attribute events. + +Unsupported: many-to-many (``secondary``), inheritance mapping, eager loading options, +cascade configuration (save-update and backref population only; deleting a parent does +not touch its children), composite primary keys in ``Session.get``, hybrid properties. +""" + +import datetime as _dt +import typing as _typing + +from sqlalchemy import (Column, ForeignKey, Integer, String, Float, Boolean, DateTime, Date, LargeBinary, + MetaData, Table, TypeEngine, select, exc, _is_type, _to_type) + +__all__ = ["DeclarativeBase", "Mapped", "mapped_column", "relationship", "Session", "sessionmaker", + "declarative_base", "registry"] + +_PY_TYPES = {int: Integer, str: String, float: Float, bool: Boolean, bytes: LargeBinary, + _dt.datetime: DateTime, _dt.date: Date} +_PY_NAMES = {"int": Integer, "str": String, "float": Float, "bool": Boolean, "bytes": LargeBinary, + "datetime": DateTime, "datetime.datetime": DateTime, "date": Date, "datetime.date": Date} + + +class _MappedAnnotation: + def __init__(self, arg): + self.arg = arg + + +class Mapped: + def __class_getitem__(cls, item): + return _MappedAnnotation(item) + + +WriteOnlyMapped = Mapped +DynamicMapped = Mapped + + +def _name_of(t): + if isinstance(t, str): + return t.strip().strip("'\"") + fa = getattr(t, "__forward_arg__", None) + if fa is not None: + return fa + return getattr(t, "__name__", None) + + +def _strip(s, prefix): + for p in prefix: + if s.startswith(p + "[") and s.endswith("]"): + return s[len(p) + 1:-1].strip() + return None + + +def _analyze(ann): + """Returns (python type or class name, optional flag, is_list) for an annotation.""" + if isinstance(ann, _MappedAnnotation): + ann = ann.arg + elif isinstance(ann, str): + inner = _strip(ann.strip(), ("Mapped", "orm.Mapped", "WriteOnlyMapped", "DynamicMapped")) + if inner is None: + return None, False, False + ann = inner + if isinstance(ann, str): + s = ann.strip() + inner = _strip(s, ("Optional", "typing.Optional")) + if inner is not None: + t, _, lst = _analyze_str(inner) + return t, True, lst + if s.endswith("| None"): + t, _, lst = _analyze_str(s[:-6].strip()) + return t, True, lst + return _analyze_str(s) + origin = _typing.get_origin(ann) + args = _typing.get_args(ann) + if origin is _typing.Union or (args and type(None) in args and origin is not list): + rest = [a for a in args if a is not type(None)] + t, _, lst = _analyze(rest[0]) if rest else (None, False, False) + return t, True, lst + if origin in (list, set): + return _name_of(args[0]) if args else None, False, True + return ann, False, False + + +def _analyze_str(s): + inner = _strip(s, ("List", "list", "typing.List", "Set", "set")) + if inner is not None: + return _name_of(inner), False, True + return _name_of(s), False, False + + +def _sql_type(pytype): + if pytype is None: + return None + if isinstance(pytype, str): + t = _PY_NAMES.get(pytype) + else: + t = _PY_TYPES.get(pytype) + return t() if t is not None else None + + +class _MappedColumnSpec: + def __init__(self, args, kw): + self.args = args + self.kw = kw + + def _make(self, key, pytype, optional): + args = list(self.args) + name = key + if args and isinstance(args[0], str): + name = args.pop(0) + if not any(_is_type(a) for a in args): + t = _sql_type(pytype) + if t is not None: + args.insert(0, t) + kw = dict(self.kw) + if "nullable" not in kw: + kw["nullable"] = optional and not kw.get("primary_key", False) + col = Column(name, *args, **kw) + col.key = key + return col + + +def mapped_column(*args, **kw): + for k in ("init", "repr", "kw_only", "insert_default", "sort_order", "deferred", "compare"): + if k == "insert_default" and k in kw: + kw["default"] = kw.pop(k) + else: + kw.pop(k, None) + return _MappedColumnSpec(args, kw) + + +class _ColumnAttr: + def __init__(self, key, column): + self.key = key + self.column = column + + def __get__(self, obj, owner=None): + if obj is None: + return self.column + d = obj.__dict__ + if self.key in d: + return d[self.key] + st = d.get("_sa_state") + if st is not None and st.session is not None and st.persistent: + st.session._load_expired(obj) + # obj.__dict__ may be a snapshot taken when read: read it again after the load. + return obj.__dict__.get(self.key) + return None + + def __set__(self, obj, value): + obj.__dict__[self.key] = value + + +_ONETOMANY = "ONETOMANY" +_MANYTOONE = "MANYTOONE" + + +class InstrumentedList: + def __init__(self, owner, prop): + self._owner = owner + self._prop = prop + self._items = [] + + def _raw_add(self, x): + for i in self._items: + if i is x: + return + self._items.append(x) + + def append(self, x): + self._raw_add(x) + self._prop._back_set(self._owner, x) + + def extend(self, xs): + for x in xs: + self.append(x) + + def insert(self, i, x): + self._items.insert(i, x) + self._prop._back_set(self._owner, x) + + def remove(self, x): + for i in range(len(self._items)): + if self._items[i] is x: + del self._items[i] + self._prop._back_set(None, x) + return + raise ValueError("list.remove(x): x not in list") + + def pop(self, i=-1): + x = self._items.pop(i) + self._prop._back_set(None, x) + return x + + def clear(self): + for x in list(self._items): + self.remove(x) + + def __iter__(self): + return iter(list(self._items)) + + def __len__(self): + return len(self._items) + + def __bool__(self): + return len(self._items) > 0 + + def __getitem__(self, i): + return self._items[i] + + def __contains__(self, x): + return any(i is x for i in self._items) + + def __eq__(self, other): + if isinstance(other, InstrumentedList): + return self._items == other._items + return self._items == other + + def __repr__(self): + return repr(self._items) + + +class RelationshipProperty: + def __init__(self, argument=None, back_populates=None, uselist=None, lazy="select", order_by=None, + foreign_keys=None, cascade=None, backref=None, viewonly=False, secondary=None): + if secondary is not None: + raise NotImplementedError("many-to-many relationships (secondary=) are not supported") + self.argument = argument + self.back_populates = back_populates + self.uselist = uselist + self.order_by = order_by + self.key = None + self.parent = None + self._target_name = None + self._list_ann = None + self._ready = False + + def _setup(self): + if self._ready: + return + arg = self.argument + if arg is None: + arg = self._target_name + if callable(arg) and not isinstance(arg, type): + arg = arg() + if isinstance(arg, str): + arg = self.parent._sa_registry[arg] + self.target = arg + pt = self.parent.__table__ + tt = arg.__table__ + self.direction = None + for c in pt.c: + for fk in c.foreign_keys: + if fk.references(tt): + self.direction = _MANYTOONE + self.local = c + self.remote = tt.c[fk._target()[1]] + if self.direction is None: + for c in tt.c: + for fk in c.foreign_keys: + if fk.references(pt): + self.direction = _ONETOMANY + self.local = pt.c[fk._target()[1]] + self.remote = c + if self.direction is None: + raise exc.ArgumentError("Could not determine join condition for relationship %s.%s" + % (self.parent.__name__, self.key)) + if self.uselist is None: + self.uselist = self.direction == _ONETOMANY if self._list_ann is None else self._list_ann + self._ready = True + + def _other(self): + self._setup() + if self.back_populates: + return self.target.__sa_mapper__.relationships[self.back_populates] + return None + + def _join_info(self): + self._setup() + return self.target.__table__, self.local == self.remote + + def __get__(self, obj, owner=None): + if obj is None: + return self + d = obj.__dict__ + if self.key in d: + return d[self.key] + self._setup() + st = d.get("_sa_state") + if st is not None and st.session is not None and st.persistent: + value = st.session._lazy_load(obj, self) + elif self.uselist: + value = InstrumentedList(obj, self) + else: + value = None + d[self.key] = value + return value + + def __set__(self, obj, value): + self._setup() + if self.uselist: + lst = InstrumentedList(obj, self) + obj.__dict__[self.key] = lst + for x in value: + lst.append(x) + return + old = obj.__dict__.get(self.key) + obj.__dict__[self.key] = value + other = self._other() + if other is None: + return + if other.uselist: + if old is not None and old is not value: + ol = old.__dict__.get(other.key) + if isinstance(ol, InstrumentedList): + ol._items = [i for i in ol._items if i is not obj] + if value is not None: + getattr(value, other.key)._raw_add(obj) + elif value is not None: + value.__dict__[other.key] = obj + + def _back_set(self, owner, child): + """``child`` was added to (owner) or removed from (None) this one-to-many collection.""" + other = self._other() + if other is not None: + child.__dict__[other.key] = owner + + +def relationship(argument=None, **kw): + return RelationshipProperty(argument, **kw) + + +class _State: + def __init__(self): + self.session = None + self.persistent = False + self.snapshot = {} + + +def _state(obj): + d = obj.__dict__ + st = d.get("_sa_state") + if st is None: + st = _State() + d["_sa_state"] = st + return st + + +class Mapper: + def __init__(self, cls, table, attrs, relationships): + self.class_ = cls + self.table = table + self.attrs = attrs + self.columns = [c for k, c in attrs] + self.relationships = relationships + self.pk = [(k, c) for k, c in attrs if c.primary_key] + + def _identity(self, obj): + return tuple(obj.__dict__.get(k) for k, c in self.pk) + + def _populate(self, obj, values): + d = obj.__dict__ + st = _state(obj) + for i in range(len(self.attrs)): + k, c = self.attrs[i] + v = c.type.result(values[i]) + d[k] = v + st.snapshot[k] = v + + def _load_row(self, session, values): + key = [] + for i in range(len(self.attrs)): + if self.attrs[i][1].primary_key: + key.append(values[i]) + key = (self.class_, tuple(key)) + if session is not None: + obj = session._identity.get(key) + if obj is not None: + if any(k not in obj.__dict__ for k, c in self.attrs): + self._populate(obj, values) + return obj + obj = self.class_.__new__(self.class_) + st = _state(obj) + self._populate(obj, values) + if session is not None: + st.session = session + st.persistent = True + session._identity[key] = obj + return obj + + +class DeclarativeMeta(type): + def __new__(mcs, name, bases, ns, **kw): + cls = type.__new__(mcs, name, bases, ns) + if ns.get("_sa_root"): + return cls + if not any(b.__dict__.get("_sa_registry") is not None for b in cls.__mro__[1:]) \ + or "_sa_registry" in ns: + if "metadata" not in ns: + cls.metadata = MetaData() + cls._sa_registry = {} + cls.registry = cls._sa_registry + return cls + if "__tablename__" not in ns and "__table__" not in ns: + return cls + _map_class(cls, ns) + return cls + + +def _map_class(cls, ns): + ann = ns.get("__annotations__", {}) + attrs = [] + rels = {} + for key in ann: + a = ann[key] + v = ns.get(key) + if isinstance(v, RelationshipProperty): + pyt, optional, is_list = _analyze(a) + v._target_name = pyt if isinstance(pyt, str) else _name_of(pyt) + v._list_ann = is_list if pyt is not None else None + v.key = key + v.parent = cls + rels[key] = v + continue + pyt, optional, is_list = _analyze(a) + if isinstance(v, _MappedColumnSpec): + col = v._make(key, pyt, optional) + elif isinstance(v, Column): + col = v + if col.name is None: + col.name = col.key = key + elif v is None and (isinstance(a, _MappedAnnotation) or (isinstance(a, str) and "Mapped" in a)): + t = _sql_type(pyt) + if t is None: + raise exc.ArgumentError("Could not locate SQLAlchemy type for %s.%s" % (cls.__name__, key)) + col = Column(key, t, nullable=optional) + else: + continue + attrs.append((key, col)) + for key in ns: + v = ns[key] + if key not in ann: + if isinstance(v, _MappedColumnSpec): + attrs.append((key, v._make(key, None, False))) + elif isinstance(v, Column): + if v.name is None: + v.name = v.key = key + attrs.append((key, v)) + elif isinstance(v, RelationshipProperty): + v.key = key + v.parent = cls + rels[key] = v + table = Table(cls.__tablename__, cls.metadata, *[c for k, c in attrs]) + cls.__table__ = table + cls.__sa_mapper__ = Mapper(cls, table, attrs, rels) + for k, c in attrs: + setattr(cls, k, _ColumnAttr(k, c)) + cls._sa_registry[cls.__name__] = cls + + +class DeclarativeBase(metaclass=DeclarativeMeta): + _sa_root = True + + def __init__(self, **kw): + m = type(self).__sa_mapper__ + _state(self) + for k in kw: + if not any(k == a for a, c in m.attrs) and k not in m.relationships: + raise TypeError("%r is an invalid keyword argument for %s" % (k, type(self).__name__)) + setattr(self, k, kw[k]) + + def __repr__(self): + m = type(self).__sa_mapper__ + return "<%s %s>" % (type(self).__name__, m._identity(self)) + + +DeclarativeBaseNoMeta = DeclarativeBase + + +class MappedAsDataclass: + pass + + +def declarative_base(metadata=None): + ns = {"_sa_registry": {}} + if metadata is not None: + ns["metadata"] = metadata + return DeclarativeMeta("Base", (DeclarativeBase,), ns) + + +class _SessionTx: + def __init__(self, session): + self.session = session + + def __enter__(self): + return self.session + + def __exit__(self, t, v, tb): + if t is None: + self.session.commit() + else: + self.session.rollback() + return False + + +class Session: + def __init__(self, bind=None, autoflush=True, expire_on_commit=True, **kw): + self.bind = bind + self.autoflush = autoflush + self.expire_on_commit = expire_on_commit + self._conn = None + self._identity = {} + self._new = [] + self._deleted = [] + self._tx_inserted = [] + self._tx_deleted = [] + + # -- plumbing + def connection(self): + if self._conn is None: + self._conn = self.bind.connect() + return self._conn + + def __enter__(self): + return self + + def __exit__(self, t, v, tb): + self.close() + return False + + def begin(self): + return _SessionTx(self) + + def close(self): + if self._conn is not None: + self._conn.close() + self._conn = None + self.expunge_all() + + def expunge_all(self): + for obj in list(self._identity.values()) + self._new: + _state(obj).session = None + self._identity = {} + self._new = [] + self._deleted = [] + + def expunge(self, obj): + st = _state(obj) + st.session = None + self._new = [o for o in self._new if o is not obj] + for k in list(self._identity): + if self._identity[k] is obj: + del self._identity[k] + + # -- unit of work + def add(self, obj, _warn=True): + st = _state(obj) + if st.persistent and st.session is self: + return + if any(o is obj for o in self._new): + return + st.session = self + self._new.append(obj) + self._cascade(obj) + + def add_all(self, objs): + for o in objs: + self.add(o) + + def _cascade(self, obj): + m = type(obj).__sa_mapper__ + for key in m.relationships: + v = obj.__dict__.get(key) + if v is None: + continue + if isinstance(v, InstrumentedList): + for x in v._items: + self._cascade_one(x) + else: + self._cascade_one(v) + + def _cascade_one(self, x): + st = _state(x) + if st.persistent and st.session is self: + return + self.add(x) + + def delete(self, obj): + st = _state(obj) + if not st.persistent: + self._new = [o for o in self._new if o is not obj] + return + if not any(o is obj for o in self._deleted): + self._deleted.append(obj) + + def __contains__(self, obj): + st = _state(obj) + return st.session is self + + def _sync(self, obj): + m = type(obj).__sa_mapper__ + for key in m.relationships: + rel = m.relationships[key] + rel._setup() + v = obj.__dict__.get(key) + if v is None: + continue + if rel.direction == _MANYTOONE and not isinstance(v, InstrumentedList): + obj.__dict__[_attr_key(m, rel.local)] = v.__dict__.get(_attr_key(rel.target.__sa_mapper__, rel.remote)) + elif rel.direction == _ONETOMANY and isinstance(v, InstrumentedList): + pk = obj.__dict__.get(_attr_key(m, rel.local)) + tm = rel.target.__sa_mapper__ + for child in v._items: + child.__dict__[_attr_key(tm, rel.remote)] = pk + + def flush(self, objects=None): + for obj in list(self._identity.values()): + self._cascade(obj) + for obj in list(self._new): + self._cascade(obj) + conn = self.connection() + order = {} + tables = None + if self._new: + tables = type(self._new[0]).metadata.sorted_tables + for i in range(len(tables)): + order[tables[i].name] = i + new = sorted(self._new, key=lambda o: order.get(type(o).__table__.name, 0)) + self._new = [] + for obj in new: + self._sync(obj) + self._insert(conn, obj) + self._sync(obj) + for obj in list(self._identity.values()): + if any(o is obj for o in self._deleted): + continue + self._sync(obj) + self._update(conn, obj) + dels = self._deleted + self._deleted = [] + if dels: + tables = type(dels[0]).metadata.sorted_tables + for i in range(len(tables)): + order[tables[i].name] = i + dels = sorted(dels, key=lambda o: -order.get(type(o).__table__.name, 0)) + for obj in dels: + m = type(obj).__sa_mapper__ + where = [c == obj.__dict__.get(k) for k, c in m.pk] + conn.execute(m.table.delete().where(*where)) + key = (type(obj), m._identity(obj)) + if key in self._identity: + del self._identity[key] + _state(obj).persistent = False + self._tx_deleted.append(obj) + + def _insert(self, conn, obj): + m = type(obj).__sa_mapper__ + st = _state(obj) + d = obj.__dict__ + values = {} + for k, c in m.attrs: + v = d.get(k) + if v is None and c.default is not None: + v = c._default_value() + d[k] = v + if v is not None or k in d: + values[c.key] = v + res = conn.execute(m.table.insert().values(**values)) + for k, c in m.pk: + if d.get(k) is None: + d[k] = res.lastrowid + for k, c in m.attrs: + if k not in d: + d[k] = None + st.snapshot[k] = d[k] + st.persistent = True + st.session = self + self._identity[(type(obj), m._identity(obj))] = obj + self._tx_inserted.append(obj) + + def _update(self, conn, obj): + m = type(obj).__sa_mapper__ + st = _state(obj) + d = obj.__dict__ + changes = {} + for k, c in m.attrs: + if k in d and (k not in st.snapshot or st.snapshot[k] != d[k]): + changes[c.key] = d[k] + if not changes: + return + where = [c == st.snapshot.get(k, d.get(k)) for k, c in m.pk] + conn.execute(m.table.update().where(*where).values(**changes)) + for k in changes: + st.snapshot[k] = d[k] + + def commit(self): + self.flush() + if self._conn is not None: + self._conn.commit() + self._tx_inserted = [] + self._tx_deleted = [] + if self.expire_on_commit: + self.expire_all() + + def rollback(self): + if self._conn is not None: + self._conn.rollback() + for obj in self._tx_inserted: + m = type(obj).__sa_mapper__ + key = (type(obj), m._identity(obj)) + if self._identity.get(key) is obj: + del self._identity[key] + st = _state(obj) + st.persistent = False + st.session = None + for k, c in m.pk: + if c.autoincrement and isinstance(c.type, Integer): + obj.__dict__[k] = None + for obj in self._tx_deleted: + st = _state(obj) + st.persistent = True + self._identity[(type(obj), type(obj).__sa_mapper__._identity(obj))] = obj + for obj in self._new: + _state(obj).session = None + self._new = [] + self._deleted = [] + self._tx_inserted = [] + self._tx_deleted = [] + self.expire_all() + + def expire_all(self): + for obj in list(self._identity.values()): + self.expire(obj) + + def expire(self, obj, attribute_names=None): + m = type(obj).__sa_mapper__ + d = obj.__dict__ + pk = [k for k, c in m.pk] + for k, c in m.attrs: + if k not in pk and (attribute_names is None or k in attribute_names): + if k in d: + del d[k] + for k in m.relationships: + if attribute_names is None or k in attribute_names: + if k in d: + del d[k] + + def refresh(self, obj, attribute_names=None): + self.expire(obj, attribute_names) + self._load_expired(obj) + + def _load_expired(self, obj): + m = type(obj).__sa_mapper__ + st = _state(obj) + where = [c == st.snapshot.get(k, obj.__dict__.get(k)) for k, c in m.pk] + stmt = select(*m.columns).where(*where) + row = stmt._execute_on(self.connection(), None).first() + if row is None: + raise exc.InvalidRequestError("Instance has been deleted, or its row is otherwise not present.") + d = obj.__dict__ + for i in range(len(m.attrs)): + k = m.attrs[i][0] + d[k] = row[i] + st.snapshot[k] = row[i] + + def _lazy_load(self, obj, rel): + m = type(obj).__sa_mapper__ + tm = rel.target.__sa_mapper__ + if rel.direction == _ONETOMANY: + lst = InstrumentedList(obj, rel) + pk = getattr(obj, _attr_key(m, rel.local)) + stmt = select(rel.target).where(rel.remote == pk) + if rel.order_by is not None: + stmt = stmt.order_by(rel.order_by) + for x in stmt._execute_on(self.connection(), None, self).scalars(): + lst._items.append(x) + if rel.uselist: + return lst + return lst._items[0] if lst._items else None + fk = getattr(obj, _attr_key(m, rel.local)) + if fk is None: + return None + key = (rel.target, (fk,)) + if key in self._identity: + return self._identity[key] + stmt = select(rel.target).where(rel.remote == fk) + return stmt._execute_on(self.connection(), None, self).scalars().first() + + # -- queries + def _autoflush(self): + if self.autoflush: + self.flush() + + def execute(self, statement, params=None, **kw): + self._autoflush() + return statement._execute_on(self.connection(), params, self) + + def scalars(self, statement, params=None, **kw): + return self.execute(statement, params).scalars() + + def scalar(self, statement, params=None, **kw): + return self.execute(statement, params).scalar() + + def get(self, entity, ident, **kw): + if isinstance(ident, dict): + ident = tuple(ident.values()) + if not isinstance(ident, tuple): + ident = (ident,) + obj = self._identity.get((entity, ident)) + if obj is not None: + return obj + m = entity.__sa_mapper__ + where = [m.pk[i][1] == ident[i] for i in range(len(m.pk))] + return select(entity).where(*where)._execute_on(self.connection(), None, self).scalars().first() + + def get_one(self, entity, ident, **kw): + obj = self.get(entity, ident) + if obj is None: + raise exc.NoResultFound("No row was found when one was required") + return obj + + def merge(self, obj): + self.add(obj) + return obj + + @property + def new(self): + return list(self._new) + + @property + def deleted(self): + return list(self._deleted) + + @property + def dirty(self): + out = [] + for obj in self._identity.values(): + st = _state(obj) + m = type(obj).__sa_mapper__ + for k, c in m.attrs: + if k in obj.__dict__ and k in st.snapshot and st.snapshot[k] != obj.__dict__[k]: + out.append(obj) + break + return out + + +def _attr_key(mapper, col): + for k, c in mapper.attrs: + if c is col: + return k + return col.key + + +class sessionmaker: + def __init__(self, bind=None, **kw): + self.bind = bind + self.kw = kw + + def __call__(self, **kw): + args = dict(self.kw) + args.update(kw) + return Session(self.bind, **args) + + def begin(self): + return _SessionTx(self()) + + +def registry(): + return {} diff --git a/maven/python-transpiler/src/test/resources/conformance/instance_dict_pop.out b/maven/python-transpiler/src/test/resources/conformance/instance_dict_pop.out new file mode 100644 index 00000000000..dbbc37d791c --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/instance_dict_pop.out @@ -0,0 +1,6 @@ +1 5 False False {'y': 2, 'z': 3} +('z', 3) False ['y'] +False {} +4 {'w': 4} +KeyError 'nope' +False {'w': 4} diff --git a/maven/python-transpiler/src/test/resources/conformance/instance_dict_pop.py b/maven/python-transpiler/src/test/resources/conformance/instance_dict_pop.py new file mode 100644 index 00000000000..3f5a3cc454b --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/instance_dict_pop.py @@ -0,0 +1,21 @@ +class A: + pass + + +a = A() +a.x = 1 +a.y = 2 +a.z = 3 +print(a.__dict__.pop("x", None), a.__dict__.pop("q", 5), "x" in a.__dict__, hasattr(a, "x"), a.__dict__) +print(a.__dict__.popitem(), hasattr(a, "z"), sorted(vars(a))) +a.__dict__.clear() +print(hasattr(a, "y"), a.__dict__) +a.w = 4 +print(a.w, a.__dict__) +try: + a.__dict__.pop("nope") +except KeyError as e: + print("KeyError", e) +a.v = 9 +del a.__dict__["v"] +print(hasattr(a, "v"), a.__dict__) diff --git a/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_core.out b/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_core.out new file mode 100644 index 00000000000..3f8d9a763c7 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_core.out @@ -0,0 +1,34 @@ +(2, 'b', 2) b 2 2 +(3, 'c', 3) c 3 3 +3 +['a', 'b', 'c'] +[{'id': 1, 'name': 'a'}, {'id': 2, 'name': 'b'}, {'id': 3, 'name': 'c'}] +('c', 3) +None +a +MultipleResultsFound +NoResultFound +3 +['posts', 'users'] ['id', 'name', 'bio', 'score', 'active', 'created', 'born'] name +pk 1 +ann hi 1.5 True 2024-01-02 03:04:05 1990-05-06 datetime date +True +['cid', 'ann'] +['bob', 'cid'] +['cid'] +['bob', 'cid'] +['cid'] +['ann'] +[('ann', 'p1'), ('ann', 'p2'), ('bob', 'p3')] +[('ann', 'p1'), ('ann', 'p2')] +3 +True +[(1, 2), (2, 1)] +4.0 +deleted 1 +updated 2 +3 +p2 +(4, 4) +IntegrityError +1 diff --git a/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_core.py b/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_core.py new file mode 100644 index 00000000000..c19aeea529c --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_core.py @@ -0,0 +1,104 @@ +import datetime +from sqlalchemy import (create_engine, text, MetaData, Table, Column, Integer, String, Text, Float, + Boolean, DateTime, Date, ForeignKey, select, insert, update, delete, func, + and_, or_, not_) +from sqlalchemy import exc + +engine = create_engine("sqlite://") +with engine.connect() as conn: + conn.execute(text("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT, n INTEGER)")) + conn.execute(text("INSERT INTO t (name, n) VALUES (:name, :n)"), [{"name": "a", "n": 1}, {"name": "b", "n": 2}]) + conn.execute(text("INSERT INTO t (name, n) VALUES (:name, :n)"), {"name": "c", "n": 3}) + conn.commit() + r = conn.execute(text("SELECT id, name, n FROM t WHERE n >= :lo ORDER BY id"), {"lo": 2}) + for row in r: + print(tuple(row), row.name, row[2], row.n) + print(conn.execute(text("SELECT count(*) FROM t")).scalar()) + print(conn.execute(text("SELECT name FROM t ORDER BY name")).scalars().all()) + print([dict(m) for m in conn.execute(text("SELECT id, name FROM t ORDER BY id")).mappings()]) + print(tuple(conn.execute(text("SELECT name, n FROM t ORDER BY n DESC")).first())) + print(conn.execute(text("SELECT name FROM t WHERE n = 9")).one_or_none()) + print(conn.execute(text("SELECT name FROM t WHERE n = 1")).one().name) + try: + conn.execute(text("SELECT name FROM t")).one() + except exc.MultipleResultsFound: + print("MultipleResultsFound") + try: + conn.execute(text("SELECT name FROM t WHERE n = 9")).one() + except exc.NoResultFound: + print("NoResultFound") + conn.execute(text("DELETE FROM t WHERE n = 3")) + conn.rollback() + print(conn.execute(text("SELECT count(*) FROM t")).scalar_one()) + +metadata = MetaData() +users = Table("users", metadata, + Column("id", Integer, primary_key=True), + Column("name", String(50), nullable=False), + Column("bio", Text), + Column("score", Float, default=1.5), + Column("active", Boolean, default=True), + Column("created", DateTime), + Column("born", Date)) +posts = Table("posts", metadata, + Column("id", Integer, primary_key=True), + Column("user_id", Integer, ForeignKey("users.id")), + Column("title", String(100)), + Column("likes", Integer, default=0)) +metadata.create_all(engine) +print(sorted(metadata.tables), [c.name for c in users.columns], users.c.name.name) + +with engine.begin() as conn: + r = conn.execute(insert(users).values(name="ann", bio="hi", created=datetime.datetime(2024, 1, 2, 3, 4, 5), + born=datetime.date(1990, 5, 6))) + print("pk", r.inserted_primary_key[0]) + conn.execute(insert(users), [{"name": "bob", "score": 3.0, "active": False}, {"name": "cid", "score": 2.5, "active": True}]) + conn.execute(insert(posts), [{"user_id": 1, "title": "p1", "likes": 5}, {"user_id": 1, "title": "p2", "likes": 7}, + {"user_id": 2, "title": "p3", "likes": 1}]) + +with engine.connect() as conn: + row = conn.execute(select(users).where(users.c.id == 1)).one() + print(row.name, row.bio, row.score, row.active, row.created, row.born, + type(row.created).__name__, type(row.born).__name__) + print(conn.execute(select(users.c.name, users.c.active).order_by(users.c.id)).all() + == [("ann", True), ("bob", False), ("cid", True)]) + q = select(users.c.name).where(and_(users.c.score > 1.0, users.c.active == True)).order_by(users.c.name.desc()) + print(conn.execute(q).scalars().all()) + q = select(users.c.name).where(or_(users.c.name == "bob", users.c.name.like("c%"))).order_by(users.c.name) + print(conn.execute(q).scalars().all()) + print(conn.execute(select(users.c.name).where(not_(users.c.name.in_(["ann", "bob"])))).scalars().all()) + print(conn.execute(select(users.c.name).where(users.c.bio.is_(None)).order_by(users.c.id)).scalars().all()) + print(conn.execute(select(users.c.name).where(users.c.name != "ann").order_by(users.c.id) + .limit(1).offset(1)).scalars().all()) + print(conn.execute(select(users.c.name).where(users.c.score < 2.0, users.c.id >= 1)).scalars().all()) + j = select(users.c.name, posts.c.title).join(posts, users.c.id == posts.c.user_id).order_by(posts.c.title) + print([tuple(r) for r in conn.execute(j)]) + j = (select(users.c.name, posts.c.title).select_from(users.join(posts)) + .where(posts.c.likes > 4).order_by(posts.c.id)) + print([tuple(r) for r in conn.execute(j)]) + print(conn.execute(select(func.count()).select_from(posts)).scalar()) + print(conn.execute(select(func.count(posts.c.id), func.sum(posts.c.likes), func.max(posts.c.likes), + func.min(posts.c.likes))).one() == (3, 13, 7, 1)) + g = (select(posts.c.user_id, func.count(posts.c.id).label("cnt")) + .group_by(posts.c.user_id).order_by(posts.c.user_id)) + print([(r.user_id, r.cnt) for r in conn.execute(g)]) + conn.execute(update(users).where(users.c.name == "bob").values(score=users.c.score + 1)) + print(conn.execute(select(users.c.score).where(users.c.name == "bob")).scalar()) + r = conn.execute(delete(posts).where(posts.c.likes < 2)) + print("deleted", r.rowcount) + r = conn.execute(update(posts).values(likes=0)) + print("updated", r.rowcount) + conn.rollback() + print(conn.execute(select(func.count()).select_from(posts)).scalar()) + print(conn.execute(select(posts.c.title).order_by(posts.c.likes.desc())).first()[0]) + r = conn.execute(insert(posts).values(title="r", likes=4).returning(posts.c.id, posts.c.likes)) + print(tuple(r.one())) + try: + conn.execute(insert(users).values(id=1, name="dup")) + except exc.IntegrityError: + print("IntegrityError") + conn.rollback() + +metadata.drop_all(engine) +with engine.connect() as conn: + print(conn.execute(text("SELECT count(*) FROM sqlite_master WHERE type='table'")).scalar()) diff --git a/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_orm.out b/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_orm.out new file mode 100644 index 00000000000..0a870d8cd3f --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_orm.out @@ -0,0 +1,22 @@ +['address', 'user_account'] user_account ['id', 'email', 'user_id'] +None True +1 2 3 40 None +[2, 3] 2 +1 spongebob Spongebob Squarepants 18 +2 sandy Sandy Cheeks 18 +True ['sandy@sqlalchemy.org', 'sandy@squirrelpower.org'] +sandy True +None +3 +[('spongebob', 'spongebob@sqlalchemy.org'), ('sandy', 'sandy@sqlalchemy.org'), ('sandy', 'sandy@squirrelpower.org')] +['sandy'] +['patrick', 'sandy'] +Patrick Star ['pat@star.org'] +['sandy@squirrelpower.org'] +['sandy@sqlalchemy.org', 'sandy@squirrelpower.org'] +True 4 +3 +['sandy', 'patrick'] +['sandy@sqlalchemy.org', 'sandy@squirrelpower.org', 'pat@star.org'] +patrick 40 1 patrick +[] diff --git a/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_orm.py b/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_orm.py new file mode 100644 index 00000000000..d71644a2610 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_orm.py @@ -0,0 +1,94 @@ +from typing import List, Optional +from sqlalchemy import create_engine, ForeignKey, String, select, func +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship, Session + + +class Base(DeclarativeBase): + pass + + +class User(Base): + __tablename__ = "user_account" + id: Mapped[int] = mapped_column(primary_key=True) + name: Mapped[str] = mapped_column(String(30)) + fullname: Mapped[Optional[str]] + age: Mapped[int] = mapped_column(default=18) + addresses: Mapped[List["Address"]] = relationship(back_populates="user") + + +class Address(Base): + __tablename__ = "address" + id: Mapped[int] = mapped_column(primary_key=True) + email: Mapped[str] = mapped_column(nullable=False) + user_id: Mapped[int] = mapped_column(ForeignKey("user_account.id")) + user: Mapped["User"] = relationship(back_populates="addresses") + + +engine = create_engine("sqlite:///:memory:") +Base.metadata.create_all(engine) +print(sorted(Base.metadata.tables), User.__table__.name, [c.name for c in Address.__table__.columns]) + +with Session(engine) as session: + spongebob = User(name="spongebob", fullname="Spongebob Squarepants", + addresses=[Address(email="spongebob@sqlalchemy.org")]) + sandy = User(name="sandy", fullname="Sandy Cheeks") + sandy.addresses.append(Address(email="sandy@sqlalchemy.org")) + sandy.addresses.append(Address(email="sandy@squirrelpower.org")) + patrick = User(name="patrick", age=40) + print(spongebob.id, sandy.addresses[0].user is sandy) + session.add_all([spongebob, sandy]) + session.add(patrick) + session.flush() + print(spongebob.id, sandy.id, patrick.id, patrick.age, patrick.fullname) + print(sorted(a.id for a in sandy.addresses), sandy.addresses[1].user_id) + session.commit() + +with Session(engine) as session: + stmt = select(User).where(User.name.in_(["spongebob", "sandy"])).order_by(User.id) + for user in session.scalars(stmt): + print(user.id, user.name, user.fullname, user.age) + u = session.execute(select(User).where(User.name == "sandy")).scalar_one() + print(u is session.get(User, 2), [a.email for a in u.addresses]) + a = session.scalars(select(Address).where(Address.email.like("%squirrel%"))).one() + print(a.user.name, a.user is u) + print(session.get(User, 99)) + print(session.scalar(select(func.count()).select_from(User))) + rows = session.execute(select(User.name, Address.email).join(User.addresses).order_by(Address.id)).all() + print([tuple(r) for r in rows]) + rows = session.execute(select(User.name).join(Address, User.id == Address.user_id) + .where(Address.email.like("sandy%")).distinct()).scalars().all() + print(rows) + print(session.scalars(select(User.name).order_by(User.age.desc(), User.name).limit(2)).all()) + + patrick = session.get(User, 3) + patrick.fullname = "Patrick Star" + patrick.addresses.append(Address(email="pat@star.org")) + session.commit() + print(patrick.fullname, [x.email for x in patrick.addresses]) + + sandy = session.get(User, 2) + session.delete(sandy.addresses[0]) + session.flush() + session.refresh(sandy) + print([x.email for x in sandy.addresses]) + session.rollback() + print(sorted(x.email for x in session.get(User, 2).addresses)) + + tmp = User(name="tmp") + session.add(tmp) + session.flush() + print(tmp.id is not None, session.scalar(select(func.count(User.id)))) + session.rollback() + print(session.scalar(select(func.count(User.id)))) + + session.delete(session.scalars(select(Address).where(Address.user_id == 1)).one()) + session.flush() + session.delete(session.get(User, 1)) + session.commit() + print(session.scalars(select(User.name).order_by(User.id)).all()) + print(session.scalars(select(Address.email).order_by(Address.id)).all()) + +with Session(engine) as s2: + u = s2.scalars(select(User).where(User.age > 20)).first() + print(u.name, u.age, len(u.addresses), u.addresses[0].user.name) + print(s2.scalars(select(User).where(User.fullname.is_(None))).all()) From 07100680e2c14f2dd607133cf78d4a20042d4f1f Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 10:40:44 +0200 Subject: [PATCH 053/112] Python: WebSocket endpoints run on a real asyncio loop, through middleware Each connection is a task on one shared event loop thread; server and test client callbacks feed it with call_soon_threadsafe, so endpoints can sleep, spawn tasks and use wait_for. asyncio gains per-thread running loops, call_soon_threadsafe with an idle loop woken by other threads, and run_coroutine_threadsafe. WebSocket connections go through the middleware stack (HTTP middleware passes them through, TrustedHostMiddleware denies a bad host with 400). The generated server waits for accept before the upgrade, so a refusal answers the handshake and the accepted subprotocol is the negotiated one. Co-Authored-By: Claude Opus 5.5 --- .../codename1/python/runtime/PyServer.java | 11 +- .../transpiler/api/PythonTranspiler.java | 154 ++++++++++++++---- .../python/site/fastapi/applications.py | 48 ++++-- .../python/site/starlette/middleware/base.py | 3 + .../python/site/starlette/middleware/cors.py | 3 + .../python/site/starlette/middleware/gzip.py | 3 + .../python/site/starlette/testclient.py | 48 ++++-- .../python/site/starlette/websockets.py | 111 ++++++++----- .../codename1/python/site/uvicorn/__init__.py | 50 +++--- .../com/codename1/python/stdlib/asyncio.py | 122 +++++++++++--- .../conformance/fastapi_ws_async.out | 12 ++ .../resources/conformance/fastapi_ws_async.py | 147 +++++++++++++++++ 12 files changed, 574 insertions(+), 138 deletions(-) create mode 100644 maven/python-transpiler/src/test/resources/conformance/fastapi_ws_async.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/fastapi_ws_async.py diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyServer.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyServer.java index f98d41958bc..61f9541a37c 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyServer.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyServer.java @@ -71,12 +71,17 @@ public interface WebSocketSink { void binary(byte[] value); void close(int code, String reason); + + /// The endpoint refused the handshake before accepting; `status` is + /// the HTTP status it answered with (403 when it simply closed). + void deny(int status); } /// Opens one websocket connection in Python; answers the connection /// object [#webSocketText], [#webSocketBinary] and [#webSocketClosed] - /// take. Messages the endpoint sends, including any it sends while - /// opening, reach `sink`. + /// take. The endpoint runs on Python's websocket event loop thread, so + /// this returns at once and `sink` is called later, from that thread: + /// first `accept` or `deny`, then the messages the endpoint sends. public static Object openWebSocket(Object webSockets, String target, String[] headers, final WebSocketSink sink) { Object[] req = request("GET", target, headers, null); @@ -88,6 +93,8 @@ public static Object openWebSocket(Object webSockets, String target, String[] he sink.text(Py.toStr(a[1])); } else if ("bytes".equals(kind)) { sink.binary(PyBytes.like(a[1])); + } else if ("deny".equals(kind)) { + sink.deny((int) Py.asLong(a[1])); } else { sink.close((int) Py.asLong(a[1]), a[2] == Py.NONE ? "" : Py.toStr(a[2])); //NOPMD CompareObjectsWithEquals - identity is meant } diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java index b205e516d1f..6dd3e2e98b7 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java @@ -436,66 +436,152 @@ private static String backendServer() { + " }\n" + " return headers;\n" + " }\n\n" - + " /** One websocket connection driven by the Python endpoint. */\n" + + " /**\n" + + " * One websocket connection driven by the Python endpoint, which runs on\n" + + " * Python's websocket event loop thread. The handshake waits for the\n" + + " * endpoint to accept or refuse, so a refusal is answered before the\n" + + " * upgrade and an accepted subprotocol is the one negotiated. Messages\n" + + " * the endpoint sends before the server has opened the session are kept\n" + + " * and sent from onOpen, in order.\n" + + " */\n" + " static final class Socket implements WebSocket, PyServer.WebSocketSink {\n" + + " private static final long HANDSHAKE_MILLIS = 30000L;\n" + " private final Object open;\n" + " private final String target;\n" + " private final String[] headers;\n" + + " private final java.util.ArrayList early = new java.util.ArrayList();\n" + " private WebSocketSession session;\n" - + " private Object connection;\n\n" + + " private Object connection;\n" + + " /** 0 while the endpoint decides, 1 accepted, 2 refused. */\n" + + " private int state;\n" + + " private String subprotocol;\n" + + "\n" + " Socket(Object open, String target, String[] headers) {\n" + " this.open = open;\n" + " this.target = target;\n" + " this.headers = headers;\n" - + " }\n\n" + + " }\n" + + "\n" + + " /** Runs the endpoint up to its accept; false when it refused or never answered. */\n" + + " boolean handshake() {\n" + + " // Opened outside the monitor: the loop thread holds the GIL while\n" + + " // it calls the sink, and the sink takes the monitor.\n" + + " Object c = PyServer.openWebSocket(open, target, headers, this);\n" + + " boolean accepted;\n" + + " synchronized (this) {\n" + + " connection = c;\n" + + " long end = System.currentTimeMillis() + HANDSHAKE_MILLIS;\n" + + " while (state == 0) {\n" + + " long left = end - System.currentTimeMillis();\n" + + " if (left <= 0) {\n" + + " break;\n" + + " }\n" + + " try {\n" + + " wait(left);\n" + + " } catch (InterruptedException e) {\n" + + " break;\n" + + " }\n" + + " }\n" + + " accepted = state == 1;\n" + + " }\n" + + " if (!accepted) {\n" + + " PyServer.webSocketClosed(c, 1006, \"\");\n" + + " }\n" + + " return accepted;\n" + + " }\n" + + "\n" + + " public synchronized String[] getSubprotocols() {\n" + + " return subprotocol == null ? null : new String[] {subprotocol};\n" + + " }\n" + + "\n" + " public void onOpen(WebSocketSession s) {\n" - + " session = s;\n" - + " connection = PyServer.openWebSocket(open, target, headers, this);\n" - + " }\n\n" - + " public void onText(WebSocketSession s, String message) {\n" - + " if (connection != null) {\n" - + " PyServer.webSocketText(connection, message);\n" + + " synchronized (this) {\n" + + " session = s;\n" + + " for (int i = 0; i < early.size(); i++) {\n" + + " deliver(early.get(i));\n" + + " }\n" + + " early.clear();\n" + " }\n" - + " }\n\n" + + " }\n" + + "\n" + + " public void onText(WebSocketSession s, String message) {\n" + + " PyServer.webSocketText(connection, message);\n" + + " }\n" + + "\n" + " public void onBinary(WebSocketSession s, byte[] message, int offset, int length) {\n" - + " if (connection != null) {\n" - + " byte[] copy = new byte[length];\n" - + " System.arraycopy(message, offset, copy, 0, length);\n" - + " PyServer.webSocketBinary(connection, copy);\n" - + " }\n" - + " }\n\n" + + " byte[] copy = new byte[length];\n" + + " System.arraycopy(message, offset, copy, 0, length);\n" + + " PyServer.webSocketBinary(connection, copy);\n" + + " }\n" + + "\n" + " public void onClose(WebSocketSession s, int code, String reason) {\n" - + " if (connection != null) {\n" - + " PyServer.webSocketClosed(connection, code, reason);\n" + + " PyServer.webSocketClosed(connection, code, reason);\n" + + " }\n" + + "\n" + + " public synchronized void accept(String subprotocol) {\n" + + " this.subprotocol = subprotocol;\n" + + " state = 1;\n" + + " notifyAll();\n" + + " }\n" + + "\n" + + " public synchronized void deny(int status) {\n" + + " if (state == 0) {\n" + + " state = 2;\n" + + " notifyAll();\n" + " }\n" - + " }\n\n" - + " public void accept(String subprotocol) {\n" - + " }\n\n" - + " public void text(String value) {\n" - + " try {\n" - + " session.sendText(value);\n" - + " } catch (java.io.IOException e) {\n" - + " session.abort();\n" + + " }\n" + + "\n" + + " public synchronized void text(String value) {\n" + + " outgoing(value);\n" + + " }\n" + + "\n" + + " public synchronized void binary(byte[] value) {\n" + + " outgoing(value);\n" + + " }\n" + + "\n" + + " public synchronized void close(int code, String reason) {\n" + + " if (state == 0) {\n" + + " // Closed before accepting: the upgrade is refused.\n" + + " state = 2;\n" + + " notifyAll();\n" + + " return;\n" + + " }\n" + + " outgoing(new Object[] {Integer.valueOf(code), reason});\n" + + " }\n" + + "\n" + + " private void outgoing(Object message) {\n" + + " if (session == null) {\n" + + " early.add(message);\n" + + " } else {\n" + + " deliver(message);\n" + " }\n" - + " }\n\n" - + " public void binary(byte[] value) {\n" + + " }\n" + + "\n" + + " private void deliver(Object message) {\n" + " try {\n" - + " session.sendBinary(value);\n" + + " if (message instanceof String) {\n" + + " session.sendText((String) message);\n" + + " } else if (message instanceof byte[]) {\n" + + " session.sendBinary((byte[]) message);\n" + + " } else if (message instanceof Object[]) {\n" + + " Object[] c = (Object[]) message;\n" + + " Object code = c[0];\n" + + " session.close(code instanceof Integer ? ((Integer) code).intValue() : 1000, String.valueOf(c[1]));\n" + + " }\n" + " } catch (java.io.IOException e) {\n" + " session.abort();\n" + " }\n" - + " }\n\n" - + " public void close(int code, String reason) {\n" - + " session.close(code, reason);\n" + " }\n" - + " }\n\n" + + " }\n" + + "\n" + " @Override\n" + " public void serve(final Object dispatch, String host, int port, final Object webSockets) {\n" + " HttpServer.WebSocketRoutes routes = webSockets == null ? null : new HttpServer.WebSocketRoutes() {\n" + " public void register(HttpServer.WebSocketRegistry registry) {\n" + " registry.fallback(new HttpServer.WebSocketHandler() {\n" + " public WebSocket open(HttpServer.Request r) {\n" - + " return new Socket(webSockets, r.getTarget(), headers(r));\n" + + " Socket socket = new Socket(webSockets, r.getTarget(), headers(r));\n" + + " return socket.handshake() ? socket : null;\n" + " }\n" + " });\n" + " }\n" diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/applications.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/applications.py index a086bab2191..d870c60e4b5 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/applications.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/applications.py @@ -194,6 +194,9 @@ def _route(self, req): raise StarletteHTTPException(404, "Not Found") async def _core(self, req): + if req.scope.get("type") == "websocket": + await self._run_ws(req._route, req) + return None try: if self.openapi_url and req._path == self.openapi_url and req.method == "GET": resp = JSONResponse(self.openapi()) @@ -229,28 +232,45 @@ def dispatch(self, method, path, query, headers, body): return self._dispatch_for_client(method, path, query, headers, body, False) # -- websockets - def websocket_open(self, path, query, headers, sink): - """Starts a websocket connection; returns its session, or None with no route. - - ``sink(message)`` receives the ASGI-shaped messages the endpoint sends - (``websocket.accept``, ``websocket.send``, ``websocket.close``). The - caller feeds client messages with ``session.deliver``. - """ + def websocket_route(self, path): + """The websocket route serving ``path`` and its path parameters, or (None, None).""" path = unquote(path) - route = None - vals = None for r in self.routes: if isinstance(r, APIWebSocketRoute): vals = r.match(path) if vals is not None: - route = r - break - session = _Session(sink) - ws = WebSocket(path, query, headers, session, self) + return r, vals + return None, None + + def websocket_open(self, path, query, headers, sink, on_done=None): + """Starts a websocket connection on the shared loop; returns its session. + + ``sink(message)`` receives the ASGI-shaped messages the endpoint sends + (``websocket.accept``, ``websocket.send``, ``websocket.close``, and + ``websocket.http.response`` when middleware refuses the handshake + with a response), on the loop thread. The caller feeds client + messages with ``session.deliver`` from any thread. The connection + goes through the middleware stack first, as Starlette's does. + """ + route, vals = self.websocket_route(path) + session = _Session(sink, on_done) + ws = WebSocket(unquote(path), query, headers, session, self) + ws.app = self ws.path_params = vals or {} - session.start(self._run_ws(route, ws)) + ws._route = route + session.start(self._run_ws_stack(ws)) return session + async def _run_ws_stack(self, ws): + from starlette.websockets import WebSocketState + resp = await self._build_stack()(ws) + if isinstance(resp, Response) and ws.application_state == WebSocketState.CONNECTING: + # A middleware answered the handshake with an HTTP response + # (TrustedHostMiddleware's 400): the upgrade is refused with it. + ws.application_state = WebSocketState.DISCONNECTED + ws._session.outgoing({"type": "websocket.http.response", "status": resp.status_code, + "headers": list(resp.headers.items()), "body": resp.body}) + async def _run_ws(self, route, ws): from fastapi.exceptions import WebSocketException, WebSocketRequestValidationError if route is None: diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/middleware/base.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/middleware/base.py index 3d74299d581..8cdb231c0e9 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/middleware/base.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/middleware/base.py @@ -15,6 +15,9 @@ async def call_next(self, request): return resp async def __call__(self, request): + if request.scope.get("type") == "websocket": + # Starlette's middleware passes a websocket scope straight through. + return await self.app(request) return await self.dispatch_func(request, self.call_next) async def dispatch(self, request, call_next): diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/middleware/cors.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/middleware/cors.py index a2ec9c8d4b0..9b002cbb800 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/middleware/cors.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/middleware/cors.py @@ -54,6 +54,9 @@ def is_allowed_origin(self, origin): return origin in self.allow_origins async def __call__(self, request): + if request.scope.get("type") == "websocket": + # Starlette's middleware passes a websocket scope straight through. + return await self.app(request) headers = request.headers origin = headers.get("origin") if origin is None: diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/middleware/gzip.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/middleware/gzip.py index 5add9710c2c..fe93d8567a0 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/middleware/gzip.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/middleware/gzip.py @@ -12,6 +12,9 @@ def __init__(self, app, minimum_size=500, compresslevel=9): self.compresslevel = compresslevel async def __call__(self, request): + if request.scope.get("type") == "websocket": + # Starlette's middleware passes a websocket scope straight through. + return await self.app(request) resp = await self.app(request) if "content-encoding" in resp.headers: return resp diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/testclient.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/testclient.py index e4abb0fd486..28c3418f30d 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/testclient.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/testclient.py @@ -4,6 +4,7 @@ """A test client that calls the app's pure dispatch core.""" import gzip as _gzip import json as _json +import queue as _queue from urllib.parse import urlencode, urljoin from starlette.datastructures import Headers, MutableHeaders, URL @@ -185,7 +186,17 @@ def _send(self, method, url, params, headers, json, data, content, cookies): return resp +class WebSocketDenialResponse(TestResponse, Exception): + """Raised when the app answers a websocket handshake with an HTTP response.""" + __test__ = False + + def __init__(self, status_code, headers, content, url): + TestResponse.__init__(self, status_code, headers, content, url) + Exception.__init__(self) + + class WebSocketTestSession: + """A websocket connection to the app's loop; this thread blocks on its queue.""" __test__ = False def __init__(self, app, path, query, headers): @@ -195,11 +206,14 @@ def __init__(self, app, path, query, headers): self.headers = headers self.accepted_subprotocol = None self.extra_headers = None - self._out = [] + self._out = _queue.SimpleQueue() self._session = None def _sink(self, message): - self._out.append(message) + self._out.put(message) + + def _finished(self, session): + self._out.put(None) def _raise_error(self): s = self._session @@ -209,26 +223,37 @@ def _raise_error(self): raise e def __enter__(self): - self._session = self.app.websocket_open(self.path, self.query, self.headers, self._sink) - self._raise_error() + self._session = self.app.websocket_open(self.path, self.query, self.headers, self._sink, + self._finished) m = self.receive() + if m["type"] == "websocket.http.response": + self._wait() + raise WebSocketDenialResponse(m["status"], m["headers"], m["body"], self.path) if m["type"] == "websocket.close": + self._wait() from starlette.websockets import WebSocketDisconnect raise WebSocketDisconnect(m.get("code", 1000), m.get("reason", "")) self.accepted_subprotocol = m.get("subprotocol") self.extra_headers = m.get("headers") return self + def _wait(self): + s = self._session + if not s.wait(5): + s.cancel() + s.wait(5) + self._raise_error() + def __exit__(self, *a): s = self._session if not s.done: s.deliver({"type": "websocket.disconnect", "code": 1000}) - self._raise_error() + self._wait() return False def send(self, message): - self._session.deliver(message) self._raise_error() + self._session.deliver(message) def send_text(self, data): self.send({"type": "websocket.receive", "text": data}) @@ -247,10 +272,13 @@ def close(self, code=1000, reason=None): self.send({"type": "websocket.disconnect", "code": code, "reason": reason}) def receive(self): - self._raise_error() - if not self._out: - raise RuntimeError("the websocket endpoint is waiting for a message and sent nothing") - return self._out.pop(0) + m = self._out.get() + if m is None: + # The endpoint has finished; keep the marker for later receives. + self._out.put(None) + self._raise_error() + return {"type": "websocket.close", "code": 1000, "reason": ""} + return m def _raise_on_close(self, m): if m["type"] == "websocket.close": diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/websockets.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/websockets.py index 9a82534dfa6..4cbdb8757f6 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/websockets.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/websockets.py @@ -3,15 +3,18 @@ # exception, as the rest of the Codename One Python runtime. """Server-side websockets without ASGI. -The endpoint coroutine is driven by ``_Session`` one step at a time rather -than by an event loop: whenever it waits for a message that has not arrived, -it yields a ``_Suspend`` and the session returns to its caller (the test -client, or the platform server's frame callback). A message delivered later -resumes it. An endpoint that awaits anything else that really suspends -- a -timer, another task -- is not supported by this driver. +Every connection runs as a task on one shared asyncio event loop, which a +daemon thread runs. The platform server and the test client call in from +their own threads: ``_Session.deliver`` hands a client message to the loop +with ``call_soon_threadsafe``, and the endpoint's ``receive`` awaits an +``asyncio.Queue``. The endpoint can therefore await anything that really +suspends -- timers, other tasks, ``wait_for`` -- and send from any task. +Messages it sends reach the session's sink on the loop thread. """ +import asyncio import enum import json as _json +import threading from starlette.datastructures import URL from starlette.requests import HTTPConnection @@ -31,11 +34,6 @@ def __init__(self, code=1000, reason=None): self.reason = reason or "" -class _Suspend: - def __await__(self): - yield self - - class WebSocket(HTTPConnection): def __init__(self, path, query, headers, session, app=None): HTTPConnection.__init__(self, "GET", path, query, headers, b"", app) @@ -52,10 +50,7 @@ def url(self): async def receive(self): if self.client_state == WebSocketState.DISCONNECTED: raise RuntimeError('Cannot call "receive" once a disconnect message has been received.') - inbox = self._session.inbox - while not inbox: - await _Suspend() - m = inbox.pop(0) + m = await self._session.inbox.get() if m["type"] == "websocket.disconnect": self.client_state = WebSocketState.DISCONNECTED return m @@ -144,41 +139,79 @@ async def close(self, code=1000, reason=None): await self.send({"type": "websocket.close", "code": code, "reason": reason or ""}) +_loop = None +_loop_lock = threading.Lock() + + +def _ws_loop(): + """The event loop every websocket connection runs on, started on first use.""" + global _loop + with _loop_lock: + if _loop is None: + loop = asyncio.new_event_loop() + loop._inbox() + t = threading.Thread(target=loop.run_forever, name="websockets", daemon=True) + t.start() + _loop = loop + return _loop + + class _Session: - """One connection: the inbox the endpoint reads and the coroutine it runs in. + """One connection: the queue the endpoint reads and the task it runs in. - ``sink(message)`` receives every message the endpoint sends. ``deliver`` - queues a client message and runs the endpoint until it waits again. + ``sink(message)`` receives every message the endpoint sends, on the loop + thread. ``deliver`` queues a client message from any thread. + ``on_done(session)``, when given, runs on the loop thread once the + endpoint has finished; ``error`` then holds what it raised, if anything. """ - def __init__(self, sink): - self.inbox = [{"type": "websocket.connect"}] + def __init__(self, sink, on_done=None): self.sink = sink - self.coro = None + self.on_done = on_done + self.inbox = None + self.task = None self.done = False self.error = None + self.loop = _ws_loop() + self.finished = threading.Event() def outgoing(self, message): self.sink(message) def start(self, coro): - self.coro = coro - self.pump() + self.loop.call_soon_threadsafe(self._begin, coro) + + def _begin(self, coro): + self.inbox = asyncio.Queue() + self.inbox.put_nowait({"type": "websocket.connect"}) + self.task = self.loop.create_task(coro) + self.task.add_done_callback(self._end) + + def _end(self, task): + if not task.cancelled(): + e = task.exception() + if e is not None: + self.error = e + self.done = True + try: + if self.on_done is not None: + self.on_done(self) + finally: + self.finished.set() def deliver(self, message): - self.inbox.append(message) - self.pump() - - def pump(self): - while not self.done: - try: - y = self.coro.send(None) - except StopIteration: - self.done = True - return - except BaseException as e: # noqa - self.done = True - self.error = e - return - if isinstance(y, _Suspend) and not self.inbox: - return + self.loop.call_soon_threadsafe(self._put, message) + + def _put(self, message): + if not self.done: + self.inbox.put_nowait(message) + + def cancel(self): + """Cancels the endpoint task from any thread.""" + def stop(): + if self.task is not None and not self.done: + self.task.cancel() + self.loop.call_soon_threadsafe(stop) + + def wait(self, timeout=None): + return self.finished.wait(timeout) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/uvicorn/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/uvicorn/__init__.py index 2e45a28a21e..9228052efe8 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/site/uvicorn/__init__.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/uvicorn/__init__.py @@ -5,39 +5,33 @@ class _Connection: - """One upgraded connection, as the platform server drives it.""" + """One upgraded connection, as the platform server drives it. + + The endpoint runs as a task on the shared websocket loop; these methods + are called on the server's threads and only hand the message over. + """ def __init__(self, session): self.session = session - def _check(self): - s = self.session - if s.error is not None: - e = s.error - s.error = None - raise e - def text(self, value): - if not self.session.done: - self.session.deliver({"type": "websocket.receive", "text": value}) - self._check() + self.session.deliver({"type": "websocket.receive", "text": value}) def binary(self, value): - if not self.session.done: - self.session.deliver({"type": "websocket.receive", "bytes": bytes(value)}) - self._check() + self.session.deliver({"type": "websocket.receive", "bytes": bytes(value)}) def closed(self, code, reason): - if not self.session.done: - self.session.deliver({"type": "websocket.disconnect", "code": code, "reason": reason}) - self._check() + self.session.deliver({"type": "websocket.disconnect", "code": code, "reason": reason}) def _websockets(app): def open_(path, query, headers, send): + state = {"accepted": False, "closed": False} + def sink(m): t = m["type"] if t == "websocket.accept": + state["accepted"] = True send("accept", m.get("subprotocol"), None) elif t == "websocket.send": if m.get("text") is not None: @@ -45,10 +39,26 @@ def sink(m): else: send("bytes", m["bytes"], None) elif t == "websocket.close": + state["closed"] = True send("close", m.get("code", 1000), m.get("reason") or "") - conn = _Connection(app.websocket_open(path, query, headers, sink)) - conn._check() - return conn + elif t == "websocket.http.response": + state["closed"] = True + send("deny", m["status"], None) + + def done(session): + e = session.error + if e is not None: + session.error = None + import traceback + traceback.print_exception(e) + if not state["closed"]: + state["closed"] = True + if state["accepted"]: + send("close", 1011 if e is not None else 1000, "") + else: + send("deny", 500 if e is not None else 403, None) + + return _Connection(app.websocket_open(path, query, headers, sink, done)) return open_ diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/asyncio.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/asyncio.py index dad66f7236c..c1d4fa40f01 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/asyncio.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/asyncio.py @@ -5,9 +5,13 @@ The loop runs callbacks, timers and tasks in the thread that called run(). There is no selector: a loop with nothing ready sleeps until its next -timer. Networking, subprocesses and executors are not part of this module. +timer, or until another thread hands it work through +``call_soon_threadsafe`` / ``run_coroutine_threadsafe``. Each thread has its +own running loop. Networking, subprocesses and executors are not part of +this module. """ +import _thread import time as _time from heapq import heappush as _heappush, heappop as _heappop @@ -17,7 +21,7 @@ "Task", "TimeoutError", "all_tasks", "as_completed", "create_task", "current_task", "ensure_future", "gather", "get_event_loop", "get_running_loop", "iscoroutine", "iscoroutinefunction", "new_event_loop", "run", "set_event_loop", "shield", "sleep", "wait", "wait_for", "FIRST_COMPLETED", "FIRST_EXCEPTION", - "ALL_COMPLETED", "timeout", + "ALL_COMPLETED", "timeout", "run_coroutine_threadsafe", ] FIRST_COMPLETED = "FIRST_COMPLETED" @@ -25,6 +29,10 @@ ALL_COMPLETED = "ALL_COMPLETED" +# Since Python 3.11 asyncio's TimeoutError is the builtin one. +TimeoutError = TimeoutError + + class CancelledError(BaseException): pass @@ -296,6 +304,9 @@ def __init__(self): self._stopping = False self._running = False self._closed = False + # Work handed over by other threads, created on first use; its + # semaphore is also what an idle loop blocks on. + self._threadsafe = None def time(self): return _time.monotonic() @@ -305,7 +316,18 @@ def call_soon(self, callback, *args, context=None): self._ready.append(h) return h - call_soon_threadsafe = call_soon + def call_soon_threadsafe(self, callback, *args, context=None): + h = Handle(callback, args, self) + self._inbox().put(h) + return h + + def _inbox(self): + q = self._threadsafe + if q is None: + import queue + q = queue.SimpleQueue() + self._threadsafe = q + return q def call_later(self, delay, callback, *args, context=None): return self.call_at(self.time() + delay, callback, *args) @@ -330,6 +352,8 @@ def is_closed(self): def stop(self): self._stopping = True + if self._threadsafe is not None: + self._threadsafe.put(None) def close(self): if self._running: @@ -343,9 +367,9 @@ def run_forever(self): raise RuntimeError("This event loop is already running") if self._closed: raise RuntimeError("Event loop is closed") - global _running_loop - previous = _running_loop - _running_loop = self + me = _thread.get_ident() + previous = _running_loops.get(me) + _running_loops[me] = self self._running = True try: while True: @@ -355,7 +379,10 @@ def run_forever(self): finally: self._stopping = False self._running = False - _running_loop = previous + if previous is None: + _running_loops.pop(me, None) + else: + _running_loops[me] = previous def run_until_complete(self, future): future = ensure_future(future, loop=self) @@ -373,10 +400,21 @@ def _run_once(self): scheduled = self._scheduled while scheduled and scheduled[0][2]._cancelled: _heappop(scheduled) - if not self._ready and scheduled and not self._stopping: - wait = scheduled[0][0] - self.time() - if wait > 0: - _time.sleep(wait) + q = self._threadsafe + if q is not None: + self._drain(q, False, None) + if not self._ready and not self._stopping: + if q is not None: + if scheduled: + wait = scheduled[0][0] - self.time() + if wait > 0: + self._drain(q, True, wait) + else: + self._drain(q, True, None) + elif scheduled: + wait = scheduled[0][0] - self.time() + if wait > 0: + _time.sleep(wait) now = self.time() while scheduled and scheduled[0][0] <= now: h = _heappop(scheduled)[2] @@ -388,21 +426,42 @@ def _run_once(self): if not h._cancelled: h._run() + def _drain(self, q, block, timeout): + import queue + try: + h = q.get(block, timeout) + except queue.Empty: + return + while True: + if h is not None: + self._ready.append(h) + try: + h = q.get(False) + except queue.Empty: + return -_running_loop = None + +# The loop each thread is running, keyed by thread id. +_running_loops = {} + + +def _get_running(): + return _running_loops.get(_thread.get_ident()) _event_loop = None def get_running_loop(): - if _running_loop is None: + loop = _get_running() + if loop is None: raise RuntimeError("no running event loop") - return _running_loop + return loop def get_event_loop(): global _event_loop - if _running_loop is not None: - return _running_loop + running = _get_running() + if running is not None: + return running if _event_loop is None or _event_loop.is_closed(): _event_loop = BaseEventLoop() return _event_loop @@ -450,7 +509,7 @@ def create_task(coro, *, name=None, context=None): def run(main, *, debug=None): - if _running_loop is not None: + if _get_running() is not None: raise RuntimeError("asyncio.run() cannot be called from a running event loop") if not iscoroutine(main): raise ValueError("a coroutine was expected, got " + repr(main)) @@ -489,7 +548,7 @@ def _set_result_unless_cancelled(fut, result): def gather(*coros_or_futures, return_exceptions=False): - loop = get_event_loop() if _running_loop is None else _running_loop + loop = get_event_loop() children = [ensure_future(c, loop=loop) for c in coros_or_futures] outer = loop.create_future() if not children: @@ -602,7 +661,7 @@ def check(f): def as_completed(fs, *, timeout=None): todo = [ensure_future(f) for f in fs] - loop = get_event_loop() if _running_loop is None else _running_loop + loop = get_event_loop() queue = Queue() def on_done(f): @@ -918,3 +977,28 @@ def task_done(self): async def join(self): if self._unfinished > 0: await self._finished.wait() + + +def run_coroutine_threadsafe(coro, loop): + """Runs ``coro`` on ``loop`` from another thread; answers a concurrent future.""" + if not iscoroutine(coro): + raise TypeError("A coroutine object is required") + from concurrent.futures import Future as _CFuture + result = _CFuture() + + def copy(task): + if task.cancelled(): + result.cancel() + return + e = task.exception() + if e is not None: + result.set_exception(e) + else: + result.set_result(task.result()) + + def start(): + if result.set_running_or_notify_cancel(): + loop.create_task(coro).add_done_callback(copy) + + loop.call_soon_threadsafe(start) + return result diff --git a/maven/python-transpiler/src/test/resources/conformance/fastapi_ws_async.out b/maven/python-transpiler/src/test/resources/conformance/fastapi_ws_async.out new file mode 100644 index 00000000000..bc35dfab321 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/fastapi_ws_async.out @@ -0,0 +1,12 @@ +{'ok': True} ['tagger /plain', 'http /plain'] +slept a +slept b +{'tick': 0} {'tick': 1} {'tick': 2} done +timed out +got late +closed 1000 +['a', 'b'] +raised endpoint failed +['sleepy gone 1000', 'ticker gone'] +host example.com +bad host refused WebSocketDenialResponse 400 Invalid host header diff --git a/maven/python-transpiler/src/test/resources/conformance/fastapi_ws_async.py b/maven/python-transpiler/src/test/resources/conformance/fastapi_ws_async.py new file mode 100644 index 00000000000..d107221937e --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/fastapi_ws_async.py @@ -0,0 +1,147 @@ +import asyncio + +from fastapi import FastAPI, Request, WebSocket, WebSocketDisconnect +from fastapi.middleware.cors import CORSMiddleware +from fastapi.middleware.gzip import GZipMiddleware +from fastapi.middleware.trustedhost import TrustedHostMiddleware +from fastapi.testclient import TestClient +from starlette.middleware.base import BaseHTTPMiddleware + +app = FastAPI() +seen = [] + + +@app.middleware("http") +async def http_only(request: Request, call_next): + seen.append("http " + request.url.path) + return await call_next(request) + + +class Tagger(BaseHTTPMiddleware): + async def dispatch(self, request, call_next): + seen.append("tagger " + request.url.path) + return await call_next(request) + + +app.add_middleware(Tagger) +app.add_middleware(GZipMiddleware, minimum_size=10) +app.add_middleware(CORSMiddleware, allow_origins=["*"]) + + +@app.websocket("/sleepy") +async def sleepy(websocket: WebSocket): + await websocket.accept() + while True: + try: + text = await websocket.receive_text() + except WebSocketDisconnect as e: + seen.append("sleepy gone %d" % e.code) + return + await asyncio.sleep(0.05) + await websocket.send_text("slept " + text) + + +@app.websocket("/ticker") +async def ticker(websocket: WebSocket): + await websocket.accept() + n = int(await websocket.receive_text()) + + async def push(): + for i in range(n): + await asyncio.sleep(0.02) + await websocket.send_json({"tick": i}) + await websocket.send_text("done") + + task = asyncio.create_task(push()) + await task + try: + await websocket.receive_text() + except WebSocketDisconnect: + seen.append("ticker gone") + + +@app.websocket("/timeout") +async def timeout(websocket: WebSocket): + await websocket.accept() + for _ in range(2): + try: + text = await asyncio.wait_for(websocket.receive_text(), 0.1) + await websocket.send_text("got " + text) + except asyncio.TimeoutError: + await websocket.send_text("timed out") + await websocket.close(code=1000) + + +@app.websocket("/both") +async def both(websocket: WebSocket): + await websocket.accept() + results = await asyncio.gather(asyncio.sleep(0.01, "a"), asyncio.sleep(0.02, "b")) + await websocket.send_json(results) + await websocket.close() + + +@app.websocket("/boom") +async def boom(websocket: WebSocket): + await websocket.accept() + await asyncio.sleep(0.01) + raise ValueError("endpoint failed") + + +@app.get("/plain") +def plain(): + return {"ok": True} + + +client = TestClient(app) +print(client.get("/plain").json(), seen) +del seen[:] + +with client.websocket_connect("/sleepy") as ws: + ws.send_text("a") + ws.send_text("b") + print(ws.receive_text()) + print(ws.receive_text()) + +with client.websocket_connect("/ticker") as ws: + ws.send_text("3") + print(ws.receive_json(), ws.receive_json(), ws.receive_json(), ws.receive_text()) + +with client.websocket_connect("/timeout") as ws: + print(ws.receive_text()) + ws.send_text("late") + print(ws.receive_text()) + try: + ws.receive_text() + except WebSocketDisconnect as e: + print("closed", e.code) + +with client.websocket_connect("/both") as ws: + print(ws.receive_json()) + +try: + with client.websocket_connect("/boom") as ws: + ws.receive_text() +except ValueError as e: + print("raised", e) +print(sorted(seen)) + +# Trusted hosts apply to websockets too +app2 = FastAPI() +app2.add_middleware(TrustedHostMiddleware, allowed_hosts=["example.com"]) + + +@app2.websocket("/ws") +async def ws2(websocket: WebSocket): + await websocket.accept() + await websocket.send_text("host " + websocket.headers["host"]) + await websocket.close() + + +c2 = TestClient(app2) +with c2.websocket_connect("/ws", headers={"host": "example.com"}) as ws: + print(ws.receive_text()) +try: + with c2.websocket_connect("/ws", headers={"host": "evil.com"}) as ws: + print("connected?") +except Exception as e: + print("bad host refused", type(e).__name__, e.status_code, e.text) From 20a146ea3e1d9c1c4c3dad45495095d70721f830 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 10:56:02 +0200 Subject: [PATCH 054/112] Python: SQLAlchemy relationships, eager loading, inheritance and subqueries Many-to-many through secondary=, cascades (save-update, delete, delete-orphan, FK nulling on parent delete, passive_deletes), selectinload/joinedload and lazy=selectin/joined, composite primary keys in Session.get, single-table and joined-table inheritance, aliased(), any()/has(); Core subqueries, correlated scalar/EXISTS subqueries, IN (SELECT), CTEs, unions, case(), cast(); echo=True logging through sqlalchemy.engine.Engine; results streamed from the cursor with yield_per/partitions. Conformance outputs from CPython with SQLAlchemy 2.1.6. Co-Authored-By: Claude Opus 5.5 --- .../python/site/sqlalchemy/__init__.py | 528 +++++++++++++-- .../python/site/sqlalchemy/orm/__init__.py | 635 +++++++++++++++--- .../conformance/sqlalchemy_query.out | 26 + .../resources/conformance/sqlalchemy_query.py | 114 ++++ .../conformance/sqlalchemy_relations.out | 22 + .../conformance/sqlalchemy_relations.py | 204 ++++++ 6 files changed, 1380 insertions(+), 149 deletions(-) create mode 100644 maven/python-transpiler/src/test/resources/conformance/sqlalchemy_query.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/sqlalchemy_query.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/sqlalchemy_relations.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/sqlalchemy_relations.py diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/__init__.py index 890b8dad28a..87c928a1247 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/__init__.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/__init__.py @@ -1,15 +1,21 @@ """A SQLAlchemy 2.x subset for Codename One: Core over the DB-API ``sqlite3`` module. Only the SQLite dialect exists. Statements compile to SQL with ``?`` placeholders and run -through ``sqlite3``; every result is fetched eagerly. The ORM lives in ``sqlalchemy.orm`` +through ``sqlite3``; results are read from the cursor as they are iterated. The ORM lives in ``sqlalchemy.orm`` and plugs in through two hooks: a mapped class carries ``__sa_mapper__`` (selecting it yields objects) and a relationship attribute answers ``_join_info()`` (joining on it). -Unsupported: other dialects, connection pooling options, reflection, events, ``echo`` -output, subqueries/CTEs/unions, ``Enum``/``JSON``/``Numeric`` decimal handling. +Subqueries (FROM, scalar, ``EXISTS``, ``IN``), CTEs, unions, aliases, ``case`` and ``cast`` +are supported; a scalar or ``EXISTS`` subquery correlates to the enclosing query's tables +as SQLAlchemy does. ``echo=True`` logs each statement through the ``sqlalchemy.engine.Engine`` +logger. + +Unsupported: other dialects, connection pooling options, reflection, events, recursive +CTEs, ``Enum``/``JSON``/``Numeric`` decimal handling. """ import datetime as _dt +import logging as _logging import sqlite3 as _sqlite3 from . import exc @@ -164,6 +170,10 @@ def __init__(self, qualify=True): self.params = [] self.qualify = qualify self.columns_clause = False + # Tables of the enclosing queries, which a correlated subquery leaves out of its FROM. + self.outer = [] + self.correlate = False + self.ctes = [] class ClauseElement: @@ -218,6 +228,8 @@ def __ge__(self, other): return self._op(">=", other) def __add__(self, other): + if isinstance(self.type, String): + return Grouping(self._op("||", other), self.type) return Grouping(self._op("+", other), self.type) def __sub__(self, other): @@ -239,9 +251,13 @@ def __invert__(self): return not_(self) def in_(self, values): + if isinstance(values, (Select, CompoundSelect, ScalarSelect)): + return InExpression(self, [_SubqueryList(values)], False) return InExpression(self, [_lit(v, self.type) for v in values], False) def not_in(self, values): + if isinstance(values, (Select, CompoundSelect, ScalarSelect)): + return InExpression(self, [_SubqueryList(values)], True) return InExpression(self, [_lit(v, self.type) for v in values], True) notin_ = not_in @@ -444,7 +460,9 @@ def __init__(self, name, args): self.args = [_lit(a) for a in args] if name == "count": self.type = Integer() - elif self.args and name in ("sum", "max", "min", "abs", "coalesce", "lower", "upper"): + elif name in ("length", "char_length"): + self.type = Integer() + elif self.args and name in ("sum", "max", "min", "abs", "coalesce", "lower", "upper", "trim"): self.type = self.args[0].type elif name == "avg": self.type = Float() @@ -728,9 +746,12 @@ def _join_condition(left_tables, right): def _from_obj(x): + al = getattr(x, "_sa_aliased", None) + if al is not None: + return al.alias m = getattr(x, "__sa_mapper__", None) if m is not None: - return m.table + return m.local_from return x @@ -778,18 +799,23 @@ def drop_all(self, bind, tables=None, checkfirst=True): class _Slot: """One entry of a select's columns clause: a column expression, or a mapped entity.""" - def __init__(self, elem=None, mapper=None): + def __init__(self, elem=None, mapper=None, cols=None, from_=None, key=None): self.elem = elem self.mapper = mapper if mapper is not None: - self.key = mapper.class_.__name__ - self.width = len(mapper.columns) + self.key = key if key is not None else mapper.class_.__name__ + self.cols = cols if cols is not None else mapper.columns + self.from_ = from_ if from_ is not None else mapper.local_from + self.width = len(self.cols) else: self.key = elem.key if elem.key is not None else elem.name self.width = 1 def _slots(ent): + al = getattr(ent, "_sa_aliased", None) + if al is not None: + return [al._slot()] m = getattr(ent, "__sa_mapper__", None) if m is not None: return [_Slot(mapper=m)] @@ -814,9 +840,13 @@ def __init__(self, *entities): self._limit = None self._offset = None self._distinct = False + self._options = [] + self._exec_opts = {} def _clone(self): s = Select() + s._options = list(self._options) + s._exec_opts = dict(self._exec_opts) s._slots = list(self._slots) s._where = list(self._where) s._order = list(self._order) @@ -904,21 +934,48 @@ def _first_table(self): self._column_tables(acc) return acc[0] + def options(self, *opts): + s = self._clone() + s._options.extend(opts) + return s + + def execution_options(self, **kw): + s = self._clone() + s._exec_opts.update(kw) + return s + def _column_tables(self, acc): for sl in self._slots: if sl.mapper is not None: - _add_unique(acc, sl.mapper.table) + _add_unique(acc, sl.from_) else: sl.elem._tables(acc) - def _froms(self): + def _all_where(self): + out = list(self._where) + for sl in self._slots: + if sl.mapper is not None and sl.mapper.poly_filter is not None: + f = sl.mapper.poly_filter(sl.cols) + if f is not None: + out.append(f) + return out + + def _froms(self, ctx=None): + out = self._froms_all() + if ctx is not None and ctx.correlate and ctx.outer: + kept = [b for b in out if not any(b is o for o in ctx.outer)] + if kept: + return kept + return out + + def _froms_all(self): if self._from: base = list(self._from) else: base = [] self._column_tables(base) if not base: - for w in self._where: + for w in self._all_where(): w._tables(base) joined = [] for target, _, _ in self._joins: @@ -938,17 +995,32 @@ def _froms(self): return out def _compile(self, ctx): + froms = self._froms(ctx) + saved_outer = ctx.outer + saved_corr = ctx.correlate + mine = list(saved_outer) + for f in froms: + mine.extend(f._all_tables()) + for target, _, _ in self._joins: + mine.extend(target._all_tables()) + ctx.outer = mine + ctx.correlate = False ctx.columns_clause = True cols = [] for sl in self._slots: if sl.mapper is not None: - for c in sl.mapper.columns: + for c in sl.cols: cols.append(c._compile(ctx)) else: cols.append(sl.elem._compile(ctx)) ctx.columns_clause = False sql = "SELECT " + ("DISTINCT " if self._distinct else "") + ", ".join(cols) - froms = self._froms() + sql = self._compile_rest(ctx, sql, froms) + ctx.outer = saved_outer + ctx.correlate = saved_corr + return sql + + def _compile_rest(self, ctx, sql, froms): if froms or self._joins: if not froms: froms = [self._joins[0][0]] @@ -965,8 +1037,9 @@ def _compile(self, ctx): seen.append(target) parts[0] = parts[0] + jsql sql += " \nFROM " + ", ".join(parts) - if self._where: - sql += " \nWHERE " + " AND ".join(w._compile(ctx) for w in self._where) + where = self._all_where() + if where: + sql += " \nWHERE " + " AND ".join(w._compile(ctx) for w in where) if self._group: sql += " GROUP BY " + ", ".join(g._compile(ctx) for g in self._group) if self._having: @@ -983,15 +1056,310 @@ def _compile(self, ctx): def _execute_on(self, conn, params, session=None): ctx = _Ctx() - sql = self._compile(ctx) + sql = _compile_top(self, ctx) cur = conn._run(sql, ctx.params) - return CursorResult._from_cursor(cur, self._slots, session) + res = CursorResult._from_cursor(cur, self._slots, session) + res._yield_per = self._exec_opts.get("yield_per") + return res + + def __str__(self): + return _compile_top(self, _Ctx()) def subquery(self, name=None): - raise NotImplementedError("subqueries are not supported") + return Alias(self, name) + + alias = subquery + + def cte(self, name=None, recursive=False): + if recursive: + raise NotImplementedError("recursive CTEs are not supported") + return CTE(self, name) def scalar_subquery(self): - raise NotImplementedError("subqueries are not supported") + return ScalarSelect(self) + + as_scalar = scalar_subquery + + def label(self, name): + return ScalarSelect(self).label(name) + + def exists(self): + return Exists(self) + + def union(self, *others): + return CompoundSelect("UNION", (self,) + others) + + def union_all(self, *others): + return CompoundSelect("UNION ALL", (self,) + others) + + def _first(self): + return self + + def _all_tables(self): + return [] + + +_anon = [0] + + +def _anon_name(prefix): + _anon[0] += 1 + return "%s_%d" % (prefix, _anon[0]) + + +def _compile_top(stmt, ctx): + """Compiles a statement, putting the WITH clause of every CTE it reached ahead of it.""" + body_ctx = _Ctx() + body = stmt._compile(body_ctx) + if not body_ctx.ctes: + ctx.params.extend(body_ctx.params) + return body + parts = [] + for c in body_ctx.ctes: + inner = _Ctx() + parts.append("%s AS \n(%s)" % (c.name, c.element._compile(inner))) + ctx.params.extend(inner.params) + ctx.params.extend(body_ctx.params) + return "WITH " + ", \n".join(parts) + " \n" + body + + +class Alias(FromClause): + """A named FROM element over a table (an alias) or a select (a subquery).""" + + def __init__(self, element, name=None): + self.element = element + self.name = name if name is not None else _anon_name( + element.name if isinstance(element, Table) else "anon") + self.c = self.columns = ColumnCollection() + if isinstance(element, Table): + src = list(element.c) + else: + src = [] + for sl in element._first()._slots: + if sl.mapper is not None: + src.extend(sl.cols) + else: + src.append(sl.elem) + for col in src: + nm = col.key if col.key is not None else col.name + c = Column(nm, col.type, primary_key=getattr(col, "primary_key", False)) + c.table = self + c._proxied = col + self.c._add(c) + + def corresponding_column(self, col): + for c in self.c: + if getattr(c, "_proxied", None) is col: + return c + return None + + def _compile(self, ctx): + if isinstance(self.element, Table): + return "%s AS %s" % (self.element.name, self.name) + inner = _Ctx() + inner.params = ctx.params + inner.ctes = ctx.ctes + return "(%s) AS %s" % (self.element._compile(inner), self.name) + + def _tables(self, acc): + _add_unique(acc, self) + + def _all_tables(self): + return [self] + + @property + def foreign_keys(self): + return [] + + def select(self): + return select(self) + + +class CTE(Alias): + def _compile(self, ctx): + if not any(c is self for c in ctx.ctes): + ctx.ctes.append(self) + return self.name + + +class ScalarSelect(ColumnElement): + def __init__(self, sel): + self.sel = sel + sl = sel._first()._slots[0] + self.type = sl.elem.type if sl.elem is not None else NullType() + + def _compile(self, ctx): + cc = ctx.columns_clause + corr = ctx.correlate + ctx.columns_clause = False + ctx.correlate = True + s = "(%s)" % self.sel._compile(ctx) + ctx.columns_clause = cc + ctx.correlate = corr + return s + + def _first(self): + return self.sel._first() + + +class _SubqueryList(ScalarSelect): + def _compile(self, ctx): + s = ScalarSelect._compile(self, ctx) + return s[1:-1] + + +class Exists(ColumnElement): + def __init__(self, sel): + self.sel = sel + self.type = Boolean() + + def where(self, *clauses): + return Exists(self.sel.where(*clauses)) + + def select_from(self, *froms): + return Exists(self.sel.select_from(*froms)) + + def _compile(self, ctx): + cc = ctx.columns_clause + corr = ctx.correlate + ctx.columns_clause = False + ctx.correlate = True + s = "EXISTS (%s)" % self.sel._compile(ctx) + ctx.columns_clause = cc + ctx.correlate = corr + return s + + def select(self): + return select(self) + + +def exists(*args): + if args: + a = args[0] + if isinstance(a, (Select, CompoundSelect)): + return Exists(a) + return Exists(Select(*args)) + return Exists(Select(TextClause("*"))) + + +class CompoundSelect(ClauseElement): + def __init__(self, keyword, selects): + self.keyword = keyword + self.selects = list(selects) + self._order = [] + self._limit = None + self._exec_opts = {} + + def _first(self): + return self.selects[0]._first() + + @property + def _slots(self): + return self._first()._slots + + def order_by(self, *clauses): + c = CompoundSelect(self.keyword, self.selects) + c._order = self._order + [x for x in clauses if x is not None] + c._limit = self._limit + return c + + def limit(self, n): + c = self.order_by() + c._limit = n + return c + + def _compile(self, ctx): + sql = (" %s " % self.keyword).join(x._compile(ctx) for x in self.selects) + if self._order: + q = ctx.qualify + ctx.qualify = False + sql += " ORDER BY " + ", ".join(o._compile(ctx) for o in self._order) + ctx.qualify = q + if self._limit is not None: + sql += " LIMIT ?" + ctx.params.append(self._limit) + return sql + + def __str__(self): + return _compile_top(self, _Ctx()) + + def _execute_on(self, conn, params, session=None): + ctx = _Ctx() + sql = _compile_top(self, ctx) + return CursorResult._from_cursor(conn._run(sql, ctx.params), self._slots, session) + + def subquery(self, name=None): + return Alias(self, name) + + def cte(self, name=None, recursive=False): + return CTE(self, name) + + def scalar_subquery(self): + return ScalarSelect(self) + + def _all_tables(self): + return [] + + +def union(*selects): + return CompoundSelect("UNION", selects) + + +def union_all(*selects): + return CompoundSelect("UNION ALL", selects) + + +def alias(element, name=None): + return Alias(element, name) + + +class Case(ColumnElement): + def __init__(self, whens, else_): + self.whens = [(w[0], _lit(w[1])) for w in whens] + self.else_ = None if else_ is None else _lit(else_) + self.type = self.whens[0][1].type if self.whens else NullType() + + def _compile(self, ctx): + cc = ctx.columns_clause + ctx.columns_clause = False + out = "CASE" + for cond, val in self.whens: + out += " WHEN %s THEN %s" % (cond._compile(ctx), val._compile(ctx)) + if self.else_ is not None: + out += " ELSE " + self.else_._compile(ctx) + ctx.columns_clause = cc + return out + " END" + + def _tables(self, acc): + for cond, val in self.whens: + cond._tables(acc) + val._tables(acc) + + +def case(*whens, **kw): + if len(whens) == 1 and isinstance(whens[0], dict): + raise NotImplementedError("case() with a value= mapping is not supported") + return Case(whens, kw.get("else_")) + + +class Cast(ColumnElement): + def __init__(self, expr, typ): + self.expr = _lit(expr) + self.type = _to_type(typ) + + def _compile(self, ctx): + cc = ctx.columns_clause + ctx.columns_clause = False + out = "CAST(%s AS %s)" % (self.expr._compile(ctx), self.type.ddl()) + ctx.columns_clause = cc + return out + + def _tables(self, acc): + self.expr._tables(acc) + + +def cast(expr, type_): + return Cast(expr, type_) def select(*entities): @@ -1226,18 +1594,29 @@ def _only(rows, allow_none): return rows[0] +_END = object() + + class _Rows: + """Rows read on demand from an iterator: a cursor is only fetched as far as it is consumed.""" + + _yield_per = None + def __init__(self, rows): - self._rows = rows - self._pos = 0 + self._it = iter(rows) + + def _next(self): + return next(self._it, _END) def _take(self): - r = self._rows[self._pos:] - self._pos = len(self._rows) - return r + return list(self._it) def __iter__(self): - return iter(self._take()) + while True: + r = self._next() + if r is _END: + return + yield r def all(self): return self._take() @@ -1245,14 +1624,36 @@ def all(self): fetchall = all def fetchone(self): - if self._pos < len(self._rows): - self._pos += 1 - return self._rows[self._pos - 1] - return None + r = self._next() + return None if r is _END else r - def fetchmany(self, size=1): - r = self._rows[self._pos:self._pos + size] - self._pos += len(r) + def fetchmany(self, size=None): + if size is None: + size = self._yield_per or 1 + out = [] + while len(out) < size: + r = self._next() + if r is _END: + break + out.append(r) + return out + + def partitions(self, size=None): + if size is None: + size = self._yield_per or 100 + while True: + p = self.fetchmany(size) + if not p: + return + yield p + + def yield_per(self, num): + self._yield_per = num + return self + + def _derive(self, cls, gen): + r = cls(gen) + r._yield_per = self._yield_per return r def first(self): @@ -1268,12 +1669,11 @@ def one_or_none(self): def unique(self): seen = [] out = [] - for r in self._rows[self._pos:]: - if not any(r is s or r == s for s in seen): + for r in self._take(): + if not any(r is x or r == x for x in seen): seen.append(r) out.append(r) - self._rows = out - self._pos = 0 + self._it = iter(out) return self @@ -1298,37 +1698,25 @@ def _from_cursor(cur, slots, session=None): desc = cur.description if desc is None: return CursorResult([], [], cur.rowcount, cur.lastrowid) - raw = cur.fetchall() if slots is None: keys = [d[0] for d in desc] km = dict((k, i) for i, k in enumerate(keys)) - return CursorResult([Row(r, km) for r in raw], keys, cur.rowcount, cur.lastrowid) + return CursorResult((Row(r, km) for r in _cursor_rows(cur)), keys, cur.rowcount, cur.lastrowid) keys = [s.key for s in slots] km = {} for i, k in enumerate(keys): if k is not None and k not in km: km[k] = i - rows = [] - for r in raw: - vals = [] - pos = 0 - for s in slots: - if s.mapper is not None: - vals.append(s.mapper._load_row(session, r[pos:pos + s.width])) - else: - vals.append(s.elem.type.result(r[pos])) - pos += s.width - rows.append(Row(vals, km)) - return CursorResult(rows, keys, cur.rowcount, cur.lastrowid) + return CursorResult(_load_rows(cur, slots, km, session), keys, cur.rowcount, cur.lastrowid) def keys(self): return list(self._keys) def scalars(self, index=0): - return ScalarResult([r[index] for r in self._take()]) + return self._derive(ScalarResult, (r[index] for r in self)) def mappings(self): - return MappingResult([r._mapping for r in self._take()]) + return self._derive(MappingResult, (r._mapping for r in self)) def scalar(self): r = self._take() @@ -1351,6 +1739,28 @@ def close(self): Result = CursorResult +def _cursor_rows(cur): + while True: + batch = cur.fetchmany(64) + if not batch: + return + for r in batch: + yield r + + +def _load_rows(cur, slots, km, session): + for r in _cursor_rows(cur): + vals = [] + pos = 0 + for sl in slots: + if sl.mapper is not None: + vals.append(sl.mapper._load_row(session, r[pos:pos + sl.width], sl.cols)) + else: + vals.append(sl.elem.type.result(r[pos])) + pos += sl.width + yield Row(vals, km) + + # ----------------------------------------------------------------------------- engine class Connection: @@ -1360,6 +1770,10 @@ def __init__(self, engine): self.closed = False def _run(self, sql, params, many=False): + if self.engine.echo: + log = _logging.getLogger("sqlalchemy.engine.Engine") + log.info("%s", sql) + log.info("%r", tuple(params) if isinstance(params, list) and not many else params) cur = self._dbapi.cursor() try: if many: @@ -1490,6 +1904,16 @@ def dispose(self): def create_engine(url, echo=False, **kwargs): if not isinstance(url, URL): url = make_url(url) + if echo: + # As SQLAlchemy does: INFO on the engine logger, and a stdout handler when nothing + # handles that logger yet. + log = _logging.getLogger("sqlalchemy.engine.Engine") + log.setLevel(_logging.DEBUG if echo == "debug" else _logging.INFO) + if not log.handlers: + import sys as _sys + h = _logging.StreamHandler(_sys.stdout) + h.setFormatter(_logging.Formatter("%(asctime)s %(levelname)s %(name)s %(message)s")) + log.addHandler(h) return Engine(url, echo) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/orm/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/orm/__init__.py index 792550028d2..f670fb9dde9 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/orm/__init__.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/orm/__init__.py @@ -1,5 +1,5 @@ """SQLAlchemy 2.0 declarative ORM subset: DeclarativeBase, Mapped, mapped_column, -relationship (one-to-many / many-to-one, lazy "select" loading) and Session. +relationship (one-to-many, many-to-one, many-to-many through ``secondary``) and Session. Mapping is done by a metaclass reading the class body's ``__annotations__``; nothing is evaluated with ``exec``. Instance state lives in ``obj.__dict__``: a column attribute that @@ -7,9 +7,14 @@ ``expire_on_commit``, ``refresh`` and ``rollback`` work. Changes are found at flush time by comparing the attribute values with a snapshot taken at load, so no attribute events. -Unsupported: many-to-many (``secondary``), inheritance mapping, eager loading options, -cascade configuration (save-update and backref population only; deleting a parent does -not touch its children), composite primary keys in ``Session.get``, hybrid properties. +Also: cascades (save-update, delete, delete-orphan; deleting a parent nulls its children's +foreign keys unless ``passive_deletes``), ``selectinload``/``joinedload`` and ``lazy=`` eager +strategies (both run one extra ``IN`` query per relationship rather than a JOIN), composite +primary keys, single-table and joined-table inheritance with ``polymorphic_on``, +``aliased()`` and ``any()``/``has()`` on relationships. + +Unsupported: chained loader options, concrete-table inheritance, joins along a many-to-many +relationship, hybrid properties. """ import datetime as _dt @@ -17,9 +22,12 @@ from sqlalchemy import (Column, ForeignKey, Integer, String, Float, Boolean, DateTime, Date, LargeBinary, MetaData, Table, TypeEngine, select, exc, _is_type, _to_type) +from sqlalchemy import Select as _Select, Exists as _Exists, TextClause as _TextClause, Join as _Join, \ + CursorResult as _CursorResult, Alias as _Alias, _Slot __all__ = ["DeclarativeBase", "Mapped", "mapped_column", "relationship", "Session", "sessionmaker", - "declarative_base", "registry"] + "declarative_base", "registry", "selectinload", "joinedload", "subqueryload", "lazyload", + "aliased"] _PY_TYPES = {int: Integer, str: String, float: Float, bool: Boolean, bytes: LargeBinary, _dt.datetime: DateTime, _dt.date: Date} @@ -47,7 +55,15 @@ def _name_of(t): fa = getattr(t, "__forward_arg__", None) if fa is not None: return fa - return getattr(t, "__name__", None) + n = getattr(t, "__name__", None) + if n is None and t is not None: + # A forward reference whose type object carries no name (Optional["Author"]): its + # repr still quotes the class name. + r = repr(t) + for q in ("'", '"'): + if r.count(q) >= 2: + return r.split(q)[1] + return n def _strip(s, prefix): @@ -159,6 +175,7 @@ def __set__(self, obj, value): _ONETOMANY = "ONETOMANY" _MANYTOONE = "MANYTOONE" +_MANYTOMANY = "MANYTOMANY" class InstrumentedList: @@ -166,6 +183,8 @@ def __init__(self, owner, prop): self._owner = owner self._prop = prop self._items = [] + # A many-to-many collection's members as the association table holds them. + self._db = [] def _raw_add(self, x): for i in self._items: @@ -173,9 +192,16 @@ def _raw_add(self, x): return self._items.append(x) + def _raw_remove(self, x): + for i in range(len(self._items)): + if self._items[i] is x: + del self._items[i] + return True + return False + def append(self, x): self._raw_add(x) - self._prop._back_set(self._owner, x) + self._prop._back_set(self._owner, x, True) def extend(self, xs): for x in xs: @@ -183,19 +209,23 @@ def extend(self, xs): def insert(self, i, x): self._items.insert(i, x) - self._prop._back_set(self._owner, x) + self._prop._back_set(self._owner, x, True) def remove(self, x): - for i in range(len(self._items)): - if self._items[i] is x: - del self._items[i] - self._prop._back_set(None, x) - return - raise ValueError("list.remove(x): x not in list") + if not self._raw_remove(x): + raise ValueError("list.remove(x): x not in list") + self._removed(x) + + def _removed(self, x): + self._prop._back_set(self._owner, x, False) + if self._prop.delete_orphan: + sess = _state(self._owner).session + if sess is not None: + sess._orphans.append((x, self._prop)) def pop(self, i=-1): x = self._items.pop(i) - self._prop._back_set(None, x) + self._removed(x) return x def clear(self): @@ -228,17 +258,26 @@ def __repr__(self): class RelationshipProperty: def __init__(self, argument=None, back_populates=None, uselist=None, lazy="select", order_by=None, - foreign_keys=None, cascade=None, backref=None, viewonly=False, secondary=None): - if secondary is not None: - raise NotImplementedError("many-to-many relationships (secondary=) are not supported") + foreign_keys=None, cascade=None, backref=None, viewonly=False, secondary=None, + passive_deletes=False, **kw): self.argument = argument self.back_populates = back_populates self.uselist = uselist self.order_by = order_by + self.lazy = lazy + self.secondary = secondary + self.passive_deletes = passive_deletes + parts = [x.strip() for x in (cascade if cascade is not None else "save-update, merge").split(",")] + if "all" in parts: + parts.extend(["save-update", "merge", "refresh-expire", "expunge", "delete"]) + self.cascade = parts + self.cascade_delete = "delete" in parts + self.delete_orphan = "delete-orphan" in parts self.key = None self.parent = None self._target_name = None self._list_ann = None + self._ann = None self._ready = False def _setup(self): @@ -252,27 +291,54 @@ def _setup(self): if isinstance(arg, str): arg = self.parent._sa_registry[arg] self.target = arg + if arg is None: + raise exc.ArgumentError("Could not determine the target class of relationship %s.%s (%r)" + % (self.parent.__name__ if self.parent is not None else None, self.key, + self._ann)) pt = self.parent.__table__ tt = arg.__table__ self.direction = None - for c in pt.c: - for fk in c.foreign_keys: - if fk.references(tt): - self.direction = _MANYTOONE - self.local = c - self.remote = tt.c[fk._target()[1]] - if self.direction is None: - for c in tt.c: + sec = self.secondary + if isinstance(sec, str): + sec = self.parent.metadata.tables[sec] + elif sec is not None and not isinstance(sec, Table) and callable(sec): + sec = sec() + self.secondary = sec + if sec is not None: + self.sec_local = None + self.sec_remote = None + for c in sec.c: for fk in c.foreign_keys: - if fk.references(pt): - self.direction = _ONETOMANY + if self.sec_local is None and fk.references(pt): + self.sec_local = c self.local = pt.c[fk._target()[1]] - self.remote = c + elif fk.references(tt): + self.sec_remote = c + self.remote = tt.c[fk._target()[1]] + if self.sec_local is not None and self.sec_remote is not None: + self.direction = _MANYTOMANY + else: + for c in pt.c: + for fk in c.foreign_keys: + if fk.references(tt): + self.direction = _MANYTOONE + self.local = c + self.remote = tt.c[fk._target()[1]] + if self.direction is None: + for c in tt.c: + for fk in c.foreign_keys: + if fk.references(pt): + self.direction = _ONETOMANY + self.local = pt.c[fk._target()[1]] + self.remote = c if self.direction is None: raise exc.ArgumentError("Could not determine join condition for relationship %s.%s" % (self.parent.__name__, self.key)) + if isinstance(self.order_by, str): + cname, aname = self.order_by.split(".") + self.order_by = getattr(self.parent._sa_registry[cname], aname) if self.uselist is None: - self.uselist = self.direction == _ONETOMANY if self._list_ann is None else self._list_ann + self.uselist = self.direction != _MANYTOONE if self._list_ann is None else self._list_ann self._ready = True def _other(self): @@ -283,8 +349,34 @@ def _other(self): def _join_info(self): self._setup() + if self.direction == _MANYTOMANY: + raise NotImplementedError("joining along a many-to-many relationship is not supported") return self.target.__table__, self.local == self.remote + def _exists(self, criterion): + sel = _Select(_TextClause("1")) + tf = self.target.__sa_mapper__.local_from + if self.direction == _MANYTOMANY: + sel = sel.select_from(self.secondary, tf).where(self.local == self.sec_local, + self.sec_remote == self.remote) + else: + sel = sel.select_from(tf).where(self.local == self.remote) + if criterion is not None: + sel = sel.where(criterion) + return _Exists(sel) + + def any(self, criterion=None, **kw): + self._setup() + if self.direction == _MANYTOONE: + raise exc.InvalidRequestError("'any()' not implemented for scalar attributes. Use has().") + return self._exists(criterion) + + def has(self, criterion=None, **kw): + self._setup() + if self.direction != _MANYTOONE: + raise exc.InvalidRequestError("'has()' not implemented for collections. Use any().") + return self._exists(criterion) + def __get__(self, obj, owner=None): if obj is None: return self @@ -306,6 +398,9 @@ def __set__(self, obj, value): self._setup() if self.uselist: lst = InstrumentedList(obj, self) + if self.direction == _MANYTOMANY: + old = self.__get__(obj) + lst._db = list(old._db) obj.__dict__[self.key] = lst for x in value: lst.append(x) @@ -325,17 +420,75 @@ def __set__(self, obj, value): elif value is not None: value.__dict__[other.key] = obj - def _back_set(self, owner, child): - """``child`` was added to (owner) or removed from (None) this one-to-many collection.""" + def _back_set(self, owner, child, add): + """``child`` was added to or removed from ``owner``'s collection: update the other side.""" other = self._other() - if other is not None: - child.__dict__[other.key] = owner + if other is None: + return + if self.direction == _MANYTOMANY: + lst = child.__dict__.get(other.key) + if add: + if lst is None: + lst = getattr(child, other.key) + lst._raw_add(owner) + elif lst is not None: + lst._raw_remove(owner) + return + child.__dict__[other.key] = owner if add else None def relationship(argument=None, **kw): return RelationshipProperty(argument, **kw) +class _LoaderOption: + def __init__(self, rel, strategy): + self.rel = rel + self.strategy = strategy + + +def selectinload(attr, *args, **kw): + return _LoaderOption(attr, "selectin") + + +def joinedload(attr, *args, **kw): + return _LoaderOption(attr, "joined") + + +def subqueryload(attr, *args, **kw): + return _LoaderOption(attr, "selectin") + + +def lazyload(attr, *args, **kw): + return _LoaderOption(attr, "select") + + +class AliasedClass: + def __init__(self, cls, name=None): + m = cls.__sa_mapper__ + if len(m.tables) > 1: + raise NotImplementedError("aliased() of a joined-inheritance class is not supported") + self._m = m + self.alias = _Alias(m.table, name) + self._sa_aliased = self + + def _slot(self): + return _Slot(mapper=self._m, cols=[self.alias.corresponding_column(c) for c in self._m.columns], + from_=self.alias, key=self.alias.name) + + def __getattr__(self, name): + if name.startswith("_"): + raise AttributeError(name) + for k, c in self._m.attrs: + if k == name: + return self.alias.corresponding_column(c) + raise AttributeError(name) + + +def aliased(element, alias=None, name=None, **kw): + return AliasedClass(element, name) + + class _State: def __init__(self): self.session = None @@ -353,13 +506,47 @@ def _state(obj): class Mapper: - def __init__(self, cls, table, attrs, relationships): + def __init__(self, cls, table, attrs, relationships, inherits=None, args=None, inherit_cols=None): + args = args or {} self.class_ = cls self.table = table self.attrs = attrs self.columns = [c for k, c in attrs] self.relationships = relationships - self.pk = [(k, c) for k, c in attrs if c.primary_key] + self.inherits = inherits + self.base = inherits.base if inherits is not None else self + self.identity_class = self.base.class_ + # (subclass table column, parent column) pairs linking joined-inheritance tables. + self.inherit_cols = (inherits.inherit_cols if inherits is not None else []) + (inherit_cols or []) + if inherits is not None: + self.pk = inherits.pk + self.tables = list(inherits.tables) + else: + self.pk = [(k, c) for k, c in attrs if c.primary_key] + self.tables = [] + if not any(t is table for t in self.tables): + self.tables.append(table) + if inherits is not None and table is not inherits.table: + cond = None + for sub, par in (inherit_cols or []): + cond = sub == par + self.local_from = _Join(inherits.local_from, table, cond, False) + else: + self.local_from = inherits.local_from if inherits is not None else table + if inherits is None: + self.poly_map = {} + self.polymorphic_on = args.get("polymorphic_on") + else: + self.polymorphic_on = inherits.polymorphic_on + self.polymorphic_identity = args.get("polymorphic_identity") + if self.polymorphic_identity is not None: + self.base.poly_map[self.polymorphic_identity] = self + + def poly_filter(self, cols): + if self.inherits is None or self.polymorphic_on is None or self.table is not self.inherits.table: + return None + ids = [m.polymorphic_identity for m in self.base.poly_map.values() if issubclass(m.class_, self.class_)] + return self.polymorphic_on.in_(ids) def _identity(self, obj): return tuple(obj.__dict__.get(k) for k, c in self.pk) @@ -373,22 +560,38 @@ def _populate(self, obj, values): d[k] = v st.snapshot[k] = v - def _load_row(self, session, values): - key = [] - for i in range(len(self.attrs)): - if self.attrs[i][1].primary_key: - key.append(values[i]) - key = (self.class_, tuple(key)) - if session is not None: - obj = session._identity.get(key) - if obj is not None: - if any(k not in obj.__dict__ for k, c in self.attrs): - self._populate(obj, values) - return obj - obj = self.class_.__new__(self.class_) + def _load_row(self, session, values, cols=None): + if cols is None: + cols = self.columns + origs = [getattr(c, "_proxied", c) for c in cols] + ident = [] + for k, pc in self.pk: + for i in range(len(origs)): + if origs[i] is pc: + ident.append(values[i]) + break + m = self + if self.polymorphic_on is not None: + for i in range(len(origs)): + if origs[i] is self.polymorphic_on: + sub = self.base.poly_map.get(values[i]) + if sub is not None: + m = sub + break + key = (self.identity_class, tuple(ident)) + obj = session._identity.get(key) if session is not None else None + fresh = obj is None + if fresh: + obj = m.class_.__new__(m.class_) st = _state(obj) - self._populate(obj, values) - if session is not None: + d = obj.__dict__ + for i in range(len(origs)): + k = _attr_key(m, origs[i]) + if fresh or k not in d: + v = origs[i].type.result(values[i]) + d[k] = v + st.snapshot[k] = v + if fresh and session is not None: st.session = session st.persistent = True session._identity[key] = obj @@ -407,16 +610,18 @@ def __new__(mcs, name, bases, ns, **kw): cls._sa_registry = {} cls.registry = cls._sa_registry return cls - if "__tablename__" not in ns and "__table__" not in ns: + parent = getattr(cls, "__sa_mapper__", None) + if parent is None and "__tablename__" not in ns and "__table__" not in ns: return cls - _map_class(cls, ns) + _map_class(cls, ns, parent) return cls -def _map_class(cls, ns): +def _map_class(cls, ns, parent=None): ann = ns.get("__annotations__", {}) attrs = [] rels = {} + specs = [] for key in ann: a = ann[key] v = ns.get(key) @@ -424,6 +629,15 @@ def _map_class(cls, ns): pyt, optional, is_list = _analyze(a) v._target_name = pyt if isinstance(pyt, str) else _name_of(pyt) v._list_ann = is_list if pyt is not None else None + v._ann = a + if v._target_name is None and isinstance(a, _MappedAnnotation): + # Optional["Author"] whose typing arguments lost the forward reference: the + # repr still quotes the class name. + r = repr(a.arg) + for q in ("'", '"'): + if r.count(q) >= 2: + v._target_name = r.split(q)[1] + break v.key = key v.parent = cls rels[key] = v @@ -431,6 +645,7 @@ def _map_class(cls, ns): pyt, optional, is_list = _analyze(a) if isinstance(v, _MappedColumnSpec): col = v._make(key, pyt, optional) + specs.append((v, col)) elif isinstance(v, Column): col = v if col.name is None: @@ -447,7 +662,9 @@ def _map_class(cls, ns): v = ns[key] if key not in ann: if isinstance(v, _MappedColumnSpec): - attrs.append((key, v._make(key, None, False))) + col = v._make(key, None, False) + specs.append((v, col)) + attrs.append((key, col)) elif isinstance(v, Column): if v.name is None: v.name = v.key = key @@ -456,11 +673,45 @@ def _map_class(cls, ns): v.key = key v.parent = cls rels[key] = v - table = Table(cls.__tablename__, cls.metadata, *[c for k, c in attrs]) + args = dict(ns.get("__mapper_args__", {})) + inherit_cols = [] + own = [k for k, c in attrs] + if parent is None: + table = Table(cls.__tablename__, cls.metadata, *[c for k, c in attrs]) + all_attrs = attrs + all_rels = rels + else: + all_rels = dict(parent.relationships) + all_rels.update(rels) + if "__tablename__" not in ns and "__table__" not in ns: + # Single-table inheritance: the subclass's columns join the parent's table. + table = parent.table + for k, c in attrs: + c.table = table + table.c._add(c) + all_attrs = parent.attrs + attrs + else: + table = Table(ns["__tablename__"], cls.metadata, *[c for k, c in attrs]) + pkeys = [k for k, c in parent.attrs] + for k, c in attrs: + for pk_, pc in parent.attrs: + if k == pk_: + inherit_cols.append((c, pc)) + all_attrs = parent.attrs + [(k, c) for k, c in attrs if k not in pkeys] + on = args.get("polymorphic_on") + if on is not None: + if isinstance(on, str): + on = [c for k, c in all_attrs if k == on][0] + elif isinstance(on, _MappedColumnSpec): + on = [c for sp, c in specs if sp is on][0] + elif isinstance(on, _ColumnAttr): + on = on.column + args["polymorphic_on"] = on cls.__table__ = table - cls.__sa_mapper__ = Mapper(cls, table, attrs, rels) - for k, c in attrs: - setattr(cls, k, _ColumnAttr(k, c)) + cls.__sa_mapper__ = Mapper(cls, table, all_attrs, all_rels, parent, args, inherit_cols) + for k, c in all_attrs: + if k in own: + setattr(cls, k, _ColumnAttr(k, c)) cls._sa_registry[cls.__name__] = cls @@ -520,6 +771,7 @@ def __init__(self, bind=None, autoflush=True, expire_on_commit=True, **kw): self._deleted = [] self._tx_inserted = [] self._tx_deleted = [] + self._orphans = [] # -- plumbing def connection(self): @@ -596,8 +848,32 @@ def delete(self, obj): if not st.persistent: self._new = [o for o in self._new if o is not obj] return - if not any(o is obj for o in self._deleted): - self._deleted.append(obj) + if any(o is obj for o in self._deleted): + return + self._deleted.append(obj) + m = type(obj).__sa_mapper__ + for key in m.relationships: + rel = m.relationships[key] + rel._setup() + if rel.direction != _ONETOMANY: + continue + if key not in obj.__dict__ and rel.passive_deletes: + # The database's ON DELETE takes care of rows that were never loaded. + continue + v = getattr(obj, key) + if isinstance(v, InstrumentedList): + children = list(v._items) + else: + children = [] if v is None else [v] + other = rel._other() + rk = _attr_key(rel.target.__sa_mapper__, rel.remote) + for c in children: + if rel.cascade_delete: + self.delete(c) + else: + c.__dict__[rk] = None + if other is not None: + c.__dict__[other.key] = None def __contains__(self, obj): st = _state(obj) @@ -619,9 +895,20 @@ def _sync(self, obj): for child in v._items: child.__dict__[_attr_key(tm, rel.remote)] = pk + def _is_deleted(self, obj): + return any(o is obj for o in self._deleted) + def flush(self, objects=None): + orphans = self._orphans + self._orphans = [] + for x, prop in orphans: + other = prop._other() + if other is not None and x.__dict__.get(other.key) is not None: + continue + self.delete(x) for obj in list(self._identity.values()): - self._cascade(obj) + if not self._is_deleted(obj): + self._cascade(obj) for obj in list(self._new): self._cascade(obj) conn = self.connection() @@ -631,72 +918,142 @@ def flush(self, objects=None): tables = type(self._new[0]).metadata.sorted_tables for i in range(len(tables)): order[tables[i].name] = i - new = sorted(self._new, key=lambda o: order.get(type(o).__table__.name, 0)) + new = sorted(self._new, key=lambda o: order.get(type(o).__sa_mapper__.tables[0].name, 0)) self._new = [] for obj in new: self._sync(obj) self._insert(conn, obj) self._sync(obj) + live = [] for obj in list(self._identity.values()): - if any(o is obj for o in self._deleted): + if self._is_deleted(obj): continue + live.append(obj) self._sync(obj) self._update(conn, obj) + done = [] + for obj in live: + self._sync_m2m(conn, obj, done) dels = self._deleted self._deleted = [] if dels: tables = type(dels[0]).metadata.sorted_tables for i in range(len(tables)): order[tables[i].name] = i - dels = sorted(dels, key=lambda o: -order.get(type(o).__table__.name, 0)) + dels = sorted(dels, key=lambda o: -order.get(type(o).__sa_mapper__.tables[0].name, 0)) for obj in dels: m = type(obj).__sa_mapper__ - where = [c == obj.__dict__.get(k) for k, c in m.pk] - conn.execute(m.table.delete().where(*where)) - key = (type(obj), m._identity(obj)) + for key in m.relationships: + rel = m.relationships[key] + rel._setup() + if rel.direction == _MANYTOMANY: + conn.execute(rel.secondary.delete().where( + rel.sec_local == obj.__dict__.get(_attr_key(m, rel.local)))) + for t in reversed(m.tables): + conn.execute(t.delete().where(*self._pk_where(m, t, obj))) + key = (m.identity_class, m._identity(obj)) if key in self._identity: del self._identity[key] _state(obj).persistent = False self._tx_deleted.append(obj) + def _sync_m2m(self, conn, obj, done): + m = type(obj).__sa_mapper__ + for key in m.relationships: + rel = m.relationships[key] + rel._setup() + if rel.direction != _MANYTOMANY: + continue + lst = obj.__dict__.get(key) + if not isinstance(lst, InstrumentedList): + continue + mine = obj.__dict__.get(_attr_key(m, rel.local)) + for x in lst._items: + if not any(x is y for y in lst._db): + self._assoc(conn, rel, mine, x, True, done) + for y in lst._db: + if not any(x is y for x in lst._items): + self._assoc(conn, rel, mine, y, False, done) + lst._db = list(lst._items) + + def _assoc(self, conn, rel, mine, other, add, done): + theirs = other.__dict__.get(_attr_key(rel.target.__sa_mapper__, rel.remote)) + pair = sorted([(rel.sec_local.name, mine), (rel.sec_remote.name, theirs)]) + sig = (rel.secondary.name, add, pair[0], pair[1]) + if sig in done: + return + done.append(sig) + if add: + conn.execute(rel.secondary.insert().values({rel.sec_local.key: mine, rel.sec_remote.key: theirs})) + else: + conn.execute(rel.secondary.delete().where(rel.sec_local == mine, rel.sec_remote == theirs)) + + def _pk_where(self, m, t, obj): + st = _state(obj) + d = obj.__dict__ + if t is m.tables[0]: + return [c == st.snapshot.get(k, d.get(k)) for k, c in m.pk] + out = [] + for sub, par in m.inherit_cols: + if sub.table is t: + k = _attr_key(m, par) + out.append(sub == st.snapshot.get(k, d.get(k))) + return out + def _insert(self, conn, obj): m = type(obj).__sa_mapper__ st = _state(obj) d = obj.__dict__ - values = {} - for k, c in m.attrs: - v = d.get(k) - if v is None and c.default is not None: - v = c._default_value() - d[k] = v - if v is not None or k in d: - values[c.key] = v - res = conn.execute(m.table.insert().values(**values)) - for k, c in m.pk: - if d.get(k) is None: - d[k] = res.lastrowid + if m.polymorphic_on is not None and m.polymorphic_identity is not None: + dk = _attr_key(m, m.polymorphic_on) + if d.get(dk) is None: + d[dk] = m.polymorphic_identity + for t in m.tables: + values = {} + for k, c in m.attrs: + if c.table is not t: + continue + v = d.get(k) + if v is None and c.default is not None: + v = c._default_value() + d[k] = v + if v is not None or k in d: + values[c.key] = v + for sub, par in m.inherit_cols: + if sub.table is t: + values[sub.key] = d.get(_attr_key(m, par)) + res = conn.execute(t.insert().values(values)) + if t is m.tables[0]: + for k, c in m.pk: + if d.get(k) is None: + d[k] = res.lastrowid for k, c in m.attrs: if k not in d: d[k] = None st.snapshot[k] = d[k] st.persistent = True st.session = self - self._identity[(type(obj), m._identity(obj))] = obj + self._identity[(m.identity_class, m._identity(obj))] = obj self._tx_inserted.append(obj) def _update(self, conn, obj): m = type(obj).__sa_mapper__ st = _state(obj) d = obj.__dict__ - changes = {} + changed = [] for k, c in m.attrs: if k in d and (k not in st.snapshot or st.snapshot[k] != d[k]): - changes[c.key] = d[k] - if not changes: + changed.append((k, c)) + if not changed: return - where = [c == st.snapshot.get(k, d.get(k)) for k, c in m.pk] - conn.execute(m.table.update().where(*where).values(**changes)) - for k in changes: + for t in m.tables: + ch = {} + for k, c in changed: + if c.table is t: + ch[c.key] = d[k] + if ch: + conn.execute(t.update().where(*self._pk_where(m, t, obj)).values(ch)) + for k, c in changed: st.snapshot[k] = d[k] def commit(self): @@ -713,7 +1070,7 @@ def rollback(self): self._conn.rollback() for obj in self._tx_inserted: m = type(obj).__sa_mapper__ - key = (type(obj), m._identity(obj)) + key = (m.identity_class, m._identity(obj)) if self._identity.get(key) is obj: del self._identity[key] st = _state(obj) @@ -725,11 +1082,13 @@ def rollback(self): for obj in self._tx_deleted: st = _state(obj) st.persistent = True - self._identity[(type(obj), type(obj).__sa_mapper__._identity(obj))] = obj + m = type(obj).__sa_mapper__ + self._identity[(m.identity_class, m._identity(obj))] = obj for obj in self._new: _state(obj).session = None self._new = [] self._deleted = [] + self._orphans = [] self._tx_inserted = [] self._tx_deleted = [] self.expire_all() @@ -759,7 +1118,7 @@ def _load_expired(self, obj): m = type(obj).__sa_mapper__ st = _state(obj) where = [c == st.snapshot.get(k, obj.__dict__.get(k)) for k, c in m.pk] - stmt = select(*m.columns).where(*where) + stmt = select(*m.columns).select_from(m.local_from).where(*where) row = stmt._execute_on(self.connection(), None).first() if row is None: raise exc.InvalidRequestError("Instance has been deleted, or its row is otherwise not present.") @@ -772,6 +1131,17 @@ def _load_expired(self, obj): def _lazy_load(self, obj, rel): m = type(obj).__sa_mapper__ tm = rel.target.__sa_mapper__ + if rel.direction == _MANYTOMANY: + lst = InstrumentedList(obj, rel) + pk = getattr(obj, _attr_key(m, rel.local)) + stmt = select(rel.target).select_from(tm.local_from, rel.secondary).where( + rel.sec_local == pk, rel.remote == rel.sec_remote) + if rel.order_by is not None: + stmt = stmt.order_by(rel.order_by) + for x in stmt._execute_on(self.connection(), None, self).scalars(): + lst._items.append(x) + lst._db = list(lst._items) + return lst if rel.direction == _ONETOMANY: lst = InstrumentedList(obj, rel) pk = getattr(obj, _attr_key(m, rel.local)) @@ -786,12 +1156,83 @@ def _lazy_load(self, obj, rel): fk = getattr(obj, _attr_key(m, rel.local)) if fk is None: return None - key = (rel.target, (fk,)) + key = (tm.identity_class, (fk,)) if key in self._identity: return self._identity[key] stmt = select(rel.target).where(rel.remote == fk) return stmt._execute_on(self.connection(), None, self).scalars().first() + # -- eager loading + def _eager_rels(self, stmt): + out = [] + if not isinstance(stmt, _Select): + return out + for i in range(len(stmt._slots)): + sl = stmt._slots[i] + if sl.mapper is None: + continue + for key in sl.mapper.relationships: + rel = sl.mapper.relationships[key] + strategy = rel.lazy + for o in stmt._options: + if o.rel is rel: + strategy = o.strategy + if strategy in ("selectin", "joined", "subquery", "immediate"): + out.append((i, rel)) + return out + + def _run(self, statement, params): + res = statement._execute_on(self.connection(), params, self) + eager = self._eager_rels(statement) + if not eager: + return res + rows = res.all() + for i, rel in eager: + self._eager_load([r[i] for r in rows], rel) + out = _CursorResult(rows, res._keys, res.rowcount, res.lastrowid) + out._yield_per = res._yield_per + return out + + def _eager_load(self, objs, rel): + """Loads ``rel`` for every object at once, with one ``IN`` query.""" + rel._setup() + todo = [] + for o in objs: + if o is not None and rel.key not in o.__dict__ and not any(o is x for x in todo): + todo.append(o) + if not todo: + return + tm = rel.target.__sa_mapper__ + if rel.direction == _MANYTOMANY: + for o in todo: + o.__dict__[rel.key] = self._lazy_load(o, rel) + return + rk = _attr_key(tm, rel.remote) + keys = [o.__dict__.get(_attr_key(type(o).__sa_mapper__, rel.local)) for o in todo] + wanted = [] + for k in keys: + if k is not None and k not in wanted: + wanted.append(k) + groups = {} + if wanted: + stmt = select(rel.target).where(rel.remote.in_(wanted)) + if rel.direction == _ONETOMANY and rel.order_by is not None: + stmt = stmt.order_by(rel.order_by) + for x in stmt._execute_on(self.connection(), None, self).scalars(): + k = x.__dict__.get(rk) + if k not in groups: + groups[k] = [] + groups[k].append(x) + for i in range(len(todo)): + o = todo[i] + found = groups.get(keys[i], []) + if rel.direction == _ONETOMANY: + lst = InstrumentedList(o, rel) + lst._items = list(found) + o.__dict__[rel.key] = lst if rel.uselist else (found[0] if found else None) + else: + o.__dict__[rel.key] = found[0] if found else None + # -- queries def _autoflush(self): if self.autoflush: @@ -799,7 +1240,7 @@ def _autoflush(self): def execute(self, statement, params=None, **kw): self._autoflush() - return statement._execute_on(self.connection(), params, self) + return self._run(statement, params) def scalars(self, statement, params=None, **kw): return self.execute(statement, params).scalars() @@ -808,16 +1249,16 @@ def scalar(self, statement, params=None, **kw): return self.execute(statement, params).scalar() def get(self, entity, ident, **kw): + m = entity.__sa_mapper__ if isinstance(ident, dict): - ident = tuple(ident.values()) + ident = tuple(ident[k] for k, c in m.pk) if not isinstance(ident, tuple): ident = (ident,) - obj = self._identity.get((entity, ident)) - if obj is not None: + obj = self._identity.get((m.identity_class, ident)) + if obj is not None and isinstance(obj, entity): return obj - m = entity.__sa_mapper__ where = [m.pk[i][1] == ident[i] for i in range(len(m.pk))] - return select(entity).where(*where)._execute_on(self.connection(), None, self).scalars().first() + return self._run(select(entity).where(*where), None).scalars().first() def get_one(self, entity, ident, **kw): obj = self.get(entity, ident) diff --git a/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_query.out b/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_query.out new file mode 100644 index 00000000000..8742e7c8e2e --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_query.out @@ -0,0 +1,26 @@ +True sqlalchemy.engine.Engine False +[('Ann', 2), ('bob', 1), ('Cid', 0), ('dee', 0)] +[('Ann', 2), ('bob', 1)] +['Ann', 'bob'] +['Cid', 'dee'] +['Ann', 'bob'] +['Ann', 'bob'] +['Cid'] +['Ann', 'Cid', 'bob'] +['Cid', 'dee'] +[19, 19, 25, 25] +['Ann', 'dee'] +[('Cid', 'Ann'), ('bob', 'Ann')] +Cid +[('Ann', 'adult'), ('bob', 'adult'), ('Cid', 'senior'), ('dee', 'teen')] +['30y', '25y', '41y', '19y'] +13 +[('ann', 'ANN', 3), ('bob', 'BOB', 3), ('cid', 'CID', 3), ('dee', 'DEE', 3)] +[-1, 1, 1, -1] +['Ann'] +['Ann', 'bob'] +['ann@a.com', 'ann@b.org'] +[['Ann', 'bob'], ['Cid', 'dee']] +[[30, 25, 41], [19]] +Ann ['bob', 'Cid', 'dee'] +True diff --git a/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_query.py b/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_query.py new file mode 100644 index 00000000000..c6e614151cb --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_query.py @@ -0,0 +1,114 @@ +import logging +from typing import List, Optional +from sqlalchemy import create_engine, ForeignKey, String, Integer, select, func, exists, union, \ + union_all, case, cast, and_, or_, literal +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship, Session, aliased + + +class Base(DeclarativeBase): + pass + + +class User(Base): + __tablename__ = "users" + id: Mapped[int] = mapped_column(primary_key=True) + name: Mapped[str] = mapped_column(String(30)) + age: Mapped[int] + boss_id: Mapped[Optional[int]] = mapped_column(ForeignKey("users.id")) + addresses: Mapped[List["Address"]] = relationship(back_populates="user") + + +class Address(Base): + __tablename__ = "addresses" + id: Mapped[int] = mapped_column(primary_key=True) + email: Mapped[str] + user_id: Mapped[int] = mapped_column(ForeignKey("users.id")) + user: Mapped["User"] = relationship(back_populates="addresses") + + +class Capture(logging.Handler): + def __init__(self): + logging.Handler.__init__(self) + self.records = [] + + def emit(self, record): + self.records.append(record) + + +cap = Capture() +log = logging.getLogger("sqlalchemy.engine.Engine") +log.addHandler(cap) +log.propagate = False + +engine = create_engine("sqlite://", echo=True) +Base.metadata.create_all(engine) +s = Session(engine) +ann = User(name="Ann", age=30, addresses=[Address(email="ann@a.com"), Address(email="ann@b.org")]) +s.add(ann) +s.flush() +s.add_all([User(name="bob", age=25, boss_id=ann.id, addresses=[Address(email="bob@a.com")]), + User(name="Cid", age=41, boss_id=ann.id), User(name="dee", age=19)]) +s.commit() +print(len(cap.records) > 0, cap.records[0].name, any("SELECT" in r.getMessage() for r in cap.records)) +log.removeHandler(cap) +engine.echo = False +n = len(cap.records) + +# scalar subquery (correlated) +cnt = select(func.count(Address.id)).where(Address.user_id == User.id).scalar_subquery() +print(s.execute(select(User.name, cnt.label("n")).order_by(User.id)).all()) + +# subquery in FROM +sub = select(Address.user_id, func.count(Address.id).label("c")).group_by(Address.user_id).subquery() +print(s.execute(select(User.name, sub.c.c).join(sub, User.id == sub.c.user_id).order_by(User.name)).all()) + +# exists +print(s.scalars(select(User.name).where(exists().where(Address.user_id == User.id)).order_by(User.id)).all()) +print(s.scalars(select(User.name).where(~select(Address.id).where(Address.user_id == User.id).exists()) + .order_by(User.id)).all()) + +# in_ with subquery, between, ilike +print(s.scalars(select(User.name).where(User.id.in_(select(Address.user_id).where(Address.email.like("%a.com")))) + .order_by(User.id)).all()) +print(s.scalars(select(User.name).where(User.age.between(20, 35)).order_by(User.id)).all()) +print(s.scalars(select(User.name).where(User.name.ilike("c%"))).all()) + +# cte +old = select(User.id, User.name).where(User.age > 24).cte("old") +print(s.execute(select(old.c.name).order_by(old.c.name)).scalars().all()) + +# union / union_all +u = union(select(User.name).where(User.age < 20), select(User.name).where(User.age > 40)) +print(sorted(s.execute(u).scalars().all())) +ua = union_all(select(User.age).where(User.age < 26), select(User.age).where(User.age < 26)) +print(sorted(s.execute(ua).scalars().all())) +print(sorted(s.execute(select(User.name).where(User.age < 20).union(select(User.name).where(User.age == 30))).scalars())) + +# aliased self join +boss = aliased(User) +print(s.execute(select(User.name, boss.name).join(boss, User.boss_id == boss.id).order_by(User.name)).all()) +print(s.scalars(select(boss).where(boss.age > 40)).one().name) + +# case, cast, functions +grp = case((User.age < 20, "teen"), (User.age < 35, "adult"), else_="senior") +print(s.execute(select(User.name, grp).order_by(User.id)).all()) +print(s.execute(select(cast(User.age, String) + "y").order_by(User.id)).scalars().all()) +print(s.execute(select(cast("12", Integer) + 1)).scalar()) +print(s.execute(select(func.lower(User.name), func.upper(User.name), func.length(User.name)) + .order_by(User.id)).all()) +print(s.execute(select(func.coalesce(User.boss_id, -1)).order_by(User.id)).scalars().all()) + +# relationship any / has +print(s.scalars(select(User.name).where(User.addresses.any(Address.email.like("%.org")))).all()) +print(s.scalars(select(User.name).where(User.addresses.any()).order_by(User.id)).all()) +print(s.scalars(select(Address.email).where(Address.user.has(User.age > 26)).order_by(Address.id)).all()) + +# streaming +res = s.execute(select(User.name).order_by(User.id).execution_options(yield_per=2)) +print([[r[0] for r in p] for p in res.partitions()]) +res = s.execute(select(User.age).order_by(User.id)) +print([list(p) for p in res.scalars().partitions(3)]) +res = s.scalars(select(User).order_by(User.id).execution_options(yield_per=1)) +print(next(iter(res)).name, [u.name for u in res]) +print(len(cap.records) == n) +s.close() diff --git a/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_relations.out b/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_relations.out new file mode 100644 index 00000000000..de45ce569db --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_relations.out @@ -0,0 +1,22 @@ +['db', 'py'] ['one', 'two'] +3 +[('one', ['py']), ('two', ['py', 'web'])] +['py', 'web'] 0 +1 +3 2 +['b2', 'b3'] +0 [('n1', None), ('n2', None)] +2 +[1, 1] +['x', 'y'] +['z'] +cy +70 80 None +[75, 80, 90] +[('Manager', 'm1', 'manager'), ('Engineer', 'e1', 'engineer'), ('Engineer', 'e2', 'engineer'), ('Employee', 'x', 'employee')] +['py', 'java'] +md ['id', 'name', 'type', 'manager_data', 'language'] +[('Truck', 6, 'truck'), ('Vehicle', 2, 'vehicle')] +10 6 1 +12 8 +1 0 diff --git a/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_relations.py b/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_relations.py new file mode 100644 index 00000000000..4b36fbe03b8 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_relations.py @@ -0,0 +1,204 @@ +from typing import List, Optional +from sqlalchemy import create_engine, ForeignKey, String, Table, Column, Integer, select, func +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship, Session, \ + selectinload, joinedload + + +class Base(DeclarativeBase): + pass + + +post_tags = Table("post_tags", Base.metadata, + Column("post_id", ForeignKey("post.id"), primary_key=True), + Column("tag_id", ForeignKey("tag.id"), primary_key=True)) + + +class Post(Base): + __tablename__ = "post" + id: Mapped[int] = mapped_column(primary_key=True) + title: Mapped[str] = mapped_column(String(50)) + tags: Mapped[List["Tag"]] = relationship(secondary=post_tags, back_populates="posts") + + +class Tag(Base): + __tablename__ = "tag" + id: Mapped[int] = mapped_column(primary_key=True) + name: Mapped[str] = mapped_column(String(20)) + posts: Mapped[List["Post"]] = relationship(secondary=post_tags, back_populates="tags") + + +class Author(Base): + __tablename__ = "author" + id: Mapped[int] = mapped_column(primary_key=True) + name: Mapped[str] + books: Mapped[List["Book"]] = relationship(back_populates="author", cascade="all, delete-orphan", + order_by="Book.id") + notes: Mapped[List["Note"]] = relationship(back_populates="author") + memos: Mapped[List["Memo"]] = relationship(passive_deletes=True, lazy="selectin") + + +class Book(Base): + __tablename__ = "book" + id: Mapped[int] = mapped_column(primary_key=True) + title: Mapped[str] + author_id: Mapped[int] = mapped_column(ForeignKey("author.id")) + author: Mapped["Author"] = relationship(back_populates="books") + + +class Note(Base): + __tablename__ = "note" + id: Mapped[int] = mapped_column(primary_key=True) + text: Mapped[str] + author_id: Mapped[Optional[int]] = mapped_column(ForeignKey("author.id")) + author: Mapped[Optional["Author"]] = relationship(back_populates="notes", lazy="joined") + + +class Memo(Base): + __tablename__ = "memo" + id: Mapped[int] = mapped_column(primary_key=True) + author_id: Mapped[Optional[int]] = mapped_column(ForeignKey("author.id", ondelete="CASCADE")) + + +class Enrollment(Base): + __tablename__ = "enrollment" + student: Mapped[str] = mapped_column(String(20), primary_key=True) + course: Mapped[str] = mapped_column(String(20), primary_key=True) + grade: Mapped[int] + + +class Employee(Base): + __tablename__ = "employee" + id: Mapped[int] = mapped_column(primary_key=True) + name: Mapped[str] + type: Mapped[str] + __mapper_args__ = {"polymorphic_on": "type", "polymorphic_identity": "employee"} + + +class Manager(Employee): + manager_data: Mapped[Optional[str]] + __mapper_args__ = {"polymorphic_identity": "manager"} + + +class Engineer(Employee): + language: Mapped[Optional[str]] + __mapper_args__ = {"polymorphic_identity": "engineer"} + + +class Vehicle(Base): + __tablename__ = "vehicle" + id: Mapped[int] = mapped_column(primary_key=True) + kind: Mapped[str] + wheels: Mapped[int] + __mapper_args__ = {"polymorphic_on": "kind", "polymorphic_identity": "vehicle"} + + +class Truck(Vehicle): + __tablename__ = "truck" + id: Mapped[int] = mapped_column(ForeignKey("vehicle.id"), primary_key=True) + payload: Mapped[int] + __mapper_args__ = {"polymorphic_identity": "truck"} + + +engine = create_engine("sqlite://") +Base.metadata.create_all(engine) + +# ---- many-to-many +with Session(engine) as s: + py, db, web = Tag(name="py"), Tag(name="db"), Tag(name="web") + p1 = Post(title="one", tags=[py, db]) + p2 = Post(title="two", tags=[py]) + s.add_all([p1, p2, web]) + s.commit() + print(sorted(t.name for t in p1.tags), [p.title for p in py.posts]) + p2.tags.append(web) + p1.tags.remove(db) + s.commit() + print(s.scalar(select(func.count()).select_from(post_tags))) + print(sorted((p.title, sorted(t.name for t in p.tags)) for p in s.scalars(select(Post)))) + print(sorted(t.name for t in web.posts[0].tags), len(db.posts)) + s.delete(p2) + s.commit() + print(s.scalar(select(func.count()).select_from(post_tags))) + +# ---- cascades +with Session(engine) as s: + a = Author(name="ann", books=[Book(title="b1"), Book(title="b2"), Book(title="b3")], + notes=[Note(text="n1"), Note(text="n2")]) + s.add(a) + s.commit() + print(s.scalar(select(func.count(Book.id))), s.scalar(select(func.count(Note.id)))) + a.books.remove(a.books[0]) + s.commit() + print([b.title for b in s.scalars(select(Book).order_by(Book.id))]) + s.delete(a) + s.commit() + print(s.scalar(select(func.count(Book.id))), [(n.text, n.author_id) for n in s.scalars(select(Note).order_by(Note.id))]) + +with Session(engine) as s: + a = Author(name="bob") + s.add(a) + s.flush() + s.add_all([Memo(author_id=a.id), Memo(author_id=a.id)]) + s.commit() + aid = a.id +with Session(engine) as s: + a = s.get(Author, aid) + print(len(a.memos)) + s.expire(a, ["memos"]) + s.delete(a) + s.commit() + print(s.scalars(select(Memo.author_id)).all()) + +# ---- eager loading +with Session(engine) as s: + a = Author(name="cy", books=[Book(title="x"), Book(title="y")], notes=[Note(text="z")]) + s.add(a) + s.commit() +with Session(engine) as s: + authors = s.scalars(select(Author).where(Author.name == "cy").options(selectinload(Author.books))).all() + s.expunge_all() + print([b.title for b in authors[0].books]) +with Session(engine) as s: + authors = s.scalars(select(Author).where(Author.name == "cy").options(joinedload(Author.notes))).unique().all() + s.expunge_all() + print([n.text for n in authors[0].notes]) +with Session(engine) as s: + notes = s.scalars(select(Note).where(Note.text == "z")).all() + s.expunge_all() + print(notes[0].author.name) + +# ---- composite primary keys +with Session(engine) as s: + s.add_all([Enrollment(student="al", course="math", grade=90), + Enrollment(student="al", course="art", grade=70), + Enrollment(student="bo", course="math", grade=80)]) + s.commit() + e = s.get(Enrollment, ("al", "art")) + print(e.grade, s.get(Enrollment, {"course": "math", "student": "bo"}).grade, s.get(Enrollment, ("zz", "art"))) + e.grade = 75 + s.commit() + print(s.scalars(select(Enrollment.grade).order_by(Enrollment.grade)).all()) + +# ---- single-table inheritance +with Session(engine) as s: + s.add_all([Manager(name="m1", manager_data="md"), Engineer(name="e1", language="py"), + Engineer(name="e2", language="java"), Employee(name="x")]) + s.commit() + print([(type(e).__name__, e.name, e.type) for e in s.scalars(select(Employee).order_by(Employee.id))]) + print([e.language for e in s.scalars(select(Engineer).order_by(Engineer.id))]) + print(s.scalars(select(Manager)).one().manager_data, [c.name for c in Employee.__table__.columns]) + +# ---- joined-table inheritance +with Session(engine) as s: + s.add_all([Truck(wheels=6, payload=10), Vehicle(wheels=2)]) + s.commit() + print([(type(v).__name__, v.wheels, v.kind) for v in s.scalars(select(Vehicle).order_by(Vehicle.id))]) + t = s.scalars(select(Truck)).one() + print(t.payload, t.wheels, t.id) + t.payload = 12 + t.wheels = 8 + s.commit() + print(s.get(Truck, t.id).payload, s.get(Vehicle, t.id).wheels) + s.delete(t) + s.commit() + print(s.scalar(select(func.count(Vehicle.id))), s.scalar(select(func.count(Truck.payload)))) From 88d45c926dd286efe221c98e54a51954c4631594 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 10:58:38 +0200 Subject: [PATCH 055/112] Backend: refuse a websocket upgrade with any status, add 101 headers; Python: pure ASGI middleware HttpServer.refuseUpgrade(status, body | Response) lets a WebSocketHandler refuse with its own status instead of the 404 a null answers, and WebSocket.getHandshakeHeaders() (a default method, forwarded by Backend's wrapper) adds headers to the 101. Existing handlers are unchanged. The generated BackendServer uses both: a missing route or a close before accept refuses with 403, a middleware response with its own status, headers and body, an endpoint failure before accept with 500, and accept(headers=...) reaches the handshake. User-written pure ASGI middleware classes are adapted to the request-based stack for http and websocket scopes (starlette.middleware._asgi). Co-Authored-By: Claude Opus 5.5 --- .../backend/WebSocketReviewFixesTest.java | 70 +++++++ .../codename1/python/runtime/PyServer.java | 31 +++- .../transpiler/api/PythonTranspiler.java | 55 +++++- .../python/site/fastapi/applications.py | 7 +- .../site/starlette/middleware/__init__.py | 4 +- .../python/site/starlette/middleware/_asgi.py | 175 ++++++++++++++++++ .../python/site/starlette/middleware/base.py | 3 + .../python/site/starlette/middleware/cors.py | 3 + .../python/site/starlette/middleware/gzip.py | 3 + .../site/starlette/middleware/trustedhost.py | 3 + .../python/site/starlette/websockets.py | 16 ++ .../codename1/python/site/uvicorn/__init__.py | 18 +- .../conformance/fastapi_asgi_middleware.out | 11 ++ .../conformance/fastapi_asgi_middleware.py | 142 ++++++++++++++ .../src/com/codename1/backend/Backend.java | 17 +- .../src/com/codename1/backend/HttpServer.java | 115 +++++++++++- .../src/com/codename1/backend/WebSocket.java | 12 ++ 17 files changed, 664 insertions(+), 21 deletions(-) create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/middleware/_asgi.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/fastapi_asgi_middleware.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/fastapi_asgi_middleware.py diff --git a/maven/backend/src/test/java/com/codename1/backend/WebSocketReviewFixesTest.java b/maven/backend/src/test/java/com/codename1/backend/WebSocketReviewFixesTest.java index 9f51a506944..4c4ae009e7b 100644 --- a/maven/backend/src/test/java/com/codename1/backend/WebSocketReviewFixesTest.java +++ b/maven/backend/src/test/java/com/codename1/backend/WebSocketReviewFixesTest.java @@ -103,6 +103,76 @@ public WebSocket open(HttpServer.Request request) { } } + @Test + @DisplayName("a router refuses with its own status, and an endpoint adds 101 headers") + void refusalStatusAndHandshakeHeaders() throws Exception { + Backend backend = Backend.builder().port(0).quiet() + .webSockets(new Backend.WebSocketEndpoints() { + public void register(HttpServer.WebSocketRegistry registry, + DataSource dataSource, + com.codename1.backend.orm.EntityManager entities) { + registry.fallback(new HttpServer.WebSocketHandler() { + public WebSocket open(HttpServer.Request request) { + if (request.getTarget().startsWith("/bad")) { + return HttpServer.refuseUpgrade(200, "not a refusal"); + } + if (!request.getTarget().startsWith("/ok")) { + return HttpServer.refuseUpgrade(403, "go away"); + } + return new WebSocket() { + public void onOpen(WebSocketSession s) { + } + + public void onText(WebSocketSession s, String m) { + } + + public void onBinary(WebSocketSession s, byte[] m, + int o, int l) { + } + + public java.util.Map getHandshakeHeaders() { + java.util.Map h = new java.util.LinkedHashMap(); + h.put("X-Room", "lobby"); + h.put("Sec-WebSocket-Accept", "forged"); + return h; + } + }; + } + }); + } + }) + .start(); + try { + int port = backend.getServer().getPort(); + RawWebSocketClient refused = new RawWebSocketClient(port, "/no", null); + try { + assertTrue(refused.getStatusLine().startsWith("HTTP/1.1 403"), + refused.getStatusLine()); + assertEquals("go away", refused.readRefusal()); + } finally { + refused.close(); + } + RawWebSocketClient bad = new RawWebSocketClient(port, "/bad", null); + try { + assertTrue(bad.getStatusLine().startsWith("HTTP/1.1 500"), + bad.getStatusLine()); + } finally { + bad.close(); + } + // The constructor checks Sec-WebSocket-Accept, so a forged one written + // by the endpoint would fail it. + RawWebSocketClient ok = new RawWebSocketClient(port, "/ok", null); + try { + assertTrue(ok.getStatusLine().startsWith("HTTP/1.1 101"), ok.getStatusLine()); + assertEquals("lobby", ok.getResponseHeader("x-room")); + } finally { + ok.close(); + } + } finally { + backend.stop(); + } + } + @Test @DisplayName("a fallback router authenticates a handshake by its session") void aFallbackRouterSeesTheSession() throws Exception { diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyServer.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyServer.java index 61f9541a37c..01f45322f66 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyServer.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyServer.java @@ -63,8 +63,9 @@ public void serve(Object dispatch, String host, int port, Object webSockets) { /// What a server does with the messages a Python websocket endpoint sends. public interface WebSocketSink { - /// The endpoint accepted; the handshake has already completed. - void accept(String subprotocol); + /// The endpoint accepted. `headers` are the extra handshake response + /// headers it asked for, name then value, never null. + void accept(String subprotocol, String[] headers); void text(String value); @@ -73,8 +74,10 @@ public interface WebSocketSink { void close(int code, String reason); /// The endpoint refused the handshake before accepting; `status` is - /// the HTTP status it answered with (403 when it simply closed). - void deny(int status); + /// the HTTP status it answered with (403 when it simply closed, 500 + /// when it failed), `headers` the response's headers, name then value, + /// and `body` its body. Neither is ever null. + void deny(int status, String[] headers, byte[] body); } /// Opens one websocket connection in Python; answers the connection @@ -88,13 +91,16 @@ public static Object openWebSocket(Object webSockets, String target, String[] he PyBuiltin send = PyBuiltin.positional("send", a -> { String kind = Py.toStr(a[0]); if ("accept".equals(kind)) { - sink.accept(a[1] == Py.NONE ? null : Py.toStr(a[1])); //NOPMD CompareObjectsWithEquals - identity is meant + sink.accept(a[1] == Py.NONE ? null : Py.toStr(a[1]), strings(a[2])); //NOPMD CompareObjectsWithEquals - identity is meant } else if ("text".equals(kind)) { sink.text(Py.toStr(a[1])); } else if ("bytes".equals(kind)) { sink.binary(PyBytes.like(a[1])); } else if ("deny".equals(kind)) { - sink.deny((int) Py.asLong(a[1])); + Object detail = a[2]; + Object[] parts = detail instanceof PyList ? ((PyList) detail).toArray() : null; + sink.deny((int) Py.asLong(a[1]), parts == null ? new String[0] : strings(parts[0]), + parts == null ? new byte[0] : PyBytes.like(parts[1])); } else { sink.close((int) Py.asLong(a[1]), a[2] == Py.NONE ? "" : Py.toStr(a[2])); //NOPMD CompareObjectsWithEquals - identity is meant } @@ -108,6 +114,19 @@ public static Object openWebSocket(Object webSockets, String target, String[] he } } + /// A Python list of strings as an array; anything else is empty. + private static String[] strings(Object list) { + if (!(list instanceof PyList)) { + return new String[0]; + } + Object[] items = ((PyList) list).toArray(); + String[] out = new String[items.length]; + for (int i = 0; i < items.length; i++) { + out[i] = Py.toStr(items[i]); + } + return out; + } + /// Delivers a text message to a connection [#openWebSocket] opened. public static void webSocketText(Object connection, String value) { event(connection, "text", value); diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java index 6dd3e2e98b7..e26a9fb9e42 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java @@ -455,6 +455,8 @@ private static String backendServer() { + " /** 0 while the endpoint decides, 1 accepted, 2 refused. */\n" + " private int state;\n" + " private String subprotocol;\n" + + " private String[] acceptHeaders;\n" + + " private WebSocket refusal;\n" + "\n" + " Socket(Object open, String target, String[] headers) {\n" + " this.open = open;\n" @@ -462,8 +464,8 @@ private static String backendServer() { + " this.headers = headers;\n" + " }\n" + "\n" - + " /** Runs the endpoint up to its accept; false when it refused or never answered. */\n" - + " boolean handshake() {\n" + + " /** Runs the endpoint up to its accept; answers this socket, or the refusal. */\n" + + " WebSocket handshake() {\n" + " // Opened outside the monitor: the loop thread holds the GIL while\n" + " // it calls the sink, and the sink takes the monitor.\n" + " Object c = PyServer.openWebSocket(open, target, headers, this);\n" @@ -486,14 +488,43 @@ private static String backendServer() { + " }\n" + " if (!accepted) {\n" + " PyServer.webSocketClosed(c, 1006, \"\");\n" + + " synchronized (this) {\n" + + " return refusal != null ? refusal\n" + + " : HttpServer.refuseUpgrade(500, \"Internal Server Error\");\n" + + " }\n" + " }\n" - + " return accepted;\n" + + " return this;\n" + " }\n" + "\n" + " public synchronized String[] getSubprotocols() {\n" + " return subprotocol == null ? null : new String[] {subprotocol};\n" + " }\n" + "\n" + + " public synchronized java.util.Map getHandshakeHeaders() {\n" + + " if (acceptHeaders == null || acceptHeaders.length == 0) {\n" + + " return null;\n" + + " }\n" + + " return pairs(acceptHeaders);\n" + + " }\n" + + "\n" + + " static java.util.Map pairs(String[] h) {\n" + + " Map out = new LinkedHashMap();\n" + + " for (int i = 0; i + 1 < h.length; i += 2) {\n" + + " Object had = out.get(h[i]);\n" + + " if (had == null) {\n" + + " out.put(h[i], h[i + 1]);\n" + + " } else {\n" + + " java.util.List all = had instanceof java.util.List ? (java.util.List) had : new java.util.ArrayList();\n" + + " if (all != had) {\n" + + " all.add(had);\n" + + " }\n" + + " all.add(h[i + 1]);\n" + + " out.put(h[i], all);\n" + + " }\n" + + " }\n" + + " return out;\n" + + " }\n" + + "\n" + " public void onOpen(WebSocketSession s) {\n" + " synchronized (this) {\n" + " session = s;\n" @@ -518,14 +549,26 @@ private static String backendServer() { + " PyServer.webSocketClosed(connection, code, reason);\n" + " }\n" + "\n" - + " public synchronized void accept(String subprotocol) {\n" + + " public synchronized void accept(String subprotocol, String[] headers) {\n" + " this.subprotocol = subprotocol;\n" + + " this.acceptHeaders = headers;\n" + " state = 1;\n" + " notifyAll();\n" + " }\n" + "\n" - + " public synchronized void deny(int status) {\n" + + " public synchronized void deny(int status, String[] headers, byte[] body) {\n" + " if (state == 0) {\n" + + " String type = null;\n" + + " for (int i = 0; i + 1 < headers.length; i += 2) {\n" + + " if (\"content-type\".equalsIgnoreCase(headers[i])) {\n" + + " type = headers[i + 1];\n" + + " }\n" + + " }\n" + + " Map extra = pairs(headers);\n" + + " extra.remove(\"content-type\");\n" + + " extra.remove(\"content-length\");\n" + + " refusal = HttpServer.refuseUpgrade(new HttpServer.Response(status,\n" + + " type == null ? \"text/plain; charset=utf-8\" : type, body, extra));\n" + " state = 2;\n" + " notifyAll();\n" + " }\n" @@ -581,7 +624,7 @@ private static String backendServer() { + " registry.fallback(new HttpServer.WebSocketHandler() {\n" + " public WebSocket open(HttpServer.Request r) {\n" + " Socket socket = new Socket(webSockets, r.getTarget(), headers(r));\n" - + " return socket.handshake() ? socket : null;\n" + + " return socket.handshake();\n" + " }\n" + " });\n" + " }\n" diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/applications.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/applications.py index d870c60e4b5..1fd7275901c 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/applications.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/applications.py @@ -88,7 +88,12 @@ def _build_stack(self): app = self._core for m in reversed(self.user_middleware): cls, args, kwargs = m - app = cls(app, *args, **kwargs) + if getattr(cls, "_cn1_request_based", False): + app = cls(app, *args, **kwargs) + else: + # A pure ASGI middleware: adapted to the request stack. + from starlette.middleware import _asgi + app = _asgi.wrap(cls, app, args, kwargs) self._stack = app return self._stack diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/middleware/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/middleware/__init__.py index 0a7c2d2efa8..df68d2221de 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/middleware/__init__.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/middleware/__init__.py @@ -7,7 +7,9 @@ taking a request and answering a response. Every middleware class takes that app first and implements ``async __call__(request)``; the application builds the stack with the last one added outermost, as Starlette does. -Websocket connections do not pass through it. +Websocket connections pass through it too, with the websocket as the +request. A middleware without ``_cn1_request_based`` is taken to be pure +ASGI (``__call__(scope, receive, send)``) and adapted by ``_asgi``. """ diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/middleware/_asgi.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/middleware/_asgi.py new file mode 100644 index 00000000000..f44a803a9da --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/middleware/_asgi.py @@ -0,0 +1,175 @@ +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License version 2 with the Classpath +# exception, as the rest of the Codename One Python runtime. +"""Pure ASGI middleware over the request-based stack. + +The stack here passes request objects, not ASGI messages. A user-written +middleware of the shape ``M(app)`` with ``async __call__(scope, receive, +send)`` still works: :func:`wrap` builds the ASGI scope from the request, +hands the middleware an ASGI ``app`` that runs the rest of the stack, and +turns what the middleware sends back into a response. A websocket keeps +its connection object; the middleware's ``receive`` and ``send`` are +installed on it for as long as the app inside runs. +""" +from starlette.responses import Response + + +def _latin1(v): + return v if isinstance(v, bytes) else str(v).encode("latin-1") + + +def _text(v): + return v.decode("latin-1") if isinstance(v, (bytes, bytearray)) else str(v) + + +def _scope(conn): + kind = conn.scope.get("type", "http") + scope = { + "type": kind, + "asgi": {"version": "3.0", "spec_version": "2.4"}, + "http_version": "1.1", + "scheme": "ws" if kind == "websocket" else "http", + "path": conn._path, + "raw_path": conn._path.encode("utf-8"), + "query_string": conn._query.encode("latin-1"), + "root_path": "", + "headers": list(conn.headers.raw), + "client": conn.client, + "server": ("testserver", 80), + "state": {}, + "app": conn.app, + "_cn1_conn": conn, + } + if kind == "websocket": + protos = conn.headers.get("sec-websocket-protocol") + scope["subprotocols"] = [p.strip() for p in protos.split(",")] if protos else [] + else: + scope["method"] = conn.method + return scope + + +def _sync_headers(conn, scope): + # A middleware may hand the app a scope with other headers. + from starlette.datastructures import Headers + hs = scope.get("headers") + if hs is not None: + conn.headers = Headers([(_text(k), _text(v)) for k, v in hs]) + + +def _inner(app): + """The request-based ``app`` as an ASGI app.""" + direct = getattr(app, "_cn1_asgi_app", None) + if direct is not None: + # Two ASGI middlewares in a row talk ASGI to each other directly, so + # the outer one sees the inner one's messages as they are sent. + return direct + + async def asgi(scope, receive, send): + conn = scope["_cn1_conn"] + _sync_headers(conn, scope) + if scope["type"] == "websocket": + saved = (conn._receive_hook, conn._send_hook) + conn._receive_hook, conn._send_hook = receive, send + try: + resp = await app(conn) + finally: + conn._receive_hook, conn._send_hook = saved + from starlette.websockets import WebSocketState + if isinstance(resp, Response) and conn.application_state == WebSocketState.CONNECTING: + # A refusal from a request-based middleware inside. + await send({"type": "websocket.http.response.start", "status": resp.status_code, + "headers": resp.headers.raw}) + await send({"type": "websocket.http.response.body", "body": resp.body}) + return + body = b"" + while True: + m = await receive() + if m["type"] != "http.request": + break + body += m.get("body", b"") + if not m.get("more_body", False): + break + conn._body = body + resp = await app(conn) + conn._cn1_inner_response = resp + await send({"type": "http.response.start", "status": resp.status_code, + "headers": resp.headers.raw}) + await send({"type": "http.response.body", "body": resp.body}) + return asgi + + +def wrap(cls, app, args, kwargs): + """``cls`` (a pure ASGI middleware) around the request-based ``app``.""" + mw = cls(_inner(app), *args, **kwargs) + + async def call(conn): + scope = _scope(conn) + if scope["type"] == "websocket": + await _websocket(mw, conn, scope) + return None + return await _http(mw, conn, scope) + call._cn1_asgi_app = mw + return call + + +async def _http(mw, conn, scope): + state = {"read": False} + + async def receive(): + if state["read"]: + return {"type": "http.disconnect"} + state["read"] = True + return {"type": "http.request", "body": conn._body, "more_body": False} + + out = {"status": None, "headers": [], "body": b""} + + async def send(m): + t = m["type"] + if t == "http.response.start": + out["status"] = m["status"] + out["headers"] = list(m.get("headers") or []) + elif t == "http.response.body": + out["body"] += bytes(m.get("body", b"")) + + await mw(scope, receive, send) + if out["status"] is None: + raise RuntimeError("No response returned.") + resp = Response(out["body"], status_code=out["status"]) + del resp.headers._list[:] + for k, v in out["headers"]: + resp.headers.append(_text(k), _text(v)) + inner = getattr(conn, "_cn1_inner_response", None) + if inner is not None: + resp.background = inner.background + return resp + + +async def _websocket(mw, conn, scope): + from starlette.websockets import WebSocketState + receive = conn._receive_hook or conn._raw_receive + outermost = conn._send_hook is None + raw_send = conn._send_hook or conn._raw_send + denial = {} + + async def send(m): + t = m["type"] + if t == "websocket.http.response.start": + denial["status"] = m["status"] + denial["headers"] = [(_text(k), _text(v)) for k, v in m.get("headers") or []] + denial["body"] = b"" + return + if t == "websocket.http.response.body": + denial["body"] += bytes(m.get("body", b"")) + if not m.get("more_body", False) and outermost \ + and conn.application_state == WebSocketState.CONNECTING: + conn.application_state = WebSocketState.DISCONNECTED + conn._session.outgoing({"type": "websocket.http.response", "status": denial["status"], + "headers": denial["headers"], "body": denial["body"]}) + elif not m.get("more_body", False): + await raw_send({"type": "websocket.http.response.start", "status": denial["status"], + "headers": denial["headers"]}) + await raw_send({"type": "websocket.http.response.body", "body": denial["body"]}) + return + await raw_send(m) + + await mw(scope, receive, send) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/middleware/base.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/middleware/base.py index 8cdb231c0e9..0232ed8e56e 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/middleware/base.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/middleware/base.py @@ -5,6 +5,9 @@ class BaseHTTPMiddleware: + # Request based (see starlette.middleware); not wrapped as ASGI. + _cn1_request_based = True + def __init__(self, app, dispatch=None): self.app = app self.dispatch_func = self.dispatch if dispatch is None else dispatch diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/middleware/cors.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/middleware/cors.py index 9b002cbb800..eae2b202218 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/middleware/cors.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/middleware/cors.py @@ -11,6 +11,9 @@ class CORSMiddleware: + # Request based (see starlette.middleware); not wrapped as ASGI. + _cn1_request_based = True + def __init__(self, app, allow_origins=(), allow_methods=("GET",), allow_headers=(), allow_credentials=False, allow_origin_regex=None, expose_headers=(), max_age=600, allow_private_network=False): diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/middleware/gzip.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/middleware/gzip.py index fe93d8567a0..2203e8751c4 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/middleware/gzip.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/middleware/gzip.py @@ -6,6 +6,9 @@ class GZipMiddleware: + # Request based (see starlette.middleware); not wrapped as ASGI. + _cn1_request_based = True + def __init__(self, app, minimum_size=500, compresslevel=9): self.app = app self.minimum_size = minimum_size diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/middleware/trustedhost.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/middleware/trustedhost.py index 44f33b46f73..d04c1885ab7 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/middleware/trustedhost.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/middleware/trustedhost.py @@ -8,6 +8,9 @@ class TrustedHostMiddleware: + # Request based (see starlette.middleware); not wrapped as ASGI. + _cn1_request_based = True + def __init__(self, app, allowed_hosts=None, www_redirect=True): if allowed_hosts is None: allowed_hosts = ["*"] diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/websockets.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/websockets.py index 4cbdb8757f6..fa719e4f01e 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/websockets.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/websockets.py @@ -41,6 +41,11 @@ def __init__(self, path, query, headers, session, app=None): self._session = session self.client_state = WebSocketState.CONNECTING self.application_state = WebSocketState.CONNECTING + # Set while a pure ASGI middleware wraps this connection: the + # receive/send it handed the app inside it, which the endpoint's + # receive and send must go through. + self._receive_hook = None + self._send_hook = None @property def url(self): @@ -48,6 +53,11 @@ def url(self): return URL("ws://" + host + self._path + ("?" + self._query if self._query else "")) async def receive(self): + if self._receive_hook is not None: + return await self._receive_hook() + return await self._raw_receive() + + async def _raw_receive(self): if self.client_state == WebSocketState.DISCONNECTED: raise RuntimeError('Cannot call "receive" once a disconnect message has been received.') m = await self._session.inbox.get() @@ -56,6 +66,12 @@ async def receive(self): return m async def send(self, message): + if self._send_hook is not None: + await self._send_hook(message) + else: + await self._raw_send(message) + + async def _raw_send(self, message): t = message["type"] if self.application_state == WebSocketState.DISCONNECTED: raise RuntimeError('Cannot call "send" once a close message has been sent.') diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/uvicorn/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/uvicorn/__init__.py index 9228052efe8..7da44c80dca 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/site/uvicorn/__init__.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/uvicorn/__init__.py @@ -24,6 +24,15 @@ def closed(self, code, reason): self.session.deliver({"type": "websocket.disconnect", "code": code, "reason": reason}) +def _flat(headers): + """ASGI header pairs as a flat list of strings, name then value.""" + out = [] + for k, v in headers or []: + out.append(k.decode("latin-1") if isinstance(k, (bytes, bytearray)) else str(k)) + out.append(v.decode("latin-1") if isinstance(v, (bytes, bytearray)) else str(v)) + return out + + def _websockets(app): def open_(path, query, headers, send): state = {"accepted": False, "closed": False} @@ -32,18 +41,22 @@ def sink(m): t = m["type"] if t == "websocket.accept": state["accepted"] = True - send("accept", m.get("subprotocol"), None) + send("accept", m.get("subprotocol"), _flat(m.get("headers"))) elif t == "websocket.send": if m.get("text") is not None: send("text", m["text"], None) else: send("bytes", m["bytes"], None) + elif t == "websocket.close" and not state["accepted"]: + # Closed before accepting: uvicorn refuses the upgrade with 403. + state["closed"] = True + send("deny", 403, None) elif t == "websocket.close": state["closed"] = True send("close", m.get("code", 1000), m.get("reason") or "") elif t == "websocket.http.response": state["closed"] = True - send("deny", m["status"], None) + send("deny", m["status"], [_flat(m.get("headers")), bytes(m.get("body") or b"")]) def done(session): e = session.error @@ -56,6 +69,7 @@ def done(session): if state["accepted"]: send("close", 1011 if e is not None else 1000, "") else: + # As uvicorn: a close before accept is a 403, a failure 500. send("deny", 500 if e is not None else 403, None) return _Connection(app.websocket_open(path, query, headers, sink, done)) diff --git a/maven/python-transpiler/src/test/resources/conformance/fastapi_asgi_middleware.out b/maven/python-transpiler/src/test/resources/conformance/fastapi_asgi_middleware.out new file mode 100644 index 00000000000..cd74453a494 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/fastapi_asgi_middleware.out @@ -0,0 +1,11 @@ +200 {'hi': 1} v1 base +['scope http /hello', 'start 200', "gate body b''"] +200 {'len': 6} v1 +['scope http /echo', 'start 200', "gate body b'abcdef'"] +404 {'detail': 'Not Found'} v1 +echo hey [(b'x-room', b'lobby')] +['scope websocket /ws/chat', 'ws in websocket.connect', 'ws out websocket.accept', 'ws in websocket.receive', 'ws out websocket.send', 'ws out websocket.close'] +WebSocketDenialResponse 401 no entry gate +['scope websocket /ws/secret'] +disconnect 1000 +['scope websocket /ws/none', 'ws out websocket.close'] diff --git a/maven/python-transpiler/src/test/resources/conformance/fastapi_asgi_middleware.py b/maven/python-transpiler/src/test/resources/conformance/fastapi_asgi_middleware.py new file mode 100644 index 00000000000..b1aafc96c99 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/fastapi_asgi_middleware.py @@ -0,0 +1,142 @@ +from fastapi import FastAPI, Request, WebSocket +from fastapi.testclient import TestClient +from starlette.middleware.base import BaseHTTPMiddleware +from starlette.websockets import WebSocketDisconnect + +log = [] + + +class AddHeader: + """Pure ASGI middleware: adds a header to every HTTP response.""" + + def __init__(self, app, value="asgi"): + self.app = app + self.value = value + + async def __call__(self, scope, receive, send): + log.append("scope " + scope["type"] + " " + scope["path"]) + if scope["type"] != "http": + await self.app(scope, receive, send) + return + + async def send_wrapper(message): + if message["type"] == "http.response.start": + headers = list(message.get("headers", [])) + headers.append((b"x-asgi", self.value.encode("latin-1"))) + message = dict(message) + message["headers"] = headers + log.append("start %d" % message["status"]) + await send(message) + + await self.app(scope, receive, send_wrapper) + + +class Gate: + """Pure ASGI middleware: refuses /ws/secret, and counts websocket messages.""" + + def __init__(self, app): + self.app = app + + async def __call__(self, scope, receive, send): + if scope["type"] == "websocket" and scope["path"] == "/ws/secret": + await send({"type": "websocket.http.response.start", "status": 401, + "headers": [(b"content-type", b"text/plain"), (b"x-why", b"gate")]}) + await send({"type": "websocket.http.response.body", "body": b"no entry"}) + return + if scope["type"] == "websocket": + async def counting_receive(): + m = await receive() + log.append("ws in " + m["type"]) + return m + + async def counting_send(m): + log.append("ws out " + m["type"]) + await send(m) + + await self.app(scope, counting_receive, counting_send) + return + body = [] + + async def read(): + m = await receive() + body.append(m.get("body", b"")) + return m + + await self.app(scope, read, send) + log.append("gate body %r" % b"".join(body)) + + +app = FastAPI() +app.add_middleware(Gate) +app.add_middleware(AddHeader, value="v1") + + +class Tag(BaseHTTPMiddleware): + async def dispatch(self, request, call_next): + resp = await call_next(request) + resp.headers["x-tag"] = "base" + return resp + + +app.add_middleware(Tag) + + +@app.get("/hello") +def hello(): + return {"hi": 1} + + +@app.post("/echo") +async def echo(request: Request): + return {"len": len(await request.body())} + + +@app.websocket("/ws/chat") +async def chat(ws: WebSocket): + await ws.accept(headers=[(b"x-room", b"lobby")]) + msg = await ws.receive_text() + await ws.send_text("echo " + msg) + await ws.close() + + +@app.websocket("/ws/secret") +async def secret(ws: WebSocket): + await ws.accept() + + +client = TestClient(app) + +r = client.get("/hello") +print(r.status_code, r.json(), r.headers.get("x-asgi"), r.headers.get("x-tag")) +print(log) +del log[:] + +r = client.post("/echo", content=b"abcdef") +print(r.status_code, r.json(), r.headers.get("x-asgi")) +print(log) +del log[:] + +r = client.get("/nope") +print(r.status_code, r.json(), r.headers.get("x-asgi")) +del log[:] + +with client.websocket_connect("/ws/chat") as ws: + ws.send_text("hey") + print(ws.receive_text(), ws.extra_headers) +print(log) +del log[:] + +try: + with client.websocket_connect("/ws/secret") as ws: + print("connected?") +except Exception as e: + print(type(e).__name__, e.status_code, e.text, e.headers.get("x-why")) +print(log) +del log[:] + +try: + with client.websocket_connect("/ws/none") as ws: + print("connected?") +except WebSocketDisconnect as e: + print("disconnect", e.code) +print(log) diff --git a/vm/backend/src/com/codename1/backend/Backend.java b/vm/backend/src/com/codename1/backend/Backend.java index 59b4ebd3c2e..d507a26e33d 100644 --- a/vm/backend/src/com/codename1/backend/Backend.java +++ b/vm/backend/src/com/codename1/backend/Backend.java @@ -675,7 +675,11 @@ public WebSocket open(HttpServer.Request request) throws Exception { } try { WebSocket endpoint = router.open(request); - return endpoint == null ? null + // A refusal is not an endpoint: it is never opened, + // and wrapped it would no longer read as one. + return endpoint == null + || endpoint instanceof HttpServer.UpgradeRefusal + ? endpoint : new TaskBound(endpoint, tasks, tracer, inFlight); } finally { if (sessions != null) { @@ -851,6 +855,17 @@ public String[] getSubprotocols() { unbind(previous); } } + + /// Forwarded for the same reason as [#getSubprotocols]. + @Override + public java.util.Map getHandshakeHeaders() { + Object[] previous = bind(); + try { + return endpoint.getHandshakeHeaders(); + } finally { + unbind(previous); + } + } } /// The handler a Backend puts in front of the server's handlers: it sets up diff --git a/vm/backend/src/com/codename1/backend/HttpServer.java b/vm/backend/src/com/codename1/backend/HttpServer.java index cdca56654ae..a9267a61e93 100644 --- a/vm/backend/src/com/codename1/backend/HttpServer.java +++ b/vm/backend/src/com/codename1/backend/HttpServer.java @@ -1729,10 +1729,56 @@ private Response fallback(Request request, int status, String text) { /// /// The same "null means not mine" the Handler chain uses, so a generated router /// can answer for the routes it knows and leave the rest alone. + /// + /// A router that wants a refusal other than 404 returns [#refuseUpgrade] + /// instead of null: the handshake is then answered with that response. public interface WebSocketHandler { WebSocket open(Request request) throws Exception; } + /// An endpoint a [WebSocketHandler] returns to refuse the upgrade with + /// `status` and `body` rather than the 404 a null answers -- a 403 for a + /// client that is not allowed in, a 400 for a handshake missing what the + /// endpoint needs. The status must be 400 or higher; anything else is + /// answered as a 500, because a refusal that reads as a success would leave + /// the client waiting for a session that never starts. The body is sent as + /// `text/plain` and may be null. + public static WebSocket refuseUpgrade(int status, String body) { + return new UpgradeRefusal(Response.text(status, body == null ? "" : body)); + } + + /// [#refuseUpgrade(int, String)] with a whole response, so the refusal can + /// carry its own headers and content type. + public static WebSocket refuseUpgrade(Response response) { + if (response == null) { + throw new IllegalArgumentException("a refusal needs a response"); + } + return new UpgradeRefusal(response); + } + + /// The marker [#refuseUpgrade] answers. It is never opened: the server sees + /// it before the handshake and writes its response instead. + static final class UpgradeRefusal implements WebSocket { + final Response response; + + UpgradeRefusal(Response response) { + this.response = response; + } + + @Override + public void onOpen(WebSocketSession session) { + } + + @Override + public void onText(WebSocketSession session, String message) { + } + + @Override + public void onBinary(WebSocketSession session, byte[] message, int offset, + int length) { + } + } + /// Where websocket routes come from. /// /// The server ASKS for them, once, while it is starting and before the listener @@ -4821,6 +4867,18 @@ private boolean tryUpgradeMarked(Conn conn, int fd, long session, Request reques .withHeaders(upgrade == null ? null : upgrade.extraHeaders)); return false; } + if (endpoint instanceof UpgradeRefusal) { + Response refused = ((UpgradeRefusal) endpoint).response; + if (refused.status < 400 || refused.status > 599) { + System.err.println("websocket refusal with status " + refused.status + + " answered as 500"); + refused = Response.text(500, "internal error"); + } + writeRefusedUpgrade(conn, fd, session, refused.withHeaders( + mergeHeaders(refused.extraHeaders, + upgrade == null ? null : upgrade.extraHeaders))); + return false; + } String subprotocol; try { @@ -4837,6 +4895,20 @@ private boolean tryUpgradeMarked(Conn conn, int fd, long session, Request reques return false; } + Map endpointHeaders; + try { + // Application code too, and asked before the borrow ends for the same + // reason: a failure is still answerable with a status. + endpointHeaders = endpoint.getHandshakeHeaders(); + } catch (RuntimeException err) { + System.err.println("websocket endpoint getHandshakeHeaders failed: " + err); + writeRefusedUpgrade(conn, fd, session, Response.text(500, "internal error") + .withHeaders(upgrade == null ? null : upgrade.extraHeaders)); + return false; + } + Map handshakeExtra = mergeHeaders(upgrade == null ? null : upgrade.extraHeaders, + endpointHeaders); + // Snapshot what the session needs BEFORE the buffer the Request was parsed // from stops being the one this connection holds. getHeaders() builds // Strings, which survive; the slices behind them do not. @@ -4862,7 +4934,8 @@ private boolean tryUpgradeMarked(Conn conn, int fd, long session, Request reques conn.releaseIdleMemory(); try { - writeHandshakeResponse(conn, WebSocketHandshake.accept(key), subprotocol, upgrade); + writeHandshakeResponse(conn, WebSocketHandshake.accept(key), subprotocol, + handshakeExtra); } catch (IOException err) { trace("fd=" + fd + " handshake write failed: " + err); // Handled here, not by the caller: this path reports the connection @@ -5277,7 +5350,7 @@ private void sweepDeferredCloses() { /// fresh connection two writes are two segments, and the client waits a round /// trip before it can send anything. private void writeHandshakeResponse(Conn conn, String accept, String subprotocol, - Response upgrade) + Map extraHeaders) throws IOException { conn.reset(); conn.put("HTTP/1.1 101 Switching Protocols\r\n"); @@ -5295,8 +5368,8 @@ private void writeHandshakeResponse(Conn conn, String accept, String subprotocol conn.put(subprotocol); conn.put("\r\n"); } - if (upgrade != null && upgrade.extraHeaders != null) { - java.util.Iterator entries = upgrade.extraHeaders.entrySet().iterator(); + if (extraHeaders != null) { + java.util.Iterator entries = extraHeaders.entrySet().iterator(); while (entries.hasNext()) { Map.Entry entry = (Map.Entry) entries.next(); String name = String.valueOf(entry.getKey()); @@ -5324,6 +5397,40 @@ && isHeaderSafe(text) && !isServerOwnedHeader(name) writeTo(conn.fd, conn.session, conn.out, 0, conn.outLength); } + /// Both header maps in one, `first`'s entries before `second`'s; a name in + /// both keeps every value. Null when neither has any. + private static Map mergeHeaders(Map first, Map second) { + if (first == null || first.isEmpty()) { + return second; + } + if (second == null || second.isEmpty()) { + return first; + } + Map merged = new LinkedHashMap(first); + java.util.Iterator entries = second.entrySet().iterator(); + while (entries.hasNext()) { + Map.Entry entry = (Map.Entry) entries.next(); + Object had = merged.get(entry.getKey()); + if (had == null) { + merged.put(entry.getKey(), entry.getValue()); + continue; + } + List values = new ArrayList(); + addValues(values, had); + addValues(values, entry.getValue()); + merged.put(entry.getKey(), values); + } + return merged; + } + + private static void addValues(List into, Object value) { + if (value instanceof List) { + into.addAll((List) value); + } else { + into.add(value); + } + } + /// 426, naming the version this server speaks. RFC 6455 4.2.2 requires it. private void writeUpgradeRequired(Conn conn) { try { diff --git a/vm/backend/src/com/codename1/backend/WebSocket.java b/vm/backend/src/com/codename1/backend/WebSocket.java index b09c485e133..4e54acfa0a3 100644 --- a/vm/backend/src/com/codename1/backend/WebSocket.java +++ b/vm/backend/src/com/codename1/backend/WebSocket.java @@ -114,4 +114,16 @@ default void onError(WebSocketSession session, Exception error) { default String[] getSubprotocols() { return null; } + + /// Extra headers for the 101 handshake response, or null for none. + /// + /// Asked once, after [#getSubprotocols] and before the 101 is written. Keys + /// are header names; a value is a String or a `List` of them, one header line + /// each. A name the handshake owns -- `Upgrade`, `Connection`, + /// `Sec-WebSocket-Accept`, `Sec-WebSocket-Protocol`, + /// `Sec-WebSocket-Extensions`, or one the server always writes -- and a value + /// carrying a CR or an LF are left out, never written. + default java.util.Map getHandshakeHeaders() { + return null; + } } From ef7298d52cc1af32251462dcc7d0527355b20136 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 11:06:26 +0200 Subject: [PATCH 056/112] Python: typing.ForwardRef, and SQLAlchemy PostgreSQL/MySQL/MariaDB dialects over the backend drivers typing keeps a string inside Optional[...]/List[...] as ForwardRef, as CPython does, so the ORM no longer parses reprs to find a relationship target. _cn1db (PyDb in the runtime, BackendDb generated for server builds over com.codename1.backend.Database) gives SQLAlchemy a DB-API for server URLs; without the backend, opening one names the scheme. make_url parses full URLs; each engine gets a dialect for quoting, column DDL (SERIAL / AUTO_INCREMENT), LIMIT/OFFSET, booleans and RETURNING of generated keys on PostgreSQL. Co-Authored-By: Claude Opus 5.5 --- .../com/codename1/python/runtime/PyDb.java | 53 +++ .../codename1/python/runtime/StdModules.java | 44 +- .../transpiler/api/PythonTranspiler.java | 93 ++++ .../python/site/sqlalchemy/__init__.py | 418 ++++++++++++++++-- .../python/site/sqlalchemy/_cn1dbapi.py | 137 ++++++ .../site/sqlalchemy/dialects/__init__.py | 1 + .../site/sqlalchemy/dialects/mariadb.py | 3 + .../python/site/sqlalchemy/dialects/mysql.py | 3 + .../site/sqlalchemy/dialects/postgresql.py | 3 + .../python/site/sqlalchemy/dialects/sqlite.py | 3 + .../codename1/python/site/sqlalchemy/exc.py | 8 + .../python/site/sqlalchemy/orm/__init__.py | 18 +- .../com/codename1/python/stdlib/typing.py | 34 +- .../conformance/sqlalchemy_dialects.out | 23 + .../conformance/sqlalchemy_dialects.py | 42 ++ .../conformance/typing_forward_ref.out | 10 + .../conformance/typing_forward_ref.py | 21 + 17 files changed, 852 insertions(+), 62 deletions(-) create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/PyDb.java create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/_cn1dbapi.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/dialects/__init__.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/dialects/mariadb.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/dialects/mysql.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/dialects/postgresql.py create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/dialects/sqlite.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/sqlalchemy_dialects.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/sqlalchemy_dialects.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/typing_forward_ref.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/typing_forward_ref.py diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyDb.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyDb.java new file mode 100644 index 00000000000..65a7eeef3b2 --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyDb.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +import java.io.IOException; + +/// The server databases (PostgreSQL, MySQL, MariaDB) behind the `_cn1db` +/// module, which SQLAlchemy's non-SQLite dialects run on. +/// +/// The runtime does not depend on the backend: a build that has it generates +/// an implementation over `com.codename1.backend.Database` and installs it, +/// the way `BackendServer` is installed for `_cn1server`. Without one, opening +/// a URL raises an error that names its scheme. +public abstract class PyDb { + private static PyDb current; + + /// The installed driver, or null when this build has none. + static PyDb get() { + return current; + } + + /// Replaces the driver. + public static void install(PyDb db) { + current = db; + } + + /// Opens a connection to the database `url` names + /// (`postgresql://user:pw@host:port/db`, `mysql://...`, `mariadb://...`). + public abstract Connection open(String url) throws IOException; + + /// One open connection. `run` answers rows for a query and the change + /// count (and, where the engine has one, the generated id) otherwise. + public abstract static class Connection extends PySql.Connection { + /// The engine the server reported: `postgresql`, `mysql` or `mariadb`. + public abstract String dialect(); + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java index a341fda878e..34b548c31e0 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java @@ -92,7 +92,11 @@ static boolean provide(String name) { return true; } if ("_cn1sql".equals(name)) { - PyModule.registerModule(sql()); + PyModule.registerModule(sql("_cn1sql", false)); + return true; + } + if ("_cn1db".equals(name)) { + PyModule.registerModule(sql("_cn1db", true)); return true; } if ("_cn1http".equals(name)) { @@ -360,11 +364,35 @@ private static PyModule fileSystem() { /// a tuple; `begin`, `commit`, `rollback` and `close` of a handle. An /// engine failure raises `OSError` with the engine's message, which the /// `sqlite3` module turns into its own exceptions. - private static PyModule sql() { - PyModule m = new PyModule("_cn1sql", new String[0]); + /// + /// `_cn1db` is the same over `PyDb`, for a server database URL, plus + /// `dialect(handle)`; where the build has no driver, `open` raises + /// `OSError` naming the URL's scheme. + private static PyModule sql(String moduleName, boolean server) { + PyModule m = new PyModule(moduleName, new String[0]); + if (server) { + m.setAttr("available", PyBuiltin.positional("available", a -> Py.bool(PyDb.get() != null))); + m.setAttr("dialect", PyBuiltin.positional("dialect", a -> { + PySql.Connection c = sqlConnection(a[0]); + return Py.str(c instanceof PyDb.Connection ? ((PyDb.Connection) c).dialect() : "sqlite"); + })); + } m.setAttr("open", PyBuiltin.positional("open", a -> { try { - PySql.Connection c = PySql.get().open(Py.toStr(a[0])); + String target = Py.toStr(a[0]); + PySql.Connection c; + if (server) { + PyDb db = PyDb.get(); + if (db == null) { + int colon = target.indexOf(':'); + String scheme = colon > 0 ? target.substring(0, colon) : target; + throw Py.error(Py.OS_ERROR, "No driver for '" + scheme + "' databases in this build: " + + "server databases need the Codename One backend (a server build)"); + } + c = db.open(target); + } else { + c = PySql.get().open(target); + } long h = ++sqlHandles; SQL.put(h, c); return PyInt.of(h); @@ -377,7 +405,10 @@ private static PyModule sql() { Object[] ps = Py.toArray(a[2]); Object[] params = new Object[ps.length]; for (int i = 0; i < ps.length; i++) { - params[i] = sqlParam(ps[i], i); + // A bool stays a bool for a server database: a PostgreSQL boolean column + // refuses an integer. SQLite stores it as the integer it is. + params[i] = server && ps[i] instanceof PyBool + ? Boolean.valueOf(ps[i] == Py.TRUE) : sqlParam(ps[i], i); //NOPMD CompareObjectsWithEquals - identity is meant } PySql.Result r; try { @@ -480,6 +511,9 @@ private static Object sqlValue(Object v) { if (v instanceof String) { return Py.str((String) v); } + if (v instanceof Boolean) { + return Py.bool(((Boolean) v).booleanValue()); + } return v; } diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java index e26a9fb9e42..6094a2496e2 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java @@ -185,6 +185,7 @@ public static Map transpile(Map sources, String } if (server) { out.put(PACKAGE.replace('.', '/') + "/BackendServer.java", backendServer()); + out.put(PACKAGE.replace('.', '/') + "/BackendDb.java", backendDb()); } out.put(PACKAGE.replace('.', '/') + "/PythonModules.java", registry(classes, mainModule, interop != null, server, nativeModules)); @@ -403,6 +404,97 @@ public static String className(String module) { return sb.toString(); } + /// The driver behind `_cn1db`, over the backend's `Database`: what + /// SQLAlchemy's PostgreSQL, MySQL and MariaDB dialects run on. `?` + /// placeholders are bound by the backend for every engine. Row values are + /// narrowed to what the runtime hands Python (long, double, string, bytes, + /// boolean); anything else -- a timestamp, a decimal -- arrives as its text. + private static String backendDb() { + return "// Generated by the Codename One Python transpiler.\n" + + "package " + PACKAGE + ";\n\n" + + "import com.codename1.backend.Database;\n" + + "import com.codename1.python.runtime.PyDb;\n" + + "import com.codename1.python.runtime.PySql;\n" + + "import java.io.IOException;\n" + + "import java.util.Iterator;\n" + + "import java.util.List;\n" + + "import java.util.Map;\n\n" + + "final class BackendDb extends PyDb {\n" + + " static void install() {\n" + + " PyDb.install(new BackendDb());\n" + + " }\n\n" + + " @Override\n" + + " public PyDb.Connection open(String url) throws IOException {\n" + + " return new Conn(Database.open(url));\n" + + " }\n\n" + + " static Object value(Object v) {\n" + + " if (v == null || v instanceof Long || v instanceof Double || v instanceof String\n" + + " || v instanceof byte[] || v instanceof Boolean) {\n" + + " return v;\n" + + " }\n" + + " if (v instanceof Integer || v instanceof Short || v instanceof Byte) {\n" + + " return Long.valueOf(((Number) v).longValue());\n" + + " }\n" + + " if (v instanceof Float) {\n" + + " return Double.valueOf(((Float) v).doubleValue());\n" + + " }\n" + + " return String.valueOf(v);\n" + + " }\n\n" + + " static final class Conn extends PyDb.Connection {\n" + + " private final Database db;\n\n" + + " Conn(Database db) {\n" + + " this.db = db;\n" + + " }\n\n" + + " @Override\n" + + " public String dialect() {\n" + + " return db.dialect().getName();\n" + + " }\n\n" + + " @Override\n" + + " public PySql.Result run(String sql, Object[] params, boolean query) throws IOException {\n" + + " PySql.Result r = new PySql.Result();\n" + + " if (!query) {\n" + + " r.changes = db.execute(sql, params);\n" + + " r.lastRowId = db.lastInsertId();\n" + + " return r;\n" + + " }\n" + + " List rows = db.query(sql, params);\n" + + " for (int i = 0; i < rows.size(); i++) {\n" + + " Map row = (Map) rows.get(i);\n" + + " if (i == 0) {\n" + + " r.columns = new String[row.size()];\n" + + " int c = 0;\n" + + " for (Iterator it = row.keySet().iterator(); it.hasNext();) {\n" + + " r.columns[c++] = String.valueOf(it.next());\n" + + " }\n" + + " }\n" + + " Object[] values = new Object[r.columns.length];\n" + + " for (int c = 0; c < values.length; c++) {\n" + + " values[c] = value(row.get(r.columns[c]));\n" + + " }\n" + + " r.rows.add(values);\n" + + " }\n" + + " return r;\n" + + " }\n\n" + + " @Override\n" + + " public void begin() throws IOException {\n" + + " db.beginTransaction();\n" + + " }\n\n" + + " @Override\n" + + " public void commit() throws IOException {\n" + + " db.commitTransaction();\n" + + " }\n\n" + + " @Override\n" + + " public void rollback() throws IOException {\n" + + " db.rollbackTransaction();\n" + + " }\n\n" + + " @Override\n" + + " public void close() {\n" + + " db.close();\n" + + " }\n" + + " }\n" + + "}\n"; + } + /// The bridge from `_cn1server.serve` to the backend's `HttpServer`. One /// worker thread: Python code runs one request at a time, as under the /// GIL, and the server's event loop still accepts and parses @@ -734,6 +826,7 @@ private static String registry(Map classes, String mainModule, b } if (server) { sb.append(" BackendServer.install();\n"); + sb.append(" BackendDb.install();\n"); } if (classes.containsKey("_cn1_http")) { sb.append(" com.codename1.python.runtime.Cn1Http.installDefault();\n"); diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/__init__.py index 87c928a1247..c494a8cfb66 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/__init__.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/__init__.py @@ -1,6 +1,7 @@ """A SQLAlchemy 2.x subset for Codename One: Core over the DB-API ``sqlite3`` module. -Only the SQLite dialect exists. Statements compile to SQL with ``?`` placeholders and run +SQLite runs over ``sqlite3``; PostgreSQL, MySQL and MariaDB URLs run over the backend's own +drivers (``_cn1dbapi``), in a server build. Statements compile to SQL with ``?`` placeholders and run through ``sqlite3``; results are read from the cursor as they are iterated. The ORM lives in ``sqlalchemy.orm`` and plugs in through two hooks: a mapped class carries ``__sa_mapper__`` (selecting it yields objects) and a relationship attribute answers ``_join_info()`` (joining on it). @@ -10,7 +11,7 @@ as SQLAlchemy does. ``echo=True`` logs each statement through the ``sqlalchemy.engine.Engine`` logger. -Unsupported: other dialects, connection pooling options, reflection, events, recursive +Unsupported: Oracle and SQL Server dialects, connection pooling options, reflection, events, recursive CTEs, ``Enum``/``JSON``/``Numeric`` decimal handling. """ @@ -98,7 +99,7 @@ class Boolean(TypeEngine): def bind(self, value): if value is None: return None - return 1 if value else 0 + return bool(value) def result(self, value): return None if value is None else bool(value) @@ -165,8 +166,22 @@ def _is_type(a): # ----------------------------------------------------------------------------- compiler +# The dialect statements compile for: the executing engine's, pushed by Connection.execute. +_active = [] + +class _Compiled: + def __init__(self, sql, params, dialect): + self.string = sql + self.positiontup = list(params) + self.dialect = dialect + + def __str__(self): + return self.string + + class _Ctx: def __init__(self, qualify=True): + self.dialect = _active[-1] self.params = [] self.qualify = qualify self.columns_clause = False @@ -182,6 +197,26 @@ class ClauseElement: def _compile(self, ctx): raise NotImplementedError(type(self).__name__) + def compile(self, bind=None, dialect=None, **kw): + """The statement as the given dialect (or the bind's) would run it, with ``?`` + placeholders: ``str()`` of the result is the SQL, ``positiontup`` the values.""" + if dialect is None and bind is not None: + dialect = getattr(bind, "dialect", None) + _active.append(dialect if dialect is not None else _active[0]) + try: + ctx = _Ctx() + sql = _compile_top(self, ctx) if isinstance(self, (Select, CompoundSelect)) else self._compile_any(ctx) + finally: + _active.pop() + return _Compiled(sql, ctx.params, ctx.dialect) + + def _compile_any(self, ctx): + if isinstance(self, _DML): + if isinstance(self, Insert): + return self._sql(self._row(None), ctx) + raise exc.CompileError("compile() of UPDATE or DELETE is not supported") + return self._compile(ctx) + def _tables(self, acc): pass @@ -428,12 +463,23 @@ def not_(clause): return UnaryExpression(clause, "NOT", None, Boolean()) +class _BoolConst(ColumnElement): + def __init__(self, value): + self.value = value + self.type = Boolean() + + def _compile(self, ctx): + if ctx.dialect.supports_native_boolean: + return "true" if self.value else "false" + return "1" if self.value else "0" + + def true(): - return TextClause("1") + return _BoolConst(True) def false(): - return TextClause("0") + return _BoolConst(False) class Label(ColumnElement): @@ -563,9 +609,10 @@ def __init__(self, *args, **kw): self.table = None def _compile(self, ctx): + q = ctx.dialect.quote if ctx.qualify and self.table is not None: - return "%s.%s" % (self.table.name, self.name) - return self.name + return "%s.%s" % (q(self.table.name), q(self.name)) + return q(self.name) def _tables(self, acc): if self.table is not None: @@ -659,7 +706,7 @@ def foreign_keys(self): return out def _compile(self, ctx): - return self.name + return ctx.dialect.quote(self.name) def _tables(self, acc): _add_unique(acc, self) @@ -668,26 +715,30 @@ def _all_tables(self): return [self] def _create_sql(self): + d = _active[-1] + q = d.quote parts = [] + pk = self.primary_key for c in self.c: - s = "%s %s" % (c.name, c.type.ddl()) - if not c.nullable: + s = "%s %s" % (q(c.name), d.column_ddl(c, self)) + if not c.nullable or c in pk: s += " NOT NULL" + if d.autoincrement_suffix and d._is_autoincrement(c, self): + s += d.autoincrement_suffix parts.append(s) - pk = self.primary_key if pk: - parts.append("PRIMARY KEY (%s)" % ", ".join(c.name for c in pk)) + parts.append("PRIMARY KEY (%s)" % ", ".join(q(c.name) for c in pk)) for c in self.c: if c.unique: - parts.append("UNIQUE (%s)" % c.name) + parts.append("UNIQUE (%s)" % q(c.name)) for c in self.c: for fk in c.foreign_keys: tname, cname = fk._target() - s = "FOREIGN KEY(%s) REFERENCES %s (%s)" % (c.name, tname, cname) + s = "FOREIGN KEY(%s) REFERENCES %s (%s)" % (q(c.name), q(tname), q(cname)) if fk.ondelete: s += " ON DELETE " + fk.ondelete parts.append(s) - return "CREATE TABLE IF NOT EXISTS %s (\n\t%s\n)" % (self.name, ", \n\t".join(parts)) + return "CREATE TABLE IF NOT EXISTS %s (\n\t%s\n)" % (q(self.name), ", \n\t".join(parts)) def create(self, bind, checkfirst=True): with bind.begin() as conn: @@ -1047,11 +1098,7 @@ def _compile_rest(self, ctx, sql, froms): if self._order: sql += " ORDER BY " + ", ".join(o._compile(ctx) for o in self._order) if self._limit is not None or self._offset is not None: - sql += " \nLIMIT ?" - ctx.params.append(self._limit if self._limit is not None else -1) - if self._offset is not None: - sql += " OFFSET ?" - ctx.params.append(self._offset) + sql += ctx.dialect.limit_clause(ctx, self._limit, self._offset) return sql def _execute_on(self, conn, params, session=None): @@ -1454,12 +1501,13 @@ def _row(self, params): def _sql(self, row, ctx): cols = [c for c in self.table.c if c.key in row] if not cols: - return "INSERT INTO %s DEFAULT VALUES" % self.table.name + self._returning_sql(ctx) + return "INSERT INTO %s DEFAULT VALUES" % ctx.dialect.quote(self.table.name) + self._returning_sql(ctx) vals = [] for c in cols: vals.append(_lit(row[c.key], c.type)._compile(ctx)) + q = ctx.dialect.quote return "INSERT INTO %s (%s) VALUES (%s)" % ( - self.table.name, ", ".join(c.name for c in cols), ", ".join(vals)) + self._returning_sql(ctx) + q(self.table.name), ", ".join(q(c.name) for c in cols), ", ".join(vals)) + self._returning_sql(ctx) def _execute_on(self, conn, params, session=None): if isinstance(params, list) and len(params) > 1: @@ -1477,7 +1525,16 @@ def _execute_on(self, conn, params, session=None): params = params[0] if params else None row = self._row(params) ctx = _Ctx() - cur = conn._run(self._sql(row, ctx), ctx.params) + sql = self._sql(row, ctx) + gen = [c for c in self.table.primary_key if row.get(c.key) is None] + if gen and not self._returning and ctx.dialect.insert_returning_key: + # No lastrowid on PostgreSQL: the generated key comes back through RETURNING. + cur = conn._run(sql + " RETURNING " + ctx.dialect.quote(gen[0].name), ctx.params) + got = cur.fetchone() + cur.lastrowid = got[0] if got else None + cur.description = None + else: + cur = conn._run(sql, ctx.params) res = self._result(cur) pk = [] for c in self.table.primary_key: @@ -1497,14 +1554,14 @@ def _execute_on(self, conn, params, session=None): if c.key in vals: v = vals[c.key] sets.append("%s=%s" % (c.name, _lit(v, c.type)._compile(ctx))) - sql = "UPDATE %s SET %s" % (self.table.name, ", ".join(sets)) + self._where_sql(ctx) + self._returning_sql(ctx) + sql = "UPDATE %s SET %s" % (ctx.dialect.quote(self.table.name), ", ".join(sets)) + self._where_sql(ctx) + self._returning_sql(ctx) return self._result(conn._run(sql, ctx.params)) class Delete(_DML): def _execute_on(self, conn, params, session=None): ctx = _Ctx() - sql = "DELETE FROM " + self.table.name + self._where_sql(ctx) + self._returning_sql(ctx) + sql = "DELETE FROM " + ctx.dialect.quote(self.table.name) + self._where_sql(ctx) + self._returning_sql(ctx) return self._result(conn._run(sql, ctx.params)) @@ -1787,7 +1844,11 @@ def _run(self, sql, params, many=False): def execute(self, statement, parameters=None, execution_options=None): if isinstance(statement, str): raise exc.ArgumentError("Textual SQL expression should be explicitly declared as text()") - return statement._execute_on(self, parameters) + _active.append(self.engine.dialect) + try: + return statement._execute_on(self, parameters) + finally: + _active.pop() def exec_driver_sql(self, sql, parameters=()): return CursorResult._from_cursor(self._run(sql, parameters), None) @@ -1847,43 +1908,322 @@ def rollback(self): self.conn.rollback() +_RESERVED = frozenset(( + "all", "and", "as", "asc", "between", "both", "by", "case", "check", "column", "constraint", + "create", "cross", "default", "delete", "desc", "distinct", "drop", "else", "end", "except", + "false", "for", "foreign", "from", "grant", "group", "having", "in", "index", "inner", "insert", + "intersect", "into", "is", "join", "key", "leading", "left", "like", "limit", "not", "null", + "offset", "on", "or", "order", "outer", "primary", "references", "right", "select", "set", + "table", "then", "to", "trailing", "true", "union", "unique", "update", "user", "using", + "values", "when", "where", "with")) + + +def _plain_identifier(name): + if not name or not (name[0] == "_" or "a" <= name[0] <= "z"): + return False + for ch in name: + if not (ch == "_" or "a" <= ch <= "z" or "0" <= ch <= "9" or ch == "$"): + return False + return name not in _RESERVED + + +class _IdentifierPreparer: + def __init__(self, q): + self.initial_quote = q + self.final_quote = q + + def quote(self, name): + if name is None or _plain_identifier(name): + return name + q = self.initial_quote + return q + name.replace(q, q + q) + q + + class _Dialect: + """What differs between engines: quoting, column DDL, LIMIT/OFFSET, booleans and + how a generated key comes back. Statements always run with ``?`` placeholders (the + drivers here bind them); ``paramstyle`` reports the one SQLAlchemy's DB-API uses.""" + name = "sqlite" driver = "pysqlite" + paramstyle = "qmark" + supports_native_boolean = False + insert_returning = True + insert_returning_key = False + autoincrement_suffix = "" + _quote_char = '"' + + def __init__(self, **kw): + self.identifier_preparer = _IdentifierPreparer(self._quote_char) + + def quote(self, name): + return self.identifier_preparer.quote(name) + + def _is_autoincrement(self, c, table): + pk = table.primary_key + if len(pk) != 1 or pk[0] is not c or c.foreign_keys: + return False + if getattr(c, "autoincrement", "auto") is False: + return False + return isinstance(c.type, Integer) + + def column_ddl(self, c, table): + return c.type.ddl() + + def limit_clause(self, ctx, limit, offset): + ctx.params.append(limit if limit is not None else -1) + if offset is None: + return " \nLIMIT ?" + ctx.params.append(offset) + return " \nLIMIT ? OFFSET ?" + + +class SQLiteDialect(_Dialect): + pass + + +def _server_type(t, names): + for cls, ddl in names: + if isinstance(t, cls): + return ddl(t) if callable(ddl) else ddl + return t.ddl() + + +def _varchar(dialect): + def ddl(t): + n = getattr(t, "length", None) + if n is None: + if dialect == "postgresql": + return "VARCHAR" + raise exc.CompileError("VARCHAR requires a length on dialect %s" % dialect) + return "VARCHAR(%d)" % n + return ddl + + +class PGDialect(_Dialect): + name = "postgresql" + driver = "psycopg2" + paramstyle = "pyformat" + supports_native_boolean = True + insert_returning_key = True + + def column_ddl(self, c, table): + t = c.type + if self._is_autoincrement(c, table): + if isinstance(t, BigInteger): + return "BIGSERIAL" + if isinstance(t, SmallInteger): + return "SMALLSERIAL" + return "SERIAL" + return _server_type(t, ( + (Text, "TEXT"), (String, _varchar("postgresql")), (BigInteger, "BIGINT"), + (SmallInteger, "SMALLINT"), (Integer, "INTEGER"), (Boolean, "BOOLEAN"), + (DateTime, "TIMESTAMP WITHOUT TIME ZONE"), (Date, "DATE"), (Numeric, "NUMERIC"), + (Float, "FLOAT"), (LargeBinary, "BYTEA"))) + + def limit_clause(self, ctx, limit, offset): + sql = "" + if limit is not None: + sql += " \nLIMIT ?" + ctx.params.append(limit) + if offset is not None: + sql += " OFFSET ?" + ctx.params.append(offset) + return sql + + +class MySQLDialect(_Dialect): + name = "mysql" + driver = "mysqldb" + paramstyle = "format" + supports_native_boolean = False + insert_returning = False + autoincrement_suffix = " AUTO_INCREMENT" + _quote_char = "`" + + def column_ddl(self, c, table): + return _server_type(c.type, ( + (Text, "TEXT"), (String, _varchar(self.name)), (BigInteger, "BIGINT"), + (SmallInteger, "SMALLINT"), (Integer, "INTEGER"), (Boolean, "BOOL"), + (DateTime, "DATETIME"), (Date, "DATE"), (Numeric, "NUMERIC"), (Float, "FLOAT"), + (LargeBinary, "BLOB"))) + + def limit_clause(self, ctx, limit, offset): + if offset is None: + ctx.params.append(limit) + return " \nLIMIT ?" + ctx.params.append(offset) + ctx.params.append(limit if limit is not None else 18446744073709551615) + return " \nLIMIT ?, ?" + + +class MariaDBDialect(MySQLDialect): + name = "mariadb" + insert_returning = True + + +_DIALECTS = {"sqlite": SQLiteDialect, "postgresql": PGDialect, "postgres": PGDialect, + "mysql": MySQLDialect, "mariadb": MariaDBDialect} +_DRIVERS = {"sqlite": "pysqlite", "postgresql": "psycopg", "mysql": "mysqldb", "mariadb": "mysqldb"} + + +def _dialect_for(url): + if url.get_backend_name() not in _DIALECTS: + raise exc.NoSuchModuleError("Can't load plugin: sqlalchemy.dialects:" + url.drivername.replace("+", ".")) + d = _DIALECTS[url.get_backend_name()]() + drv = url.get_driver_name() + d.driver = drv + if drv in ("psycopg", "psycopg2", "pymysql", "asyncpg") and d.name != "sqlite": + d.paramstyle = "pyformat" + return d + + +_active.append(SQLiteDialect()) + + +def _unquote(s): + out = [] + i = 0 + while i < len(s): + if s[i] == "%" and i + 2 < len(s): + try: + out.append(chr(int(s[i + 1:i + 3], 16))) + i += 3 + continue + except ValueError: + pass + out.append(s[i]) + i += 1 + return "".join(out) class URL: - def __init__(self, s, database): - self._s = s - self.drivername = s.split(":", 1)[0] + def __init__(self, drivername, username=None, password=None, host=None, port=None, + database=None, query=None): + self.drivername = drivername + self.username = username + self.password = password + self.host = host + self.port = port self.database = database + self.query = dict(query or {}) + + @classmethod + def create(cls, drivername, username=None, password=None, host=None, port=None, + database=None, query=None): + return cls(drivername, username, password, host, port, database, query) + + def get_backend_name(self): + return self.drivername.split("+", 1)[0] + + def get_driver_name(self): + if "+" in self.drivername: + return self.drivername.split("+", 1)[1] + return _DRIVERS.get(self.get_backend_name(), "") + + def get_dialect(self): + return _DIALECTS[self.get_backend_name()] + + def set(self, **kw): + a = dict(drivername=self.drivername, username=self.username, password=self.password, + host=self.host, port=self.port, database=self.database, query=self.query) + a.update(kw) + return URL(**a) + + def render_as_string(self, hide_password=True): + s = self.drivername + "://" + if self.username is not None: + s += self.username + if self.password is not None: + s += ":" + ("***" if hide_password else self.password) + s += "@" + if self.host is not None: + s += "[" + self.host + "]" if ":" in self.host else self.host + if self.port is not None: + s += ":" + str(self.port) + if self.database is not None: + s += "/" + self.database + if self.query: + s += "?" + "&".join("%s=%s" % (k, self.query[k]) for k in sorted(self.query)) + return s + + def _backend_url(self): + # The backend's drivers take postgresql:// | mysql:// | mariadb:// and no driver suffix. + u = self.set(drivername="postgresql" if self.get_backend_name() == "postgres" + else self.get_backend_name()) + return u.render_as_string(hide_password=False) def __str__(self): - return self._s + return self.render_as_string(hide_password=True) + + def __repr__(self): + return self.render_as_string(hide_password=True) + + def __eq__(self, other): + return isinstance(other, URL) and str(self) == str(other) and self.password == other.password + + def __hash__(self): + return hash(str(self)) def make_url(s): - if not s.startswith("sqlite"): - raise exc.ArgumentError("Only sqlite:// URLs are supported, got %r" % s) - rest = s.split("://", 1)[1] if "://" in s else "" - db = rest[1:] if rest.startswith("/") else rest - q = db.find("?") + if isinstance(s, URL): + return s + if "://" not in s: + raise exc.ArgumentError("Could not parse SQLAlchemy URL from given URL string") + name, rest = s.split("://", 1) + backend = name.split("+", 1)[0] + query = {} + q = rest.find("?") if q >= 0: - db = db[:q] - return URL(s, db if db else None) + for part in rest[q + 1:].split("&"): + if part: + k, _, v = part.partition("=") + query[k] = _unquote(v) + rest = rest[:q] + if backend == "sqlite": + db = rest[1:] if rest.startswith("/") else rest + return URL(name, database=db if db else None, query=query) + user = password = host = port = None + slash = rest.find("/") + netloc, db = (rest, None) if slash < 0 else (rest[:slash], rest[slash + 1:] or None) + at = netloc.rfind("@") + if at >= 0: + cred, netloc = netloc[:at], netloc[at + 1:] + user, sep, pw = cred.partition(":") + user = _unquote(user) + password = _unquote(pw) if sep else None + if netloc.startswith("["): + end = netloc.find("]") + host, after = netloc[1:end], netloc[end + 1:] + if after.startswith(":"): + port = int(after[1:]) + elif ":" in netloc: + host, p = netloc.rsplit(":", 1) + port = int(p) + else: + host = netloc or None + return URL(name, user, password, host, port, db, query) class Engine: def __init__(self, url, echo=False): self.url = url self.echo = echo - self.dialect = _Dialect() + self.dialect = _dialect_for(url) self._dbapi = None def _raw_connection(self): # One DB-API connection for the engine (SQLAlchemy's SingletonThreadPool for SQLite). if self._dbapi is None: - self._dbapi = _sqlite3.connect(self.url.database or ":memory:") + if self.dialect.name == "sqlite": + self._dbapi = _sqlite3.connect(self.url.database or ":memory:") + else: + from . import _cn1dbapi + self._dbapi = _cn1dbapi.connect(self.url._backend_url()) + if self._dbapi.server_dialect == "mariadb" and self.dialect.name == "mysql": + # The server says which it is, whatever the URL scheme claimed. + self.dialect = MariaDBDialect() return self._dbapi def raw_connection(self): diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/_cn1dbapi.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/_cn1dbapi.py new file mode 100644 index 00000000000..fdcdafd35ef --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/_cn1dbapi.py @@ -0,0 +1,137 @@ +"""A minimal DB-API 2.0 driver over the backend's server databases (``_cn1db``). + +It is what the PostgreSQL, MySQL and MariaDB dialects connect through: the backend's own +wire-protocol drivers, reached from Python. Placeholders are ``?`` on every engine; the +backend binds them. Errors subclass the ``sqlite3`` ones, so the engine maps both the same +way into ``sqlalchemy.exc``. +""" + +import sqlite3 as _sqlite3 + +import _cn1db + +paramstyle = "qmark" + + +class Error(_sqlite3.Error): + pass + + +class OperationalError(Error, _sqlite3.OperationalError): + pass + + +class IntegrityError(Error, _sqlite3.IntegrityError): + pass + + +_INTEGRITY = ("duplicate", "unique", "foreign key", "violates", "cannot be null", "not-null") + + +def _error(e): + msg = str(e) + low = msg.lower() + for k in _INTEGRITY: + if k in low: + return IntegrityError(msg) + return OperationalError(msg) + + +class Cursor: + def __init__(self, conn): + self.connection = conn + self.description = None + self.rowcount = -1 + self.lastrowid = None + self.arraysize = 1 + self._rows = [] + self._at = 0 + + def execute(self, sql, params=()): + c = self.connection + if c._handle is None: + raise OperationalError("Cannot operate on a closed database.") + head = sql.lstrip()[:16].upper() + query = head.startswith("SELECT") or head.startswith("WITH") or " RETURNING " in sql.upper() + if not c._in_tx and not head.startswith("CREATE") and not head.startswith("DROP"): + _cn1db.begin(c._handle) + c._in_tx = True + try: + cols, rows, changes, rowid = _cn1db.run(c._handle, sql, tuple(params or ()), query) + except OSError as e: + raise _error(e) + self._rows = list(rows) + self._at = 0 + self.description = tuple((n, None, None, None, None, None, None) for n in cols) if query else None + self.rowcount = len(self._rows) if query else changes + self.lastrowid = rowid if rowid else None + return self + + def executemany(self, sql, seq): + total = 0 + for p in seq: + self.execute(sql, p) + total += max(self.rowcount, 0) + self.rowcount = total + return self + + def fetchone(self): + if self._at >= len(self._rows): + return None + r = self._rows[self._at] + self._at += 1 + return r + + def fetchmany(self, size=None): + n = size or self.arraysize + out = self._rows[self._at:self._at + n] + self._at += len(out) + return out + + def fetchall(self): + out = self._rows[self._at:] + self._at = len(self._rows) + return out + + def close(self): + self._rows = [] + + def __iter__(self): + return iter(self.fetchall()) + + +class Connection: + def __init__(self, url): + try: + self._handle = _cn1db.open(url) + except OSError as e: + raise _error(e) + self._in_tx = False + self.server_dialect = _cn1db.dialect(self._handle) + + @property + def in_transaction(self): + return self._in_tx + + def cursor(self): + return Cursor(self) + + def commit(self): + if self._in_tx: + self._in_tx = False + _cn1db.commit(self._handle) + + def rollback(self): + if self._in_tx: + self._in_tx = False + _cn1db.rollback(self._handle) + + def close(self): + if self._handle is not None: + self.rollback() + _cn1db.close(self._handle) + self._handle = None + + +def connect(url): + return Connection(url) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/dialects/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/dialects/__init__.py new file mode 100644 index 00000000000..ece3e581537 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/dialects/__init__.py @@ -0,0 +1 @@ +"""The dialects: ``sqlite``, ``postgresql``, ``mysql`` and ``mariadb``.""" diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/dialects/mariadb.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/dialects/mariadb.py new file mode 100644 index 00000000000..48e011a5570 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/dialects/mariadb.py @@ -0,0 +1,3 @@ +"""The mariadb dialect; ``dialect()`` makes one, for ``compile(dialect=...)``.""" + +from .. import MariaDBDialect as dialect diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/dialects/mysql.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/dialects/mysql.py new file mode 100644 index 00000000000..8444572a5d9 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/dialects/mysql.py @@ -0,0 +1,3 @@ +"""The mysql dialect; ``dialect()`` makes one, for ``compile(dialect=...)``.""" + +from .. import MySQLDialect as dialect diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/dialects/postgresql.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/dialects/postgresql.py new file mode 100644 index 00000000000..a448e13cee8 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/dialects/postgresql.py @@ -0,0 +1,3 @@ +"""The postgresql dialect; ``dialect()`` makes one, for ``compile(dialect=...)``.""" + +from .. import PGDialect as dialect diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/dialects/sqlite.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/dialects/sqlite.py new file mode 100644 index 00000000000..457c1171d28 --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/dialects/sqlite.py @@ -0,0 +1,3 @@ +"""The sqlite dialect; ``dialect()`` makes one, for ``compile(dialect=...)``.""" + +from .. import SQLiteDialect as dialect diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/exc.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/exc.py index e50208ad47e..a7978c55d17 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/exc.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/exc.py @@ -9,6 +9,14 @@ class ArgumentError(SQLAlchemyError): pass +class CompileError(SQLAlchemyError): + pass + + +class NoSuchModuleError(ArgumentError): + pass + + class InvalidRequestError(SQLAlchemyError): pass diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/orm/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/orm/__init__.py index f670fb9dde9..2cba79b74c4 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/orm/__init__.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/orm/__init__.py @@ -55,15 +55,7 @@ def _name_of(t): fa = getattr(t, "__forward_arg__", None) if fa is not None: return fa - n = getattr(t, "__name__", None) - if n is None and t is not None: - # A forward reference whose type object carries no name (Optional["Author"]): its - # repr still quotes the class name. - r = repr(t) - for q in ("'", '"'): - if r.count(q) >= 2: - return r.split(q)[1] - return n + return getattr(t, "__name__", None) def _strip(s, prefix): @@ -630,14 +622,6 @@ def _map_class(cls, ns, parent=None): v._target_name = pyt if isinstance(pyt, str) else _name_of(pyt) v._list_ann = is_list if pyt is not None else None v._ann = a - if v._target_name is None and isinstance(a, _MappedAnnotation): - # Optional["Author"] whose typing arguments lost the forward reference: the - # repr still quotes the class name. - r = repr(a.arg) - for q in ("'", '"'): - if r.count(q) >= 2: - v._target_name = r.split(q)[1] - break v.key = key v.parent = cls rels[key] = v diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/typing.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/typing.py index b9f8f89021c..1b00ff76a14 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/typing.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/typing.py @@ -29,6 +29,34 @@ class _Final: __slots__ = () +class ForwardRef(_Final): + """A class named by a string inside a subscript (`Optional["Author"]`), kept as + CPython keeps it so `get_args` hands the name back.""" + + __slots__ = ("__forward_arg__", "__forward_module__") + + def __init__(self, arg, is_argument=True, module=None, *, is_class=False): + if not isinstance(arg, str): + raise TypeError("Forward reference must be a string -- got " + repr(arg)) + self.__forward_arg__ = arg + self.__forward_module__ = module + + def __eq__(self, other): + if not isinstance(other, ForwardRef): + return NotImplemented + return self.__forward_arg__ == other.__forward_arg__ + + def __hash__(self): + return hash(("ForwardRef", self.__forward_arg__)) + + def __repr__(self): + return "ForwardRef(" + repr(self.__forward_arg__) + ")" + + +def _ref(p): + return ForwardRef(p) if isinstance(p, str) else p + + class _SpecialForm(_Final): def __init__(self, name, getitem=None): self._name = name @@ -131,7 +159,7 @@ def _union(form, params): params = (params,) flat = [] for p in params: - p = type(None) if p is None else p + p = type(None) if p is None else _ref(p) if isinstance(p, _UnionGenericAlias): items = p.__args__ else: @@ -219,6 +247,10 @@ def __getitem__(self, params): if args is ...: return _CallableGenericAlias(self.__origin__, (..., result), self._name) return _CallableGenericAlias(self.__origin__, tuple(args) + (result,), self._name) + if isinstance(params, tuple): + params = tuple(_ref(p) for p in params) + else: + params = _ref(params) return _GenericAlias(self.__origin__, params, self._name) def __call__(self, *args, **kwargs): diff --git a/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_dialects.out b/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_dialects.out new file mode 100644 index 00000000000..c214d28256e --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_dialects.out @@ -0,0 +1,23 @@ +postgresql://scott:***@db.example:5433/app?sslmode=require postgresql postgresql psycopg + scott tiger db.example 5433 app {'sslmode': 'require'} +postgresql+psycopg://u@localhost/app postgresql+psycopg postgresql psycopg + u None localhost None app {} +mysql://root:***@127.0.0.1/shop mysql mysql mysqldb + root pw 127.0.0.1 None shop {} +mysql+pymysql://root:***@[::1]:3307/shop?charset=utf8mb4 mysql+pymysql mysql pymysql + root p@ss ::1 3307 shop {'charset': 'utf8mb4'} +mariadb://u:***@host/db mariadb mariadb mysqldb + u p host None db {} +sqlite:///file.db sqlite sqlite pysqlite + None None None None file.db {} +sqlite:// sqlite sqlite pysqlite + None None None None None {} +mysql://a:b@h/d +postgresql://a@h/other +postgresql True +mysql False +sqlite False +sqlite pysqlite qmark +(1,) +[('a', True)] +[('b',)] diff --git a/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_dialects.py b/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_dialects.py new file mode 100644 index 00000000000..1152ae89304 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_dialects.py @@ -0,0 +1,42 @@ +from sqlalchemy import (create_engine, make_url, MetaData, Table, Column, Integer, String, + Boolean, select, insert, true, exc) +from sqlalchemy.dialects import postgresql, mysql, sqlite + +for s in ("postgresql://scott:tiger@db.example:5433/app?sslmode=require", + "postgresql+psycopg://u@localhost/app", + "mysql://root:pw@127.0.0.1/shop", + "mysql+pymysql://root:p%40ss@[::1]:3307/shop?charset=utf8mb4", + "mariadb://u:p@host/db", + "sqlite:///file.db", + "sqlite://"): + u = make_url(s) + print(str(u), u.drivername, u.get_backend_name(), u.get_driver_name()) + print(" ", u.username, u.password, u.host, u.port, u.database, dict(u.query)) + +print(make_url("mysql://a:b@h/d").render_as_string(hide_password=False)) +print(make_url("postgresql://a@h/d").set(database="other")) + +try: + make_url("oracle2://x@y/z") +except exc.ArgumentError as e: + print("ArgumentError", type(e).__name__) + +for d in (postgresql.dialect(), mysql.dialect(), sqlite.dialect()): + print(d.name, d.supports_native_boolean) + +e = create_engine("sqlite://") +print(e.dialect.name, e.dialect.driver, e.dialect.paramstyle) + +md = MetaData() +t = Table("t", md, Column("id", Integer, primary_key=True), Column("name", String(20)), + Column("ok", Boolean)) + +# Behaviour on SQLite is unchanged. +eng = create_engine("sqlite://") +md.create_all(eng) +with eng.begin() as conn: + r = conn.execute(insert(t).values(name="a", ok=True)) + print(r.inserted_primary_key) + conn.execute(insert(t).values(name="b", ok=False)) + print(conn.execute(select(t.c.name, t.c.ok).where(t.c.ok == true())).all()) + print(conn.execute(select(t.c.name).order_by(t.c.id).offset(1)).all()) diff --git a/maven/python-transpiler/src/test/resources/conformance/typing_forward_ref.out b/maven/python-transpiler/src/test/resources/conformance/typing_forward_ref.out new file mode 100644 index 00000000000..bf896402323 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/typing_forward_ref.out @@ -0,0 +1,10 @@ +(ForwardRef('Author'), ) +Author True +True +(ForwardRef('Book'),) +(ForwardRef('A'), ) +True False +1 +ForwardRef('Node') +(, ) +Author diff --git a/maven/python-transpiler/src/test/resources/conformance/typing_forward_ref.py b/maven/python-transpiler/src/test/resources/conformance/typing_forward_ref.py new file mode 100644 index 00000000000..cb4ed71af34 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/typing_forward_ref.py @@ -0,0 +1,21 @@ +from typing import Optional, List, Union, ForwardRef, get_args, get_origin + +a = get_args(Optional["Author"]) +print(a) +print(a[0].__forward_arg__, a[1] is type(None)) +print(get_origin(Optional["Author"]) is Union) +print(get_args(List["Book"])) +print(get_args(Union["A", int])) +print(ForwardRef("X") == ForwardRef("X"), ForwardRef("X") == ForwardRef("Y")) +print(len({ForwardRef("X"), ForwardRef("X")})) +print(repr(ForwardRef("Node"))) +print(get_args(Optional[int])) + + +class Mapped: + def __class_getitem__(cls, item): + return ("Mapped", item) + + +m = Mapped[Optional["Author"]] +print(get_args(m[1])[0].__forward_arg__) From a3a44859232708cbdca61dec3386c7e18a7e8c55 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 11:10:59 +0200 Subject: [PATCH 057/112] Python backend: end-to-end server test, true streaming, GIL release in serve - scripts/python-backend/e2e-test.sh builds the FastAPI (new fastapi-chat sample) and Flask samples with build.sh --jvm, serves them on the backend's HttpServer and drives them with a stdlib client: routes, both kinds of middleware, websockets (echo, background sender, 403 refusal, ASGI-middleware 401, subprotocol + handshake headers) and streaming. Wired into python-project.yml. - _cn1server.serve held the GIL while waiting for the server to stop, so the first request deadlocked; it now releases it for the wait. - Flask generator bodies and StreamingResponse (sync and async) stream chunk by chunk through Response.stream; Flask pushes the request context around each step; a pure ASGI middleware that forwards the body message unread keeps the stream. Test clients still read the body whole. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/python-project.yml | 11 + .../codename1/python/runtime/PyServer.java | 73 ++++- .../codename1/python/runtime/StdModules.java | 15 +- .../transpiler/api/PythonTranspiler.java | 42 +++ .../python/site/fastapi/applications.py | 15 +- .../codename1/python/site/flask/__init__.py | 56 +++- .../python/site/starlette/middleware/_asgi.py | 46 +++- .../python/site/starlette/responses.py | 65 ++++- scripts/python-backend/build.sh | 17 +- scripts/python-backend/e2e-test.sh | 50 ++++ scripts/python-backend/e2e_client.py | 256 ++++++++++++++++++ .../samples/fastapi-chat/src/main.py | 153 +++++++++++ .../samples/flask-todo/src/main.py | 20 +- 13 files changed, 797 insertions(+), 22 deletions(-) create mode 100755 scripts/python-backend/e2e-test.sh create mode 100755 scripts/python-backend/e2e_client.py create mode 100644 scripts/python-backend/samples/fastapi-chat/src/main.py diff --git a/.github/workflows/python-project.yml b/.github/workflows/python-project.yml index c8ed9bfe713..72d1bc6cd0a 100644 --- a/.github/workflows/python-project.yml +++ b/.github/workflows/python-project.yml @@ -27,6 +27,8 @@ on: - 'maven/integration-tests/python-project-test.sh' - 'maven/integration-tests/inc/**' - 'scripts/python-samples/**' + - 'scripts/python-backend/**' + - 'vm/backend/src/**' - '.github/workflows/python-project.yml' push: branches: [ master ] @@ -41,6 +43,8 @@ on: - 'maven/integration-tests/python-project-test.sh' - 'maven/integration-tests/inc/**' - 'scripts/python-samples/**' + - 'scripts/python-backend/**' + - 'vm/backend/src/**' - '.github/workflows/python-project.yml' concurrency: @@ -92,6 +96,13 @@ jobs: - name: Unit tests working-directory: maven run: JAVA_HOME="$JAVA17_HOME" mvn -B -pl build-engine,codenameone-maven-plugin test -Dtest='Python*' -Dsurefire.failIfNoSpecifiedTests=false + # The real server path: Flask and FastAPI samples served by the + # backend's HttpServer on a JVM and driven over sockets -- HTTP, + # middleware, websockets and streamed responses. + - name: Python backend end-to-end + run: | + (cd maven && mvn -B -q -f ../vm/pom.xml -pl JavaAPI -am install -DskipTests) + bash scripts/python-backend/e2e-test.sh "$HOME/.m2/repository" - name: Refresh archetype catalog run: | cd maven diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyServer.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyServer.java index 01f45322f66..bf52e6f3323 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyServer.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyServer.java @@ -199,7 +199,67 @@ private static Response call0(Object dispatch, Object[] request) { headers[2 * i] = Py.toStr(p.get(0)); headers[2 * i + 1] = Py.toStr(p.get(1)); } - return new Response((int) Py.asLong(t.get(0)), headers, PyBytes.like(t.get(2))); + Object body = t.get(2); + byte[] bytes = PyBytes.like(body); + if (bytes != null || body == Py.NONE) { //NOPMD CompareObjectsWithEquals - identity is meant + return new Response((int) Py.asLong(t.get(0)), headers, bytes == null ? new byte[0] : bytes, null); + } + // Anything else is an iterable of chunks, sent as each is produced. + return new Response((int) Py.asLong(t.get(0)), headers, new byte[0], Py.iter(body)); + } + + /// The next chunk of a streamed response, or an empty array when it is + /// done. Runs the application's generator, under the GIL, up to its next + /// `yield`; empty chunks are skipped, so empty means the end and never + /// "nothing yet". + public static byte[] nextChunk(Response response) { + if (response.stream == null) { + return new byte[0]; + } + Gil.enter(); + try { + while (true) { + Object item = Py.next(response.stream); + if (item == null) { + response.stream = null; + return new byte[0]; + } + byte[] chunk = PyBytes.like(item); + if (chunk == null) { + chunk = Py.toStr(item).getBytes("UTF-8"); + } + if (chunk.length > 0) { + return chunk; + } + } + } catch (java.io.UnsupportedEncodingException e) { + throw Py.error(Py.OS_ERROR, "UTF-8 is unsupported"); + } finally { + Gil.exit(); + } + } + + /// Stops a streamed response early -- the client went away -- by closing + /// the application's generator, so its `finally` blocks run. + public static void closeStream(Response response) { + Object it = response.stream; + response.stream = null; + if (it == null) { + return; + } + Gil.enter(); + try { + Object close = Py.obj(it).getAttrOrNull("close"); + if (close != null) { + Py.obj(close).call(new Object[0], Py.NO_KW); + } + } catch (PyException e) { + // The client is gone; a generator that fails while closing has + // nobody left to answer. + e.printStackTrace(); + } finally { + Gil.exit(); + } } /// What `dispatch` answered. @@ -207,12 +267,21 @@ public static final class Response { public final int status; /// Name, value, name, value... public final String[] headers; + /// Empty when the body is streamed. public final byte[] body; + /// The chunk iterator of a streamed body, until it is exhausted. + Object stream; - Response(int status, String[] headers, byte[] body) { + Response(int status, String[] headers, byte[] body, Object stream) { this.status = status; this.headers = headers; this.body = body; + this.stream = stream; + } + + /// Whether the body comes from [#nextChunk] rather than [#body]. + public boolean isStreamed() { + return stream != null; } /// The value of the first header named `name`, ignoring case. diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java index 34b548c31e0..1ccd62848fa 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java @@ -68,8 +68,19 @@ static boolean provide(String name) { throw Py.typeError("serve(dispatch, host, port[, websockets])"); } Object ws = a.length == 4 && a[3] != Py.NONE ? a[3] : null; //NOPMD CompareObjectsWithEquals - identity is meant - PyServer.get().serve(a[0], a[1] == Py.NONE ? null : Py.toStr(a[1]), //NOPMD CompareObjectsWithEquals - identity is meant - (int) Py.asLong(a[2]), ws); + String host = a[1] == Py.NONE ? null : Py.toStr(a[1]); //NOPMD CompareObjectsWithEquals - identity is meant + int port = (int) Py.asLong(a[2]); + PyServer server = PyServer.get(); + // Serving is a wait: the calling thread blocks until the server + // stops, and every request is answered on a worker thread that + // takes the GIL itself. Holding it here deadlocked the first + // request, which no test without a real server could see. + int saved = Gil.release(); + try { + server.serve(a[0], host, port, ws); + } finally { + Gil.restore(saved); + } return Py.NONE; })); PyModule.registerModule(m); diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java index 6094a2496e2..84a77f7783e 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java @@ -709,6 +709,40 @@ private static String backendServer() { + " }\n" + " }\n" + "\n" + + " /**\n" + + " * A streamed body: each chunk the application yields is written and\n" + + " * flushed as it is produced (chunked on HTTP/1.1). A client that goes\n" + + " * away, or a response the server never writes (HEAD), closes the\n" + + " * generator so its finally blocks run.\n" + + " */\n" + + " static final class Chunks implements HttpServer.BodyWriter, java.io.Closeable {\n" + + " private final PyServer.Response res;\n" + + "\n" + + " Chunks(PyServer.Response res) {\n" + + " this.res = res;\n" + + " }\n" + + "\n" + + " public void writeTo(java.io.OutputStream out) throws java.io.IOException {\n" + + " boolean done = false;\n" + + " try {\n" + + " byte[] chunk;\n" + + " while ((chunk = PyServer.nextChunk(res)).length > 0) {\n" + + " out.write(chunk);\n" + + " out.flush();\n" + + " }\n" + + " done = true;\n" + + " } finally {\n" + + " if (!done) {\n" + + " PyServer.closeStream(res);\n" + + " }\n" + + " }\n" + + " }\n" + + "\n" + + " public void close() {\n" + + " PyServer.closeStream(res);\n" + + " }\n" + + " }\n" + + "\n" + " @Override\n" + " public void serve(final Object dispatch, String host, int port, final Object webSockets) {\n" + " HttpServer.WebSocketRoutes routes = webSockets == null ? null : new HttpServer.WebSocketRoutes() {\n" @@ -736,6 +770,14 @@ private static String backendServer() { + " extra.put(res.headers[i], res.headers[i + 1]);\n" + " }\n" + " }\n" + + " if (res.isStreamed()) {\n" + + " HttpServer.Response streamed = HttpServer.Response.stream(res.status, type, new Chunks(res));\n" + + " for (Iterator it = extra.entrySet().iterator(); it.hasNext();) {\n" + + " Map.Entry e = (Map.Entry) it.next();\n" + + " streamed.header((String) e.getKey(), (String) e.getValue());\n" + + " }\n" + + " return streamed;\n" + + " }\n" + " return new HttpServer.Response(res.status, type, res.body, extra);\n" + " }\n" + " }, (com.codename1.backend.Tls) null, routes);\n" diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/applications.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/applications.py index 1fd7275901c..5e604d52cef 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/applications.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/applications.py @@ -234,7 +234,20 @@ def _dispatch_for_client(self, method, path, query, headers, body, raise_excepti return resp.status_code, resp.headers.items(), data def dispatch(self, method, path, query, headers, body): - return self._dispatch_for_client(method, path, query, headers, body, False) + # The server's entry: a streaming body is handed over as its chunk + # iterator, which the server sends as each chunk is produced. The test + # client goes through _dispatch_for_client and reads it whole. + try: + resp = self._handle(method, path, query, headers, body) + except Exception: + resp = PlainTextResponse("Internal Server Error", status_code=500) + if method.upper() == "HEAD": + return resp.status_code, resp.headers.items(), b"" + take = getattr(resp, "take_stream", None) + stream = take() if take is not None else None + if stream is not None: + return resp.status_code, resp.headers.items(), stream + return resp.status_code, resp.headers.items(), resp.body # -- websockets def websocket_route(self, path): diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/flask/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/flask/__init__.py index 9271afc9fc0..c97c8f56662 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/site/flask/__init__.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/flask/__init__.py @@ -249,16 +249,18 @@ def __init__(self, response=None, status=None, headers=None, mimetype=None, cont self.status_code = status else: self.status = status + self._iter = None self._data = b"" self.is_streamed = False if response is not None: if isinstance(response, (str, bytes, bytearray)): self.set_data(response) else: - # A generator body is drained here, while the request context is - # still pushed: the native server has no chunked-transfer API, so a - # streamed response is buffered and sent whole. - self.set_data(b"".join([_b(c) for c in response])) + # A generator body stays a generator: the server sends each + # chunk as it is yielded (chunked, no Content-Length). Reading + # the data -- the test client, an after_request hook -- drains + # it once and keeps the bytes. + self._iter = iter(response) self.is_streamed = True self._cookies = [] @@ -279,6 +281,24 @@ def status(self, value): self.status_code = int(value.split(None, 1)[0]) self._status_text = (self.status_code, value) + @property + def _data(self): + it = self._iter + if it is not None: + self._iter = None + self._bytes = b"".join([_b(c) for c in it]) + return self._bytes + + @_data.setter + def _data(self, value): + self._iter = None + self._bytes = value + + def _take_stream(self): + it = self._iter + self._iter = None + return it + def set_data(self, value): self._data = _b(value) self.headers["Content-Length"] = str(len(self._data)) @@ -1158,6 +1178,9 @@ def dispatch(self, method, path, query, headers, body): if method == "HEAD": hdrs = resp.headers.items() return resp.status_code, hdrs, b"" + stream = resp._take_stream() + if stream is not None: + return resp.status_code, resp.headers.items(), _in_context(ctx, stream) if "Content-Length" not in resp.headers and resp.status_code not in (204, 304): resp.headers["Content-Length"] = str(len(resp._data)) return resp.status_code, resp.headers.items(), resp._data @@ -1315,7 +1338,8 @@ class TestResponse(Response): def __init__(self, body, status, headers): Response.__init__(self) self.headers = Headers(headers) - self._data = bytes(body) + # A streamed body arrives as its chunk generator; the client reads it whole. + self._data = bytes(body) if isinstance(body, (bytes, bytearray)) else b"".join(list(body)) self.status_code = status @@ -1522,11 +1546,29 @@ def render_template_string(source, **context): return app.jinja_env.from_string(source).render(app._template_context(context)) +_END = object() + + +def _in_context(ctx, chunks): + """Yields `chunks` as bytes, with the request context pushed around + each step, so a streamed body can still read `request` and `session` -- + what stream_with_context promises, given to every streamed body.""" + while True: + _stack.append(ctx) + try: + c = next(chunks, _END) + finally: + _stack.pop() + if c is _END: + return + yield _b(c) + + def stream_with_context(generator_or_function): """Keeps the request context around a generator body. - Response drains a generator while the request context is still pushed - (the native server has no chunked API), so this only wraps a function. + The server pushes the request context around every step of a streamed + body, so a generator needs no wrapping; this only wraps a function. """ if hasattr(generator_or_function, "__next__") or not callable(generator_or_function): return generator_or_function diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/middleware/_asgi.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/middleware/_asgi.py index f44a803a9da..6173c061119 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/middleware/_asgi.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/middleware/_asgi.py @@ -94,10 +94,40 @@ async def asgi(scope, receive, send): conn._cn1_inner_response = resp await send({"type": "http.response.start", "status": resp.status_code, "headers": resp.headers.raw}) + if getattr(resp, "_iterator", None) is not None: + # A streamed body is offered unread: a middleware that passes the + # message on leaves it to stream to the client chunk by chunk, + # and one that reads message["body"] gets the whole of it. + await send(_StreamBody(resp)) + return await send({"type": "http.response.body", "body": resp.body}) return asgi +class _Pending: + """The body of a streamed response nobody has read yet.""" + + def __init__(self, resp): + self.resp = resp + + +class _StreamBody(dict): + def __init__(self, resp): + dict.__init__(self, type="http.response.body", body=_Pending(resp), more_body=False) + + def _real(self, key, value): + if key == "body" and isinstance(value, _Pending): + value = value.resp.body + dict.__setitem__(self, "body", value) + return value + + def __getitem__(self, key): + return self._real(key, dict.__getitem__(self, key)) + + def get(self, key, default=None): + return self._real(key, dict.get(self, key, default)) + + def wrap(cls, app, args, kwargs): """``cls`` (a pure ASGI middleware) around the request-based ``app``.""" mw = cls(_inner(app), *args, **kwargs) @@ -121,7 +151,7 @@ async def receive(): state["read"] = True return {"type": "http.request", "body": conn._body, "more_body": False} - out = {"status": None, "headers": [], "body": b""} + out = {"status": None, "headers": [], "body": b"", "stream": None} async def send(m): t = m["type"] @@ -129,11 +159,23 @@ async def send(m): out["status"] = m["status"] out["headers"] = list(m.get("headers") or []) elif t == "http.response.body": - out["body"] += bytes(m.get("body", b"")) + raw = dict.get(m, "body", b"") if isinstance(m, dict) else m.get("body", b"") + if isinstance(raw, _Pending) and not out["body"]: + # Passed on unread: the inner stream goes out as it is. + out["stream"] = raw.resp + else: + out["body"] += bytes(m.get("body", b"")) await mw(scope, receive, send) if out["status"] is None: raise RuntimeError("No response returned.") + stream = out["stream"] + if stream is not None and getattr(stream, "_iterator", None) is not None: + stream.status_code = out["status"] + del stream.headers._list[:] + for k, v in out["headers"]: + stream.headers.append(_text(k), _text(v)) + return stream resp = Response(out["body"], status_code=out["status"]) del resp.headers._list[:] for k, v in out["headers"]: diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/responses.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/responses.py index 8e4a054d70a..f155a2adf41 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/responses.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/responses.py @@ -39,7 +39,8 @@ def _init_headers(self, headers): have_len = True if k.lower() == "content-type": have_type = True - body = self.body + # A stream pending is not measured: it goes out chunked. + body = None if getattr(self, "_iterator", None) is not None else self.body if body is not None and not have_len and not (self.status_code < 200 or self.status_code in (204, 304)): self.headers.append("content-length", str(len(body))) ct = self.media_type @@ -94,8 +95,62 @@ def __init__(self, url, status_code=307, headers=None, background=None): class StreamingResponse(Response): + """A body produced as it is sent. + + Served, each chunk the iterator yields goes to the client as it is + yielded (chunked). Read through ``body`` -- the test client, a + middleware -- the iterator is drained once and kept. + """ + def __init__(self, content, status_code=200, headers=None, media_type=None, background=None): - if hasattr(content, "__iter__") and not isinstance(content, (str, bytes)): - content = b"".join([c if isinstance(c, (bytes, bytearray)) else str(c).encode("utf-8") - for c in content]) - Response.__init__(self, content, status_code, headers, media_type, background) + self.status_code = status_code + if media_type is not None: + self.media_type = media_type + self.background = background + if isinstance(content, (str, bytes, bytearray)) or content is None: + self._iterator = None + self._body = self.render(content) + else: + self._iterator = _sync_chunks(content, self.charset) + self._body = None + self.headers = MutableHeaders() + self._init_headers(headers) + + @property + def body(self): + if self._iterator is not None: + it = self._iterator + self._iterator = None + self._body = b"".join(list(it)) + return self._body + + @body.setter + def body(self, value): + self._iterator = None + self._body = value + + def take_stream(self): + """The pending chunk iterator, once, or None when there is none.""" + it = self._iterator + self._iterator = None + return it + + +def _sync_chunks(content, charset): + if hasattr(content, "__aiter__"): + content = _drive_async(content.__aiter__()) + for c in content: + if isinstance(c, (bytes, bytearray)): + yield bytes(c) + else: + yield str(c).encode(charset) + + +def _drive_async(it): + from fastapi.routing import run_coro + while True: + try: + item = run_coro(it.__anext__()) + except StopAsyncIteration: + return + yield item diff --git a/scripts/python-backend/build.sh b/scripts/python-backend/build.sh index f38970af2ee..34c7ea96d0d 100755 --- a/scripts/python-backend/build.sh +++ b/scripts/python-backend/build.sh @@ -3,7 +3,12 @@ # binary: Python -> Java -> bytecode, then vm/backend/build.sh translates it # with the backend runtime and its HTTP server into C and compiles it. # -# build.sh [maven repo] +# build.sh [--jvm] [maven repo] +# +# --jvm stops before the native build and writes as a launcher that +# runs the same application on a JVM (JAVA17_HOME) over the backend jar's +# JavaSE runtime: no clang, no translator, seconds to build. What +# e2e-test.sh drives. # # The application serves itself the usual way -- `app.run(port=...)` or # `uvicorn.run(app, ...)` -- and the transpiler bridges that to the backend's @@ -19,6 +24,8 @@ # Needs JAVA17_HOME (javac for the generated code) and JDK_8_HOME or # JAVA_HOME (the backend build and the translator), and clang. set -e +JVM=0 +if [ "$1" = "--jvm" ]; then JVM=1; shift; fi HERE="$(cd "$(dirname "$0")" && pwd)" REPO="$(cd "$HERE/../.." && pwd)" APP="$(cd "${1:?usage: build.sh [maven repo]}" && pwd)" @@ -67,6 +74,14 @@ JAVA # Line numbers and source file of the Python, for stack traces and debuggers. "$J17/bin/java" -cp "$ENGINE:$ASM" com.codename1.maven.PythonLineMapper "$CLASSES" +if [ "$JVM" = 1 ]; then + mkdir -p "$(dirname "$OUT")" + printf '#!/bin/sh\nexec "%s/bin/java" -cp "%s:%s" com.pybackend.PyBackendMain "$@"\n' \ + "$J17" "$CLASSES" "$BACKEND" > "$OUT" + chmod +x "$OUT" + exit 0 +fi + # The backend build compiles a demo directory with JDK 8; the application is # already compiled, so it is an empty one and the classes come in through # CN1_BACKEND_EXTRA_CLASSES. diff --git a/scripts/python-backend/e2e-test.sh b/scripts/python-backend/e2e-test.sh new file mode 100755 index 00000000000..f437509b4d6 --- /dev/null +++ b/scripts/python-backend/e2e-test.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# End-to-end test of the REAL server path of a Python backend: builds the +# FastAPI and Flask samples (build.sh --jvm, the same generated server glue +# a native build uses, on the backend's JavaSE runtime), starts each, and +# drives it over real sockets with e2e_client.py -- HTTP routes, both kinds +# of middleware, websockets and streamed responses. +# +# e2e-test.sh [maven repo] +# +# Needs JAVA17_HOME, python3, and the inputs build.sh lists in its header. +set -e +HERE="$(cd "$(dirname "$0")" && pwd)" +M2="${1:-$(cd "$HERE/../.." && pwd)/.m2-repo}" +WORK="$(mktemp -d "${TMPDIR:-/tmp}/cn1-py-e2e.XXXXXX")" +PIDS=() +cleanup() { + for p in "${PIDS[@]}"; do kill "$p" 2>/dev/null || true; done +} +trap cleanup EXIT + +free_port() { + python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1",0)); print(s.getsockname()[1])' +} + +run_sample() { + local name="$1" kind="$2" + "$HERE/build.sh" --jvm "$HERE/samples/$name" "$WORK/$name" "$M2" + local port + port="$(free_port)" + "$WORK/$name" "$port" > "$WORK/$name.log" 2>&1 & + local pid=$! + PIDS+=("$pid") + for _ in $(seq 1 100); do + if grep -q "serving on port" "$WORK/$name.log"; then break; fi + if ! kill -0 "$pid" 2>/dev/null; then break; fi + sleep 0.2 + done + grep -q "serving on port" "$WORK/$name.log" || { echo "$name did not start:"; cat "$WORK/$name.log"; exit 1; } + local status=0 + python3 "$HERE/e2e_client.py" "$kind" "$port" || status=$? + kill "$pid" 2>/dev/null || true + wait "$pid" 2>/dev/null || true + if [ "$status" != 0 ]; then + echo "--- $name server log"; tail -50 "$WORK/$name.log"; exit "$status" + fi +} + +run_sample fastapi-chat fastapi +run_sample flask-todo flask +echo "python backend e2e: passed" diff --git a/scripts/python-backend/e2e_client.py b/scripts/python-backend/e2e_client.py new file mode 100755 index 00000000000..9f0f04e0049 --- /dev/null +++ b/scripts/python-backend/e2e_client.py @@ -0,0 +1,256 @@ +#!/usr/bin/env python3 +"""Drives a running Python backend sample over real sockets, stdlib only. + + e2e_client.py fastapi|flask + +Exits non-zero, naming the check, on the first failure. +""" +import base64 +import json +import os +import socket +import struct +import sys +import time +import urllib.error +import urllib.request + +HOST = "127.0.0.1" +PORT = 0 +FAILED = [] + + +def check(name, cond, detail=""): + print(("ok " if cond else "FAIL ") + name + ("" if cond else " -- " + str(detail))) + if not cond: + FAILED.append(name) + + +def http(method, path, body=None, headers=None): + req = urllib.request.Request("http://%s:%d%s" % (HOST, PORT, path), data=body, + method=method, headers=headers or {}) + try: + with urllib.request.urlopen(req, timeout=10) as r: + return r.status, r.headers, r.read() + except urllib.error.HTTPError as e: + return e.code, e.headers, e.read() + + +def timed_stream(path): + """The (seconds, bytes) of each read of a streamed body, raw HTTP/1.1.""" + s = socket.create_connection((HOST, PORT), timeout=10) + s.sendall(("GET %s HTTP/1.1\r\nHost: x\r\nConnection: close\r\n\r\n" % path).encode()) + start = time.time() + data = b"" + arrivals = [] + while True: + b = s.recv(65536) + if not b: + break + data += b + arrivals.append((time.time() - start, b)) + s.close() + head, _, body = data.partition(b"\r\n\r\n") + return head.decode("latin-1"), dechunk(body), arrivals + + +def dechunk(body): + out = b"" + while body: + line, _, rest = body.partition(b"\r\n") + n = int(line.split(b";")[0], 16) + if n == 0: + break + out += rest[:n] + body = rest[n + 2:] + return out + + +def first_arrival_of(arrivals, needle): + seen = b"" + for t, b in arrivals: + seen += b + if needle in seen: + return t + return None + + +class WS: + """A minimal RFC 6455 client: masked frames out, unmasked in.""" + + def __init__(self, path, protocols=None): + self.sock = socket.create_connection((HOST, PORT), timeout=10) + key = base64.b64encode(os.urandom(16)).decode() + lines = ["GET %s HTTP/1.1" % path, "Host: %s:%d" % (HOST, PORT), "Upgrade: websocket", + "Connection: Upgrade", "Sec-WebSocket-Key: " + key, "Sec-WebSocket-Version: 13"] + if protocols: + lines.append("Sec-WebSocket-Protocol: " + protocols) + self.sock.sendall(("\r\n".join(lines) + "\r\n\r\n").encode()) + self.buf = b"" + while b"\r\n\r\n" not in self.buf: + b = self.sock.recv(4096) + if not b: + break + self.buf += b + head, _, self.buf = self.buf.partition(b"\r\n\r\n") + rows = head.decode("latin-1").split("\r\n") + self.status = int(rows[0].split()[1]) + self.headers = {} + for r in rows[1:]: + k, _, v = r.partition(":") + self.headers[k.strip().lower()] = v.strip() + if self.status != 101: + n = int(self.headers.get("content-length", "0")) + while len(self.buf) < n: + b = self.sock.recv(4096) + if not b: + break + self.buf += b + self.body = self.buf[:n] + + def _read(self, n): + while len(self.buf) < n: + b = self.sock.recv(4096) + if not b: + raise EOFError("connection closed") + self.buf += b + out, self.buf = self.buf[:n], self.buf[n:] + return out + + def send(self, text, opcode=1): + payload = text.encode() if isinstance(text, str) else text + mask = os.urandom(4) + n = len(payload) + if n < 126: + hdr = struct.pack("!BB", 0x80 | opcode, 0x80 | n) + else: + hdr = struct.pack("!BBH", 0x80 | opcode, 0x80 | 126, n) + self.sock.sendall(hdr + mask + bytes(c ^ mask[i % 4] for i, c in enumerate(payload))) + + def recv(self): + """(opcode, payload) of the next frame.""" + b0, b1 = self._read(2) + n = b1 & 0x7F + if n == 126: + n = struct.unpack("!H", self._read(2))[0] + elif n == 127: + n = struct.unpack("!Q", self._read(8))[0] + return b0 & 0x0F, self._read(n) + + def recv_text(self): + op, p = self.recv() + if op == 8: + raise EOFError("close %d" % (struct.unpack("!H", p[:2])[0] if len(p) >= 2 else 1005)) + return p.decode() + + def close(self): + try: + self.send(struct.pack("!H", 1000), opcode=8) + except OSError: + pass + self.sock.close() + + +def streaming(path, marker_first, marker_last): + head, body, arrivals = timed_stream(path) + check(path + " 200", head.startswith("HTTP/1.1 200"), head.split("\r\n")[0]) + check(path + " is chunked", "transfer-encoding: chunked" in head.lower(), head) + check(path + " has no content-length", "content-length" not in head.lower(), head) + first = first_arrival_of(arrivals, marker_first) + last = first_arrival_of(arrivals, marker_last) + check(path + " first chunk before the generator finishes", + first is not None and last is not None and first < 0.9 and last >= 1.8, + "first %s last %s" % (first, last)) + return body + + +def fastapi(): + st, h, b = http("GET", "/health") + check("GET /health 200", st == 200, st) + check("GET /health body", json.loads(b) == {"ok": True}, b) + check("http middleware header", h.get("x-handled-by") == "timing", dict(h)) + check("pure ASGI middleware header", h.get("x-asgi") == "stamped", dict(h)) + st, h, b = http("POST", "/messages", json.dumps({"text": "hi"}).encode(), + {"Content-Type": "application/json"}) + check("POST /messages 201", st == 201, (st, b)) + st, h, b = http("GET", "/messages") + check("GET /messages", st == 200 and json.loads(b) == {"messages": ["hi"]}, b) + st, h, b = http("GET", "/blocked") + check("middleware 401", st == 401 and b"blocked" in b, (st, b)) + st, h, b = http("GET", "/nope") + check("404", st == 404, st) + + body = streaming("/stream", b"chunk 0", b"chunk 2") + check("/stream body", body == b"chunk 0\nchunk 1\nchunk 2\n", body) + body = streaming("/astream", b"async 0", b"async 2") + check("/astream body", body == b"async 0\nasync 1\nasync 2\n", body) + + ws = WS("/ws/echo") + check("ws echo 101", ws.status == 101, ws.status) + ws.send("hello") + check("ws echo reply", ws.recv_text() == "echo: hello") + ws.send("again") + check("ws echo second reply", ws.recv_text() == "echo: again") + ws.close() + + ws = WS("/ws/ticker") + check("ws ticker 101", ws.status == 101, ws.status) + ticks = [ws.recv_text() for _ in range(3)] + check("ws ticker background sender", ticks == ["tick 0", "tick 1", "tick 2"], ticks) + ws.send("stop") + closed = None + try: + for _ in range(20): + ws.recv_text() + except EOFError as e: + closed = str(e) + check("ws ticker closes 1000 on stop", closed == "close 1000", closed) + ws.sock.close() + + ws = WS("/ws/forbidden") + check("ws close-before-accept is 403", ws.status == 403, ws.status) + ws.sock.close() + + ws = WS("/ws/private") + check("ws ASGI middleware refuses 401", ws.status == 401, ws.status) + check("ws refusal body", getattr(ws, "body", b"") == b"no token", getattr(ws, "body", None)) + ws.sock.close() + ws = WS("/ws/private?token=secret") + check("ws private with token 101", ws.status == 101, ws.status) + check("ws private welcome", ws.recv_text() == "welcome") + ws.sock.close() + + ws = WS("/ws/proto", protocols="chat.v0, chat.v1") + check("ws proto 101", ws.status == 101, ws.status) + check("ws subprotocol negotiated", ws.headers.get("sec-websocket-protocol") == "chat.v1", ws.headers) + check("ws accept header", ws.headers.get("x-room") == "lobby", ws.headers) + check("ws proto message", ws.recv_text() == "proto ok") + ws.sock.close() + + ws = WS("/ws/missing") + check("ws unknown route refused", ws.status == 403, ws.status) + ws.sock.close() + + st, h, b = http("GET", "/health") + check("still serving after websockets", st == 200, st) + + +def flask(): + st, h, b = http("GET", "/") + check("GET / answers", st in (200, 404), st) + body = streaming("/stream?who=bob", b"bob 0", b"bob 2") + check("flask /stream body with request context", body == b"bob 0\nbob 1\nbob 2\n", body) + + +def main(): + global PORT + which, PORT = sys.argv[1], int(sys.argv[2]) + {"fastapi": fastapi, "flask": flask}[which]() + if FAILED: + print("%d check(s) failed: %s" % (len(FAILED), ", ".join(FAILED))) + sys.exit(1) + print("all checks passed") + + +if __name__ == "__main__": + main() diff --git a/scripts/python-backend/samples/fastapi-chat/src/main.py b/scripts/python-backend/samples/fastapi-chat/src/main.py new file mode 100644 index 00000000000..c89da041f97 --- /dev/null +++ b/scripts/python-backend/samples/fastapi-chat/src/main.py @@ -0,0 +1,153 @@ +"""A FastAPI chat service: HTTP routes, both kinds of middleware, websockets +and a streamed response. Served by the Codename One backend; driven end to +end by scripts/python-backend/e2e-test.sh. + + python main.py [port] +""" +import asyncio +import sys +import time + +from fastapi import FastAPI, WebSocket, WebSocketDisconnect +from fastapi.responses import StreamingResponse, JSONResponse + +app = FastAPI(title="chat") +messages = [] + + +class StampMiddleware: + """Pure ASGI middleware: tags every HTTP response, refuses websockets + to /ws/private without a token, and leaves the rest alone.""" + + def __init__(self, app): + self.app = app + + async def __call__(self, scope, receive, send): + if scope["type"] == "websocket" and scope["path"] == "/ws/private": + query = scope.get("query_string", b"") + if b"token=secret" not in query: + await send({"type": "websocket.http.response.start", "status": 401, + "headers": [(b"content-type", b"text/plain")]}) + await send({"type": "websocket.http.response.body", "body": b"no token"}) + return + if scope["type"] != "http": + await self.app(scope, receive, send) + return + + async def stamped(message): + if message["type"] == "http.response.start": + headers = list(message.get("headers", [])) + headers.append((b"x-asgi", b"stamped")) + message = dict(message) + message["headers"] = headers + await send(message) + await self.app(scope, receive, stamped) + + +app.add_middleware(StampMiddleware) + + +@app.middleware("http") +async def timing(request, call_next): + if request.url.path == "/blocked": + return JSONResponse({"detail": "blocked by middleware"}, status_code=401) + response = await call_next(request) + response.headers["x-handled-by"] = "timing" + return response + + +@app.get("/health") +def health(): + return {"ok": True} + + +@app.post("/messages", status_code=201) +def post_message(body: dict): + messages.append(body.get("text", "")) + return {"count": len(messages)} + + +@app.get("/messages") +def list_messages(): + return {"messages": messages} + + +@app.get("/stream") +def stream(): + def gen(): + for i in range(3): + yield "chunk %d\n" % i + if i < 2: + time.sleep(1) + return StreamingResponse(gen(), media_type="text/plain") + + +@app.get("/astream") +async def astream(): + async def gen(): + for i in range(3): + yield "async %d\n" % i + if i < 2: + await asyncio.sleep(1) + return StreamingResponse(gen(), media_type="text/plain") + + +@app.websocket("/ws/echo") +async def echo(ws: WebSocket): + await ws.accept() + try: + while True: + text = await ws.receive_text() + await ws.send_text("echo: " + text) + except WebSocketDisconnect: + pass + + +@app.websocket("/ws/ticker") +async def ticker(ws: WebSocket): + await ws.accept() + stop = asyncio.Event() + + async def sender(): + n = 0 + while not stop.is_set(): + await ws.send_text("tick %d" % n) + n += 1 + await asyncio.sleep(0.2) + + task = asyncio.create_task(sender()) + try: + while True: + text = await ws.receive_text() + if text == "stop": + break + except WebSocketDisconnect: + pass + stop.set() + await task + await ws.close(1000) + + +@app.websocket("/ws/forbidden") +async def forbidden(ws: WebSocket): + await ws.close() + + +@app.websocket("/ws/private") +async def private(ws: WebSocket): + await ws.accept() + await ws.send_text("welcome") + await ws.close() + + +@app.websocket("/ws/proto") +async def proto(ws: WebSocket): + await ws.accept(subprotocol="chat.v1", headers=[(b"x-room", b"lobby")]) + await ws.send_text("proto ok") + await ws.close() + + +if __name__ == "__main__": + import uvicorn + port = int(sys.argv[1]) if len(sys.argv) > 1 else 8080 + uvicorn.run(app, host="127.0.0.1", port=port) diff --git a/scripts/python-backend/samples/flask-todo/src/main.py b/scripts/python-backend/samples/flask-todo/src/main.py index 0a59bb2649b..ac2b8c42d91 100644 --- a/scripts/python-backend/samples/flask-todo/src/main.py +++ b/scripts/python-backend/samples/flask-todo/src/main.py @@ -1,7 +1,9 @@ # A small JSON API in plain Flask. Built by scripts/python-backend/build.sh # into one native executable that serves through the backend's HTTP server; # `python3 main.py` with Flask installed runs the same file unchanged. -from flask import Flask, jsonify, request, abort +import time + +from flask import Flask, Response, jsonify, request, abort, stream_with_context app = Flask(__name__) todos = {} @@ -52,5 +54,19 @@ def fib(n): return jsonify({"n": n, "fib": str(a)}) +@app.route("/stream") +def stream(): + """Streamed as it is produced: one line a second, naming the request.""" + who = request.args.get("who", "anon") + + def gen(): + for i in range(3): + yield "%s %d\n" % (request.args.get("who", who), i) + if i < 2: + time.sleep(1) + return Response(stream_with_context(gen()), mimetype="text/plain") + + if __name__ == "__main__": - app.run(host="127.0.0.1", port=8080) + import sys + app.run(host="127.0.0.1", port=int(sys.argv[1]) if len(sys.argv) > 1 else 8080) From 27576bf77e720583c7a4dc388db489b946abd9ae Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 11:14:10 +0200 Subject: [PATCH 058/112] Guide: Python WebSockets run on asyncio, streams are chunked Co-Authored-By: Claude Opus 5.5 --- docs/developer-guide/Python-Interop.asciidoc | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/developer-guide/Python-Interop.asciidoc b/docs/developer-guide/Python-Interop.asciidoc index b2382cfef50..9ebc96766c2 100644 --- a/docs/developer-guide/Python-Interop.asciidoc +++ b/docs/developer-guide/Python-Interop.asciidoc @@ -321,15 +321,17 @@ dependencies, `response_model` and OpenAPI generation are supported, and Flask sessions (signed cookies), `flash`, Jinja templates (`extends`, `block`, `include`, loops, conditions and the common filters, with autoescaping), multipart file uploads and streamed responses work as well. A streamed -response is sent once its generator finishes. Jinja macros aren't supported yet, and a session cookie +response is sent chunk by chunk as its generator yields. Jinja macros aren't supported yet, and a session cookie written by CPython's Flask isn't readable here, or the reverse. FastAPI middleware works as in Starlette: `@app.middleware("http")`, `BaseHTTPMiddleware` subclasses, `CORSMiddleware`, `GZipMiddleware` and `TrustedHostMiddleware`. WebSocket endpoints are supported, including -`TestClient.websocket_connect`. A WebSocket endpoint can't yet wait on -something that suspends, such as `asyncio.sleep`, while its socket is open, -and middleware doesn't apply to WebSocket connections. +`TestClient.websocket_connect`. A WebSocket endpoint runs on a real asyncio +event loop, so it can await `asyncio.sleep`, tasks and queues while its socket +is open, and pure ASGI middleware sees WebSocket connections as it does in +Starlette. Denying a connection answers with the status the endpoint or +middleware chose, such as 403. === Compared with CPython, PyPy and GraalPy From 9dac7de13588b8a195967837ea657ab0462c6fe9 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 11:17:48 +0200 Subject: [PATCH 059/112] Python: run the SQLAlchemy conformance programs against real PostgreSQL and MySQL - The _cn1db bridge (BackendDb) is generated for any backend build whose sources reach _cn1db, not only for one that also serves HTTP: a batch program using SQLAlchemy over PostgreSQL had no driver at all. - create_all compiles DDL for the bind's dialect (it used SQLite's, so PostgreSQL got INTEGER keys with no SERIAL); drop_all quotes names. - The server DB-API driver binds text()'s :name parameters by position. - The conformance programs take the engine URL from argv, drop their tables first and spell raw auto-numbered DDL per engine. - scripts/python-backend/sqlalchemy-realdb-test.sh runs them on servers. Co-Authored-By: Claude Opus 5.5 --- .../transpiler/api/PythonTranspiler.java | 24 +++-- .../python/site/sqlalchemy/__init__.py | 14 ++- .../python/site/sqlalchemy/_cn1dbapi.py | 39 ++++++++ .../resources/conformance/sqlalchemy_core.py | 10 +- .../resources/conformance/sqlalchemy_orm.py | 4 +- .../resources/conformance/sqlalchemy_query.py | 6 +- .../conformance/sqlalchemy_relations.py | 4 +- .../python-backend/sqlalchemy-realdb-test.sh | 91 +++++++++++++++++++ 8 files changed, 175 insertions(+), 17 deletions(-) create mode 100755 scripts/python-backend/sqlalchemy-realdb-test.sh diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java index 84a77f7783e..c95fc744613 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java @@ -135,19 +135,23 @@ public static Map transpile(Map sources, String } Set stdlib = addStdlib(sources, trees); boolean server = false; + boolean db = false; for (String text : sources.values()) { - if (text.contains("_cn1server")) { - server = true; - break; - } + server |= text.contains("_cn1server"); + db |= text.contains("_cn1db"); } - if (server) { + if (server || db) { // A web framework is bundled: serve it over the backend's HTTP // server when the build has one (a backend build). Elsewhere // serving raises, and the frameworks' test clients still work. try (com.codename1.python.transpiler.codegen.ClassPathReader cp = new com.codename1.python.transpiler.codegen.ClassPathReader(classPath)) { - server = cp.find("com/codename1/backend/HttpServer") != null; + // The database bridge is wanted by any program reaching + // _cn1db (SQLAlchemy's server dialects): a batch job or a + // worker as much as a web application. + boolean backend = cp.find("com/codename1/backend/HttpServer") != null; + server &= backend; + db = backend && (db || server); } catch (IOException e) { throw new IllegalStateException("cannot read the class path: " + e.getMessage(), e); } @@ -185,9 +189,11 @@ public static Map transpile(Map sources, String } if (server) { out.put(PACKAGE.replace('.', '/') + "/BackendServer.java", backendServer()); + } + if (db) { out.put(PACKAGE.replace('.', '/') + "/BackendDb.java", backendDb()); } - out.put(PACKAGE.replace('.', '/') + "/PythonModules.java", registry(classes, mainModule, interop != null, server, + out.put(PACKAGE.replace('.', '/') + "/PythonModules.java", registry(classes, mainModule, interop != null, server, db, nativeModules)); return out; } @@ -850,7 +856,7 @@ static String withLineMap(String java, String pyFile) { + java.substring(brace + 1); } - private static String registry(Map classes, String mainModule, boolean interop, boolean server, + private static String registry(Map classes, String mainModule, boolean interop, boolean server, boolean db, Map nativeModules) { StringBuilder sb = new StringBuilder(); sb.append("// Generated by the Codename One Python transpiler.\n"); @@ -868,6 +874,8 @@ private static String registry(Map classes, String mainModule, b } if (server) { sb.append(" BackendServer.install();\n"); + } + if (db) { sb.append(" BackendDb.install();\n"); } if (classes.containsKey("_cn1_http")) { diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/__init__.py index c494a8cfb66..ef0597cf468 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/__init__.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/__init__.py @@ -833,16 +833,22 @@ def sorted_tables(self): return done def create_all(self, bind, tables=None, checkfirst=True): + # The DDL is the bind's dialect's (SERIAL, AUTO_INCREMENT, its types), not SQLite's. with bind.begin() as conn: - for t in self.sorted_tables: - if tables is None or t in tables: - conn._run(t._create_sql(), ()) + _active.append(conn.engine.dialect) + try: + for t in self.sorted_tables: + if tables is None or t in tables: + conn._run(t._create_sql(), ()) + finally: + _active.pop() def drop_all(self, bind, tables=None, checkfirst=True): with bind.begin() as conn: + q = conn.engine.dialect.quote for t in reversed(self.sorted_tables): if tables is None or t in tables: - conn._run("DROP TABLE IF EXISTS " + t.name, ()) + conn._run("DROP TABLE IF EXISTS " + q(t.name), ()) # ----------------------------------------------------------------------------- statements diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/_cn1dbapi.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/_cn1dbapi.py index fdcdafd35ef..6d3d867de6a 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/_cn1dbapi.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/_cn1dbapi.py @@ -21,6 +21,10 @@ class OperationalError(Error, _sqlite3.OperationalError): pass +class ProgrammingError(Error, _sqlite3.ProgrammingError): + pass + + class IntegrityError(Error, _sqlite3.IntegrityError): pass @@ -37,6 +41,39 @@ def _error(e): return OperationalError(msg) +def _named(sql, params): + """``:name`` binds (``text()``, the named style sqlite3 takes) as ``?`` and a tuple: the + backend's drivers bind by position. Quoted text, identifiers and PostgreSQL's ``::`` + casts are left alone.""" + out = [] + values = [] + i = 0 + n = len(sql) + while i < n: + ch = sql[i] + if ch in "'\"`": + j = sql.find(ch, i + 1) + j = n - 1 if j < 0 else j + out.append(sql[i:j + 1]) + i = j + 1 + continue + if ch == ":" and i + 1 < n and (sql[i + 1].isalpha() or sql[i + 1] == "_") \ + and (i == 0 or sql[i - 1] != ":"): + j = i + 1 + while j < n and (sql[j].isalnum() or sql[j] == "_"): + j += 1 + name = sql[i + 1:j] + if name not in params: + raise ProgrammingError("You did not supply a value for binding parameter :" + name + ".") + values.append(params[name]) + out.append("?") + i = j + continue + out.append(ch) + i += 1 + return "".join(out), tuple(values) + + class Cursor: def __init__(self, conn): self.connection = conn @@ -48,6 +85,8 @@ def __init__(self, conn): self._at = 0 def execute(self, sql, params=()): + if isinstance(params, dict): + sql, params = _named(sql, params) c = self.connection if c._handle is None: raise OperationalError("Cannot operate on a closed database.") diff --git a/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_core.py b/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_core.py index c19aeea529c..214520cabab 100644 --- a/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_core.py +++ b/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_core.py @@ -1,12 +1,17 @@ +import sys import datetime from sqlalchemy import (create_engine, text, MetaData, Table, Column, Integer, String, Text, Float, Boolean, DateTime, Date, ForeignKey, select, insert, update, delete, func, and_, or_, not_) from sqlalchemy import exc -engine = create_engine("sqlite://") +engine = create_engine(sys.argv[1] if len(sys.argv) > 1 else "sqlite://") +# Raw DDL is the one place the SQL is the engine's own: an auto-numbered key is spelled per engine. +AUTO_ID = {"postgresql": "SERIAL PRIMARY KEY", "mysql": "INTEGER PRIMARY KEY AUTO_INCREMENT", + "mariadb": "INTEGER PRIMARY KEY AUTO_INCREMENT"}.get(engine.dialect.name, "INTEGER PRIMARY KEY") with engine.connect() as conn: - conn.execute(text("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT, n INTEGER)")) + conn.execute(text("DROP TABLE IF EXISTS t")) + conn.execute(text("CREATE TABLE t (id " + AUTO_ID + ", name TEXT, n INTEGER)")) conn.execute(text("INSERT INTO t (name, n) VALUES (:name, :n)"), [{"name": "a", "n": 1}, {"name": "b", "n": 2}]) conn.execute(text("INSERT INTO t (name, n) VALUES (:name, :n)"), {"name": "c", "n": 3}) conn.commit() @@ -45,6 +50,7 @@ Column("user_id", Integer, ForeignKey("users.id")), Column("title", String(100)), Column("likes", Integer, default=0)) +metadata.drop_all(engine) # a server database keeps tables between runs metadata.create_all(engine) print(sorted(metadata.tables), [c.name for c in users.columns], users.c.name.name) diff --git a/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_orm.py b/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_orm.py index d71644a2610..07082afdc31 100644 --- a/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_orm.py +++ b/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_orm.py @@ -1,3 +1,4 @@ +import sys from typing import List, Optional from sqlalchemy import create_engine, ForeignKey, String, select, func from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship, Session @@ -24,7 +25,8 @@ class Address(Base): user: Mapped["User"] = relationship(back_populates="addresses") -engine = create_engine("sqlite:///:memory:") +engine = create_engine(sys.argv[1] if len(sys.argv) > 1 else "sqlite://") +Base.metadata.drop_all(engine) # a server database keeps tables between runs Base.metadata.create_all(engine) print(sorted(Base.metadata.tables), User.__table__.name, [c.name for c in Address.__table__.columns]) diff --git a/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_query.py b/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_query.py index c6e614151cb..d7e7ed6940b 100644 --- a/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_query.py +++ b/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_query.py @@ -1,3 +1,4 @@ +import sys import logging from typing import List, Optional from sqlalchemy import create_engine, ForeignKey, String, Integer, select, func, exists, union, \ @@ -40,7 +41,10 @@ def emit(self, record): log.addHandler(cap) log.propagate = False -engine = create_engine("sqlite://", echo=True) +engine = create_engine(sys.argv[1] if len(sys.argv) > 1 else "sqlite://", echo=True) +engine.echo = False +Base.metadata.drop_all(engine) # a server database keeps tables between runs +engine.echo = True Base.metadata.create_all(engine) s = Session(engine) ann = User(name="Ann", age=30, addresses=[Address(email="ann@a.com"), Address(email="ann@b.org")]) diff --git a/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_relations.py b/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_relations.py index 4b36fbe03b8..64b7fab00be 100644 --- a/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_relations.py +++ b/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_relations.py @@ -1,3 +1,4 @@ +import sys from typing import List, Optional from sqlalchemy import create_engine, ForeignKey, String, Table, Column, Integer, select, func from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship, Session, \ @@ -99,7 +100,8 @@ class Truck(Vehicle): __mapper_args__ = {"polymorphic_identity": "truck"} -engine = create_engine("sqlite://") +engine = create_engine(sys.argv[1] if len(sys.argv) > 1 else "sqlite://") +Base.metadata.drop_all(engine) # a server database keeps tables between runs Base.metadata.create_all(engine) # ---- many-to-many diff --git a/scripts/python-backend/sqlalchemy-realdb-test.sh b/scripts/python-backend/sqlalchemy-realdb-test.sh new file mode 100755 index 00000000000..d63e11b5f2c --- /dev/null +++ b/scripts/python-backend/sqlalchemy-realdb-test.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +# +# Runs the SQLAlchemy conformance programs against real database servers and +# requires the output SQLite (and CPython) give. +# +# The conformance suite runs them on SQLite only, so the PostgreSQL, MySQL and +# MariaDB dialects -- which go through the backend's own drivers via _cn1db -- +# are exercised here. Each program is transpiled as a backend build (the +# backend jar on the class path generates the database bridge) and run on the +# JVM with the server URL as its argument (the programs read sys.argv). +# +# CN1_SQLA_POSTGRES=postgres://u:p@127.0.0.1:5432/db?sslmode=disable +# CN1_SQLA_MYSQL=mysql://u:p@127.0.0.1:3306/db?sslmode=disable +# CN1_SQLA_MARIADB=mariadb://... +# JAVA17_HOME= +# scripts/python-backend/sqlalchemy-realdb-test.sh [maven repo] +# +# An engine whose variable is unset is skipped with a note; at least one must +# be set. Needs codenameone-core, -python-runtime, -python-transpiler and +# -backend installed in the maven repo (default: /.m2-repo). +# +# Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. +# Licensed under the GNU General Public License v2 with Classpath Exception. +set -euo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +REPO="$(cd "$HERE/../.." && pwd)" +M2="${1:-$REPO/.m2-repo}" +J17="${JAVA17_HOME:?set JAVA17_HOME}" +CONF="$REPO/maven/python-transpiler/src/test/resources/conformance" +PROGRAMS="sqlalchemy_core sqlalchemy_orm sqlalchemy_relations sqlalchemy_query" +art() { echo "$M2/$1/$2/$3/$2-$3.jar"; } +CORE=$(art com/codenameone codenameone-core 8.0-SNAPSHOT) +RUNTIME=$(art com/codenameone codenameone-python-runtime 8.0-SNAPSHOT) +TRANSPILER=$(art com/codenameone codenameone-python-transpiler 8.0-SNAPSHOT) +BACKEND=$(art com/codenameone codenameone-backend 8.0-SNAPSHOT) +JAVAAPI=$(art com/codename1/parparvm JavaAPI 1.0-SNAPSHOT) +for j in "$CORE" "$RUNTIME" "$TRANSPILER" "$BACKEND" "$JAVAAPI"; do + [ -f "$j" ] || { echo "missing $j -- install it into $M2" >&2; exit 1; } +done + +ENGINES="" +[ -n "${CN1_SQLA_POSTGRES:-}" ] && ENGINES="$ENGINES postgresql" +[ -n "${CN1_SQLA_MYSQL:-}" ] && ENGINES="$ENGINES mysql" +[ -n "${CN1_SQLA_MARIADB:-}" ] && ENGINES="$ENGINES mariadb" +[ -n "$ENGINES" ] || { echo "set CN1_SQLA_POSTGRES, CN1_SQLA_MYSQL and/or CN1_SQLA_MARIADB" >&2; exit 2; } + +WORK="${CN1_SQLA_WORK:-$(mktemp -d "${TMPDIR:-/tmp}/sqla-realdb.XXXXXX")}" +mkdir -p "$WORK" +failed=0 +for prog in $PROGRAMS; do + APP="$WORK/$prog" + mkdir -p "$APP/src" "$APP/gen" "$APP/classes" + cp "$CONF/$prog.py" "$APP/src/main.py" + "$J17/bin/java" -cp "$TRANSPILER" com.codename1.python.transpiler.api.Main \ + -cp "$JAVAAPI:$CORE:$BACKEND" "$APP/src" "$APP/gen" main >"$APP/transpile.log" 2>&1 \ + || { cat "$APP/transpile.log" >&2; exit 1; } + [ -f "$APP/gen/com/codename1/generated/python/BackendDb.java" ] \ + || { echo "$prog: no database bridge was generated" >&2; exit 1; } + mkdir -p "$APP/gen/com/pybackend" + cat >"$APP/gen/com/pybackend/Main.java" <<'JAVA' +package com.pybackend; + +public class Main { + public static void main(String[] args) { + com.codename1.python.runtime.Py.setArgv("main", args); + System.exit(com.codename1.generated.python.PythonModules.runMain()); + } +} +JAVA + (cd "$APP/classes" && unzip -qo "$RUNTIME" 'com/*' && unzip -qo "$CORE" 'com/codename1/util/*') + "$J17/bin/javac" -nowarn --release 17 -d "$APP/classes" $(find "$REPO/scripts/python-bench/hostfs" -name '*.java') + "$J17/bin/javac" -nowarn --release 17 -cp "$APP/classes:$BACKEND" -d "$APP/classes" \ + $(find "$APP/gen" -name '*.java') + for engine in $ENGINES; do + case "$engine" in + postgresql) url="$CN1_SQLA_POSTGRES" ;; + mysql) url="$CN1_SQLA_MYSQL" ;; + mariadb) url="$CN1_SQLA_MARIADB" ;; + esac + out="$APP/$engine.out" + "$J17/bin/java" -cp "$APP/classes:$BACKEND:$CORE" com.pybackend.Main "$url" >"$out" 2>&1 || true + if diff -u "$CONF/$prog.out" "$out" >"$APP/$engine.diff"; then + echo "PASS $prog on $engine" + else + echo "FAIL $prog on $engine (diff: $APP/$engine.diff)" + head -30 "$APP/$engine.diff" + failed=1 + fi + done +done +exit $failed From b88bc79852014681fae84f8261ecb03c2ccb30dc Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 11:20:40 +0200 Subject: [PATCH 060/112] Python backend: serve requests concurrently, run background tasks after the response The generated BackendServer serves on a pool of 16 workers instead of one; each holds the GIL only while Python runs, so a slow streamed response no longer blocks every other request. FastAPI's dispatch hands background tasks to the server as a fourth tuple element, and the server runs them once the response (or a stream's last chunk) has been sent, skipping them for a client that went away. e2e covers both. Co-Authored-By: Claude Opus 5.5 --- .../codename1/python/runtime/PyServer.java | 71 +++++++++++++++++-- .../transpiler/api/PythonTranspiler.java | 27 +++++-- .../python/site/fastapi/applications.py | 24 ++++--- scripts/python-backend/e2e_client.py | 48 +++++++++++++ .../samples/fastapi-chat/src/main.py | 41 ++++++++++- 5 files changed, 188 insertions(+), 23 deletions(-) diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyServer.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyServer.java index bf52e6f3323..448e420a47d 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyServer.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyServer.java @@ -30,7 +30,15 @@ /// generated registry; an application without a server has none, and /// serving raises `OSError`. `dispatch` takes `(method, path, query, /// headers, body)`, the headers a list of pairs and the body bytes, and -/// answers `(status, headers, body)` the same way. +/// answers `(status, headers, body)` the same way, or +/// `(status, headers, body, after)` when work is to run once the response +/// has been sent (FastAPI's background tasks): the server calls +/// [#runAfter(Response)] when the last byte is out. +/// +/// Requests are served concurrently, on several threads. Each holds the +/// [Gil] only while it runs Python -- [#call], [#nextChunk], [#runAfter] -- +/// and never while it writes to a socket or waits for a chunk to drain, so a +/// slow streamed response does not hold up the requests beside it. public abstract class PyServer { private static PyServer current; @@ -180,10 +188,11 @@ public static Response call(Object dispatch, Object[] request) { private static Response call0(Object dispatch, Object[] request) { Object r = Py.obj(dispatch).call(request, Py.NO_KW); - if (!(r instanceof PyTuple) || ((PyTuple) r).size() != 3) { - throw Py.typeError("dispatch must return (status, headers, body)"); + if (!(r instanceof PyTuple) || ((PyTuple) r).size() < 3 || ((PyTuple) r).size() > 4) { + throw Py.typeError("dispatch must return (status, headers, body[, after])"); } PyTuple t = (PyTuple) r; + Object after = t.size() == 4 && t.get(3) != Py.NONE ? t.get(3) : null; //NOPMD CompareObjectsWithEquals - identity is meant Object hl = t.get(1); Object[] pairs = hl instanceof PyList ? ((PyList) hl).toArray() : hl instanceof PyTuple ? ((PyTuple) hl).items : null; @@ -202,10 +211,52 @@ private static Response call0(Object dispatch, Object[] request) { Object body = t.get(2); byte[] bytes = PyBytes.like(body); if (bytes != null || body == Py.NONE) { //NOPMD CompareObjectsWithEquals - identity is meant - return new Response((int) Py.asLong(t.get(0)), headers, bytes == null ? new byte[0] : bytes, null); + return new Response((int) Py.asLong(t.get(0)), headers, bytes == null ? new byte[0] : bytes, null, after); } // Anything else is an iterable of chunks, sent as each is produced. - return new Response((int) Py.asLong(t.get(0)), headers, new byte[0], Py.iter(body)); + return new Response((int) Py.asLong(t.get(0)), headers, new byte[0], Py.iter(body), after); + } + + /// Runs the work `dispatch` asked to run after the response, at most + /// once, under the GIL. A failure is printed: the response is already + /// out, so there is nobody left to answer it to, as in Starlette. + public static void runAfter(Response response) { + Object after = response.after.getAndSet(null); + if (after == null) { + return; + } + Gil.enter(); + try { + Py.obj(after).call(new Object[0], Py.NO_KW); + } catch (PyException e) { + e.printStackTrace(); + } finally { + Gil.exit(); + } + } + + /// Drops the work to run after a response that was never completed -- + /// the client went away -- as Starlette skips its background tasks when + /// a send fails. + public static void cancelAfter(Response response) { + response.after.set(null); + } + + /// [#runAfter(Response)] on a thread of its own, for a server that has + /// no hook for "the response is written": the caller returns and the + /// response goes out without waiting for the work, which then waits only + /// for the GIL. Does nothing when there is no such work. + public static void runAfterLater(final Response response) { + if (response.after.get() == null) { + return; + } + Thread t = new Thread(new Runnable() { + @Override + public void run() { + runAfter(response); + } + }); + t.start(); } /// The next chunk of a streamed response, or an empty array when it is @@ -271,12 +322,20 @@ public static final class Response { public final byte[] body; /// The chunk iterator of a streamed body, until it is exhausted. Object stream; + /// What [#runAfter(Response)] calls, until it has. + final java.util.concurrent.atomic.AtomicReference after; - Response(int status, String[] headers, byte[] body, Object stream) { + Response(int status, String[] headers, byte[] body, Object stream, Object after) { this.status = status; this.headers = headers; this.body = body; this.stream = stream; + this.after = new java.util.concurrent.atomic.AtomicReference(after); + } + + /// Whether there is work to run once this response has been sent. + public boolean hasAfter() { + return after.get() != null; } /// Whether the body comes from [#nextChunk] rather than [#body]. diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java index c95fc744613..72530164c81 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java @@ -501,10 +501,12 @@ private static String backendDb() { + "}\n"; } - /// The bridge from `_cn1server.serve` to the backend's `HttpServer`. One - /// worker thread: Python code runs one request at a time, as under the - /// GIL, and the server's event loop still accepts and parses - /// connections concurrently around it. + /// The bridge from `_cn1server.serve` to the backend's `HttpServer`. + /// Requests run on a pool of worker threads; each holds the GIL only + /// while Python runs (`PyServer.call`, `nextChunk`, `runAfter`), so one + /// slow streamed response -- sleeping between chunks, or draining to a + /// slow client -- leaves the other workers free. Background tasks run + /// once the response has been handed back or its last chunk written. private static String backendServer() { return "// Generated by the Codename One Python transpiler.\n" + "package " + PACKAGE + ";\n\n" @@ -516,6 +518,11 @@ private static String backendServer() { + "import java.util.LinkedHashMap;\n" + "import java.util.Map;\n\n" + "final class BackendServer extends PyServer {\n" + + " /**\n" + + " * Requests served at once. Python still runs one thread at a time\n" + + " * under the GIL; the others wait on sockets, sleeps and streams.\n" + + " */\n" + + " private static final int WORKERS = 16;\n\n" + " static void install() {\n" + " PyServer.install(new BackendServer());\n" + " }\n\n" @@ -719,7 +726,8 @@ private static String backendServer() { + " * A streamed body: each chunk the application yields is written and\n" + " * flushed as it is produced (chunked on HTTP/1.1). A client that goes\n" + " * away, or a response the server never writes (HEAD), closes the\n" - + " * generator so its finally blocks run.\n" + + " * generator so its finally blocks run. Background tasks run once the\n" + + " * last chunk is out, and not at all for a client that went away.\n" + " */\n" + " static final class Chunks implements HttpServer.BodyWriter, java.io.Closeable {\n" + " private final PyServer.Response res;\n" @@ -738,7 +746,10 @@ private static String backendServer() { + " }\n" + " done = true;\n" + " } finally {\n" - + " if (!done) {\n" + + " if (done) {\n" + + " PyServer.runAfterLater(res);\n" + + " } else {\n" + + " PyServer.cancelAfter(res);\n" + " PyServer.closeStream(res);\n" + " }\n" + " }\n" @@ -746,6 +757,7 @@ private static String backendServer() { + "\n" + " public void close() {\n" + " PyServer.closeStream(res);\n" + + " PyServer.runAfterLater(res);\n" + " }\n" + " }\n" + "\n" @@ -763,7 +775,7 @@ private static String backendServer() { + " };\n" + " HttpServer server;\n" + " try {\n" - + " server = HttpServer.start(host, port, 512, 1, new HttpServer.Handler() {\n" + + " server = HttpServer.start(host, port, 512, WORKERS, new HttpServer.Handler() {\n" + " public HttpServer.Response handle(HttpServer.Request r) {\n" + " PyServer.Response res = PyServer.call(dispatch,\n" + " PyServer.request(r.getMethod(), r.getTarget(), headers(r), r.getBodyBytes()));\n" @@ -784,6 +796,7 @@ private static String backendServer() { + " }\n" + " return streamed;\n" + " }\n" + + " PyServer.runAfterLater(res);\n" + " return new HttpServer.Response(res.status, type, res.body, extra);\n" + " }\n" + " }, (com.codename1.backend.Tls) null, routes);\n" diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/applications.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/applications.py index 5e604d52cef..58c761d3c53 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/applications.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/applications.py @@ -215,11 +215,11 @@ async def _core(self, req): resp = await self._handle_exception(req, exc) return resp - def _handle(self, method, path, query, headers, body): + def _handle(self, method, path, query, headers, body, run_background=True): req = Request(method.upper(), unquote(path), query, headers, body, self) req.app = self resp = run_coro(self._build_stack()(req)) - if resp.background is not None: + if run_background and resp.background is not None: resp.background() return resp @@ -237,17 +237,23 @@ def dispatch(self, method, path, query, headers, body): # The server's entry: a streaming body is handed over as its chunk # iterator, which the server sends as each chunk is produced. The test # client goes through _dispatch_for_client and reads it whole. + # Background tasks are handed over as a fourth element: the server + # runs them once the whole response, streamed or not, has been sent, + # as Starlette does after its final send. try: - resp = self._handle(method, path, query, headers, body) + resp = self._handle(method, path, query, headers, body, False) except Exception: resp = PlainTextResponse("Internal Server Error", status_code=500) + after = resp.background if method.upper() == "HEAD": - return resp.status_code, resp.headers.items(), b"" - take = getattr(resp, "take_stream", None) - stream = take() if take is not None else None - if stream is not None: - return resp.status_code, resp.headers.items(), stream - return resp.status_code, resp.headers.items(), resp.body + data = b"" + else: + take = getattr(resp, "take_stream", None) + stream = take() if take is not None else None + data = stream if stream is not None else resp.body + if after is not None: + return resp.status_code, resp.headers.items(), data, after + return resp.status_code, resp.headers.items(), data # -- websockets def websocket_route(self, path): diff --git a/scripts/python-backend/e2e_client.py b/scripts/python-backend/e2e_client.py index 9f0f04e0049..a9a84540ac5 100755 --- a/scripts/python-backend/e2e_client.py +++ b/scripts/python-backend/e2e_client.py @@ -11,6 +11,7 @@ import socket import struct import sys +import threading import time import urllib.error import urllib.request @@ -164,6 +165,50 @@ def streaming(path, marker_first, marker_last): return body +def concurrency(): + """A slow stream on one connection must not hold up a request on another.""" + result = {} + + def slow(): + result["stream"] = timed_stream("/stream") + t = threading.Thread(target=slow) + t.start() + time.sleep(0.3) + start = time.time() + st, h, b = http("GET", "/health") + took = time.time() - start + stream_running = t.is_alive() + t.join() + check("request beside a slow stream answers", st == 200, st) + check("request beside a slow stream is not held up", + took < 0.5 and stream_running, "took %.2fs, stream still running %s" % (took, stream_running)) + check("the slow stream still completes", + result.get("stream", ("", b"", []))[1] == b"chunk 0\nchunk 1\nchunk 2\n", result.get("stream")) + + +def background_tasks(): + """Background tasks run after the response is sent, as in Starlette.""" + http("DELETE", "/events") + start = time.time() + st, h, b = http("GET", "/background") + took = time.time() - start + check("GET /background 200", st == 200 and json.loads(b) == {"queued": True}, (st, b)) + check("response does not wait for its background task", took < 0.4, "took %.2fs" % took) + time.sleep(1.0) + st, h, b = http("GET", "/events") + check("background task ran after the handler", + json.loads(b)["events"] == ["handler", "task plain"], b) + + http("DELETE", "/events") + head, body, arrivals = timed_stream("/background-stream") + check("/background-stream body", body == b"bg 0\nbg 1\n", body) + check("/background-stream ends before its task", arrivals and arrivals[-1][0] < 0.9, arrivals) + time.sleep(1.0) + st, h, b = http("GET", "/events") + check("streamed background task ran after the last chunk", + json.loads(b)["events"] == ["stream done", "task stream"], b) + + def fastapi(): st, h, b = http("GET", "/health") check("GET /health 200", st == 200, st) @@ -234,6 +279,9 @@ def fastapi(): st, h, b = http("GET", "/health") check("still serving after websockets", st == 200, st) + concurrency() + background_tasks() + def flask(): st, h, b = http("GET", "/") diff --git a/scripts/python-backend/samples/fastapi-chat/src/main.py b/scripts/python-backend/samples/fastapi-chat/src/main.py index c89da041f97..840b401fa84 100644 --- a/scripts/python-backend/samples/fastapi-chat/src/main.py +++ b/scripts/python-backend/samples/fastapi-chat/src/main.py @@ -8,7 +8,7 @@ import sys import time -from fastapi import FastAPI, WebSocket, WebSocketDisconnect +from fastapi import BackgroundTasks, FastAPI, WebSocket, WebSocketDisconnect from fastapi.responses import StreamingResponse, JSONResponse app = FastAPI(title="chat") @@ -92,6 +92,45 @@ async def gen(): return StreamingResponse(gen(), media_type="text/plain") +events = [] + + +def _late(name): + # Slow enough that a server running it before (or while holding up) the + # response would be seen by the client's clock. + time.sleep(0.5) + events.append("task " + name) + + +@app.get("/background") +def background(tasks: BackgroundTasks): + events.append("handler") + tasks.add_task(_late, "plain") + return {"queued": True} + + +@app.get("/background-stream") +def background_stream(tasks: BackgroundTasks): + def gen(): + for i in range(2): + yield "bg %d\n" % i + time.sleep(0.3) + events.append("stream done") + tasks.add_task(_late, "stream") + return StreamingResponse(gen(), media_type="text/plain", background=tasks) + + +@app.get("/events") +def list_events(): + return {"events": events} + + +@app.delete("/events") +def clear_events(): + del events[:] + return {"events": events} + + @app.websocket("/ws/echo") async def echo(ws: WebSocket): await ws.accept() From a8ce9380994b288aee7b50a8c3d1175ac7dde5df Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 11:27:45 +0200 Subject: [PATCH 061/112] Python SQLAlchemy on real PostgreSQL: aliased self join, Decimal, error labels, os.environ Repeated result column names are labelled name_1 as SQLAlchemy does (the backend drivers key a row by column name, so a self join lost a column). Numeric binds and returns decimal.Decimal. DB-API errors name the driver's module. os.environ is filled from System.getenv, and getenv falls back to a by-name read where the VM cannot enumerate the environment. The real-DB script resets the PostgreSQL schema before each program; the query program orders by id and the passive_deletes check is engine independent. Co-Authored-By: Claude Opus 5.5 --- .../codename1/python/runtime/StdModules.java | 19 +++++ .../python/site/sqlalchemy/__init__.py | 85 ++++++++++++++++++- .../codename1/python/site/sqlalchemy/exc.py | 4 +- .../codename1/python/stdlib/os/__init__.py | 12 ++- .../conformance/sqlalchemy_query.out | 4 +- .../resources/conformance/sqlalchemy_query.py | 11 +-- .../conformance/sqlalchemy_relations.out | 2 +- .../conformance/sqlalchemy_relations.py | 6 +- .../python-backend/sqlalchemy-realdb-test.sh | 27 +++++- 9 files changed, 154 insertions(+), 16 deletions(-) diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java index 1ccd62848fa..494919a1625 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java @@ -363,6 +363,25 @@ private static PyModule fileSystem() { }); fs(m, "size", (fs, a) -> PyInt.of(fs.size(Py.toStr(a[0])))); fs(m, "mtime", (fs, a) -> Py.ofFloat(fs.lastModified(Py.toStr(a[0])) / 1000.0)); + // The process environment, for os.environ. ParparVM's System.getenv() + // map is empty (it cannot enumerate the environment), so `getenv` reads + // one variable by name as well; the JVM answers both. + m.setAttr("environ", PyBuiltin.positional("environ", a -> { + PyDict d = new PyDict(); + java.util.Map env = System.getenv(); + if (env != null) { + for (java.util.Map.Entry e : env.entrySet()) { + if (e.getKey() != null && e.getValue() != null) { + d.setItem(Py.str(e.getKey()), Py.str(e.getValue())); + } + } + } + return d; + })); + m.setAttr("getenv", PyBuiltin.positional("getenv", a -> { + String v = System.getenv(Py.toStr(a[0])); + return v == null ? Py.NONE : Py.str(v); + })); return m; } diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/__init__.py index ef0597cf468..5800c72627e 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/__init__.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/__init__.py @@ -16,6 +16,7 @@ """ import datetime as _dt +import decimal as _decimal import logging as _logging import sqlite3 as _sqlite3 @@ -90,8 +91,42 @@ def result(self, value): class Numeric(Float): + """``decimal.Decimal`` both ways, as SQLAlchemy's ``Numeric`` (``asdecimal=True``). + + A Decimal is bound as its text, which every engine casts exactly; SQLite hands back a + float, which is rounded to the column's scale the way SQLAlchemy does.""" _ddl = "NUMERIC" + def __init__(self, precision=None, scale=None, asdecimal=True, **kw): + self.precision = precision + self.scale = scale + self.asdecimal = asdecimal + + def ddl(self): + if self.precision is None: + return self._ddl + if self.scale is None: + return "%s(%d)" % (self._ddl, self.precision) + return "%s(%d, %d)" % (self._ddl, self.precision, self.scale) + + def bind(self, value): + if value is not None and isinstance(value, _decimal.Decimal): + return str(value) + return value + + def result(self, value): + if value is None: + return None + if not self.asdecimal: + return float(value) + if isinstance(value, _decimal.Decimal): + return value + if isinstance(value, float): + if self.scale is not None: + return _decimal.Decimal("%.*f" % (self.scale, value)) + return _decimal.Decimal(repr(value)) + return _decimal.Decimal(str(value)) + class Boolean(TypeEngine): _ddl = "BOOLEAN" @@ -482,6 +517,47 @@ def false(): return _BoolConst(False) +def _result_name(col_sql): + """The name a server gives a compiled result column, or None for an expression.""" + i = col_sql.rfind(" AS ") + if i >= 0: + tail = col_sql[i + 4:].strip() + if tail and all(ch.isalnum() or ch in "_\"`" for ch in tail): + return tail.strip("\"`") + if col_sql and all(ch.isalnum() or ch in "_.\"`" for ch in col_sql): + return col_sql.split(".")[-1].strip("\"`") + return None + + +def _dedupe_labels(cols): + """Labels a repeated result column ``name_1``, ``name_2``... as SQLAlchemy does. + + Rows are read by position, but the backend's drivers hand each row over keyed by + column name, so ``SELECT users.name, users_1.name`` came back one column short (an + aliased self join). Distinct names keep every column.""" + seen = set() + for c in cols: + n = _result_name(c) + if n is not None: + seen.add(n) + taken = set() + for i in range(len(cols)): + n = _result_name(cols[i]) + if n is None: + continue + if n in taken: + k = 1 + while (n + "_" + str(k)) in seen: + k += 1 + alias = n + "_" + str(k) + seen.add(alias) + j = cols[i].rfind(" AS ") + base = cols[i][:j] if j >= 0 else cols[i] + cols[i] = base + " AS " + alias + n = alias + taken.add(n) + + class Label(ColumnElement): def __init__(self, name, elem): self.name = self.key = name @@ -1071,6 +1147,7 @@ def _compile(self, ctx): else: cols.append(sl.elem._compile(ctx)) ctx.columns_clause = False + _dedupe_labels(cols) sql = "SELECT " + ("DISTINCT " if self._distinct else "") + ", ".join(cols) sql = self._compile_rest(ctx, sql, froms) ctx.outer = saved_outer @@ -1844,7 +1921,8 @@ def _run(self, sql, params, many=False): else: cur.execute(sql, params) except _sqlite3.Error as e: - raise exc._wrap(e, sql, params) + d = self.engine.dialect + raise exc._wrap(e, sql, params, _DBAPI_MODULES.get(d.driver) or _DBAPI_MODULES.get(d.name, "sqlite3")) return cur def execute(self, statement, parameters=None, execution_options=None): @@ -2265,3 +2343,8 @@ def create_engine(url, echo=False, **kwargs): def inspect(obj): return obj + +# The DB-API module SQLAlchemy names in an error, "(psycopg2.OperationalError) ...", by driver. +_DBAPI_MODULES = {"pysqlite": "sqlite3", "psycopg": "psycopg", "psycopg2": "psycopg2", "asyncpg": "asyncpg", + "mysqldb": "MySQLdb", "pymysql": "pymysql", "mariadbconnector": "mariadb", + "postgresql": "psycopg", "mysql": "MySQLdb", "mariadb": "MySQLdb"} diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/exc.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/exc.py index a7978c55d17..0ae71c6f7c4 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/exc.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/exc.py @@ -85,8 +85,8 @@ class NotSupportedError(DatabaseError): } -def _wrap(e, statement, params): +def _wrap(e, statement, params, driver="sqlite3"): """Wraps a DB-API exception the way SQLAlchemy does, keeping the original as ``orig``.""" cls = _BY_NAME.get(type(e).__name__, DBAPIError) - msg = "(sqlite3.%s) %s\n[SQL: %s]" % (type(e).__name__, e, statement) + msg = "(%s.%s) %s\n[SQL: %s]" % (driver, type(e).__name__, e, statement) return cls(msg, statement, params, e) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/os/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/os/__init__.py index a62e929f664..b0924a3e92d 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/os/__init__.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/os/__init__.py @@ -30,20 +30,28 @@ W_OK = 2 X_OK = 1 -environ = {} +environ = _cn1fs.environ() +_unset = set() error = OSError def getenv(key, default=None): - return environ.get(key, default) + if key in environ: + return environ[key] + # ParparVM cannot enumerate the environment, so environ may lack a + # variable that is still readable by name. + v = None if key in _unset else _cn1fs.getenv(key) + return default if v is None else v def putenv(key, value): environ[key] = value + _unset.discard(key) def unsetenv(key): environ.pop(key, None) + _unset.add(key) class PathLike: diff --git a/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_query.out b/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_query.out index 8742e7c8e2e..89107e5b5b7 100644 --- a/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_query.out +++ b/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_query.out @@ -6,11 +6,11 @@ True sqlalchemy.engine.Engine False ['Ann', 'bob'] ['Ann', 'bob'] ['Cid'] -['Ann', 'Cid', 'bob'] +['Ann', 'bob', 'Cid'] ['Cid', 'dee'] [19, 19, 25, 25] ['Ann', 'dee'] -[('Cid', 'Ann'), ('bob', 'Ann')] +[('bob', 'Ann'), ('Cid', 'Ann')] Cid [('Ann', 'adult'), ('bob', 'adult'), ('Cid', 'senior'), ('dee', 'teen')] ['30y', '25y', '41y', '19y'] diff --git a/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_query.py b/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_query.py index d7e7ed6940b..d4d0ef8bbb4 100644 --- a/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_query.py +++ b/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_query.py @@ -44,6 +44,7 @@ def emit(self, record): engine = create_engine(sys.argv[1] if len(sys.argv) > 1 else "sqlite://", echo=True) engine.echo = False Base.metadata.drop_all(engine) # a server database keeps tables between runs +# Every query is ordered by id: a server collation sorts "bob" and "Cid" differently from SQLite. engine.echo = True Base.metadata.create_all(engine) s = Session(engine) @@ -64,7 +65,7 @@ def emit(self, record): # subquery in FROM sub = select(Address.user_id, func.count(Address.id).label("c")).group_by(Address.user_id).subquery() -print(s.execute(select(User.name, sub.c.c).join(sub, User.id == sub.c.user_id).order_by(User.name)).all()) +print(s.execute(select(User.name, sub.c.c).join(sub, User.id == sub.c.user_id).order_by(User.id)).all()) # exists print(s.scalars(select(User.name).where(exists().where(Address.user_id == User.id)).order_by(User.id)).all()) @@ -75,11 +76,11 @@ def emit(self, record): print(s.scalars(select(User.name).where(User.id.in_(select(Address.user_id).where(Address.email.like("%a.com")))) .order_by(User.id)).all()) print(s.scalars(select(User.name).where(User.age.between(20, 35)).order_by(User.id)).all()) -print(s.scalars(select(User.name).where(User.name.ilike("c%"))).all()) +print(s.scalars(select(User.name).where(User.name.ilike("c%")).order_by(User.id)).all()) # cte old = select(User.id, User.name).where(User.age > 24).cte("old") -print(s.execute(select(old.c.name).order_by(old.c.name)).scalars().all()) +print(s.execute(select(old.c.name).order_by(old.c.id)).scalars().all()) # union / union_all u = union(select(User.name).where(User.age < 20), select(User.name).where(User.age > 40)) @@ -90,7 +91,7 @@ def emit(self, record): # aliased self join boss = aliased(User) -print(s.execute(select(User.name, boss.name).join(boss, User.boss_id == boss.id).order_by(User.name)).all()) +print(s.execute(select(User.name, boss.name).join(boss, User.boss_id == boss.id).order_by(User.id)).all()) print(s.scalars(select(boss).where(boss.age > 40)).one().name) # case, cast, functions @@ -103,7 +104,7 @@ def emit(self, record): print(s.execute(select(func.coalesce(User.boss_id, -1)).order_by(User.id)).scalars().all()) # relationship any / has -print(s.scalars(select(User.name).where(User.addresses.any(Address.email.like("%.org")))).all()) +print(s.scalars(select(User.name).where(User.addresses.any(Address.email.like("%.org"))).order_by(User.id)).all()) print(s.scalars(select(User.name).where(User.addresses.any()).order_by(User.id)).all()) print(s.scalars(select(Address.email).where(Address.user.has(User.age > 26)).order_by(Address.id)).all()) diff --git a/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_relations.out b/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_relations.out index de45ce569db..7e93688ca11 100644 --- a/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_relations.out +++ b/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_relations.out @@ -7,7 +7,7 @@ ['b2', 'b3'] 0 [('n1', None), ('n2', None)] 2 -[1, 1] +True True ['x', 'y'] ['z'] cy diff --git a/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_relations.py b/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_relations.py index 64b7fab00be..d545f5baaf0 100644 --- a/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_relations.py +++ b/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_relations.py @@ -149,7 +149,11 @@ class Truck(Vehicle): s.expire(a, ["memos"]) s.delete(a) s.commit() - print(s.scalars(select(Memo.author_id)).all()) + # passive_deletes: the ORM leaves the memos to the database's ON DELETE CASCADE and never + # nulls them. SQLite enforces no foreign keys by default and keeps them; a server deletes + # them. Either way none is left pointing at NULL or at another author. + left = s.scalars(select(Memo.author_id)).all() + print(len(left) in (0, 2), all(x == aid for x in left)) # ---- eager loading with Session(engine) as s: diff --git a/scripts/python-backend/sqlalchemy-realdb-test.sh b/scripts/python-backend/sqlalchemy-realdb-test.sh index d63e11b5f2c..8e818c4de78 100755 --- a/scripts/python-backend/sqlalchemy-realdb-test.sh +++ b/scripts/python-backend/sqlalchemy-realdb-test.sh @@ -26,8 +26,8 @@ HERE="$(cd "$(dirname "$0")" && pwd)" REPO="$(cd "$HERE/../.." && pwd)" M2="${1:-$REPO/.m2-repo}" J17="${JAVA17_HOME:?set JAVA17_HOME}" -CONF="$REPO/maven/python-transpiler/src/test/resources/conformance" -PROGRAMS="sqlalchemy_core sqlalchemy_orm sqlalchemy_relations sqlalchemy_query" +CONF="${CN1_SQLA_CONF:-$REPO/maven/python-transpiler/src/test/resources/conformance}" +PROGRAMS="${CN1_SQLA_PROGRAMS:-sqlalchemy_core sqlalchemy_orm sqlalchemy_relations sqlalchemy_query}" art() { echo "$M2/$1/$2/$3/$2-$3.jar"; } CORE=$(art com/codenameone codenameone-core 8.0-SNAPSHOT) RUNTIME=$(art com/codenameone codenameone-python-runtime 8.0-SNAPSHOT) @@ -66,6 +66,28 @@ public class Main { System.exit(com.codename1.generated.python.PythonModules.runMain()); } } +JAVA + # Each program starts from an empty database: whatever an earlier program + # (or an earlier run) left behind -- tables, sequences, types -- would + # otherwise leak into its output. + cat >"$APP/gen/com/pybackend/Reset.java" <<'JAVA' +package com.pybackend; + +import com.codename1.backend.Database; + +public class Reset { + public static void main(String[] args) throws Exception { + Database db = Database.open(args[0]); + try { + if ("postgresql".equals(db.dialect().getName())) { + db.execute("DROP SCHEMA public CASCADE", new Object[0]); + db.execute("CREATE SCHEMA public", new Object[0]); + } + } finally { + db.close(); + } + } +} JAVA (cd "$APP/classes" && unzip -qo "$RUNTIME" 'com/*' && unzip -qo "$CORE" 'com/codename1/util/*') "$J17/bin/javac" -nowarn --release 17 -d "$APP/classes" $(find "$REPO/scripts/python-bench/hostfs" -name '*.java') @@ -78,6 +100,7 @@ JAVA mariadb) url="$CN1_SQLA_MARIADB" ;; esac out="$APP/$engine.out" + "$J17/bin/java" -cp "$APP/classes:$BACKEND:$CORE" com.pybackend.Reset "$url" "$J17/bin/java" -cp "$APP/classes:$BACKEND:$CORE" com.pybackend.Main "$url" >"$out" 2>&1 || true if diff -u "$CONF/$prog.out" "$out" >"$APP/$engine.diff"; then echo "PASS $prog on $engine" From 8c72263a94b082ea3292dd4ffb9f69d028d2bbf2 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 11:38:21 +0200 Subject: [PATCH 062/112] Guide: SQLAlchemy section, its example held by a conformance program Co-Authored-By: Claude Opus 5.5 --- .../developer-guide/python-interop.py | 38 +++++++++++++++++++ docs/developer-guide/Python-Interop.asciidoc | 21 ++++++++++ .../conformance/guide_sqlalchemy.out | 1 + .../resources/conformance/guide_sqlalchemy.py | 35 +++++++++++++++++ 4 files changed, 95 insertions(+) create mode 100644 maven/python-transpiler/src/test/resources/conformance/guide_sqlalchemy.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/guide_sqlalchemy.py diff --git a/docs/demos/common/src/main/snippets/developer-guide/python-interop.py b/docs/demos/common/src/main/snippets/developer-guide/python-interop.py index f0a223a5947..0bf99987130 100644 --- a/docs/demos/common/src/main/snippets/developer-guide/python-interop.py +++ b/docs/demos/common/src/main/snippets/developer-guide/python-interop.py @@ -207,3 +207,41 @@ def int_arithmetic() -> int: checksum = CommonWorkloads.intArithmetic() // end::python-interop-python-006[] + +// tag::python-interop-python-007[] +import os +from sqlalchemy import ForeignKey, String, create_engine, select +from sqlalchemy.orm import (DeclarativeBase, Mapped, Session, mapped_column, + relationship, selectinload) + + +class Base(DeclarativeBase): + pass + + +class Author(Base): + __tablename__ = "authors" + id: Mapped[int] = mapped_column(primary_key=True) + name: Mapped[str] = mapped_column(String(80)) + books: Mapped[list["Book"]] = relationship( + back_populates="author", cascade="all, delete-orphan") + + +class Book(Base): + __tablename__ = "books" + id: Mapped[int] = mapped_column(primary_key=True) + title: Mapped[str] = mapped_column(String(200)) + author_id: Mapped[int] = mapped_column(ForeignKey("authors.id")) + author: Mapped[Author] = relationship(back_populates="books") + + +engine = create_engine(os.environ.get("DATABASE_URL", "sqlite:///library.db")) +Base.metadata.create_all(engine) + +with Session(engine) as session: + session.add(Author(name="Ursula", books=[Book(title="The Dispossessed")])) + session.commit() + query = select(Author).options(selectinload(Author.books)).order_by(Author.id) + for author in session.scalars(query): + print(author.name, [b.title for b in author.books]) +// end::python-interop-python-007[] diff --git a/docs/developer-guide/Python-Interop.asciidoc b/docs/developer-guide/Python-Interop.asciidoc index 9ebc96766c2..af19c7e326d 100644 --- a/docs/developer-guide/Python-Interop.asciidoc +++ b/docs/developer-guide/Python-Interop.asciidoc @@ -333,6 +333,27 @@ is open, and pure ASGI middleware sees WebSocket connections as it does in Starlette. Denying a connection answers with the status the endpoint or middleware chose, such as 403. +==== SQLAlchemy + +SQLAlchemy's Core and ORM run over the platform's SQLite, and on a backend +over the PostgreSQL, MySQL and MariaDB drivers the Codename One backend ships. +The engine URL picks the database, as it does under CPython: + +[source,python] +---- +include::../demos/common/src/main/snippets/developer-guide/python-interop.py[tag=python-interop-python-007,indent=0] +---- + +Declarative models with `Mapped` annotations, relationships in both +directions (including many-to-many and cascades), `selectinload` and +`joinedload`, joins, aliases, subqueries, common table expressions, unions, +single and joined table inheritance and `text()` queries are supported. +`Numeric` columns come back as `decimal.Decimal` and `DateTime` and `Date` +columns as `datetime` values on every database. A `text()` query has no +column types, so on a server database its numbers and dates come back as +the text the server sent. SQLAlchemy's own drivers, such as `psycopg`, aren't +used: the URL `postgresql+psycopg://` selects the backend's driver. + === Compared with CPython, PyPy and GraalPy The same programs, run on one Apple silicon Mac with diff --git a/maven/python-transpiler/src/test/resources/conformance/guide_sqlalchemy.out b/maven/python-transpiler/src/test/resources/conformance/guide_sqlalchemy.out new file mode 100644 index 00000000000..66e8e68e4e8 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/guide_sqlalchemy.out @@ -0,0 +1 @@ +Ursula ['The Dispossessed'] diff --git a/maven/python-transpiler/src/test/resources/conformance/guide_sqlalchemy.py b/maven/python-transpiler/src/test/resources/conformance/guide_sqlalchemy.py new file mode 100644 index 00000000000..6e099a63fff --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/guide_sqlalchemy.py @@ -0,0 +1,35 @@ +import os +from sqlalchemy import ForeignKey, String, create_engine, select +from sqlalchemy.orm import (DeclarativeBase, Mapped, Session, mapped_column, + relationship, selectinload) + + +class Base(DeclarativeBase): + pass + + +class Author(Base): + __tablename__ = "authors" + id: Mapped[int] = mapped_column(primary_key=True) + name: Mapped[str] = mapped_column(String(80)) + books: Mapped[list["Book"]] = relationship( + back_populates="author", cascade="all, delete-orphan") + + +class Book(Base): + __tablename__ = "books" + id: Mapped[int] = mapped_column(primary_key=True) + title: Mapped[str] = mapped_column(String(200)) + author_id: Mapped[int] = mapped_column(ForeignKey("authors.id")) + author: Mapped[Author] = relationship(back_populates="books") + + +engine = create_engine(os.environ.get("DATABASE_URL", "sqlite://")) +Base.metadata.create_all(engine) + +with Session(engine) as session: + session.add(Author(name="Ursula", books=[Book(title="The Dispossessed")])) + session.commit() + query = select(Author).options(selectinload(Author.books)).order_by(Author.id) + for author in session.scalars(query): + print(author.name, [b.title for b in author.books]) From 6b57f2c4e7c295bd7ac06a12a6a33b18b9edac4b Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 11:44:05 +0200 Subject: [PATCH 063/112] Python: itsdangerous-compatible Flask session cookies, Jinja macros/call/import/filter blocks Flask sessions now use an itsdangerous URLSafeTimedSerializer (salt cookie-session, hmac key derivation over SHA-1, Flask's tagged JSON, zlib with '.' prefix, base64 timestamp), so cookies interoperate with CPython Flask. Adds an itsdangerous subset module. Jinja gains macro (defaults, varargs, kwargs, caller), call blocks, import / from-import, filter blocks, filtered set blocks, loop.changed/depth and cycler. Co-Authored-By: Claude Opus 5.5 --- .../codename1/python/site/flask/__init__.py | 159 ++++++-- .../python/site/itsdangerous/__init__.py | 339 ++++++++++++++++++ .../codename1/python/site/jinja2/__init__.py | 236 +++++++++++- .../conformance/flask_session_compat.out | 14 + .../conformance/flask_session_compat.py | 87 +++++ .../resources/conformance/jinja_macros.out | 17 + .../resources/conformance/jinja_macros.py | 55 +++ 7 files changed, 866 insertions(+), 41 deletions(-) create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/itsdangerous/__init__.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/flask_session_compat.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/flask_session_compat.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/jinja_macros.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/jinja_macros.py diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/flask/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/flask/__init__.py index c97c8f56662..376d38dae18 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/site/flask/__init__.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/flask/__init__.py @@ -9,7 +9,6 @@ """ import base64 as _base64 import hashlib as _hashlib -import hmac as _hmac import io as _io import json as _json import os as _os @@ -1418,50 +1417,136 @@ def _fail(self, *a, **k): setdefault = _fail -def _b64e(b): - return _base64.urlsafe_b64encode(b).rstrip(b"=").decode("ascii") - - -def _b64d(s): - s = s.encode("ascii") - return _base64.urlsafe_b64decode(s + b"=" * (-len(s) % 4)) +_MONTHS = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"] +_DAYS = ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"] + + +def _http_date(d): + import datetime as _dt + if d.tzinfo is not None: + d = d.astimezone(_dt.timezone.utc) + return "%s, %02d %s %04d %02d:%02d:%02d GMT" % (_DAYS[d.weekday()], d.day, _MONTHS[d.month - 1], + d.year, d.hour, d.minute, d.second) + + +def _parse_http_date(s): + import datetime as _dt + parts = s.replace(",", " ").split() + day, mon, year = int(parts[1]), _MONTHS.index(parts[2]) + 1, int(parts[3]) + h, m, sec = [int(x) for x in parts[4].split(":")] + return _dt.datetime(year, mon, day, h, m, sec, tzinfo=_dt.timezone.utc) + + +class TaggedJSONSerializer: + """Flask's tagged JSON (``flask.json.tag``): the tags `` t`` (tuple), + `` b`` (bytes), `` m`` (Markup), `` u`` (UUID), `` d`` (datetime) and + `` di`` (a one-key dict whose key is itself a tag) are written and read + exactly as Flask writes them, so a session holding any of those types + round-trips with a CPython server.""" + + def tag(self, value): + import datetime as _dt + if isinstance(value, dict): + if len(value) == 1: + k = next(iter(value)) + if k in (" t", " b", " m", " u", " d", " di"): + return {" di": {k + "__": self.tag(value[k])}} + return {k: self.tag(v) for k, v in value.items()} + if isinstance(value, tuple): + return {" t": [self.tag(x) for x in value]} + if isinstance(value, list): + return [self.tag(x) for x in value] + if isinstance(value, (bytes, bytearray)): + return {" b": _base64.b64encode(bytes(value)).decode("ascii")} + if callable(getattr(value, "__html__", None)): + return {" m": str(value.__html__())} + if isinstance(value, _uuid.UUID): + return {" u": value.hex} + if isinstance(value, _dt.datetime): + return {" d": _http_date(value)} + return value + + def untag(self, value): + if isinstance(value, dict): + if len(value) == 1: + k = next(iter(value)) + v = value[k] + if k == " t": + return tuple(v) + if k == " b": + return _base64.b64decode(v) + if k == " m": + return Markup(v) + if k == " u": + return _uuid.UUID(v) + if k == " d": + return _parse_http_date(v) + if k == " di": + ik = next(iter(v)) + return {ik[:-2]: v[ik]} + return value + return value + + def _scan(self, value): + if isinstance(value, dict): + value = {k: self._scan(v) for k, v in value.items()} + elif isinstance(value, list): + value = [self._scan(x) for x in value] + return self.untag(value) + + def dumps(self, value, **kwargs): + return _json.dumps(self.tag(value), separators=(",", ":"), sort_keys=True) + + def loads(self, value): + return self._scan(_json.loads(value)) + + +session_json_serializer = TaggedJSONSerializer() class SecureCookieSessionInterface: - """Signed (not encrypted) cookie: ``base64url(json) "." base64url(HMAC-SHA1)``. - - The key is derived from the app's secret key and the salt - ``cookie-session``, but the cookie carries no timestamp, so it is not - byte-compatible with the cookie a CPython Flask server writes. - """ + """Flask's own cookie format: an ``itsdangerous.URLSafeTimedSerializer`` + with the salt ``cookie-session``, the ``hmac`` key derivation over SHA-1 + and Flask's tagged JSON, so a cookie written by a CPython Flask server + with the same ``secret_key`` opens here and the other way around. + ``SECRET_KEY_FALLBACKS`` keys verify but never sign, as in Flask.""" salt = "cookie-session" + key_derivation = "hmac" + serializer = session_json_serializer - def _key(self, app): - sk = app.secret_key - if isinstance(sk, str): - sk = sk.encode("utf-8") - return _hashlib.sha1(self.salt.encode("utf-8") + b"signer" + sk).digest() - - def _sign(self, app, payload): - return _b64e(_hmac.new(self._key(app), payload.encode("ascii"), _hashlib.sha1).digest()) + def get_signing_serializer(self, app): + import itsdangerous as _its + if not app.secret_key: + return None + keys = list(app.config.get("SECRET_KEY_FALLBACKS") or []) + keys.append(app.secret_key) + return _its.URLSafeTimedSerializer(keys, salt=self.salt, serializer=self.serializer, + signer_kwargs={"key_derivation": self.key_derivation, + "digest_method": _hashlib.sha1}) + + def _max_age(self, app): + life = app.config.get("PERMANENT_SESSION_LIFETIME", 31 * 86400) + if hasattr(life, "total_seconds"): + life = life.total_seconds() + return int(life) def open_session(self, app, req): - if not app.secret_key: + import itsdangerous as _its + s = self.get_signing_serializer(app) + if s is None: return NullSession() name = app.config.get("SESSION_COOKIE_NAME", "session") raw = req.cookies.get(name) - if raw and "." in raw: - payload, sig = raw.rsplit(".", 1) - if _hmac.compare_digest(self._sign(app, payload), sig): - try: - data = _json.loads(_b64d(payload).decode("utf-8")) - if isinstance(data, dict): - return SecureCookieSession(data) - except ValueError: - pass - s = SecureCookieSession() - s.new = True - return s + if raw: + try: + data = s.loads(raw, max_age=self._max_age(app)) + if isinstance(data, dict): + return SecureCookieSession(data) + except _its.BadSignature: + pass + sess = SecureCookieSession() + sess.new = True + return sess def save_session(self, app, sess, resp): if isinstance(sess, NullSession): @@ -1479,8 +1564,8 @@ def save_session(self, app, sess, resp): return if not sess.modified: return - payload = _b64e(_json.dumps(dict(sess), separators=(",", ":"), sort_keys=True).encode("utf-8")) - resp.set_cookie(name, payload + "." + self._sign(app, payload), httponly=True) + value = self.get_signing_serializer(app).dumps(dict(sess)) + resp.set_cookie(name, value, httponly=True) def flash(message, category="message"): diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/itsdangerous/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/itsdangerous/__init__.py new file mode 100644 index 00000000000..dcc7a843b1f --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/itsdangerous/__init__.py @@ -0,0 +1,339 @@ +"""A subset of itsdangerous, wire compatible with the real package. + +Signer, TimestampSigner, Serializer, TimedSerializer, URLSafeSerializer and +URLSafeTimedSerializer produce and accept exactly the bytes itsdangerous 2.x +does for the same secret key, salt and key derivation, so a value signed by a +CPython server verifies here and the other way around. Supported key +derivations are ``django-concat`` (the default), ``concat``, ``hmac`` and +``none``; the digest is SHA-1 unless ``digest_method`` names another hashlib +constructor. Key rotation (a list of secret keys) is supported: the last key +signs, every key verifies. +""" +import base64 as _base64 +import hashlib as _hashlib +import hmac as _hmac +import json as _json +import time as _time +import zlib as _zlib + +__all__ = ["BadData", "BadSignature", "BadTimeSignature", "BadPayload", "SignatureExpired", + "Signer", "TimestampSigner", "Serializer", "TimedSerializer", + "URLSafeSerializer", "URLSafeTimedSerializer", "base64_encode", "base64_decode"] + + +class BadData(Exception): + def __init__(self, message): + Exception.__init__(self, message) + self.message = message + + def __str__(self): + return self.message + + +class BadSignature(BadData): + def __init__(self, message, payload=None): + BadData.__init__(self, message) + self.payload = payload + + +class BadTimeSignature(BadSignature): + def __init__(self, message, payload=None, date_signed=None): + BadSignature.__init__(self, message, payload) + self.date_signed = date_signed + + +class SignatureExpired(BadTimeSignature): + pass + + +class BadHeader(BadSignature): + pass + + +class BadPayload(BadData): + def __init__(self, message, original_error=None): + BadData.__init__(self, message) + self.original_error = original_error + + +def want_bytes(s, encoding="utf-8"): + if isinstance(s, str): + return s.encode(encoding) + return s + + +def base64_encode(string): + return _base64.urlsafe_b64encode(want_bytes(string)).rstrip(b"=") + + +def base64_decode(string): + string = want_bytes(string) + try: + return _base64.urlsafe_b64decode(string + b"=" * (-len(string) % 4)) + except Exception as e: + raise BadData("Could not base64 decode the payload because of an exception") + + +def int_to_bytes(num): + out = bytearray() + while num > 0: + out.insert(0, num & 0xFF) + num >>= 8 + return bytes(out) + + +def bytes_to_int(b): + n = 0 + for c in b: + n = (n << 8) | c + return n + + +def _secret_keys(secret_key): + if isinstance(secret_key, (str, bytes)): + return [want_bytes(secret_key)] + return [want_bytes(k) for k in secret_key] + + +class Signer: + default_digest_method = staticmethod(_hashlib.sha1) + default_key_derivation = "django-concat" + + def __init__(self, secret_key, salt=b"itsdangerous.Signer", sep=b".", key_derivation=None, + digest_method=None, algorithm=None): + self.secret_keys = _secret_keys(secret_key) + self.sep = want_bytes(sep) + if salt is None: + salt = b"itsdangerous.Signer" + self.salt = want_bytes(salt) + self.key_derivation = key_derivation or self.default_key_derivation + self.digest_method = digest_method or self.default_digest_method + + @property + def secret_key(self): + return self.secret_keys[-1] + + def derive_key(self, secret_key=None): + if secret_key is None: + secret_key = self.secret_keys[-1] + else: + secret_key = want_bytes(secret_key) + kd = self.key_derivation + if kd == "concat": + return self.digest_method(self.salt + secret_key).digest() + if kd == "django-concat": + return self.digest_method(self.salt + b"signer" + secret_key).digest() + if kd == "hmac": + return _hmac.new(secret_key, self.salt, self.digest_method).digest() + if kd == "none": + return secret_key + raise TypeError("Unknown key derivation method") + + def _sig(self, key, value): + return base64_encode(_hmac.new(key, value, self.digest_method).digest()) + + def get_signature(self, value): + return self._sig(self.derive_key(), want_bytes(value)) + + def sign(self, value): + value = want_bytes(value) + return value + self.sep + self.get_signature(value) + + def verify_signature(self, value, sig): + value = want_bytes(value) + sig = want_bytes(sig) + for k in reversed(self.secret_keys): + if _hmac.compare_digest(self._sig(self.derive_key(k), value), sig): + return True + return False + + def unsign(self, signed_value): + signed_value = want_bytes(signed_value) + if self.sep not in signed_value: + raise BadSignature("No '" + self.sep.decode("utf-8") + "' found in value") + value, sig = signed_value.rsplit(self.sep, 1) + if self.verify_signature(value, sig): + return value + raise BadSignature("Signature " + repr(sig) + " does not match", payload=value) + + def validate(self, signed_value): + try: + self.unsign(signed_value) + return True + except BadSignature: + return False + + +class TimestampSigner(Signer): + def get_timestamp(self): + return int(_time.time()) + + def timestamp_to_datetime(self, ts): + import datetime as _dt + return _dt.datetime.fromtimestamp(ts, _dt.timezone.utc) + + def sign(self, value): + value = want_bytes(value) + value = value + self.sep + base64_encode(int_to_bytes(self.get_timestamp())) + return value + self.sep + self.get_signature(value) + + def unsign(self, signed_value, max_age=None, return_timestamp=False): + try: + result = Signer.unsign(self, signed_value) + sig_error = None + except BadSignature as e: + sig_error = e + result = e.payload or b"" + sep = self.sep + if sep not in result: + if sig_error: + raise sig_error + raise BadTimeSignature("timestamp missing", payload=result) + value, ts_bytes = result.rsplit(sep, 1) + ts_int = None + try: + ts_int = bytes_to_int(base64_decode(ts_bytes)) + except Exception: + pass + if sig_error is not None: + raise BadTimeSignature(str(sig_error), payload=value) + if ts_int is None: + raise BadTimeSignature("Malformed timestamp", payload=value) + if max_age is not None: + age = self.get_timestamp() - ts_int + if age > max_age: + raise SignatureExpired("Signature age " + str(age) + " > " + str(max_age) + + " seconds", payload=value) + if age < 0: + raise SignatureExpired("Signature age " + str(age) + " < 0 seconds", payload=value) + if return_timestamp: + return value, self.timestamp_to_datetime(ts_int) + return value + + def validate(self, signed_value, max_age=None): + try: + self.unsign(signed_value, max_age=max_age) + return True + except BadSignature: + return False + + +class _CompactJson: + @staticmethod + def loads(payload): + return _json.loads(payload) + + @staticmethod + def dumps(obj, **kwargs): + kwargs.setdefault("ensure_ascii", False) + kwargs.setdefault("separators", (",", ":")) + return _json.dumps(obj, **kwargs) + + +class Serializer: + default_serializer = _json + default_signer = Signer + + def __init__(self, secret_key, salt=b"itsdangerous", serializer=None, serializer_kwargs=None, + signer=None, signer_kwargs=None, fallback_signers=None): + self.secret_keys = _secret_keys(secret_key) + self.salt = want_bytes(salt) if salt is not None else None + self.serializer = serializer if serializer is not None else self.default_serializer + self.serializer_kwargs = serializer_kwargs or {} + self.signer = signer if signer is not None else self.default_signer + self.signer_kwargs = signer_kwargs or {} + + @property + def secret_key(self): + return self.secret_keys[-1] + + def load_payload(self, payload, serializer=None): + s = serializer if serializer is not None else self.serializer + try: + if isinstance(payload, bytes): + payload = payload.decode("utf-8") + return s.loads(payload) + except Exception as e: + raise BadPayload("Could not load the payload because an exception occurred on" + " unserializing the data.", original_error=e) + + def dump_payload(self, obj): + return want_bytes(self.serializer.dumps(obj, **self.serializer_kwargs)) + + def make_signer(self, salt=None): + if salt is None: + salt = self.salt + return self.signer(self.secret_keys, salt=salt, **self.signer_kwargs) + + def dumps(self, obj, salt=None): + rv = self.make_signer(salt).sign(self.dump_payload(obj)) + return rv.decode("utf-8") + + def loads(self, s, salt=None, **kwargs): + return self.load_payload(self.make_signer(salt).unsign(want_bytes(s), **kwargs)) + + def loads_unsafe(self, s, salt=None, **kwargs): + try: + return True, self.loads(s, salt, **kwargs) + except BadSignature as e: + if e.payload is None: + return False, None + try: + return False, self.load_payload(e.payload) + except BadPayload: + return False, None + + +class TimedSerializer(Serializer): + default_signer = TimestampSigner + + def loads(self, s, max_age=None, return_timestamp=False, salt=None): + signer = self.make_signer(salt) + base64d, ts = signer.unsign(want_bytes(s), max_age=max_age, return_timestamp=True) + payload = self.load_payload(base64d) + if return_timestamp: + return payload, ts + return payload + + +class URLSafeSerializerMixin: + default_serializer = _CompactJson + + def load_payload(self, payload, *args, **kwargs): + payload = want_bytes(payload) + decompress = False + if payload.startswith(b"."): + payload = payload[1:] + decompress = True + try: + json = base64_decode(payload) + except Exception as e: + raise BadPayload("Could not base64 decode the payload because of an exception", + original_error=e) + if decompress: + try: + json = _zlib.decompress(json) + except Exception as e: + raise BadPayload("Could not zlib decompress the payload before decoding the" + " payload", original_error=e) + return Serializer.load_payload(self, json, *args, **kwargs) + + def dump_payload(self, obj): + json = Serializer.dump_payload(self, obj) + is_compressed = False + compressed = _zlib.compress(json) + if len(compressed) < (len(json) - 1): + json = compressed + is_compressed = True + base64d = base64_encode(json) + if is_compressed: + base64d = b"." + base64d + return base64d + + +class URLSafeSerializer(URLSafeSerializerMixin, Serializer): + pass + + +class URLSafeTimedSerializer(URLSafeSerializerMixin, TimedSerializer): + pass diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/jinja2/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/jinja2/__init__.py index 6065bd242cb..063313ec963 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/site/jinja2/__init__.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/jinja2/__init__.py @@ -515,6 +515,21 @@ def _parse_expr(s): # ---- statements ------------------------------------------------------------ +def _params(p): + """Macro parameters after the opening parenthesis: names with optional defaults.""" + params = [] + while not p.accept("op", ")"): + n = p.expect("name")[1] + d = None + if p.accept("op", "="): + d = p.expr() + params.append((n, d)) + if not p.accept("op", ","): + p.expect("op", ")") + break + return params + + def _parse(tokens): pos = [0] blocks = {} @@ -531,6 +546,8 @@ def body(ends): nodes.append(("out", _parse_expr(val))) else: word = val.split(None, 1)[0] if val else "" + if "(" in word: + word = word[:word.index("(")] rest = val[len(word):].strip() if word in ends: return nodes, word, rest @@ -580,8 +597,12 @@ def stmt(word, rest): break e = ("tuple", items) return ("set", targets, e) + filt = None + if p.accept("op", "|"): + fp = _P("__cn1_body|" + rest.split("|", 1)[1]) + filt = fp.expr() b, _, _ = body(("endset",)) - return ("setblock", targets[0], b) + return ("setblock", targets[0], b, filt) if word == "block": name = rest.split()[0] b, _, _ = body(("endblock",)) @@ -609,6 +630,46 @@ def stmt(word, rest): p.accept("op", ",") b, _, _ = body(("endwith",)) return ("with", assigns, b) + if word == "macro": + p = _P(rest) + name = p.expect("name")[1] + p.expect("op", "(") + params = _params(p) + b, _, _ = body(("endmacro",)) + return ("macro", name, params, b) + if word == "call": + p = _P(rest) + params = [] + if p.accept("op", "("): + params = _params(p) + call = p.expr() + if call[0] != "call": + raise TemplateSyntaxError("expected call") + b, _, _ = body(("endcall",)) + return ("callblock", params, call, b) + if word == "filter": + filt = _P("__cn1_body|" + rest).expr() + b, _, _ = body(("endfilter",)) + return ("filterblock", filt, b) + if word == "import": + p = _P(rest) + src_e = p.expr(False) + p.expect("name", "as") + return ("import", src_e, p.expect("name")[1]) + if word == "from": + p = _P(rest) + src_e = p.expr(False) + p.expect("name", "import") + names = [] + while p.at("name") and p.peek()[1] not in ("with", "without"): + n = p.next()[1] + alias = n + if p.accept("name", "as"): + alias = p.expect("name")[1] + names.append((n, alias)) + if not p.accept("op", ","): + break + return ("fromimport", src_e, names) raise TemplateSyntaxError("Encountered unknown tag %r." % (word,)) nodes, _, _ = body(()) @@ -617,11 +678,130 @@ def stmt(word, rest): # ---- runtime --------------------------------------------------------------- +class _TemplateModule: + def __init__(self, name): + self._name = name + self._body = "" + + def __str__(self): + return self._body + + def __html__(self): + return Markup(self._body) + + +class _Macro: + """A ``{% macro %}`` (or the ``caller`` of a ``{% call %}``) bound to the + variables of the frame that defined it.""" + + def __init__(self, tmpl, name, params, body, fr): + self._tmpl = tmpl + self._fr = fr + self._params = params + self._body = body + self.name = name + self.arguments = tuple(p[0] for p in params) + self.catch_varargs = _uses(body, "varargs") + self.catch_kwargs = _uses(body, "kwargs") + self.caller = _uses(body, "caller") + + def __call__(self, *args, **kwargs): + fr = self._fr + vars = dict(fr.vars) + n = len(self._params) + caller = kwargs.pop("caller", None) + for i in range(n): + pname, dflt = self._params[i] + if i < len(args): + if pname in kwargs: + raise TypeError("macro %r got multiple values for argument %r" + % (self.name, pname)) + vars[pname] = args[i] + elif pname in kwargs: + vars[pname] = kwargs.pop(pname) + elif dflt is not None: + vars[pname] = self._tmpl._eval(dflt, _Frame(fr.tmpl, vars, fr.blocks, [])) + else: + vars[pname] = Undefined(hint="parameter %r was not provided" % (pname,), + name=pname) + extra = tuple(args[n:]) + if extra and not self.catch_varargs: + raise TypeError("macro %r takes not more than %d argument(s)" % (self.name, n)) + if kwargs and not self.catch_kwargs: + raise TypeError("macro %r takes no keyword argument %r" + % (self.name, next(iter(kwargs)))) + vars["varargs"] = extra + vars["kwargs"] = kwargs + if caller is not None: + vars["caller"] = caller + elif self.caller: + vars["caller"] = Undefined(name="caller") + out = [] + self._tmpl._exec_list(self._body, _Frame(fr.tmpl, vars, fr.blocks, out)) + s = "".join(out) + return Markup(s) if self._tmpl.autoescape else s + + def __repr__(self): + return "" % (self.name,) + + +def _uses(node, name): + """True when ``name`` is read anywhere inside a parsed body.""" + if isinstance(node, tuple): + if len(node) == 2 and node[0] == "name" and node[1] == name: + return True + if node and node[0] == "macro": + return False + for x in node: + if _uses(x, name): + return True + elif isinstance(node, list): + for x in node: + if _uses(x, name): + return True + return False + + +class Cycler: + def __init__(self, *items): + if not items: + raise RuntimeError("at least one item has to be provided") + self.items = items + self.pos = 0 + + def reset(self): + self.pos = 0 + + @property + def current(self): + return self.items[self.pos] + + def next(self): + rv = self.items[self.pos] + self.pos = (self.pos + 1) % len(self.items) + return rv + + __next__ = next + + +_NO_CHANGE = object() + + class _Loop: + depth = 1 + depth0 = 0 + def __init__(self, items): self._items = items self.length = len(items) self.index0 = 0 + self._last_changed = _NO_CHANGE + + def changed(self, *value): + if self._last_changed != value: + self._last_changed = value + return True + return False index = property(lambda self: self.index0 + 1) revindex = property(lambda self: self.length - self.index0) @@ -868,7 +1048,7 @@ def _render_root(self, vars, blocks, out): if self._extends is not None: fr = _Frame(self, vars, merged, []) for n in self._nodes: - if n[0] == "set": + if n[0] in ("set", "macro", "import", "fromimport"): self._exec(n, fr) parent = self._eval(self._extends, fr) if not isinstance(parent, Template): @@ -903,7 +1083,39 @@ def _exec(self, n, fr): sub = _Frame(fr.tmpl, fr.vars, fr.blocks, []) self._exec_list(n[2], sub) s = "".join(sub.out) - fr.vars[n[1]] = Markup(s) if self.autoescape else s + s = Markup(s) if self.autoescape else s + if n[3] is not None: + tmp = dict(fr.vars) + tmp["__cn1_body"] = s + s = self._eval(n[3], _Frame(fr.tmpl, tmp, fr.blocks, [])) + fr.vars[n[1]] = s + elif k == "macro": + fr.vars[n[1]] = _Macro(self, n[1], n[2], n[3], fr) + elif k == "callblock": + call = n[2] + caller = _Macro(self, "caller", n[1], n[3], fr) + f = self._eval(call[1], fr) + args = [self._eval(a, fr) for a in call[2]] + kw = dict((kk, self._eval(v, fr)) for kk, v in call[3]) + kw["caller"] = caller + fr.out.append(self._finalize(f(*args, **kw))) + elif k == "filterblock": + sub = _Frame(fr.tmpl, fr.vars, fr.blocks, []) + self._exec_list(n[2], sub) + s = "".join(sub.out) + tmp = dict(fr.vars) + tmp["__cn1_body"] = Markup(s) if self.autoescape else s + fr.out.append(self._finalize(self._eval(n[1], _Frame(fr.tmpl, tmp, fr.blocks, [])))) + elif k == "import": + fr.vars[n[2]] = self._module(self._eval(n[1], fr)) + elif k == "fromimport": + mod = self._module(self._eval(n[1], fr)) + for name, alias in n[2]: + v = getattr(mod, name, None) + if v is None or name.startswith("_"): + v = Undefined(hint="the template %r does not export the requested name %r" + % (mod._name, name), name=name) + fr.vars[alias] = v elif k == "block": self._render_block(n[1], 0, fr) elif k == "include": @@ -924,6 +1136,22 @@ def _exec(self, n, fr): vars[name] = self._eval(e, fr) self._exec_list(n[2], _Frame(fr.tmpl, vars, fr.blocks, fr.out)) + def _module(self, t): + """Run a template's top level and expose its macros and top-level + assignments as attributes, as ``{% import %}`` does.""" + if not isinstance(t, Template): + t = self.environment.get_template(t) + vars = dict(self.environment.globals) + out = [] + t._render_root(vars, {}, out) + mod = _TemplateModule(t.name) + for k, v in vars.items(): + if k not in self.environment.globals or vars[k] is not self.environment.globals[k]: + if not k.startswith("_"): + setattr(mod, k, v) + mod._body = "".join(out) + return mod + def _render_block(self, name, level, fr): chain = fr.blocks.get(name) if not chain: @@ -1094,7 +1322,7 @@ def __init__(self, loader=None, autoescape=False, keep_trailing_newline=False, self.filters = {} self.tests = {} self.globals = {"range": range, "dict": dict, "lipsum": lambda *a, **k: "", - "cycler": None, "namespace": _Namespace} + "cycler": Cycler, "namespace": _Namespace} self._cache = {} def from_string(self, source, globals=None, template_class=None): diff --git a/maven/python-transpiler/src/test/resources/conformance/flask_session_compat.out b/maven/python-transpiler/src/test/resources/conformance/flask_session_compat.out new file mode 100644 index 00000000000..5c8bddeef90 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/flask_session_compat.out @@ -0,0 +1,14 @@ +b=bytes:b'\x00hi'|m=dict:{' t': 'x'}|n=int:3|t=tuple:(1, 2)|user=str:'alice'|when=datetime:datetime.datetime(2024, 5, 6, 7, 8, 9, tzinfo=datetime.timezone.utc) +b=bytes:b'\x00hi'|big=str:'abcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabc'|m=dict:{' t': 'x'}|n=int:3|t=tuple:(1, 2)|user=str:'alice'|when=datetime:datetime.datetime(2024, 5, 6, 7, 8, 9, tzinfo=datetime.timezone.utc) +empty +eyJ0Ijp7IiB0IjpbMSwieCJdfSwidXNlciI6ImJvYiJ9.cT-zAA.zlAkaBL0o4pLDoZuWjg22ZYJQ34 +t=tuple:(1, 'x')|user=str:'bob' +True True +big=str:'xyzxyzxyzxyzxyzxyzxyzxyzxyzxyzxyzxyzxyzxyzxyzxyzxyzxyzxyzxyzxyzxyzxyzxyzxyzxyzxyzxyzxyzxyzxyzxyzxyzxyzxyzxyzxyzxyzxyzxyzxyzxyzxyzxyzxyzxyzxyzxyzxyzxyzxyzxyzxyzxyzxyzxyzxyzxyzxyzxyz'|t=tuple:(1, 'x')|user=str:'bob' +b'hello.E09YEwkmhp2f6vmJaITaXlEvFuc' b'hello' False +b'v.cT-zAA.0b4O2Yf7AClrgvw2u9RK2NBi75k' b'v' +eyJhIjpbMSwyXX0.Q066ynt7YYu3wNu2plLwXD-xis8 {'a': [1, 2]} +InBheWxvYWQi.cT-zAA.Wj158aMBUqXyg6gHZKkY0iqmMSg payload +bad BadTimeSignature +expired Signature age 100 > 10 seconds +7 diff --git a/maven/python-transpiler/src/test/resources/conformance/flask_session_compat.py b/maven/python-transpiler/src/test/resources/conformance/flask_session_compat.py new file mode 100644 index 00000000000..36fea4ab634 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/flask_session_compat.py @@ -0,0 +1,87 @@ +import itsdangerous +from datetime import timedelta +from flask import Flask, session +from itsdangerous import (Signer, TimestampSigner, URLSafeSerializer, URLSafeTimedSerializer, + BadSignature, SignatureExpired) + +# Freeze the clock so signatures are reproducible; both runtimes read it here. +itsdangerous.TimestampSigner.get_timestamp = lambda self: 1900000000 + +# Written by CPython Flask 3.1 with secret_key "conformance-secret". +REAL_A = (".eJyrVkpSsqpWUgCSSo7uGQVKtTpKuWCRlEwwVRIfD5SqUKoFSuQpWRnrKJVAxJWsog11jGKBwqXFqUVA" + "NYk5mcmpSjpK5RmpeRATgIK--Xk6CgZmCr6JlQpGBkYmCgbmVgYWVgaWCu6-IUBDAb4uIlE.auHChw" + ".Ots5ttF1cd5_5fecrJTMlAjFK4k") +REAL_Z = (".eJyrVkpSsqpWUgCSSo7uGQVKtTpKSZnpQF5iUvIgREo6SrlgB6dkgqmS-HigWyuUaoHuzlOyMtZRKoGI" + "K1lFG-oYxQKFS4tTi0D-yclMTgVqL89IzYOYABT0zc_TUTAwU_BNrFQwMjAyUTAwtzKwsDKwVHD3DQEa" + "CgDLtF3d.auHChw.po3LysCQmOeBe6wd5DPLQKiNJAI") + +app = Flask(__name__) +app.secret_key = "conformance-secret" +app.permanent_session_lifetime = timedelta(days=36500) +app.config["PERMANENT_SESSION_LIFETIME"] = timedelta(days=36500) + + +@app.route("/show") +def show(): + out = [] + for k in sorted(session.keys()): + v = session[k] + out.append("%s=%s:%r" % (k, type(v).__name__, v)) + return "|".join(out) or "empty" + + +@app.route("/set") +def set_it(): + session["user"] = "bob" + session["t"] = (1, "x") + return "ok" + + +@app.route("/big") +def big(): + session["big"] = "xyz" * 60 + return "ok" + + +def cookie_of(resp): + h = resp.headers.get("Set-Cookie") + return h.split(";")[0].split("=", 1)[1] + + +c = app.test_client() +c.set_cookie("session", REAL_A) +print(c.get("/show").get_data(as_text=True)) +c.set_cookie("session", REAL_Z) +print(c.get("/show").get_data(as_text=True)) +c.set_cookie("session", REAL_A[:-2] + "xx") +print(c.get("/show").get_data(as_text=True)) + +c2 = app.test_client() +made = cookie_of(c2.get("/set")) +print(made) +print(c2.get("/show").get_data(as_text=True)) +big_cookie = cookie_of(c2.get("/big")) +print(big_cookie.startswith("."), len(big_cookie) < 200) +print(c2.get("/show").get_data(as_text=True)) + +s = Signer("k") +print(s.sign("hello"), s.unsign(s.sign("hello")), s.validate(b"hello.bad")) +ts = TimestampSigner("k", salt="x") +v = ts.sign("v") +print(v, ts.unsign(v, max_age=5)) +ser = URLSafeSerializer("k") +print(ser.dumps({"a": [1, 2]}), ser.loads(ser.dumps({"a": [1, 2]}))) +tser = URLSafeTimedSerializer("k", salt="s") +tok = tser.dumps("payload") +print(tok, tser.loads(tok)) +try: + URLSafeTimedSerializer("other", salt="s").loads(tok) +except BadSignature as e: + print("bad", type(e).__name__) +itsdangerous.TimestampSigner.get_timestamp = lambda self: 1900000100 +try: + tser.loads(tok, max_age=10) +except SignatureExpired as e: + print("expired", e) +rot = URLSafeSerializer(["old", "new"]) +print(URLSafeSerializer("old").loads(rot.dumps(1)) if False else rot.loads(URLSafeSerializer("old").dumps(7))) diff --git a/maven/python-transpiler/src/test/resources/conformance/jinja_macros.out b/maven/python-transpiler/src/test/resources/conformance/jinja_macros.out new file mode 100644 index 00000000000..c114988b097 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/jinja_macros.out @@ -0,0 +1,17 @@ +||1.2 + + +

T

[1][2]a=b;x=3; +
inner me
+
second
+<1:A><2:B> +SHOUT ME +Hell0 W0rld F00 +captured me +[PADDED] +odd-a,even-b,odd-c,even-d +123 +1/3/3/1 2/2/3/1 3/1/3/1 +rgbr g +listing ('cls',) True True +TypeError macro 'm' takes not more than 1 argument(s) diff --git a/maven/python-transpiler/src/test/resources/conformance/jinja_macros.py b/maven/python-transpiler/src/test/resources/conformance/jinja_macros.py new file mode 100644 index 00000000000..7f6912f358a --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/jinja_macros.py @@ -0,0 +1,55 @@ +from jinja2 import Environment, DictLoader + +templates = { + "forms.html": ( + "{% macro input(name, value='', type='text', size=20) -%}\n" + "\n" + "{%- endmacro %}" + "{% macro label(text) %}{% endmacro %}" + "{% set version = '1.2' %}" + ), + "page.html": ( + "{% import 'forms.html' as forms %}" + "{{ forms.input('user') }}|{{ forms.input('pw', type='password', size=8) }}|" + "{{ forms.version }}" + ), + "from.html": ( + "{% from 'forms.html' import input as field, label %}" + "{{ label('Name') }}{{ field('n', value='') }}" + ), +} + +env = Environment(loader=DictLoader(templates)) +print(env.get_template("page.html").render()) +print(env.get_template("from.html").render()) +aenv = Environment(loader=DictLoader(templates), autoescape=True) +print(aenv.get_template("from.html").render()) + +src = """{% macro listing(title) -%} +

{{ title }}

{% for a in varargs %}[{{ a }}]{% endfor %}{% for k, v in kwargs|dictsort %}{{ k }}={{ v }};{% endfor %} +{%- endmacro -%} +{{ listing('T', 1, 2, x=3, a='b') }} +{% macro wrap(cls='box') -%} +
{{ caller() }}
+{%- endmacro -%} +{% call wrap() %}inner {{ who }}{% endcall %} +{% call wrap('big') %}second{% endcall %} +{% macro each(items) -%} +{% for i in items %}{{ caller(i, loop.index) }}{% endfor %} +{%- endmacro -%} +{% call(item, n) each(['a', 'b']) %}<{{ n }}:{{ item|upper }}>{% endcall %} +{% filter upper %}shout {{ who }}{% endfilter %} +{% filter replace('o', '0')|title %}hello world foo{% endfilter %} +{% set block_var %}captured {{ who }}{% endset %}{{ block_var }} +{% set trimmed | trim | upper %} padded {% endset %}[{{ trimmed }}] +{% for x in ['a', 'b', 'c', 'd'] %}{{ loop.cycle('odd', 'even') }}-{{ x }}{% if not loop.last %},{% endif %}{% endfor %} +{% for x in [1, 1, 2, 3, 3] %}{% if loop.changed(x) %}{{ x }}{% endif %}{% endfor %} +{% for x in 'xyz' %}{{ loop.index }}/{{ loop.revindex }}/{{ loop.length }}/{{ loop.depth }} {% endfor %} +{% set c = cycler('r', 'g', 'b') %}{% for i in range(4) %}{{ c.next() }}{% endfor %} {{ c.current }} +{{ listing.name }} {{ wrap.arguments }} {{ wrap.caller }} {{ listing.catch_varargs }} +""" +print(env.from_string(src).render(who="me")) +try: + env.from_string("{% macro m(a) %}{{ a }}{% endmacro %}{{ m(1, 2) }}").render() +except TypeError as e: + print("TypeError", e) From cd527963f77a5aed82200938123fbf55b010ff9d Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 11:44:28 +0200 Subject: [PATCH 064/112] Guide: Flask sessions interoperate with CPython, Jinja macros Co-Authored-By: Claude Opus 5.5 --- docs/developer-guide/Python-Interop.asciidoc | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/developer-guide/Python-Interop.asciidoc b/docs/developer-guide/Python-Interop.asciidoc index af19c7e326d..223a0ecdf40 100644 --- a/docs/developer-guide/Python-Interop.asciidoc +++ b/docs/developer-guide/Python-Interop.asciidoc @@ -318,11 +318,13 @@ include::../demos/common/src/main/snippets/developer-guide/python-interop.sh[tag Routing, converters, JSON bodies, `abort`, error handlers and the default error pages of Flask, and FastAPI's path, query, body and header parameters, dependencies, `response_model` and OpenAPI generation are supported, and -Flask sessions (signed cookies), `flash`, Jinja templates (`extends`, `block`, -`include`, loops, conditions and the common filters, with autoescaping), -multipart file uploads and streamed responses work as well. A streamed -response is sent chunk by chunk as its generator yields. Jinja macros aren't supported yet, and a session cookie -written by CPython's Flask isn't readable here, or the reverse. +Flask sessions, `flash`, Jinja templates (`extends`, `block`, `include`, +macros, `call`, `import`, `filter` blocks, loops, conditions and the common +filters, with autoescaping), multipart file uploads and streamed responses +work as well. A streamed response is sent chunk by chunk as its generator +yields. Session cookies use Flask's own `itsdangerous` format, so a server +built here and one running under CPython read each other's sessions when they +share a secret key. FastAPI middleware works as in Starlette: `@app.middleware("http")`, `BaseHTTPMiddleware` subclasses, `CORSMiddleware`, `GZipMiddleware` and From a35eceef9352667abd98ab5c6bb08ff79cdf281a Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 11:47:38 +0200 Subject: [PATCH 065/112] Python: virtual-thread-safe GIL, engine-neutral SQLAlchemy inspect, SQLAlchemy on PostgreSQL in CI The GIL is a compare-and-set on an owner key with a reentry count; a waiter on a backend virtual thread steps aside with VirtualThread.yieldNow instead of blocking its host in Object.wait. inspect(engine).get_table_names reads each engine's own catalog, so sqlalchemy_core no longer names sqlite_master. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/parparvm-tests.yml | 24 ++++ .../com/codename1/python/runtime/Gil.java | 120 +++++++++++++----- .../transpiler/api/PythonTranspiler.java | 16 +++ .../python/site/sqlalchemy/__init__.py | 31 +++++ .../resources/conformance/sqlalchemy_core.out | 2 +- .../resources/conformance/sqlalchemy_core.py | 4 +- .../conformance/sqlalchemy_relations.py | 3 +- 7 files changed, 162 insertions(+), 38 deletions(-) diff --git a/.github/workflows/parparvm-tests.yml b/.github/workflows/parparvm-tests.yml index d612f1a3638..52a64d4a332 100644 --- a/.github/workflows/parparvm-tests.yml +++ b/.github/workflows/parparvm-tests.yml @@ -34,6 +34,10 @@ on: - 'maven/project-model/**' - 'maven/backend/**' - 'maven/backend-test/**' + # The Python SQLAlchemy step runs these against the PostgreSQL service. + - 'maven/python-runtime/**' + - 'maven/python-transpiler/**' + - 'scripts/python-backend/sqlalchemy-realdb-test.sh' - 'scripts/hellocodenameone/pom.xml' - 'scripts/hellocodenameone/backend/**' # The core classes the backend compiles as its own (the ORM and the entity @@ -99,6 +103,10 @@ on: - 'maven/project-model/**' - 'maven/backend/**' - 'maven/backend-test/**' + # The Python SQLAlchemy step runs these against the PostgreSQL service. + - 'maven/python-runtime/**' + - 'maven/python-transpiler/**' + - 'scripts/python-backend/sqlalchemy-realdb-test.sh' - 'scripts/hellocodenameone/pom.xml' - 'scripts/hellocodenameone/backend/**' # The core classes the backend compiles as its own (the ORM and the entity @@ -450,6 +458,22 @@ jobs: # the step above just installed from this same checkout. CN1SS_BACKEND_SKIP_INSTALL: 1 + # The SQLAlchemy conformance programs against the PostgreSQL service + # above: the suite in python-transpiler runs them on SQLite only, so this + # is the one place the postgresql dialect and the _cn1db bridge meet a + # real server. PostgreSQL only; the MySQL family is run by hand. + # python-runtime compiles at release 17, hence JDK 17 for the install. + - name: Python SQLAlchemy on PostgreSQL + run: | + (cd maven && bash "$GITHUB_WORKSPACE/scripts/ci/retry.sh" mvn -B -pl python-runtime,python-transpiler \ + install -DskipTests -Dspotbugs.skip=true -Dpmd.skip=true -Dcheckstyle.skip=true \ + -Dmaven.javadoc.skip=true -Dmaven.repo.local="$GITHUB_WORKSPACE/.m2-repo") + bash scripts/python-backend/sqlalchemy-realdb-test.sh "$GITHUB_WORKSPACE/.m2-repo" + env: + JAVA_HOME: ${{ env.JDK_17_HOME }} + JAVA17_HOME: ${{ env.JDK_17_HOME }} + CN1_SQLA_POSTGRES: postgres://cn1:cn1secret@127.0.0.1:5432/cn1?sslmode=disable + - name: Run ParparVM Benchmark working-directory: vm # Each benchmark already creates its own mutator/collector threads. Run diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Gil.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Gil.java index 8ceb0cdb68b..763ccf4e2c7 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Gil.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Gil.java @@ -23,6 +23,8 @@ */ package com.codename1.python.runtime; +import java.util.concurrent.atomic.AtomicLong; + /// The global interpreter lock: Python code runs on one thread at a time. /// /// The runtime's caches, containers and module state assume a single @@ -38,45 +40,97 @@ /// Reentrant: a callback into Python from Java code that Python called runs /// on the same thread, which already holds the lock. public final class Gil { - private static final Object MONITOR = new Object(); - private static Thread owner; + /// How many failed attempts a platform thread spins through + /// `Thread.yield()` before it starts sleeping between attempts. + private static final int SPINS = 64; + + /// Who holds the lock: 0 when nobody does, a virtual thread's handle + /// (positive) or a platform thread's negated id. A key rather than the + /// `Thread`, because every virtual thread on a backend host answers the + /// host's `Thread` -- two of them would read as one reentrant owner. + private static final AtomicLong OWNER = new AtomicLong(); + + /// Reentry depth. Read and written only by the owner; ownership passes + /// through a compare-and-set on [#OWNER], which orders it. private static int depth; + private static Scheduler scheduler; + + /// What a server with virtual threads tells the lock about them. + /// + /// A virtual thread must never block in `Object.wait` or on a contended + /// monitor: that blocks its HOST thread, and with tasks queued on every + /// host the whole server wedges. So the lock never blocks -- it is a + /// compare-and-set, and a waiter steps aside through [#pause()] between + /// attempts. + public interface Scheduler { + /// The running virtual thread's handle, or 0 on a platform thread. + long current(); + + /// Steps the running virtual thread aside so its host can run + /// another; answers false (and does nothing) on a platform thread. + boolean pause(); + } + private Gil() { } + /// Installed once, before the server starts any thread. + public static void setScheduler(Scheduler s) { + scheduler = s; + } + + private static long key() { + Scheduler s = scheduler; + if (s != null) { + long vt = s.current(); + if (vt != 0) { + return vt; + } + } + return -(Thread.currentThread().getId() + 1); + } + /// Takes the lock, waiting for the thread that holds it to wait or /// leave Python. public static void enter() { - Thread t = Thread.currentThread(); - synchronized (MONITOR) { - if (owner == t) { //NOPMD CompareObjectsWithEquals - identity is meant - depth++; - return; + long me = key(); + if (OWNER.get() == me) { + depth++; + return; + } + acquire(me); + depth = 1; + } + + private static void acquire(long me) { + int spins = 0; + while (!OWNER.compareAndSet(0, me)) { + Scheduler s = scheduler; + if (s != null && s.pause()) { + continue; } - while (owner != null) { + if (spins < SPINS) { + spins++; + Thread.yield(); + } else { try { - MONITOR.wait(); + Thread.sleep(1); } catch (InterruptedException e) { // keep waiting: the lock is the only way back into Python } } - owner = t; - depth = 1; } } /// Leaves one [#enter()]. public static void exit() { - synchronized (MONITOR) { - if (owner != Thread.currentThread()) { //NOPMD CompareObjectsWithEquals - identity is meant - return; - } - depth--; - if (depth == 0) { - owner = null; - MONITOR.notifyAll(); - } + if (OWNER.get() != key()) { + return; + } + depth--; + if (depth == 0) { + OWNER.set(0); } } @@ -84,25 +138,20 @@ public static void exit() { /// [#restore(int)] takes back afterwards (0 when this thread did not /// hold it). public static int release() { - synchronized (MONITOR) { - if (owner != Thread.currentThread()) { //NOPMD CompareObjectsWithEquals - identity is meant - return 0; - } - int d = depth; - owner = null; - depth = 0; - MONITOR.notifyAll(); - return d; + if (OWNER.get() != key()) { + return 0; } + int d = depth; + depth = 0; + OWNER.set(0); + return d; } /// Takes the lock back after a wait. public static void restore(int saved) { if (saved > 0) { - enter(); - synchronized (MONITOR) { - depth = saved; - } + acquire(key()); + depth = saved; } } @@ -110,7 +159,10 @@ public static void restore(int saved) { /// does. public static void yieldNow() { int saved = release(); - Thread.yield(); + Scheduler s = scheduler; + if (s == null || !s.pause()) { + Thread.yield(); + } restore(saved); } } diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java index 72530164c81..3e04fc87980 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java @@ -524,6 +524,22 @@ private static String backendServer() { + " */\n" + " private static final int WORKERS = 16;\n\n" + " static void install() {\n" + // Handlers may run on virtual threads: the GIL must step them + // aside rather than block their host thread (see Gil.Scheduler). + + " com.codename1.python.runtime.Gil.setScheduler(new com.codename1.python.runtime.Gil.Scheduler() {\n" + + " public long current() {\n" + // The handle even while pinned in a monitor: the GIL's owner + // key must not change between enter and exit. + + " return com.codename1.backend.VirtualThread.current();\n" + + " }\n\n" + + " public boolean pause() {\n" + + " if (!com.codename1.backend.VirtualThread.isVirtual()) {\n" + + " return false;\n" + + " }\n" + + " com.codename1.backend.VirtualThread.yieldNow();\n" + + " return true;\n" + + " }\n" + + " });\n" + " PyServer.install(new BackendServer());\n" + " }\n\n" + " @Override\n" diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/__init__.py index 5800c72627e..18acdce5d85 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/__init__.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/__init__.py @@ -2341,7 +2341,38 @@ def create_engine(url, echo=False, **kwargs): return Engine(url, echo) +class Inspector: + """``inspect(engine)``: the tables the database has, asked of the engine's own catalog.""" + + def __init__(self, bind): + self.bind = bind + self.dialect = bind.engine.dialect if isinstance(bind, Connection) else bind.dialect + + def _catalog_sql(self): + name = self.dialect.name + if name == "postgresql": + return ("SELECT c.relname FROM pg_catalog.pg_class c JOIN pg_catalog.pg_namespace n " + "ON n.oid = c.relnamespace WHERE n.nspname = current_schema() " + "AND c.relkind IN ('r', 'p') ORDER BY c.relname") + if name in ("mysql", "mariadb"): + return ("SELECT table_name FROM information_schema.tables WHERE table_schema = DATABASE() " + "AND table_type = 'BASE TABLE' ORDER BY table_name") + return ("SELECT name FROM sqlite_master WHERE type = 'table' " + "AND name NOT LIKE 'sqlite~_%' ESCAPE '~' ORDER BY name") + + def get_table_names(self, schema=None): + if isinstance(self.bind, Connection): + return [str(n) for n in self.bind.exec_driver_sql(self._catalog_sql()).scalars().all()] + with self.bind.connect() as conn: + return [str(n) for n in conn.exec_driver_sql(self._catalog_sql()).scalars().all()] + + def has_table(self, table_name, schema=None): + return table_name in self.get_table_names(schema) + + def inspect(obj): + if isinstance(obj, (Engine, Connection)): + return Inspector(obj) return obj # The DB-API module SQLAlchemy names in an error, "(psycopg2.OperationalError) ...", by driver. diff --git a/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_core.out b/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_core.out index 3f8d9a763c7..32982c8ca20 100644 --- a/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_core.out +++ b/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_core.out @@ -31,4 +31,4 @@ updated 2 p2 (4, 4) IntegrityError -1 +['t'] diff --git a/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_core.py b/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_core.py index 214520cabab..69d166f2432 100644 --- a/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_core.py +++ b/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_core.py @@ -3,7 +3,7 @@ from sqlalchemy import (create_engine, text, MetaData, Table, Column, Integer, String, Text, Float, Boolean, DateTime, Date, ForeignKey, select, insert, update, delete, func, and_, or_, not_) -from sqlalchemy import exc +from sqlalchemy import exc, inspect engine = create_engine(sys.argv[1] if len(sys.argv) > 1 else "sqlite://") # Raw DDL is the one place the SQL is the engine's own: an auto-numbered key is spelled per engine. @@ -107,4 +107,4 @@ metadata.drop_all(engine) with engine.connect() as conn: - print(conn.execute(text("SELECT count(*) FROM sqlite_master WHERE type='table'")).scalar()) + print(sorted(n for n in inspect(conn).get_table_names() if n in ("t", "users", "posts"))) diff --git a/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_relations.py b/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_relations.py index d545f5baaf0..037c0daf4ef 100644 --- a/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_relations.py +++ b/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_relations.py @@ -111,7 +111,8 @@ class Truck(Vehicle): p2 = Post(title="two", tags=[py]) s.add_all([p1, p2, web]) s.commit() - print(sorted(t.name for t in p1.tags), [p.title for p in py.posts]) + # A lazy load has no ORDER BY, so the order of py.posts is the database's to pick. + print(sorted(t.name for t in p1.tags), sorted(p.title for p in py.posts)) p2.tags.append(web) p1.tags.remove(db) s.commit() From 750a2ab24a06c703917918c77fc62cf5d4b10074 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 11:57:34 +0200 Subject: [PATCH 066/112] Python: Pillow BOX/HAMMING/BICUBIC/LANCZOS resampling, bitmap-font text, P/LA/1 and Adam7 PNGs; pandas truncated repr Resampling follows Pillow's two-pass fixed-point convolution with each filter's kernel and support, premultiplied for LA and RGBA; output matches Pillow 13 byte for byte. ImageFont carries Pillow's built-in courB08 bitmap font (load_default returns it, as Pillow without FreeType does), and ImageDraw gains text, multiline_text, textbbox and textlength. PNGs now open as 1, P (palette + tRNS as info['transparency']) and LA, interlaced files decode pass by pass, and those modes convert and save. pandas reprs of frames over max_rows show head/tail 5 with dots, wide frames are fitted to 80 columns as in a non-terminal run, with the [N rows x M columns] footer; long Series print Length:. A range in a DataFrame dict is now materialized. Co-Authored-By: Claude Opus 5.5 --- NOTICE | 11 + .../com/codename1/python/runtime/PyImage.java | 179 ++++++++++--- .../com/codename1/python/site/PIL/Image.py | 182 +++++++++++-- .../codename1/python/site/PIL/ImageDraw.py | 83 +++++- .../codename1/python/site/PIL/ImageFont.py | 246 ++++++++++++++++++ .../codename1/python/site/pandas/__init__.py | 93 ++++++- .../resources/conformance/pandas_truncate.out | 153 +++++++++++ .../resources/conformance/pandas_truncate.py | 18 ++ .../conformance/pil_resample_text.out | 109 ++++++++ .../conformance/pil_resample_text.py | 87 +++++++ scripts/copyright-header-exclusions.txt | 1 + 11 files changed, 1091 insertions(+), 71 deletions(-) create mode 100644 maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/ImageFont.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/pandas_truncate.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/pandas_truncate.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/pil_resample_text.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/pil_resample_text.py diff --git a/NOTICE b/NOTICE index fd4e0d6641b..25cfa1a1698 100644 --- a/NOTICE +++ b/NOTICE @@ -244,3 +244,14 @@ math.erfc agree with CPython's. It retains FDLIBM's notice: Permission to use, copy, modify, and distribute this software is freely granted, provided that this notice is preserved. + +The module PIL.ImageFont, in maven/python-transpiler +(src/main/resources/com/codename1/python/site/PIL/ImageFont.py), is a +rewrite of the bitmap font support of Pillow's ImageFont and embeds Pillow's +built-in default font (courB08) unchanged, so that text drawn with +ImageFont.load_default_imagefont() matches Pillow pixel for pixel. It retains +Pillow's notice: + + Copyright (c) 1997-2003 by Secret Labs AB + Copyright (c) 1996-2003 by Fredrik Lundh + Licensed under the MIT-CMU (HPND) license; the full text is in the file. diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyImage.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyImage.java index 64de5863635..e7bcd245704 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyImage.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyImage.java @@ -150,7 +150,18 @@ static PyModule module() { for (int j = 0; j < 4; j++) { box[j] = Py.asDouble(c[j]); } - return wrap(resample(img(a[0]), i(a[1]), i(a[2]), box)); + Img src = img(a[0]); + boolean premultiply = a.length > 5 && Py.isTrue(a[5]); + if (premultiply) { + src = premultiply(src); + } + Img r = resample(src, i(a[1]), i(a[2]), box, a.length > 4 ? i(a[4]) : FILTER_BILINEAR); + return wrap(premultiply ? unpremultiply(r) : r); + })); + m.setAttr("palette", PyBuiltin.positional("palette", a -> wrap(palette(img(a[0]), bytes(a[1]))))); + m.setAttr("scatter", PyBuiltin.positional("scatter", a -> { + scatter(img(a[0]), img(a[1]), i(a[2]), i(a[3]), i(a[4]), i(a[5])); + return Py.NONE; })); m.setAttr("convert", PyBuiltin.positional("convert", a -> wrap(convert(img(a[0]), i(a[1]))))); m.setAttr("band", PyBuiltin.positional("band", a -> { @@ -199,8 +210,7 @@ static PyModule module() { m.setAttr("png_unfilter", PyBuiltin.positional("png_unfilter", a -> new PyBytes(unfilter(bytes(a[0]), i(a[1]), i(a[2]), i(a[3]))))); m.setAttr("png_expand", PyBuiltin.positional("png_expand", a -> - wrap(expand(bytes(a[0]), i(a[1]), i(a[2]), i(a[3]), i(a[4]), - PyBytes.like(a[5]), PyBytes.like(a[6]))))); + wrap(expand(bytes(a[0]), i(a[1]), i(a[2]), i(a[3]), i(a[4]))))); m.setAttr("png_rows", PyBuiltin.positional("png_rows", a -> { Img s = img(a[0]); int row = s.w * s.bands; @@ -380,13 +390,71 @@ private static Img affine(Img s, int w, int h, double[] a, byte[] fill) { return r; } - /// Bilinear coefficients for one axis, as Pillow's `precompute_coeffs` + static final int FILTER_LANCZOS = 1; + static final int FILTER_BILINEAR = 2; + static final int FILTER_BICUBIC = 3; + static final int FILTER_BOX = 4; + static final int FILTER_HAMMING = 5; + + private static double support(int filter) { + switch (filter) { + case FILTER_BOX: + return 0.5; + case FILTER_BICUBIC: + return 2.0; + case FILTER_LANCZOS: + return 3.0; + default: + return 1.0; + } + } + + private static double sinc(double x) { + if (x == 0.0) { + return 1.0; + } + double px = x * Math.PI; + return Math.sin(px) / px; + } + + /// Pillow's filter kernels (`Resample.c`), evaluated at signed `x`. + private static double kernel(int filter, double x) { + double t = x < 0 ? -x : x; + switch (filter) { + case FILTER_BOX: + return x > -0.5 && x <= 0.5 ? 1.0 : 0.0; + case FILTER_HAMMING: + if (t == 0.0) { + return 1.0; + } + if (t >= 1.0) { + return 0.0; + } + t = t * Math.PI; + return Math.sin(t) / t * (0.54 + 0.46 * Math.cos(t)); + case FILTER_BICUBIC: + if (t < 1.0) { + return (1.5 * t - 2.5) * t * t + 1; + } + if (t < 2.0) { + return (((t - 5) * t + 8) * t - 4) * -0.5; + } + return 0.0; + case FILTER_LANCZOS: + return x > -3.0 && x < 3.0 ? sinc(x) * sinc(x / 3.0) : 0.0; + default: + return t < 1.0 ? 1.0 - t : 0.0; + } + } + + /// Filter coefficients for one axis, as Pillow's `precompute_coeffs` /// and `normalize_coeffs_8bpc` make them. Returns `ksize`; `bounds` /// receives `xmin, count` per output position. - private static int coeffs(int inSize, double in0, double in1, int outSize, int[] bounds, int[][] out) { + private static int coeffs(int inSize, double in0, double in1, int outSize, int[] bounds, int[][] out, + int filter) { double scale = (in1 - in0) / outSize; double filterscale = scale < 1.0 ? 1.0 : scale; - double support = filterscale; + double support = support(filter) * filterscale; int ksize = (int) Math.ceil(support) * 2 + 1; int[] kk = new int[outSize * ksize]; double[] k = new double[ksize]; @@ -404,11 +472,7 @@ private static int coeffs(int inSize, double in0, double in1, int outSize, int[] } xmax -= xmin; for (int x = 0; x < xmax; x++) { - double t = (x + xmin - center + 0.5) * ss; - if (t < 0) { - t = -t; - } - double w = t < 1.0 ? 1.0 - t : 0.0; + double w = kernel(filter, (x + xmin - center + 0.5) * ss); k[x] = w; ww += w; } @@ -429,14 +493,14 @@ private static byte clip8(int v) { return (byte) (s < 0 ? 0 : s > 255 ? 255 : s); } - private static Img resample(Img s, int w, int h, double[] box) { + private static Img resample(Img s, int w, int h, double[] box, int filter) { Img cur = s; boolean horizontal = w != s.w || box[0] != 0 || box[2] != w; boolean vertical = h != s.h || box[1] != 0 || box[3] != h; if (horizontal) { int[] bounds = new int[w * 2]; int[][] kh = new int[1][]; - int ks = coeffs(cur.w, box[0], box[2], w, bounds, kh); + int ks = coeffs(cur.w, box[0], box[2], w, bounds, kh, filter); Img r = new Img(w, cur.h, cur.bands); for (int y = 0; y < cur.h; y++) { for (int x = 0; x < w; x++) { @@ -456,7 +520,7 @@ private static Img resample(Img s, int w, int h, double[] box) { if (vertical) { int[] bounds = new int[h * 2]; int[][] kv = new int[1][]; - int ks = coeffs(cur.h, box[1], box[3], h, bounds, kv); + int ks = coeffs(cur.h, box[1], box[3], h, bounds, kv, filter); Img r = new Img(cur.w, h, cur.bands); for (int y = 0; y < h; y++) { int ymin = bounds[y * 2]; @@ -479,18 +543,74 @@ private static Img resample(Img s, int w, int h, double[] box) { return cur; } + /// RGBA to RGBa (or LA to La): Pillow's `MULDIV255` premultiply. + private static Img premultiply(Img s) { + Img r = crop(s, 0, 0, s.w, s.h); + int n = s.bands; + for (int p = 0, c = s.w * s.h; p < c; p++) { + int q = p * n; + int alpha = s.d[q + n - 1] & 0xff; + for (int b = 0; b < n - 1; b++) { + int t = (s.d[q + b] & 0xff) * alpha + 128; + r.d[q + b] = (byte) (((t >> 8) + t) >> 8); + } + } + return r; + } + + /// RGBa to RGBA (or La to LA), as Pillow's `rgbA2rgba`. + private static Img unpremultiply(Img s) { + int n = s.bands; + for (int p = 0, c = s.w * s.h; p < c; p++) { + int q = p * n; + int alpha = s.d[q + n - 1] & 0xff; + if (alpha == 0 || alpha == 255) { + continue; + } + for (int b = 0; b < n - 1; b++) { + int v = (255 * (s.d[q + b] & 0xff)) / alpha; + s.d[q + b] = (byte) (v > 255 ? 255 : v); + } + } + return s; + } + + /// Maps a 1-band index image through a 256 entry RGBA table. + private static Img palette(Img s, byte[] lut) { + Img r = new Img(s.w, s.h, 4); + for (int p = 0, n = s.w * s.h; p < n; p++) { + int idx = s.d[p * s.bands] & 0xff; + System.arraycopy(lut, idx * 4, r.d, p * 4, 4); + } + return r; + } + + /// Writes `src` into `dst` at `x0 + x * dx, y0 + y * dy`: one Adam7 pass. + private static void scatter(Img dst, Img src, int x0, int y0, int dx, int dy) { + int n = dst.bands; + for (int y = 0; y < src.h; y++) { + for (int x = 0; x < src.w; x++) { + System.arraycopy(src.d, (y * src.w + x) * n, dst.d, ((y0 + y * dy) * dst.w + x0 + x * dx) * n, n); + } + } + } + private static Img convert(Img s, int bands) { Img r = new Img(s.w, s.h, bands); for (int p = 0, n = s.w * s.h; p < n; p++) { int q = p * s.bands; int cr = s.d[q] & 0xff; - int cg = s.bands == 1 ? cr : s.d[q + 1] & 0xff; - int cb = s.bands == 1 ? cr : s.d[q + 2] & 0xff; - int ca = s.bands == 4 ? s.d[q + 3] & 0xff : 255; + boolean gray = s.bands < 3; + int cg = gray ? cr : s.d[q + 1] & 0xff; + int cb = gray ? cr : s.d[q + 2] & 0xff; + int ca = s.bands == 4 ? s.d[q + 3] & 0xff : s.bands == 2 ? s.d[q + 1] & 0xff : 255; int o = p * bands; - if (bands == 1) { + if (bands <= 2) { // Pillow's L24: ITU-R 601-2 luma in 16-bit fixed point - r.d[o] = (byte) ((cr * 19595 + cg * 38470 + cb * 7471 + 0x8000) >> 16); + r.d[o] = gray ? (byte) cr : (byte) ((cr * 19595 + cg * 38470 + cb * 7471 + 0x8000) >> 16); + if (bands == 2) { + r.d[o + 1] = (byte) ca; + } } else { r.d[o] = (byte) cr; r.d[o + 1] = (byte) cg; @@ -611,33 +731,24 @@ private static int sample(byte[] raw, int rowStart, int index, int depth) { } /// Unpacks unfiltered PNG rows into an image: gray to 1 band, RGB to 3, - /// gray+alpha, RGBA and palette (to 3, or 4 with a transparency chunk). - private static Img expand(byte[] raw, int w, int h, int ctype, int depth, byte[] plte, byte[] trns) { + /// gray+alpha to 2, RGBA to 4 and palette to its 1-band indices. + private static Img expand(byte[] raw, int w, int h, int ctype, int depth) { int channels = ctype == 0 || ctype == 3 ? 1 : ctype == 2 ? 3 : ctype == 4 ? 2 : 4; int row = (w * channels * depth + 7) / 8; - int bands = ctype == 0 ? 1 : ctype == 2 ? 3 : ctype == 3 && trns == null ? 3 : 4; + int bands = ctype == 0 || ctype == 3 ? 1 : ctype == 2 ? 3 : ctype == 4 ? 2 : 4; Img r = new Img(w, h, bands); for (int y = 0; y < h; y++) { int rs = y * row; for (int x = 0; x < w; x++) { int o = (y * w + x) * bands; if (ctype == 3) { - int idx = sample(raw, rs, x, depth); - for (int b = 0; b < 3; b++) { - r.d[o + b] = plte != null && idx * 3 + b < plte.length ? plte[idx * 3 + b] : 0; - } - if (bands == 4) { - r.d[o + 3] = idx < trns.length ? trns[idx] : (byte) 255; - } + r.d[o] = (byte) sample(raw, rs, x, depth); continue; } int scale = depth < 8 ? 255 / ((1 << depth) - 1) : 1; if (ctype == 4) { - int g = sample(raw, rs, x * 2, depth); - r.d[o] = (byte) g; - r.d[o + 1] = (byte) g; - r.d[o + 2] = (byte) g; - r.d[o + 3] = (byte) sample(raw, rs, x * 2 + 1, depth); + r.d[o] = (byte) sample(raw, rs, x * 2, depth); + r.d[o + 1] = (byte) sample(raw, rs, x * 2 + 1, depth); continue; } for (int b = 0; b < channels; b++) { diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/Image.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/Image.py index 9b75c7e8e8f..ce5a9a765ea 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/Image.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/Image.py @@ -52,15 +52,29 @@ class Transpose: TRANSPOSE = Transpose.TRANSPOSE TRANSVERSE = Transpose.TRANSVERSE -_BANDS = {"L": 1, "RGB": 3, "RGBA": 4} -_BAND_NAMES = {"L": ("L",), "RGB": ("R", "G", "B"), "RGBA": ("R", "G", "B", "A")} + +class Dither: + NONE = 0 + ORDERED = 1 + RASTERIZE = 2 + FLOYDSTEINBERG = 3 + + +NONE = Dither.NONE +FLOYDSTEINBERG = Dither.FLOYDSTEINBERG + +_BANDS = {"1": 1, "L": 1, "P": 1, "LA": 2, "RGB": 3, "RGBA": 4} +_BAND_NAMES = {"1": ("1",), "L": ("L",), "P": ("P",), "LA": ("L", "A"), "RGB": ("R", "G", "B"), + "RGBA": ("R", "G", "B", "A")} +# Adam7 passes: first column, first row, column step, row step +_ADAM7 = ((0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4), (0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2)) _PNG_SIG = b"\x89PNG\r\n\x1a\n" _EXT = {".png": "PNG", ".jpg": "JPEG", ".jpeg": "JPEG", ".gif": "GIF", ".bmp": "BMP", ".webp": "WEBP"} def _bands(mode): if mode not in _BANDS: - raise ValueError("unrecognized image mode: %s (only L, RGB and RGBA are supported)" % mode) + raise ValueError("unrecognized image mode: %s (only 1, L, P, LA, RGB and RGBA are supported)" % mode) return _BANDS[mode] @@ -85,6 +99,32 @@ def _color(mode, color): return bytes([v & 255 for v in c[:n]]) +def _pack_bits(raw, w, h, filter_byte=False): + """One byte a pixel (0 or 255) to rows of eight pixels a byte, MSB first.""" + rows = bytearray() + for y in range(h): + if filter_byte: + rows.append(0) + line = raw[y * w:(y + 1) * w] + for x in range(0, w, 8): + v = 0 + for b, p in enumerate(line[x:x + 8]): + if p: + v |= 0x80 >> b + rows.append(v) + return bytes(rows) + + +def _unpack_bits(data, w, h): + stride = (w + 7) // 8 + out = bytearray(w * h) + for y in range(h): + for x in range(w): + if data[y * stride + (x >> 3)] & (0x80 >> (x & 7)): + out[y * w + x] = 255 + return bytes(out) + + def _wrap(mode, size, handle): im = Image() im.mode = mode @@ -101,6 +141,7 @@ def __init__(self): self._size = (0, 0) self._im = None self.info = {} + self._palette = None @property def size(self): @@ -130,7 +171,24 @@ def getbands(self): return _BAND_NAMES[self.mode] def _new(self, handle, size=None, mode=None): - return _wrap(mode or self.mode, size or self._size, handle) + im = _wrap(mode or self.mode, size or self._size, handle) + if im.mode == "P": + # a palette image keeps its palette and transparency index + im._palette = self._palette + im.info = dict(self.info) + return im + + def getpalette(self, rawmode="RGB"): + if self.mode != "P": + return None + return list(self._palette or b"") + + def putpalette(self, data, rawmode="RGB"): + if self.mode not in ("L", "LA", "P", "PA"): + raise ValueError("illegal image mode") + if self.mode in ("L", "LA"): + self.mode = "P" + self._palette = bytes(data) def copy(self): return self._new(_cn1image.crop(self._im, 0, 0, self.width, self.height)) @@ -142,7 +200,7 @@ def getpixel(self, xy): if y < 0: y += self.height t = _cn1image.getpixel(self._im, x, y) - return t[0] if self.mode == "L" else t + return t[0] if len(t) == 1 else t def putpixel(self, xy, value): x, y = int(xy[0]), int(xy[1]) @@ -153,7 +211,7 @@ def putpixel(self, xy, value): _cn1image.putpixel(self._im, x, y, _color(self.mode, value)) def getdata(self, band=None): - raw = self.tobytes() + raw = _cn1image.tobytes(self._im) n = _bands(self.mode) if n == 1: return list(raw) @@ -162,7 +220,7 @@ def getdata(self, band=None): return [tuple(raw[i:i + n]) for i in range(0, len(raw), n)] def histogram(self): - raw = self.tobytes() + raw = _cn1image.tobytes(self._im) n = _bands(self.mode) h = [0] * (256 * n) for i, v in enumerate(raw): @@ -171,14 +229,20 @@ def histogram(self): def getextrema(self): n = _bands(self.mode) - raw = self.tobytes() + raw = _cn1image.tobytes(self._im) ext = tuple((min(raw[b::n]), max(raw[b::n])) for b in range(n)) return ext[0] if n == 1 else ext def tobytes(self, encoder_name="raw", *args): - return _cn1image.tobytes(self._im) + raw = _cn1image.tobytes(self._im) + if self.mode == "1": + # rawmode "1": eight pixels a byte, most significant bit first, rows padded + return _pack_bits(raw, self.width, self.height) + return raw def frombytes(self, data): + if self.mode == "1": + data = _unpack_bits(data, self.width, self.height) self._im = _cn1image.frombytes(self.width, self.height, _bands(self.mode), data) def crop(self, box=None): @@ -199,10 +263,14 @@ def resize(self, size, resample=None, box=None, reducing_gap=None): resample = BICUBIC if (w, h) == self._size and tuple(box) == (0, 0) + self._size: return self.copy() + if self.mode in ("1", "P"): + resample = NEAREST if resample == NEAREST: a = [(box[2] - box[0]) / w, 0, box[0], 0, (box[3] - box[1]) / h, box[1]] return self._new(_cn1image.affine(self._im, w, h, a, bytes(4)), (w, h)) - return self._new(_cn1image.resample(self._im, w, h, tuple(float(v) for v in box)), (w, h)) + # Pillow resamples LA and RGBA premultiplied (La, RGBa), then divides back + return self._new(_cn1image.resample(self._im, w, h, tuple(float(v) for v in box), int(resample), + self.mode in ("LA", "RGBA")), (w, h)) def thumbnail(self, size, resample=BICUBIC, reducing_gap=2.0): x, y = math.floor(size[0]), math.floor(size[1]) @@ -274,7 +342,35 @@ def transform(x, y, m): def convert(self, mode=None, matrix=None, dither=None, palette=None, colors=256): if mode is None or mode == self.mode: return self.copy() - return self._new(_cn1image.convert(self._im, _bands(mode)), mode=mode) + if mode == "P": + raise ValueError("conversion to P is not supported") + handle = self._im + if self.mode == "P": + handle = _cn1image.palette(handle, self._palette_lut(mode in ("LA", "RGBA"))) + if mode == "1": + if dither != 0: + # Pillow's default here is Floyd-Steinberg dithering, which is not built in + raise ValueError("conversion to 1 is supported with dither=Dither.NONE only") + lum = handle if _bands(self.mode) == 1 and self.mode != "P" else _cn1image.convert(handle, 1) + return self._new(_cn1image.point(lum, bytes(255 if v >= 128 else 0 for v in range(256))), + mode="1") + return self._new(_cn1image.convert(handle, _bands(mode)), mode=mode) + + def _palette_lut(self, alpha): + """256 RGBA entries: the palette, with the tRNS alpha when `alpha`.""" + lut = bytearray(1024) + pal = self._palette or b"" + t = self.info.get("transparency") if alpha else None + for i in range(256): + if 3 * i + 2 < len(pal): + lut[4 * i:4 * i + 3] = pal[3 * i:3 * i + 3] + a = 255 + if isinstance(t, int): + a = 0 if i == t else 255 + elif isinstance(t, (bytes, bytearray)) and i < len(t): + a = t[i] + lut[4 * i + 3] = a + return bytes(lut) def paste(self, im, box=None, mask=None): if box is not None and len(box) == 2 and isinstance(im, Image): @@ -374,6 +470,8 @@ def new(mode, size, color=0): def frombytes(mode, size, data, decoder_name="raw", *args): w, h = int(size[0]), int(size[1]) + if mode == "1": + data = _unpack_bits(data, w, h) return _wrap(mode, (w, h), _cn1image.frombytes(w, h, _bands(mode), data)) @@ -433,10 +531,23 @@ def _chunk(kind, body): def _png_encode(im): - ctype = {"L": 0, "RGB": 2, "RGBA": 6}[im.mode] - head = struct.pack(">IIBBBBB", im.width, im.height, 8, ctype, 0, 0, 0) - body = zlib.compress(_cn1image.png_rows(im._im), 6) - return _PNG_SIG + _chunk(b"IHDR", head) + _chunk(b"IDAT", body) + _chunk(b"IEND", b"") + ctype = {"1": 0, "L": 0, "P": 3, "LA": 4, "RGB": 2, "RGBA": 6}[im.mode] + depth = 1 if im.mode == "1" else 8 + head = struct.pack(">IIBBBBB", im.width, im.height, depth, ctype, 0, 0, 0) + if depth == 1: + rows = _pack_bits(_cn1image.tobytes(im._im), im.width, im.height, True) + else: + rows = _cn1image.png_rows(im._im) + extra = b"" + if im.mode == "P": + extra = _chunk(b"PLTE", bytes(im._palette or bytes(768))) + t = im.info.get("transparency") + if isinstance(t, int): + extra += _chunk(b"tRNS", b"\xff" * t + b"\x00") + elif isinstance(t, (bytes, bytearray)): + extra += _chunk(b"tRNS", bytes(t)) + body = zlib.compress(rows, 6) + return _PNG_SIG + _chunk(b"IHDR", head) + extra + _chunk(b"IDAT", body) + _chunk(b"IEND", b"") def _png_decode(data): @@ -462,14 +573,37 @@ def _png_decode(data): if head is None: raise UnidentifiedImageError("cannot identify image file") w, h, depth, ctype, _, _, interlace = head - if interlace: - raise OSError("interlaced PNG files are not supported") channels = {0: 1, 2: 3, 3: 1, 4: 2, 6: 4}[ctype] - row = (w * channels * depth + 7) // 8 bpp = max(1, channels * depth // 8) - raw = _cn1image.png_unfilter(zlib.decompress(b"".join(idat)), row, h, bpp) - if ctype != 3: - trns = None - handle = _cn1image.png_expand(raw, w, h, ctype, depth, plte, trns) - mode = "L" if ctype == 0 else "RGB" if ctype == 2 or (ctype == 3 and trns is None) else "RGBA" - return _wrap(mode, (w, h), handle) + data = zlib.decompress(b"".join(idat)) + mode = {0: "1" if depth == 1 else "L", 2: "RGB", 3: "P", 4: "LA", 6: "RGBA"}[ctype] + if not interlace: + row = (w * channels * depth + 7) // 8 + handle = _cn1image.png_expand(_cn1image.png_unfilter(data, row, h, bpp), w, h, ctype, depth) + else: + handle = _cn1image.new(w, h, _BANDS[mode], bytes(_BANDS[mode])) + pos = 0 + for x0, y0, dx, dy in _ADAM7: + pw = (w - x0 + dx - 1) // dx + ph = (h - y0 + dy - 1) // dy + if pw <= 0 or ph <= 0: + continue + row = (pw * channels * depth + 7) // 8 + n = (row + 1) * ph + sub = _cn1image.png_expand(_cn1image.png_unfilter(data[pos:pos + n], row, ph, bpp), pw, ph, ctype, + depth) + _cn1image.scatter(handle, sub, x0, y0, dx, dy) + pos += n + im = _wrap(mode, (w, h), handle) + if ctype == 3: + im._palette = plte + if trns is not None: + # PngImagePlugin's chunk_tRNS + if ctype == 3: + simple = trns.count(b"\x00") == 1 and trns.count(b"\xff") == len(trns) - 1 + im.info["transparency"] = trns.index(b"\x00") if simple else trns + elif ctype == 0: + im.info["transparency"] = struct.unpack(">H", trns[:2])[0] + elif ctype == 2: + im.info["transparency"] = struct.unpack(">HHH", trns[:6]) + return im diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/ImageDraw.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/ImageDraw.py index 3551bf52c08..98e8b75b67c 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/ImageDraw.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/ImageDraw.py @@ -6,7 +6,7 @@ Points, rectangles and horizontal or vertical lines match Pillow pixel for pixel. Diagonal lines are Bresenham's, and ellipses and polygons are filled by pixel centre, so their edges can differ from Pillow's by a pixel. Nothing -is antialiased and there is no text. +is antialiased. Text is drawn with the built-in bitmap font (see ImageFont). """ import _cn1image @@ -169,8 +169,85 @@ def polygon(self, xy, fill=None, outline=None, width=1): a, b = q[i], q[(i + 1) % len(q)] self._segment(a[0], a[1], b[0], b[1], ink, max(1, width)) - def text(self, xy, text, fill=None, font=None, *args, **kwargs): - raise NotImplementedError("ImageDraw.text is not supported") + def _font(self, font): + if font is None: + from PIL import ImageFont + font = ImageFont.load_default() + return font + + def _spacing(self, font, spacing): + return self.textbbox((0, 0), "A", font)[3] + spacing + + def text(self, xy, text, fill=None, font=None, anchor=None, spacing=4, align="left", *args, **kwargs): + """Draws `text` with a bitmap font: each line's glyph mask is filled + with the ink at the integer position, as Pillow's basic layout does.""" + font = self._font(font) + if isinstance(text, bytes): + text = text.decode("latin-1") + if "\n" in text: + return self.multiline_text(xy, text, fill, font, anchor, spacing, align) + ink = self._ink(fill) + if ink is None: + ink = self.ink + mask = font.getmask(text, self.mode) + if mask.width == 0 or mask.height == 0: + return + x, y = int(xy[0]), int(xy[1]) + self.im.paste(ink, (x, y, x + mask.width, y + mask.height), mask) + + def multiline_text(self, xy, text, fill=None, font=None, anchor=None, spacing=4, align="left", + *args, **kwargs): + font = self._font(font) + lines = text.split("\n") + line_spacing = self._spacing(font, spacing) + widths = [self.textlength(line, font) for line in lines] + widest = max(widths) + top = xy[1] + for line, w in zip(lines, widths): + left = xy[0] + if align == "center": + left += (widest - w) / 2.0 + elif align == "right": + left += widest - w + elif align != "left": + raise ValueError('align must be "left", "center" or "right"') + self.text((left, top), line, fill, font) + top += line_spacing + + def textlength(self, text, font=None, *args, **kwargs): + if "\n" in text: + raise ValueError("can't measure length of multiline text") + return self._font(font).getlength(text) + + def textbbox(self, xy, text, font=None, anchor=None, spacing=4, align="left", *args, **kwargs): + font = self._font(font) + if "\n" in text: + return self.multiline_textbbox(xy, text, font, anchor, spacing, align) + b = font.getbbox(text) + return (b[0] + xy[0], b[1] + xy[1], b[2] + xy[0], b[3] + xy[1]) + + def multiline_textbbox(self, xy, text, font=None, anchor=None, spacing=4, align="left", + *args, **kwargs): + font = self._font(font) + lines = text.split("\n") + line_spacing = self._spacing(font, spacing) + widths = [self.textlength(line, font) for line in lines] + widest = max(widths) + bbox = None + top = xy[1] + for line, w in zip(lines, widths): + left = xy[0] + if align == "center": + left += (widest - w) / 2.0 + elif align == "right": + left += widest - w + b = self.textbbox((left, top), line, font) + if bbox is None: + bbox = list(b) + else: + bbox = [min(bbox[0], b[0]), min(bbox[1], b[1]), max(bbox[2], b[2]), max(bbox[3], b[3])] + top += line_spacing + return tuple(bbox) def Draw(im, mode=None): diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/ImageFont.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/ImageFont.py new file mode 100644 index 00000000000..36995620cad --- /dev/null +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/ImageFont.py @@ -0,0 +1,246 @@ +# +# The Python Imaging Library. +# +# PIL raster font management +# +# Copyright (c) 1997-2003 by Secret Labs AB +# Copyright (c) 1996-2003 by Fredrik Lundh +# +# See the README file for information on usage and redistribution. +# +# Pillow is licensed under the MIT-CMU (HPND) license: +# +# By obtaining, using, and/or copying this software and/or its associated +# documentation, you agree that you have read, understood, and will comply +# with the following terms and conditions: +# +# Permission to use, copy, modify and distribute this software and its +# documentation for any purpose and without fee is hereby granted, provided +# that the above copyright notice appears in all copies, and that both that +# copyright notice and this permission notice appear in supporting +# documentation, and that the name of Secret Labs AB or the author not be +# used in advertising or publicity pertaining to distribution of the software +# without specific, written prior permission. +# +# SECRET LABS AB AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS +# SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. +# IN NO EVENT SHALL SECRET LABS AB OR THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +# OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +# PERFORMANCE OF THIS SOFTWARE. +# +# Rewritten for the Codename One Python runtime: the bitmap font loader and +# renderer of Pillow's ImageFont, with Pillow's embedded default font +# (courB08, the PIL font data and its glyph sheet) kept byte for byte. +# +"""PIL raster fonts. Only Pillow's built-in bitmap font is available. + +`load_default()` returns that bitmap font: it is what Pillow returns when it +is built without FreeType. A Pillow with FreeType returns a scalable font +instead (Aileron), which this runtime does not have, so text drawn with +`load_default()` matches `ImageFont.load_default_imagefont()` in CPython. +`truetype()` raises, since no TrueType rasterizer is built in. +""" + +import base64 +import io +import struct + +from PIL import Image + + +class ImageFont: + """A PIL bitmap font: 256 glyphs cut from one mode "1" sheet.""" + + def __init__(self): + self.info = [] + self._glyphs = None + self._sheet = None + self._baseline = 0 + self._ysize = 0 + + def _load_pilfont_data(self, file, image): + if file.readline() != b"PILfont\n": + raise SyntaxError("Not a PILfont file") + file.readline() + while True: + s = file.readline() + if not s or s == b"DATA\n": + break + self.info.append(s) + data = file.read(256 * 20) + if image.mode not in ("1", "L"): + raise TypeError("invalid font image mode") + glyphs = [] + y0 = y1 = 0 + for i in range(256): + g = struct.unpack(">10h", data[i * 20:i * 20 + 20]) + glyphs.append(g) + y0 = min(y0, g[3]) + y1 = max(y1, g[5]) + self._glyphs = glyphs + self._sheet = image + self._baseline = -y0 + self._ysize = y1 - y0 + + def _width(self, text): + return sum(self._glyphs[ord(c) & 255][0] for c in text) + + def getbbox(self, text, *args, **kwargs): + return 0, 0, self._width(text), self._ysize + + def getlength(self, text, *args, **kwargs): + return self._width(text) + + def getmask(self, text, mode="", *args, **kwargs): + mask = Image.new(self._sheet.mode, (self._width(text), self._ysize), 0) + x = 0 + b = self._baseline + for c in text: + dx, dy, x0, y0, x1, y1, sx0, sy0, sx1, sy1 = self._glyphs[ord(c) & 255] + if sx1 > sx0 and sy1 > sy0: + mask.paste(self._sheet.crop((sx0, sy0, sx1, sy1)), (x0 + x, y0 + b)) + x += dx + b += dy + return mask + + +def load_default_imagefont(): + f = ImageFont() + f._load_pilfont_data(io.BytesIO(base64.b64decode(_COURB08_PIL)), + Image.open(io.BytesIO(base64.b64decode(_COURB08_PNG)))) + return f + + +def load_default(size=None): + return load_default_imagefont() + + +def truetype(font=None, size=10, index=0, encoding="", layout_engine=None): + raise OSError("cannot open resource: TrueType fonts are not supported") + + +def load(filename): + raise OSError("cannot find font file: " + str(filename)) + + +_COURB08_PIL = b""" +UElMZm9udAo7Ozs7OzsxMDsKREFUQQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAYAAAAA//8AAQAAAAAAAAABAAEA +BgAAAAH/+gADAAAAAQAAAAMABgAGAAAAAf/6AAT//QADAAAABgADAAYAAAAA//kABQABAAYAAAAL +AAgABgAAAAD/+AAFAAEACwAAABAACQAGAAAAAP/5AAUAAAAQAAAAFQAHAAYAAP////oABQAAABUA +AAAbAAYABgAAAAH/+QAE//wAGwAAAB4AAwAGAAAAAf/5AAQAAQAeAAAAIQAIAAYAAAAB//kABAAB +ACEAAAAkAAgABgAAAAD/+QAE//0AJAAAACgABAAGAAAAAP/6AAX//wAoAAAALQAFAAYAAAAB//8A +BAACAC0AAAAwAAMABgAAAAD//AAF//0AMAAAADUAAQAGAAAAAf//AAMAAAA1AAAANwABAAYAAAAB +//kABQABADcAAAA7AAgABgAAAAD/+QAFAAAAOwAAAEAABwAGAAAAAP/5AAYAAABAAAAARgAHAAYA +AAAA//kABQAAAEYAAABLAAcABgAAAAD/+QAFAAAASwAAAFAABwAGAAAAAP/5AAYAAABQAAAAVgAH +AAYAAAAA//kABQAAAFYAAABbAAcABgAAAAD/+QAFAAAAWwAAAGAABwAGAAAAAP/5AAUAAABgAAAA +ZQAHAAYAAAAA//kABQAAAGUAAABqAAcABgAAAAD/+QAFAAAAagAAAG8ABwAGAAAAAf/8AAMAAABv +AAAAcQAEAAYAAAAA//wAAwACAHEAAAB0AAYABgAAAAD/+gAE//8AdAAAAHgABQAGAAAAAP/7AAT/ +/gB4AAAAfAADAAYAAAAB//oABf//AHwAAACAAAUABgAAAAD/+gAFAAAAgAAAAIUABgAGAAAAAP/5 +AAYAAQCFAAAAiwAIAAYAAP////oABgAAAIsAAACSAAYABgAA////+gAFAAAAkgAAAJgABgAGAAAA +AP/6AAUAAACYAAAAnQAGAAYAAP////oABQAAAJ0AAACjAAYABgAA////+gAFAAAAowAAAKkABgAG +AAD////6AAUAAACpAAAArwAGAAYAAAAA//oABQAAAK8AAAC0AAYABgAA////+gAGAAAAtAAAALsA +BgAGAAAAAP/6AAQAAAC7AAAAvwAGAAYAAP////oABQAAAL8AAADFAAYABgAA////+gAGAAAAxQAA +AMwABgAGAAD////6AAUAAADMAAAA0gAGAAYAAP////oABQAAANIAAADYAAYABgAA////+gAGAAAA +2AAAAN8ABgAGAAAAAP/6AAUAAADfAAAA5AAGAAYAAP////oABQAAAOQAAADqAAYABgAAAAD/+gAF +AAEA6gAAAO8ABwAGAAD////6AAYAAADvAAAA9gAGAAYAAAAA//oABQAAAPYAAAD7AAYABgAA//// ++gAFAAAA+wAAAQEABgAGAAD////6AAYAAAEBAAABCAAGAAYAAP////oABgAAAQgAAAEPAAYABgAA +////+gAGAAABDwAAARYABgAGAAAAAP/6AAYAAAEWAAABHAAGAAYAAP////oABgAAARwAAAEjAAYA +BgAAAAD/+gAFAAABIwAAASgABgAGAAAAAf/5AAQAAQEoAAABKwAIAAYAAAAA//kABAABASsAAAEv +AAgABgAAAAH/+QAEAAEBLwAAATIACAAGAAAAAP/5AAX//AEyAAABNwADAAYAAAAAAAEABgACATcA +AAE9AAEABgAAAAH/+QAE//wBPQAAAUAAAwAGAAAAAP/7AAYAAAFAAAABRgAFAAYAAP////kABQAA +AUYAAAFMAAcABgAAAAD/+wAFAAABTAAAAVEABQAGAAAAAP/5AAYAAAFRAAABVwAHAAYAAAAA//sA +BQAAAVcAAAFcAAUABgAAAAD/+QAFAAABXAAAAWEABwAGAAAAAP/7AAYAAgFhAAABZwAHAAYAAP// +//kABQAAAWcAAAFtAAcABgAAAAD/+QAGAAABbQAAAXMABwAGAAAAAP/5AAQAAgFzAAABdwAJAAYA +AP////kABgAAAXcAAAF+AAcABgAAAAD/+QAGAAABfgAAAYQABwAGAAD////7AAUAAAGEAAABigAF +AAYAAP////sABQAAAYoAAAGQAAUABgAAAAD/+wAFAAABkAAAAZUABQAGAAD////7AAUAAgGVAAAB +mwAHAAYAAAAA//sABgACAZsAAAGhAAcABgAAAAD/+wAGAAABoQAAAacABQAGAAAAAP/7AAYAAAGn +AAABrQAFAAYAAAAA//kABgAAAa0AAAGzAAcABgAA////+wAGAAABswAAAboABQAGAAD////7AAUA +AAG6AAABwAAFAAYAAP////sABgAAAcAAAAHHAAUABgAAAAD/+wAGAAABxwAAAc0ABQAGAAD////7 +AAYAAgHNAAAB1AAHAAYAAAAA//sABQAAAdQAAAHZAAUABgAAAAH/+QAFAAEB2QAAAd0ACAAGAAAA +Av/6AAMAAQHdAAAB3gAHAAYAAAAA//kABAABAd4AAAHiAAgABgAAAAD/+wAF//0B4gAAAecAAgAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAYAAAAB +//sAAwACAecAAAHpAAcABgAAAAD/+QAFAAEB6QAAAe4ACAAGAAAAAP/5AAYAAAHuAAAB9AAHAAYA +AAAA//oABf//AfQAAAH5AAUABgAAAAD/+QAGAAAB+QAAAf8ABwAGAAAAAv/5AAMAAgH/AAACAAAJ +AAYAAAAA//kABQABAgAAAAIFAAgABgAAAAH/+gAE//sCBQAAAggAAQAGAAAAAP/5AAYAAAIIAAAC +DgAHAAYAAAAB//kABf/+Ag4AAAISAAUABgAA////+wAGAAACEgAAAhkABQAGAAAAAP/7AAX//gIZ +AAACHgADAAYAAAAA//wABf/9Ah4AAAIjAAEABgAAAAD/+QAHAAACIwAAAioABwAGAAAAAP/6AAT/ ++wIqAAACLgABAAYAAAAA//kABP/8Ai4AAAIyAAMABgAAAAD/+gAFAAACMgAAAjcABgAGAAAAAf/5 +AAT//QI3AAACOgAEAAYAAAAB//kABP/9AjoAAAI9AAQABgAAAAL/+QAE//sCPQAAAj8AAgAGAAD/ +///7AAYAAgI/AAACRgAHAAYAAAAA//kABgABAkYAAAJMAAgABgAAAAH//AAD//0CTAAAAk4AAQAG +AAAAAf//AAQAAgJOAAACUQADAAYAAAAB//kABP/9AlEAAAJUAAQABgAAAAH/+QAF//4CVAAAAlgA +BQAGAAD////7AAYAAAJYAAACXwAFAAYAAP////kABgAAAl8AAAJmAAcABgAA////+QAGAAACZgAA +Am0ABwAGAAD////5AAYAAAJtAAACdAAHAAYAAAAA//sABQACAnQAAAJ5AAcABgAA////9wAGAAAC +eQAAAoAACQAGAAD////3AAYAAAKAAAAChwAJAAYAAP////cABgAAAocAAAKOAAkABgAA////9wAG +AAACjgAAApUACQAGAAD////4AAYAAAKVAAACnAAIAAYAAP////cABgAAApwAAAKjAAkABgAA//// ++gAGAAACowAAAqoABgAGAAAAAP/6AAUAAgKqAAACrwAIAAYAAP////cABQAAAq8AAAK1AAkABgAA +////9wAFAAACtQAAArsACQAGAAD////3AAUAAAK7AAACwQAJAAYAAP////gABQAAAsEAAALHAAgA +BgAAAAD/9wAEAAACxwAAAssACQAGAAAAAP/3AAQAAALLAAACzwAJAAYAAAAA//cABAAAAs8AAALT +AAkABgAAAAD/+AAEAAAC0wAAAtcACAAGAAD////6AAUAAALXAAAC3QAGAAYAAP////cABgAAAt0A +AALkAAkABgAAAAD/9wAFAAAC5AAAAukACQAGAAAAAP/3AAUAAALpAAAC7gAJAAYAAAAA//cABQAA +Au4AAALzAAkABgAAAAD/9wAFAAAC8wAAAvgACQAGAAAAAP/4AAUAAAL4AAAC/QAIAAYAAAAA//oA +Bf//Av0AAAMCAAUABgAA////+gAGAAADAgAAAwkABgAGAAD////3AAYAAAMJAAADEAAJAAYAAP// +//cABgAAAxAAAAMXAAkABgAA////9wAGAAADFwAAAx4ACQAGAAD////4AAYAAAAAAAoABwASAAYA +AP////cABgAAAAcACgAOABMABgAA////+gAFAAAADgAKABQAEAAGAAD////6AAYAAAAUAAoAGwAQ +AAYAAAAA//gABgAAABsACgAhABIABgAAAAD/+AAGAAAAIQAKACcAEgAGAAAAAP/4AAYAAAAnAAoA +LQASAAYAAAAA//gABgAAAC0ACgAzABIABgAAAAD/+QAGAAAAMwAKADkAEQAGAAAAAP/3AAYAAAA5 +AAoAPwATAAYAAP////sABQAAAD8ACgBFAA8ABgAAAAD/+wAFAAIARQAKAEoAEQAGAAAAAP/4AAUA +AABKAAoATwASAAYAAAAA//gABQAAAE8ACgBUABIABgAAAAD/+AAFAAAAVAAKAFkAEgAGAAAAAP/5 +AAUAAABZAAoAXgARAAYAAAAA//gABgAAAF4ACgBkABIABgAAAAD/+AAGAAAAZAAKAGoAEgAGAAAA +AP/4AAYAAABqAAoAcAASAAYAAAAA//kABgAAAHAACgB2ABEABgAAAAD/+AAFAAAAdgAKAHsAEgAG +AAD////4AAYAAAB7AAoAggASAAYAAAAA//gABQAAAIIACgCHABIABgAAAAD/+AAFAAAAhwAKAIwA +EgAGAAAAAP/4AAUAAACMAAoAkQASAAYAAAAA//gABQAAAJEACgCWABIABgAAAAD/+QAFAAAAlgAK +AJsAEQAGAAAAAP/6AAX//wCbAAoAoAAPAAYAAAAA//oABQABAKAACgClABEABgAA////+AAGAAAA +pQAKAKwAEgAGAAD////4AAYAAACsAAoAswASAAYAAP////gABgAAALMACgC6ABIABgAA////+QAG +AAAAugAKAMEAEQAGAAD////4AAYAAgDBAAoAyAAUAAYAAP////kABQACAMgACgDOABMABgAA//// ++QAGAAIAzgAKANUAEw== +""" + +_COURB08_PNG = b""" +iVBORw0KGgoAAAANSUhEUgAAAx4AAAAUAQAAAAArMtZoAAAEwElEQVR4nABlAJr/AHVE4czCI/4u +Mc4b7vuds/xzjz5/3/7u/n9vMe7vnfH/9++vPn/xyf5zhxzjt8GHw8+2d83u8x27199/nxuQ6Od9 +M43/5z2I+9n9ZtmDBwMQECDRQw/eQIQohJXxpBCNVE6QCCAAAAD//wBlAJr/AgALyj1t/wINwq0g +LeNZUworuN1cjTPIzrTX6ofHWeo3v336qPzfEwRmBnHTtf95/fglZK5N0PDgfRTslpGBvz7LFc4F +IUXBWQGjQ5MGCx34EDFPwXiY4YbYxavpnhHFrk14CDAAAAD//wBlAJr/AgKqRooH2gAgPeggvUAA +Bu2WfgPoAwzRAABAAAAAAACQgLz/3Uv4Gv+gX7BJgDeeGP6AAAD1NMDzKHD7ANWr3loYbxsAD791 +NAADfcoIDyP44K/jv4Y63/Z+t98Ovt+ub4T48LAAAAD//wBlAJr/AuplMlADJAAAAGuAphWpqhMx +in0A/fRvAYBABPgBwBUgABBQ/sYAyv9g0bCHgOLoGAAAAAAAREAAwI7nr0ArYpow7aX8//9LaP/9 +SjdavWA8ePHeBIKB//81/83ndznOaXx379wAAAD//wBlAJr/AqDxW+D3AABAAbUh/QMnbQag/gAY +AYDAAACgtgD/gOqAAAB5IA/8AAAk+n9w0AAA8AAAmFRJuPo27ciC0cD5oeW4E7KA/wD3ECMAn2tt +y8PgwH8AfAxFzC0JzeAMtratAsC/ffwAAAD//wBlAJr/BGKAyCAA4AAAAvgeYTAwHd1kmQF5chkG +ABoMIHcL5xVpTfQbUqzlAAAErwAQBgAAEOClA5D9il08AEh/tUzdCBsXkbgACED+woQg8Si9VeqY +lODCn7lmF6NhnAEYgAAA/NMIAAAAAAD//2JgjLZgVGBg5Pv/Tvpc8hwGBjYGJADjHDrAwPzAjv/H +/Wf3PzCwtzcwHmBgYGcwbZz8wHaCAQMDOwMDQ8MCBgYOC3W7mp+f0w+wHOYxO3OG+e376hsMZjk3 +AAAAAP//YmCMY2A4wMAIN5e5gQETPD6AZisDAwMDgzSDAAPjByiHcQMDAwMDg1nOze1lByRu5/47 +c4859311AYNZzg0AAAAA//9iYGDBYihOIIMuwIjGL39/fwffA8b//xv/P2BPtzzHwCBjUQAAAAD/ +/yLFBrIBAAAA//9i1HhcwdhizX7u8NZNzyLbvT97bfrMf/QHI8evOwcSqGUJAAAA//9iYBB81iSw +pEE170Qrg5MIYydHqwdDQRMrAwcVrQAAAAD//2J4x7j9AAMDn8Q/BgYLBoaiAwwMjPdvMDBYM1Tv +oJodAAAAAP//Yqo/83+dxePWlxl3npsel9lvLfPcqlE9725C+acfVLMEAAAA//9i+s9gwCoaaGMR +evta/58PTEWzr21hufPjA8N+qlnBwAAAAAD//2JiWLci5v1+HmFXDqcnULE/MxgYGBj+f6CaJQAA +AAD//2Ji2FrkY3iYpYC5qDeGgeEMAwPDvwQBBoYvcTwOVLMEAAAA//9isDBgkP///0EOg9z35v// +Gc/eeW7BwPj5+QGZhANUswMAAAD//2JgqGBgYGBgqEMXlvhMPUsAAAAA//8iYDd1AAAAAP//AwDR +w7IkEbzhVQAAAABJRU5ErkJggg== +""" diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/pandas/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/pandas/__init__.py index a395be6aa00..6a518eaeb88 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/site/pandas/__init__.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/pandas/__init__.py @@ -751,9 +751,19 @@ def __repr__(self): if not self._v: return "Series([], " + ("Name: %s, " % self.name if self.name is not None else "") + \ "dtype: " + self._dt + ")" - labels = [_label(v) for v in self._index._v] + n = len(self._v) + shown = self + if n > _MAX_ROWS: + # display.max_rows exceeded: min_rows of them, half from each end + shown = self._take(list(range(_MIN_ROWS // 2)) + list(range(n - _MIN_ROWS // 2, n))) + labels = [_label(v) for v in shown._index._v] + vals = _rjust_all(_fmt_values(shown._v, shown._dt)) + if shown is not self: + # SeriesFormatter: dots centred (str.center) in the value width + dots = ".." if len(vals[_MIN_ROWS // 2 - 1]) <= 3 else "..." + vals.insert(_MIN_ROWS // 2, dots.center(len(vals[_MIN_ROWS // 2 - 1]))) + labels.insert(_MIN_ROWS // 2, "") w = max(len(x) for x in labels) - vals = _rjust_all(_fmt_values(self._v, self._dt)) lines = [] if self._index.name is not None: lines.append(str(self._index.name)) @@ -762,6 +772,8 @@ def __repr__(self): foot = [] if self.name is not None: foot.append("Name: " + str(self.name)) + if shown is not self: + foot.append("Length: %d" % n) foot.append("dtype: " + self._dt) lines.append(", ".join(foot)) return "\n".join(lines) @@ -924,6 +936,27 @@ def __getitem__(self, key): # ---- DataFrame ------------------------------------------------------------ +_MAX_ROWS = 60 +_MIN_ROWS = 10 +_WIDTH = 80 + + +def _adjoin(strcols): + return [" ".join(cells) for cells in zip(*strcols)] + + +def _fit_columns(strcols, max_len): + """StringFormatter._fit_strcols_to_terminal_width: drop middle columns + until the repr fits, and answer how many data columns remain.""" + adj_dif = max_len - _WIDTH + 1 + lens = [max(len(x) for x in col) for col in strcols] + while adj_dif > 0 and len(lens) > 1: + mid = round(len(lens) / 2) + adj_dif -= lens[mid] + 1 + del lens[mid] + return max(len(lens) - 1, 2) + + class DataFrame: def __init__(self, data=None, index=None, columns=None, dtype=None): cols = [] @@ -939,7 +972,7 @@ def __init__(self, data=None, index=None, columns=None, dtype=None): if index is None: index = v._index n = len(v) - elif isinstance(v, (list, tuple, _np.ndarray)): + elif isinstance(v, (list, tuple, range, _np.ndarray)): n = len(v) if n is None: n = len(index) if index is not None else 1 @@ -948,7 +981,7 @@ def __init__(self, data=None, index=None, columns=None, dtype=None): continue if isinstance(v, Series): vals = v._v - elif isinstance(v, (list, tuple)): + elif isinstance(v, (list, tuple, range)): vals = list(v) elif isinstance(v, _np.ndarray): vals = v.tolist() @@ -1417,17 +1450,57 @@ def __repr__(self): if not self._cols or not len(self._index): return "Empty DataFrame\nColumns: [" + ", ".join(_label(n) for n in names) + \ "]\nIndex: [" + ", ".join(_label(v) for v in self._index._v) + "]" + # pandas' DataFrameFormatter with the defaults of a run that is not + # an interactive terminal: max_rows 60 (then min_rows 10), and + # max_columns 0, which fits the columns to a width of 80 + nrows = len(self._index) + ncols = len(self._cols) + rows = list(range(nrows)) + vtrunc = nrows > _MAX_ROWS + if vtrunc: + half = _MIN_ROWS // 2 + rows = rows[:half] + rows[-half:] + strcols = self._strcols(rows, None, vtrunc, -1) + lines = _adjoin(strcols) + htrunc = False + if max(len(x) for x in lines) > _WIDTH: + fitted = _fit_columns(strcols, max(len(x) for x in lines)) + if ncols > fitted: + htrunc = True + keep = fitted // 2 + names = self._names() + strcols = self._strcols(rows, names[:keep] + names[ncols - keep:], vtrunc, keep) + lines = _adjoin(strcols) + if vtrunc or htrunc: + lines.append("") + lines.append("[%d rows x %d columns]" % (nrows, ncols)) + return "\n".join(lines) + + def _strcols(self, rows, names, vtrunc, dotcol): + """The index and every column as lists of equal-width strings.""" + frame = self._take(rows, names) named = self._index.name is not None - idx = [""] + ([str(self._index.name)] if named else []) + [_label(v) for v in self._index._v] + idx = [""] + ([str(self._index.name)] if named else []) + [_label(v) for v in frame._index._v] iw = max(len(x) for x in idx) - lines = [x.ljust(iw) for x in idx] - for c in self._cols: + strcols = [[x.ljust(iw) for x in idx]] + for c in frame._cols: head = (" " if _numeric(c._dt) else "") + _label(c.name) cells = _fmt_values(c._v, c._dt) w = max([len(head)] + [len(x) for x in cells]) - colstrs = [head.rjust(w)] + ([" " * w] if named else []) + [x.rjust(w) for x in cells] - lines = [a + " " + b for a, b in zip(lines, colstrs)] - return "\n".join(lines) + strcols.append([head.rjust(w)] + ([" " * w] if named else []) + [x.rjust(w) for x in cells]) + if dotcol >= 0: + strcols.insert(dotcol + 1, [" ..."] * len(idx)) + if vtrunc: + at = _MIN_ROWS // 2 + 1 + (1 if named else 0) + for ix, col in enumerate(strcols): + width = len(col[at - 1]) + dot_col = ix == dotcol + 1 and dotcol >= 0 + dots = "..." if width > 3 or dot_col else ".." + if ix == 0: + col.insert(at, dots.ljust(width)) + else: + col.insert(at, dots.rjust(4 if dot_col else width)) + return strcols __str__ = __repr__ diff --git a/maven/python-transpiler/src/test/resources/conformance/pandas_truncate.out b/maven/python-transpiler/src/test/resources/conformance/pandas_truncate.out new file mode 100644 index 00000000000..f3aa5a47382 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/pandas_truncate.out @@ -0,0 +1,153 @@ +0 0 +1 1 +2 2 +3 3 +4 4 + .. +95 95 +96 96 +97 97 +98 98 +99 99 +Length: 100, dtype: int64 +0 0.0 +1 1.5 +2 3.0 +3 4.5 +4 6.0 + ... +56 84.0 +57 85.5 +58 87.0 +59 88.5 +60 90.0 +Name: half, Length: 61, dtype: float64 +57 57 +58 58 +59 59 +dtype: int64 +k0 s0 +k1 s1 +k2 s2 +k3 s3 +k4 s4 + ... +k65 s65 +k66 s66 +k67 s67 +k68 s68 +k69 s69 +Length: 70, dtype: str + a b c +0 0 0.00 x0 +1 1 0.25 x1 +2 2 0.50 x2 +3 3 0.75 x3 +4 4 1.00 x4 +.. ... ... ... +195 195 48.75 x195 +196 196 49.00 x196 +197 197 49.25 x197 +198 198 49.50 x198 +199 199 49.75 x199 + +[200 rows x 3 columns] + a b c +0 0 0.00 x0 +1 1 0.25 x1 +2 2 0.50 x2 +3 3 0.75 x3 +4 4 1.00 x4 +5 5 1.25 x5 +6 6 1.50 x6 +7 7 1.75 x7 +8 8 2.00 x8 +9 9 2.25 x9 +10 10 2.50 x10 +11 11 2.75 x11 +12 12 3.00 x12 +13 13 3.25 x13 +14 14 3.50 x14 +15 15 3.75 x15 +16 16 4.00 x16 +17 17 4.25 x17 +18 18 4.50 x18 +19 19 4.75 x19 +20 20 5.00 x20 +21 21 5.25 x21 +22 22 5.50 x22 +23 23 5.75 x23 +24 24 6.00 x24 +25 25 6.25 x25 +26 26 6.50 x26 +27 27 6.75 x27 +28 28 7.00 x28 +29 29 7.25 x29 +30 30 7.50 x30 +31 31 7.75 x31 +32 32 8.00 x32 +33 33 8.25 x33 +34 34 8.50 x34 +35 35 8.75 x35 +36 36 9.00 x36 +37 37 9.25 x37 +38 38 9.50 x38 +39 39 9.75 x39 +40 40 10.00 x40 +41 41 10.25 x41 +42 42 10.50 x42 +43 43 10.75 x43 +44 44 11.00 x44 +45 45 11.25 x45 +46 46 11.50 x46 +47 47 11.75 x47 +48 48 12.00 x48 +49 49 12.25 x49 +50 50 12.50 x50 +51 51 12.75 x51 +52 52 13.00 x52 +53 53 13.25 x53 +54 54 13.50 x54 +55 55 13.75 x55 +56 56 14.00 x56 +57 57 14.25 x57 +58 58 14.50 x58 +59 59 14.75 x59 + n +0 0 +1 1 +2 2 +3 3 +4 4 +.. .. +56 56 +57 57 +58 58 +59 59 +60 60 + +[61 rows x 1 columns] + c0 c1 c2 c3 c4 c5 c6 c7 ... c17 c18 c19 c20 c21 c22 c23 c24 +0 0 1 2 3 4 5 6 7 ... 17 18 19 20 21 22 23 24 +1 0 2 4 6 8 10 12 14 ... 34 36 38 40 42 44 46 48 +2 0 3 6 9 12 15 18 21 ... 51 54 57 60 63 66 69 72 + +[3 rows x 25 columns] + col0 col1 col2 col3 col4 ... col25 col26 col27 col28 col29 +0 0 0 0 0 0 ... 0 0 0 0 0 +1 0 1 2 3 4 ... 25 26 27 28 29 +2 0 2 4 6 8 ... 50 52 54 56 58 +3 0 3 6 9 12 ... 75 78 81 84 87 +4 0 4 8 12 16 ... 100 104 108 112 116 +.. ... ... ... ... ... ... ... ... ... ... ... +75 0 75 150 225 300 ... 1875 1950 2025 2100 2175 +76 0 76 152 228 304 ... 1900 1976 2052 2128 2204 +77 0 77 154 231 308 ... 1925 2002 2079 2156 2233 +78 0 78 156 234 312 ... 1950 2028 2106 2184 2262 +79 0 79 158 237 316 ... 1975 2054 2133 2212 2291 + +[80 rows x 30 columns] + v0 v1 v2 v3 v4 v5 v6 v7 ... v12 v13 v14 v15 v16 v17 v18 v19 +0 0 1 2 3 4 5 6 7 ... 12 13 14 15 16 17 18 19 + +[1 rows x 20 columns] diff --git a/maven/python-transpiler/src/test/resources/conformance/pandas_truncate.py b/maven/python-transpiler/src/test/resources/conformance/pandas_truncate.py new file mode 100644 index 00000000000..9c1706c7b1b --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/pandas_truncate.py @@ -0,0 +1,18 @@ +# pandas: the truncated repr of long and wide frames and series with the +# default display options of a non-interactive run (max_rows 60, min_rows 10, +# max_columns 20), including the "[N rows x M columns]" footer. +import pandas as pd + +print(pd.Series(range(100))) +print(pd.Series([i * 1.5 for i in range(61)], name="half")) +print(pd.Series(range(60)).tail(3)) +print(pd.Series(["s%d" % i for i in range(70)], index=["k%d" % i for i in range(70)])) +long = pd.DataFrame({"a": range(200), "b": [i * 0.25 for i in range(200)], "c": ["x%d" % i for i in range(200)]}) +print(long) +print(long.head(60)) +print(pd.DataFrame({"n": range(61)})) +wide = pd.DataFrame({"c%d" % i: [i, i * 2, i * 3] for i in range(25)}) +print(wide) +both = pd.DataFrame({"col%d" % i: [j * i for j in range(80)] for i in range(30)}) +print(both) +print(pd.DataFrame({"v%d" % i: [i] for i in range(20)})) diff --git a/maven/python-transpiler/src/test/resources/conformance/pil_resample_text.out b/maven/python-transpiler/src/test/resources/conformance/pil_resample_text.out new file mode 100644 index 00000000000..a94e350e4c1 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/pil_resample_text.out @@ -0,0 +1,109 @@ +L NEAREST (7, 5) L (7, 5) 317e290044665d36 +L NEAREST (40, 31) L (40, 31) 88c007fa7c023c1d +L NEAREST (23, 9) L (23, 9) e11c8438b244b740 +L BOX (7, 5) L (7, 5) ad2ff5381ff2b299 +L BOX (40, 31) L (40, 31) 88c007fa7c023c1d +L BOX (23, 9) L (23, 9) d2f2635e6914c9ee +L BILINEAR (7, 5) L (7, 5) f7e9c58179c39701 +L BILINEAR (40, 31) L (40, 31) 14ded55dbb5be1df +L BILINEAR (23, 9) L (23, 9) e36664e78a7e04e3 +L HAMMING (7, 5) L (7, 5) 4f0be1d04dcf959e +L HAMMING (40, 31) L (40, 31) a1018e887bb03a10 +L HAMMING (23, 9) L (23, 9) 4b8083ab567ab493 +L BICUBIC (7, 5) L (7, 5) e6d3d9b1bb2527a8 +L BICUBIC (40, 31) L (40, 31) 837957f081a8f2f7 +L BICUBIC (23, 9) L (23, 9) f0a5fa21a29c9d3b +L LANCZOS (7, 5) L (7, 5) f8543414d53f578d +L LANCZOS (40, 31) L (40, 31) ba10448f09b60304 +L LANCZOS (23, 9) L (23, 9) 1169a5bc37a485b9 +LA NEAREST (7, 5) LA (7, 5) 4f051178fe457184 +LA NEAREST (40, 31) LA (40, 31) 4ca67303070e3a26 +LA NEAREST (23, 9) LA (23, 9) 0e0f39a6ab132eca +LA BOX (7, 5) LA (7, 5) 496d40da371758fd +LA BOX (40, 31) LA (40, 31) de9e85c9598ccdcc +LA BOX (23, 9) LA (23, 9) 7e982c8d127797ef +LA BILINEAR (7, 5) LA (7, 5) 869115d1fcdfab49 +LA BILINEAR (40, 31) LA (40, 31) f4c9e81b42481bb6 +LA BILINEAR (23, 9) LA (23, 9) d741e0a88d1e939c +LA HAMMING (7, 5) LA (7, 5) 146953cd4b9786f9 +LA HAMMING (40, 31) LA (40, 31) aa8196241eb4ec8f +LA HAMMING (23, 9) LA (23, 9) 6fca589f1505b4c2 +LA BICUBIC (7, 5) LA (7, 5) 7538d52b300d75de +LA BICUBIC (40, 31) LA (40, 31) a1e626e18f9d55f8 +LA BICUBIC (23, 9) LA (23, 9) 89cc29cc2c55af32 +LA LANCZOS (7, 5) LA (7, 5) be6a80d19140cdbd +LA LANCZOS (40, 31) LA (40, 31) fafe97297a81c45c +LA LANCZOS (23, 9) LA (23, 9) 3848d62180fa620d +RGB NEAREST (7, 5) RGB (7, 5) e33902eaccc6a4ee +RGB NEAREST (40, 31) RGB (40, 31) 47f9c9698efa1235 +RGB NEAREST (23, 9) RGB (23, 9) a5dd2f531970cbb1 +RGB BOX (7, 5) RGB (7, 5) d2f13c4a209c78a3 +RGB BOX (40, 31) RGB (40, 31) 47f9c9698efa1235 +RGB BOX (23, 9) RGB (23, 9) 8f8544a34dc17546 +RGB BILINEAR (7, 5) RGB (7, 5) 4ac791b27d4b212a +RGB BILINEAR (40, 31) RGB (40, 31) 3f6427d2172e7f5b +RGB BILINEAR (23, 9) RGB (23, 9) b45290a6a13ada03 +RGB HAMMING (7, 5) RGB (7, 5) c9faa1219542169f +RGB HAMMING (40, 31) RGB (40, 31) 7c132f71a3ba0c41 +RGB HAMMING (23, 9) RGB (23, 9) 997ac4129b854c1c +RGB BICUBIC (7, 5) RGB (7, 5) d9687d6b35a360f9 +RGB BICUBIC (40, 31) RGB (40, 31) ece63f0f00095a3e +RGB BICUBIC (23, 9) RGB (23, 9) 3be22fffafb771ba +RGB LANCZOS (7, 5) RGB (7, 5) 965bc91018afa07c +RGB LANCZOS (40, 31) RGB (40, 31) 389ff5d0f68f69f8 +RGB LANCZOS (23, 9) RGB (23, 9) b8cca520bb98a80c +RGBA NEAREST (7, 5) RGBA (7, 5) 45384e277d65b9f8 +RGBA NEAREST (40, 31) RGBA (40, 31) b1a449a70e4c4d36 +RGBA NEAREST (23, 9) RGBA (23, 9) 1066b32ad22ed4f9 +RGBA BOX (7, 5) RGBA (7, 5) 00dc8faf009a40a8 +RGBA BOX (40, 31) RGBA (40, 31) 7fab50faa04287ed +RGBA BOX (23, 9) RGBA (23, 9) 7cae7384d510a836 +RGBA BILINEAR (7, 5) RGBA (7, 5) c92def49dc9346c4 +RGBA BILINEAR (40, 31) RGBA (40, 31) 341294f199699773 +RGBA BILINEAR (23, 9) RGBA (23, 9) 287db10281b4152d +RGBA HAMMING (7, 5) RGBA (7, 5) 9bbf2f0426358506 +RGBA HAMMING (40, 31) RGBA (40, 31) 4a53513bead189c9 +RGBA HAMMING (23, 9) RGBA (23, 9) 69c680088156aa62 +RGBA BICUBIC (7, 5) RGBA (7, 5) 96d1891887bf79c7 +RGBA BICUBIC (40, 31) RGBA (40, 31) 8bcefd1f3ece5016 +RGBA BICUBIC (23, 9) RGBA (23, 9) d9335c6706d07f37 +RGBA LANCZOS (7, 5) RGBA (7, 5) e3663fa071887437 +RGBA LANCZOS (40, 31) RGBA (40, 31) 391f5cd588a24dc3 +RGBA LANCZOS (23, 9) RGBA (23, 9) 707fe438d975587c +[(24, 18, 18), (99, 29, 21), (173, 39, 24), (45, 73, 20), (120, 84, 36), (194, 94, 51)] +[(0, 0, 17), (15, 1, 17), (43, 5, 17), (68, 8, 17), (93, 12, 17), (117, 15, 17)] +(91, 54, 26) +(118, 101, 116) +(16, 6) 6fb8af0afd45cb63 +(3, 4, 69, 15) 66 +(-2, 20, 28, 46) +6c098d21954f72d6 (0, 0, 90, 40) (200, 0, 0) +................ +................ +......###....... +.......##....... +..###..####..... +.##.##.##.##.... +..####.##.##.... +.##.##.##.##.... +.##########..... +................ +................ +................ +74303965a9e21927 +IL_RGB RGB (11, 7) None 10f09e17645d5b06 + (0, 0, 0) (200, 180, 164) None + 45363c05529af79c 10f09e17645d5b06 ee3a5b55c38a9899 + RGB None True +IL_PAL P (9, 10) b'\xff\x80\x00' 3c3997f48d9bf636 + 0 2 [0, 0, 0, 255, 0, 0, 0, 255, 0, 0, 0, 255, 200, 100, 50] + 0cbb669bf69f14ce e3c91fab074d86c8 d0b1c9e23d8b4cdc + P b'\xff\x80\x00' True +IL_LA LA (5, 3) None 32e41bbf1cea4dd4 + (0, 255) (200, 135) None + 19ea97f36d2bfdc3 7b29591c95e16128 9505e021efcbc783 + LA None True +P 68df6317a6123f7f [0, 0, 0, 255, 0, 0] +1 70108ab33512ab87 0 5df160beaaac97d7 +(96, 215) 5217259aa9b776d0 297ea07acfe6cd58 558a561340bc757d +0e100e101e301e301c30 True diff --git a/maven/python-transpiler/src/test/resources/conformance/pil_resample_text.py b/maven/python-transpiler/src/test/resources/conformance/pil_resample_text.py new file mode 100644 index 00000000000..b46f9b202c7 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/pil_resample_text.py @@ -0,0 +1,87 @@ +# Pillow: BOX, BILINEAR, HAMMING, BICUBIC and LANCZOS resampling (two-pass +# convolution, premultiplied for LA and RGBA), text in the built-in bitmap +# font, and PNGs that open as P, LA and 1, interlaced (Adam7) ones included. +import hashlib +import io + +from PIL import Image, ImageDraw, ImageFont + +IL_RGB = '89504e470d0a1a0a0000000d494844520000000b000000070802000001260b109c000000ac49444154789c7d8d210e02311444a70ab7aa2488e69b35cd3adc26fc04b7ba62150255835bf90f51fd756f80e91110d59c804be010740d0143f2269964663200909b42130441962c0ff8d5a302c6cb18b0340086671778149e332f95934169cdf91bd8b5870824b414c672e7e13ec08a6399a3a424b7222f63cbf0b307c1d2c653c7b40de422f54243a27da6b1d0b1d264706a27db3f000aabbdd789f5123445bd8ade933eb3ee8a1eaa9edf9c22367ca460c6f60000000049454e44ae426082' +IL_PAL = '89504e470d0a1a0a0000000d49484452000000090000000a0403000001e32c591b0000000f504c5445000000ff000000ff000000ffc86432d7a3ae240000000374524e53ff80007f6d68780000003e49444154789c15c84b0d00200cc0d0f21140c0c0080818090246f0ef8971e83b94c4c210aed02964668316cd4327f1fc7194bac3303e05a72ace0627c803c53f065534c8b06a0000000049454e44ae426082' +IL_LA = '89504e470d0a1a0a0000000d494844520000000500000003080400000186383de50000002a49444154789c05c1410100300c02b19ba56ac20b0e6a884f5db104ca15152c9f996edf584026cae6f20123a71092c6c370780000000049454e44ae426082' + + +def _save(im): + b = io.BytesIO() + im.save(b, "PNG") + return b.getvalue() + + +def digest(im): + return hashlib.sha256(im.tobytes()).hexdigest()[:16] + + +def gradient(mode, w, h): + im = Image.new(mode, (w, h)) + n = len(im.getbands()) + for y in range(h): + for x in range(w): + v = [(x * 37 + y * 11) & 255, (x * 5 + y * 29) & 255, (x * y * 3 + 17) & 255, + (255 - x * 13 - y * 7) & 255] + if mode == "LA": + v = [v[0], v[3]] + im.putpixel((x, y), v[0] if n == 1 else tuple(v[:n])) + return im + + +# Resampling: every filter, down and up, on L, RGB and RGBA (premultiplied) +FILTERS = [("NEAREST", Image.Resampling.NEAREST), ("BOX", Image.Resampling.BOX), + ("BILINEAR", Image.Resampling.BILINEAR), ("HAMMING", Image.Resampling.HAMMING), + ("BICUBIC", Image.Resampling.BICUBIC), ("LANCZOS", Image.Resampling.LANCZOS)] +for mode in ("L", "LA", "RGB", "RGBA"): + src = gradient(mode, 23, 17) + for name, f in FILTERS: + for size in ((7, 5), (40, 31), (23, 9)): + out = src.resize(size, f) + print(mode, name, size, out.mode, out.size, digest(out)) +small = gradient("RGB", 6, 4) +print(list(small.resize((3, 2), Image.LANCZOS).getdata())) +print(list(small.resize((9, 5), Image.BICUBIC).getdata())[:6]) +print(small.resize((4, 3), Image.BICUBIC, box=(1, 0.5, 5, 3.5)).getpixel((1, 1))) +print(gradient("RGB", 40, 30).resize((8, 6)).getpixel((3, 3))) +t = gradient("RGB", 50, 20) +t.thumbnail((16, 16)) +print(t.size, digest(t)) + +# Text with the built-in bitmap font +font = ImageFont.load_default_imagefont() +im = Image.new("RGB", (90, 40), "white") +d = ImageDraw.Draw(im) +print(d.textbbox((3, 4), "Hello, PIL!", font=font), d.textlength("Hello, PIL!", font=font)) +d.text((3, 4), "Hello, PIL!", fill=(200, 0, 0), font=font) +d.text((-2, 20), "Wrap\nlines", fill="blue", font=font) +print(d.multiline_textbbox((-2, 20), "Wrap\nlines", font=font)) +print(digest(im), im.getbbox(), im.getpixel((4, 7))) +g = Image.new("L", (30, 12)) +ImageDraw.Draw(g).text((1, 0), "ab", fill=255, font=font) +for y in range(g.height): + print("".join("#" if g.getpixel((x, y)) else "." for x in range(16))) +r = Image.new("RGBA", (30, 30), (0, 0, 0, 0)) +ImageDraw.Draw(r).multiline_text((2, 2), "x\nyz", fill=(0, 255, 0, 128), font=font, align="right") +print(digest(r)) + +# Modes kept by Image.open, and interlaced files +for name in ("IL_RGB", "IL_PAL", "IL_LA"): + im = Image.open(io.BytesIO(bytes.fromhex(globals()[name]))) + print(name, im.mode, im.size, im.info.get("transparency"), digest(im)) + print(" ", im.getpixel((0, 0)), im.getpixel((im.width - 1, im.height - 1)), im.getpalette()) + print(" ", digest(im.convert("RGBA")), digest(im.convert("RGB")), digest(im.convert("L"))) + back = Image.open(io.BytesIO(_save(im))) + print(" ", back.mode, back.info.get("transparency"), digest(back) == digest(im)) +p = Image.open(io.BytesIO(bytes.fromhex(IL_PAL))) +print(p.resize((4, 4), Image.BICUBIC).mode, digest(p.crop((1, 1, 5, 5))), p.crop((0, 0, 2, 2)).getpalette()[:6]) +one = Image.open(io.BytesIO(_save(gradient("L", 12, 5).convert("1", dither=Image.Dither.NONE)))) +print(one.mode, digest(one), one.getpixel((3, 1)), digest(one.convert("L"))) +la = gradient("LA", 5, 4) +print(la.getpixel((2, 2)), digest(la.convert("RGBA")), digest(la.convert("L")), digest(gradient("RGBA", 4, 3).convert("LA"))) +print(one.tobytes().hex(), Image.frombytes("1", one.size, one.tobytes()).tobytes() == one.tobytes()) diff --git a/scripts/copyright-header-exclusions.txt b/scripts/copyright-header-exclusions.txt index 3b53078afc4..4568a1883fe 100644 --- a/scripts/copyright-header-exclusions.txt +++ b/scripts/copyright-header-exclusions.txt @@ -427,3 +427,4 @@ CodenameOne/src/com/codename1/util/TBigDecimal.java | Apache Harmony source reta CodenameOne/src/com/codename1/util/TBigInteger.java | Apache Harmony source retaining its original Apache-2.0 notice CodenameOne/src/com/codename1/util/TBitLevel.java | Apache Harmony source retaining its original Apache-2.0 notice CodenameOne/src/com/codename1/util/TDivision.java | Apache Harmony source retaining its original Apache-2.0 notice +maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/ImageFont.py | Pillow ImageFont rewrite with its embedded courB08 font, retaining the Secret Labs AB / Fredrik Lundh HPND notice From 330d9c29a994a526cd7a4478e3d31eb06fd55bb3 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 11:58:17 +0200 Subject: [PATCH 067/112] Guide: Pillow modes, filters and text; pandas shortened printing Co-Authored-By: Claude Opus 5.5 --- docs/developer-guide/Python-Interop.asciidoc | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/developer-guide/Python-Interop.asciidoc b/docs/developer-guide/Python-Interop.asciidoc index 223a0ecdf40..62080144d2c 100644 --- a/docs/developer-guide/Python-Interop.asciidoc +++ b/docs/developer-guide/Python-Interop.asciidoc @@ -199,16 +199,17 @@ the real package: | `pandas` | `Series` and `DataFrame` over the `numpy` subset: index alignment, selection with `loc`, `iloc` and masks, missing values, statistics and `describe`, `groupby` on one key, `merge`, `concat`, and - CSV and JSON reading and writing. Printing matches pandas. Long frames - print in full, and date indexes, `MultiIndex`, pivot tables, categoricals + CSV and JSON reading and writing. Printing matches pandas, including how + long and wide frames are shortened. Date indexes, `MultiIndex`, pivot tables, categoricals and rolling windows aren't supported. -| `Pillow` | `PIL.Image` in the `RGB`, `RGBA` and `L` modes: conversion, +| `Pillow` | `PIL.Image` in the `RGB`, `RGBA`, `L`, `LA`, `P` and `1` modes: conversion, cropping, resizing, rotation, pasting, channels and point operations, computed as Pillow computes them. PNG is read and written on every platform; JPEG, GIF, BMP and WebP go through Codename One's image support and need the application's display, so a server reads and writes PNG only. `ImageDraw` - draws points, lines, rectangles, ellipses and polygons, but not text. - Resizing filters other than `NEAREST` and `BILINEAR` resize bilinearly. + draws points, lines, rectangles, ellipses, polygons and text in Pillow's + bitmap font, which `ImageFont.load_default()` returns. Every resizing + filter, `LANCZOS` and `BICUBIC` included, gives Pillow's pixels. | `flask`, `fastapi`, `uvicorn` | See <>. | `typing_extensions`, `six`, `tabulate`, `idna`, `packaging` | |=== From 7d6813d85e6a1f51d5d94ba88118c93118ff9ef1 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 12:08:05 +0200 Subject: [PATCH 068/112] Python: __getattribute__ sees dunder reads, object reduce protocol drives copy/pickle, factory classes take the caller's module, live instance __dict__ Co-Authored-By: Claude Opus 5.5 --- .../codename1/python/runtime/Builtins.java | 11 ++- .../java/com/codename1/python/runtime/Py.java | 20 ++++ .../com/codename1/python/runtime/PyDict.java | 21 ++++ .../codename1/python/runtime/PyDictView.java | 4 +- .../codename1/python/runtime/PyInstance.java | 73 +++++++++++--- .../codename1/python/runtime/TypeMethods.java | 4 +- .../python/transpiler/codegen/Emitter.java | 30 ++++++ .../com/codename1/python/stdlib/copy.py | 66 ++++++++++++- .../com/codename1/python/stdlib/copyreg.py | 98 +++++++++++++++++- .../com/codename1/python/stdlib/pickle.py | 83 ++-------------- .../conformance/getattribute_dunder.out | 15 +++ .../conformance/getattribute_dunder.py | 30 ++++++ .../conformance/instance_dict_live.out | 9 ++ .../conformance/instance_dict_live.py | 22 +++++ .../test/resources/conformance/modnames_.out | 8 ++ .../resources/conformance/modnames_/main.py | 12 +++ .../resources/conformance/modnames_/shapes.py | 11 +++ .../resources/conformance/reduce_protocol.out | 34 +++++++ .../resources/conformance/reduce_protocol.py | 99 +++++++++++++++++++ 19 files changed, 553 insertions(+), 97 deletions(-) create mode 100644 maven/python-transpiler/src/test/resources/conformance/getattribute_dunder.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/getattribute_dunder.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/instance_dict_live.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/instance_dict_live.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/modnames_.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/modnames_/main.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/modnames_/shapes.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/reduce_protocol.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/reduce_protocol.py diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Builtins.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Builtins.java index c52dc1da92d..9ad3caff6e6 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Builtins.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Builtins.java @@ -296,13 +296,22 @@ static void install() { // The lookup without __getattr__: what a class's own // __getattribute__ or __getattr__ reaches for. String name = Py.toStr(a[1]); - Object v = a[0] instanceof PyObject ? ((PyObject) a[0]).getAttrOrNull(name) + Object v = a[0] instanceof PyInstance ? ((PyInstance) a[0]).genericAttrOrNull(name) + : a[0] instanceof PyObject ? ((PyObject) a[0]).getAttrOrNull(name) : Py.bindAttr(Py.typeOf(a[0]).lookup(name), a[0]); if (v == null) { throw Py.attributeError("'" + Py.typeOf(a[0]).name + "' object has no attribute '" + name + "'"); } return v; })); + // The reduction protocol's defaults are written in Python, in copyreg, + // which pickle and copy import; object only forwards to them. + o.define("__reduce_ex__", PyBuiltin.positional("__reduce_ex__", + a -> Py.call(Py.importFrom(Py.importModule("copyreg"), "_reduce_ex"), a[0], a[1]))); + o.define("__reduce__", PyBuiltin.positional("__reduce__", + a -> Py.call(Py.importFrom(Py.importModule("copyreg"), "_reduce"), a[0]))); + o.define("__getstate__", PyBuiltin.positional("__getstate__", + a -> Py.call(Py.importFrom(Py.importModule("copyreg"), "_getstate"), a[0]))); o.define("__delattr__", PyBuiltin.positional("__delattr__", a -> { if (a[0] instanceof PyInstance) { ((PyInstance) a[0]).delAttrDirect(Py.toStr(a[1])); diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Py.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Py.java index cb834ec6ba9..126665e6413 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Py.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Py.java @@ -1995,6 +1995,26 @@ public static Object typeModule(Object made, PyModule caller) { return made; } + /// What a functional class factory (namedtuple, Enum('C', ...), + /// make_dataclass) made, moved to the calling module: the library names + /// its own module, or `__main__`, where CPython reads the caller's frame. + public static Object callerModule(Object made, PyModule caller) { + if (made instanceof PyType) { + PyType t = (PyType) made; + Object m = t.getAttrOrNull("__module__"); + if (m instanceof String && FACTORY_MODULES.contains(m)) { + Object n = caller.getAttrOrNull("__name__"); + if (n instanceof String && !n.equals(m)) { + t.setAttr("__module__", n); + } + } + } + return made; + } + + private static final java.util.Set FACTORY_MODULES = new java.util.HashSet(java.util.Arrays.asList( + "__main__", "collections", "typing", "enum", "dataclasses")); + public static boolean isinstance(Object o, Object t) { if (t instanceof PyTuple) { Object[] items = ((PyTuple) t).items; diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyDict.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyDict.java index 7cef358dcb9..2a0aa0f8e71 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyDict.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyDict.java @@ -63,9 +63,16 @@ public PyType type() { } public int size() { + sync(); return size; } + /// Brings the entries up to date before a read. A plain dict holds its + /// own entries; a view over something else (an instance's `__dict__`) + /// refills itself here from what it stands for. + void sync() { + } + private static boolean keyEq(Object a, long ha, Object b, long hb) { if (a == b) { //NOPMD CompareObjectsWithEquals - identity is meant return true; @@ -216,6 +223,7 @@ void setLong(long k, Object value) { /// `self[key]`, or `null` when absent. public Object getOrNull(Object key) { + sync(); int s = find(key, Py.keyHash(key, "a dict key")); return s >= 0 ? values[index[s]] : null; } @@ -330,16 +338,19 @@ public void delItem(Object key) { @Override public boolean contains(Object x) { + sync(); return find(x, Py.keyHash(x, "a dict key")) >= 0; } @Override public int len() { + sync(); return size; } @Override public boolean isTrue() { + sync(); return size != 0; } @@ -385,6 +396,9 @@ private static void nullOut(Object[] a, int n) { /// `dict.update(other)` for a mapping or an iterable of pairs. public void update(Object other) { + if (other instanceof PyDict) { + ((PyDict) other).sync(); + } if (other instanceof PyDict) { PyDict d = (PyDict) other; for (int i = 0; i < d.used; i++) { @@ -416,6 +430,7 @@ public void update(Object other) { /// The live keys in order. public Object[] keyArray() { + sync(); Object[] r = new Object[size]; int w = 0; for (int i = 0; i < used; i++) { @@ -428,6 +443,7 @@ public Object[] keyArray() { /// Iterates over entries; `kind` 0 = keys, 1 = values, 2 = items. Object iterate(final int kind) { + sync(); final int startSize = size; return new PyJavaIter(new PyJavaIter.Source() { int i; @@ -462,6 +478,7 @@ public Object iter() { @Override public String repr() { + sync(); if (size == 0) { return "{}"; } @@ -489,6 +506,10 @@ public String repr() { @Override public Object eq(Object o) { + sync(); + if (o instanceof PyDict) { + ((PyDict) o).sync(); + } if (!(o instanceof PyDict)) { return null; } diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyDictView.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyDictView.java index 05dd13c96b7..26c94c2c881 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyDictView.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyDictView.java @@ -46,12 +46,12 @@ public Object iter() { @Override public int len() { - return d.size; + return d.len(); } @Override public boolean isTrue() { - return d.size != 0; + return d.isTrue(); } @Override diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyInstance.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyInstance.java index 081bc69a2da..391cf59656c 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyInstance.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyInstance.java @@ -168,6 +168,39 @@ String[] attributeNames() { @Override public Object getAttrOrNull(String name) { + if (cls.hasCustomGetattribute()) { + // Every explicit read goes through a class's own __getattribute__, + // dunder names included (`obj.__dict__`, getattr(obj, '__x__')); the + // implicit special-method lookup of len(obj) and friends reads the + // type and never comes here. An AttributeError it raises falls back + // to __getattr__, as CPython's slot does. + try { + return Py.obj(cls.lookup("__getattribute__")).call(new Object[] {this, name}, Py.NO_KW); + } catch (PyException e) { + if (!e.matches(Py.ATTRIBUTE_ERROR)) { + throw e; + } + Object ga = cls.lookup("__getattr__"); + if (ga == null) { + return null; + } + return Py.obj(ga).call(new Object[] {this, name}, Py.NO_KW); + } + } + Object v = genericAttrOrNull(name); + if (v != null) { + return v; + } + Object ga = cls.lookup("__getattr__"); + if (ga != null) { + return Py.obj(ga).call(new Object[] {this, name}, Py.NO_KW); + } + return null; + } + + /// `object.__getattribute__(self, name)`: the lookup without the class's + /// own `__getattribute__` or `__getattr__`; null when absent. + Object genericAttrOrNull(String name) { Object clsAttr = cls.lookup(name); if (clsAttr instanceof PyProperty) { return ((PyProperty) clsAttr).get(this); @@ -190,15 +223,10 @@ public Object getAttrOrNull(String name) { return Py.obj(get).call(new Object[] {clsAttr, this, cls}, Py.NO_KW); } if ("__dict__".equals(name)) { - // A copy of the attributes whose stores and deletes reach the - // instance too; a change made the other way after it was read - // is not seen through it. - PyDict d = new InstanceDict(this); - String[] names = attributeNames(); - for (String e : names) { - d.setItem(e, own(e)); - } - return d; + // A live view: its stores and deletes reach the instance, and it + // refills itself from the instance before every read, so a write + // made through the attribute shows in a view fetched earlier. + return new InstanceDict(this); } if ("__class__".equals(name)) { return cls; @@ -206,10 +234,6 @@ public Object getAttrOrNull(String name) { if (clsAttr != null) { return Py.bindAttr(clsAttr, this); } - Object ga = cls.lookup("__getattr__"); - if (ga != null) { - return Py.obj(ga).call(new Object[] {this, name}, Py.NO_KW); - } return null; } @@ -625,10 +649,30 @@ public double asDouble() { private static final class InstanceDict extends PyDict { private final PyInstance self; + /// Set while refilling, so the refill's own stores do not sync again. + private boolean syncing; + InstanceDict(PyInstance self) { this.self = self; } + @Override + void sync() { + if (syncing) { + return; + } + syncing = true; + try { + super.clear(); + String[] names = self.attributeNames(); + for (String e : names) { + super.setItem(e, self.own(e)); + } + } finally { + syncing = false; + } + } + @Override public void setItem(Object key, Object value) { super.setItem(key, value); @@ -642,6 +686,7 @@ void setLong(long k, Object value) { @Override public void delItem(Object key) { + sync(); // Not super.delItem: it removes through remove(), which is overridden below and // would delete the attribute a second time. if (super.remove(key) == null) { @@ -654,6 +699,7 @@ public void delItem(Object key) { // overrides `obj.__dict__.pop(name)` answered the value and left the attribute set. @Override public Object remove(Object key) { + sync(); Object v = super.remove(key); if (v != null) { self.delAttrDirect(Py.toStr(key)); @@ -663,6 +709,7 @@ public Object remove(Object key) { @Override public void clear() { + sync(); Object[] snapshot = new Object[used]; int n = 0; for (int i = 0; i < used; i++) { diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/TypeMethods.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/TypeMethods.java index d303afc1ee3..6842573e374 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/TypeMethods.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/TypeMethods.java @@ -854,7 +854,9 @@ private static PyDict d(Object o) { if (!(o instanceof PyDict)) { throw Py.typeError("descriptor requires a 'dict' object but received a '" + Py.typeOf(o).name + "'"); } - return (PyDict) o; + PyDict d = (PyDict) o; + d.sync(); + return d; } private static void installDict() { diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Emitter.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Emitter.java index b00b134cbc2..99aa7ce3b76 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Emitter.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Emitter.java @@ -2360,6 +2360,25 @@ private String joined(JoinedStr j) { return sb.append(")").toString(); } + private static final java.util.Set CLASS_FACTORIES = new java.util.HashSet(java.util.Arrays.asList( + "namedtuple", "NamedTuple", "TypedDict", "make_dataclass", "Enum", "IntEnum", "StrEnum", "Flag", "IntFlag")); + + /// A call of a functional class factory that names its class after the + /// caller's module, without an explicit `module=`. + private static boolean isClassFactory(Call c) { + String n = c.func instanceof Name ? ((Name) c.func).id + : c.func instanceof Attribute ? ((Attribute) c.func).attr : null; + if (n == null || !CLASS_FACTORIES.contains(n) || c.args.isEmpty()) { + return false; + } + for (Keyword k : c.keywords) { + if ("module".equals(k.arg)) { + return false; + } + } + return true; + } + private String call(Call c) { if (!inTypeCall && c.func instanceof Name && "type".equals(((Name) c.func).id) && c.args.size() == 3 && c.keywords.isEmpty() && sc.resolve("type") == Scope.Binding.BUILTIN) { @@ -2372,6 +2391,17 @@ private String call(Call c) { inTypeCall = false; } } + if (!inTypeCall && isClassFactory(c)) { + // namedtuple('P', ...), Enum('C', ...), make_dataclass(...): CPython + // reads __module__ off the caller's frame, which only the call site + // knows here. + inTypeCall = true; + try { + return "Py.callerModule(" + call(c) + ", M)"; + } finally { + inTypeCall = false; + } + } if (c.func instanceof Name && c.args.isEmpty() && c.keywords.isEmpty()) { String fn = ((Name) c.func).id; if (("globals".equals(fn) || "locals".equals(fn)) && sc.resolve(fn) == Scope.Binding.BUILTIN) { diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/copy.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/copy.py index 0e181ab97c9..607b79d8117 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/copy.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/copy.py @@ -3,6 +3,8 @@ # exception, as the rest of the Codename One Python runtime. """Shallow and deep copy operations.""" +import copyreg + class Error(Exception): pass @@ -30,7 +32,61 @@ def copy(x): copier = getattr(cls, "__copy__", None) if copier is not None: return copier(x) - return _reconstruct(x, None) + rv = _reduction(x, cls) + if rv is None: + return _reconstruct(x, None) + if isinstance(rv, str): + return x + return _reconstruct_from(x, None, *rv) + + +def _reduction(x, cls): + """What x reduces to, as CPython's copy asks; None for the runtime's own + reconstruction of a built-in subclass that keeps object's default.""" + reductor = copyreg.dispatch_table.get(cls) + if reductor is not None: + return reductor(x) + if getattr(cls, "__reduce_ex__", None) is not object.__reduce_ex__: + return x.__reduce_ex__(4) + if getattr(cls, "__reduce__", None) is not object.__reduce__: + return x.__reduce__() + if _builtin_base(cls) or isinstance(x, BaseException): + return None + return x.__reduce_ex__(4) + + +def _reconstruct_from(x, memo, func, args, state=None, listiter=None, dictiter=None): + deep = memo is not None + if deep and args: + args = deepcopy(args, memo) + y = func(*args) + if deep: + memo[id(x)] = y + if state is not None: + if deep: + state = deepcopy(state, memo) + if hasattr(y, "__setstate__"): + y.__setstate__(state) + else: + slotstate = None + if isinstance(state, tuple) and len(state) == 2: + state, slotstate = state + if state: + for k, v in state.items(): + setattr(y, k, v) + if slotstate: + for k, v in slotstate.items(): + setattr(y, k, v) + if listiter is not None: + for item in listiter: + y.append(deepcopy(item, memo) if deep else item) + if dictiter is not None: + for k, v in dictiter: + if deep: + k = deepcopy(k, memo) + v = deepcopy(v, memo) + y[k] = v + return y def deepcopy(x, memo=None, _nil=[]): @@ -66,7 +122,13 @@ def deepcopy(x, memo=None, _nil=[]): elif cls.__name__ in _FUNCTIONS: return x else: - y = _reconstruct(x, memo) + rv = _reduction(x, cls) + if rv is None: + y = _reconstruct(x, memo) + elif isinstance(rv, str): + return x + else: + y = _reconstruct_from(x, memo, *rv) if y is not x: memo[d] = y _keep_alive(x, memo) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/copyreg.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/copyreg.py index 2f556a650c2..0ba8e88e74d 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/copyreg.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/copyreg.py @@ -6,9 +6,10 @@ This is the registry of reduction functions (`pickle`), the reconstructors `pickle` writes for plain objects, and the extension-code registry. -`pickle` does not ask an instance for a default `__reduce_ex__`, because the -runtime's `object` has none: `pickle` builds the same reduction itself, and -`__newobj__`, `__newobj_ex__` and `_reconstructor` here are what it writes. +The runtime's `object.__reduce_ex__`, `__reduce__` and `__getstate__` answer +through `_reduce_ex`, `_reduce` and `_getstate` here, so `pickle` and `copy` +both reach the same reduction; `__newobj__`, `__newobj_ex__` and +`_reconstructor` are the reconstructors it names. """ __all__ = ["pickle", "constructor", "add_extension", "remove_extension", "clear_extension_cache"] @@ -69,6 +70,97 @@ def _slotnames(cls): return names +def _has_dict(cls): + for c in cls.__mro__: + if c is object: + continue + if "__slots__" not in c.__dict__: + return True + slots = c.__dict__["__slots__"] + if slots == "__dict__" or (not isinstance(slots, str) and "__dict__" in slots): + return True + return False + + +def _getstate(obj): + """`object.__getstate__()`: the instance dict, plus `(dict, slots)` with slots.""" + t = type(obj) + state = getattr(obj, "__dict__", None) if _has_dict(t) else None + if state is not None: + state = dict(state) if len(state) else None + slots = _slotnames(t) + if slots: + slotstate = {} + for name in slots: + try: + slotstate[name] = getattr(obj, name) + except AttributeError: + pass + if slotstate: + return (state, slotstate) + return state + + +def _reduce_ex(obj, proto): + """`object.__reduce_ex__(proto)`: a class's own `__reduce__` when it has one.""" + cls = type(obj) + if getattr(cls, "__reduce__", None) is not object.__reduce__: + return obj.__reduce__() + if proto >= 2: + return _reduce_newobj(obj, cls) + return _reduce_legacy(obj, cls, proto) + + +def _reduce(obj): + """`object.__reduce__()`: the protocol 0 reduction.""" + return _reduce_legacy(obj, type(obj), 0) + + +def _reduce_newobj(obj, cls): + getnewargs_ex = getattr(obj, "__getnewargs_ex__", None) + if getnewargs_ex is not None: + args, kwargs = getnewargs_ex() + if not isinstance(args, tuple): + raise TypeError("__getnewargs_ex__ should return a tuple of (tuple, dict)") + if kwargs: + func = __newobj_ex__ + newargs = (cls, args, kwargs) + else: + func = __newobj__ + newargs = (cls,) + args + else: + getnewargs = getattr(obj, "__getnewargs__", None) + if getnewargs is not None: + args = getnewargs() + if not isinstance(args, tuple): + raise TypeError("__getnewargs__ should return a tuple") + else: + args = () + func = __newobj__ + newargs = (cls,) + args + state = obj.__getstate__() + listitems = iter(obj) if isinstance(obj, list) else None + dictitems = iter(obj.items()) if isinstance(obj, dict) else None + return (func, newargs, state, listitems, dictitems) + + +def _reduce_legacy(obj, cls, proto): + if isinstance(obj, (list, dict, set, frozenset, bytes, bytearray, str, int, float, tuple)): + raise TypeError("cannot pickle %r object with protocol %d" % (cls.__name__, proto)) + if getattr(cls, "__getstate__", None) is object.__getstate__: + state = getattr(obj, "__dict__", None) + if state is not None and len(state) == 0: + state = None + if _slotnames(cls): + raise TypeError("a class that defines __slots__ without defining __getstate__ cannot be pickled with protocol %d" % proto) + else: + state = obj.__getstate__() + args = (cls, object, None) + if state is not None: + return (_reconstructor, args, state) + return (_reconstructor, args) + + _extension_registry = {} _inverted_registry = {} _extension_cache = {} diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/pickle.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/pickle.py index bbc9e08da78..c5880e7eb4a 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/pickle.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/pickle.py @@ -566,14 +566,14 @@ def _save_exact(self, t, obj): return True def _reduce_object(self, obj): - """What `__reduce_ex__(proto)` answers; the runtime's object has none.""" + """What `__reduce_ex__(proto)` answers for obj.""" t = type(obj) - reduce = getattr(obj, "__reduce_ex__", _NoValue) - if reduce is not _NoValue: - return reduce(self.proto) - reduce = getattr(obj, "__reduce__", _NoValue) - if reduce is not _NoValue: - return reduce() + # A class's own __reduce_ex__ or __reduce__ wins; object's default + # (copyreg._reduce_ex) comes after the built-in kinds below. + if getattr(t, "__reduce_ex__", None) is not object.__reduce_ex__: + return obj.__reduce_ex__(self.proto) + if getattr(t, "__reduce__", None) is not object.__reduce__: + return obj.__reduce__() if isinstance(obj, BaseException): state = {} extra = getattr(obj, "__dict__", None) @@ -599,74 +599,7 @@ def _reduce_object(self, obj): return (getattr, (obj.__self__, obj.__func__.__name__)) if name == "module" or name == "generator" or name == "coroutine": raise PicklingError("cannot pickle %r object" % name) - if self.proto >= 2: - return self._reduce_newobj(obj, t) - return self._reduce_legacy(obj, t) - - def _getstate(self, obj): - t = type(obj) - getstate = getattr(obj, "__getstate__", _NoValue) - if getstate is not _NoValue: - return getstate() - state = getattr(obj, "__dict__", None) - if state is not None and len(state) == 0: - state = None - slots = copyreg._slotnames(t) - if slots: - if not _has_dict(t): - state = None - slotstate = {} - for name in slots: - value = getattr(obj, name, _NoValue) - if value is not _NoValue: - slotstate[name] = value - if slotstate: - return (state, slotstate) - return state - - def _reduce_newobj(self, obj, cls): - getnewargs_ex = getattr(obj, "__getnewargs_ex__", _NoValue) - if getnewargs_ex is not _NoValue: - args, kwargs = getnewargs_ex() - if not isinstance(args, tuple): - raise TypeError("__getnewargs_ex__ should return a tuple of (tuple, dict)") - if kwargs: - func = copyreg.__newobj_ex__ - newargs = (cls, args, kwargs) - else: - func = copyreg.__newobj__ - newargs = (cls,) + args - else: - getnewargs = getattr(obj, "__getnewargs__", _NoValue) - if getnewargs is not _NoValue: - args = getnewargs() - if not isinstance(args, tuple): - raise TypeError("__getnewargs__ should return a tuple") - else: - args = () - func = copyreg.__newobj__ - newargs = (cls,) + args - state = self._getstate(obj) - listitems = iter(obj) if isinstance(obj, list) else None - dictitems = iter(obj.items()) if isinstance(obj, dict) else None - return (func, newargs, state, listitems, dictitems) - - def _reduce_legacy(self, obj, cls): - if isinstance(obj, (list, dict, set, frozenset, bytes, bytearray, str, int, float, tuple)): - raise PicklingError("can't pickle %s with protocol %d" % (_T(obj), self.proto)) - if getattr(obj, "__getstate__", _NoValue) is _NoValue: - state = getattr(obj, "__dict__", None) - if state is not None and len(state) == 0: - state = None - slots = copyreg._slotnames(cls) - if slots: - raise PicklingError("a class that defines __slots__ without defining __getstate__ cannot be pickled with protocol %d" % self.proto) - else: - state = obj.__getstate__() - args = (cls, object, None) - if state is not None: - return (copyreg._reconstructor, args, state) - return (copyreg._reconstructor, args) + return copyreg._reduce_ex(obj, self.proto) def save_pers(self, pid): if self.bin: diff --git a/maven/python-transpiler/src/test/resources/conformance/getattribute_dunder.out b/maven/python-transpiler/src/test/resources/conformance/getattribute_dunder.out new file mode 100644 index 00000000000..1a8432837bd --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/getattribute_dunder.out @@ -0,0 +1,15 @@ +get x +1 +get __dict__ +['x'] +get __dict__ +True +get __class__ +Spy +3 +get __len__ +3 +faked doc +42 +42 +True diff --git a/maven/python-transpiler/src/test/resources/conformance/getattribute_dunder.py b/maven/python-transpiler/src/test/resources/conformance/getattribute_dunder.py new file mode 100644 index 00000000000..f83c1fdbad9 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/getattribute_dunder.py @@ -0,0 +1,30 @@ +class Spy: + def __init__(self): + self.x = 1 + def __getattribute__(self, name): + print("get", name) + return object.__getattribute__(self, name) + def __len__(self): + return 3 + +s = Spy() +print(s.x) +d = s.__dict__ +print(sorted(d)) +print(getattr(s, '__dict__') == {'x': 1}) +print(s.__class__.__name__) +print(len(s)) +print(s.__len__()) + +class Fake: + def __getattribute__(self, name): + if name == '__doc__': + return 'faked doc' + if name == '__secret__': + return 42 + return object.__getattribute__(self, name) +f = Fake() +print(f.__doc__) +print(f.__secret__) +print(getattr(f, '__secret__')) +print(hasattr(f, '__secret__')) diff --git a/maven/python-transpiler/src/test/resources/conformance/instance_dict_live.out b/maven/python-transpiler/src/test/resources/conformance/instance_dict_live.out new file mode 100644 index 00000000000..bc5d07f3f66 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/instance_dict_live.out @@ -0,0 +1,9 @@ +{'a': 1, 'b': 2} +10 +20 30 +99 5 True ['a', 'b', 'x', 'y', 'z'] +{'a': 99, 'x': 10, 'y': 20, 'z': 30} +False {'a': 99, 'y': 20, 'z': 30} +True +{} False +[('q', 5)] diff --git a/maven/python-transpiler/src/test/resources/conformance/instance_dict_live.py b/maven/python-transpiler/src/test/resources/conformance/instance_dict_live.py new file mode 100644 index 00000000000..43051d67d4c --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/instance_dict_live.py @@ -0,0 +1,22 @@ +class C: + def __init__(self): + self.a = 1 +c = C() +d = c.__dict__ +c.b = 2 +print(d) +d['x'] = 10 +print(c.x) +vars(c).update(y=20, z=30) +print(c.y, c.z) +c.a = 99 +print(d['a'], len(d), 'b' in d, sorted(d.keys())) +del c.b +print(d) +del d['x'] +print(hasattr(c, 'x'), d) +print(c.__dict__ == d) +d.clear() +print(vars(c), hasattr(c, 'a')) +c.q = 5 +print(list(d.items())) diff --git a/maven/python-transpiler/src/test/resources/conformance/modnames_.out b/maven/python-transpiler/src/test/resources/conformance/modnames_.out new file mode 100644 index 00000000000..ac28bf6297a --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/modnames_.out @@ -0,0 +1,8 @@ +shapes +shapes +shapes +shapes +shapes +Point(x=1, y=2) +__main__ +Point(x=3, y=4) diff --git a/maven/python-transpiler/src/test/resources/conformance/modnames_/main.py b/maven/python-transpiler/src/test/resources/conformance/modnames_/main.py new file mode 100644 index 00000000000..52fc8f52452 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/modnames_/main.py @@ -0,0 +1,12 @@ +import shapes +from collections import namedtuple +print(shapes.Point.__module__) +print(shapes.Pair.__module__) +print(shapes.Color.__module__) +print(shapes.Box.__module__) +print(shapes.Pt3.__module__) +print(repr(shapes.Point(1, 2))) +Local = namedtuple('Local', 'a') +print(Local.__module__) +import pickle +print(pickle.loads(pickle.dumps(shapes.Point(3, 4)))) diff --git a/maven/python-transpiler/src/test/resources/conformance/modnames_/shapes.py b/maven/python-transpiler/src/test/resources/conformance/modnames_/shapes.py new file mode 100644 index 00000000000..26203b3367c --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/modnames_/shapes.py @@ -0,0 +1,11 @@ +from collections import namedtuple +from typing import NamedTuple +import enum, dataclasses +Point = namedtuple('Point', 'x y') +Pair = NamedTuple('Pair', [('a', int), ('b', int)]) +Color = enum.Enum('Color', 'RED GREEN') +Box = dataclasses.make_dataclass('Box', ['w', 'h']) +class Pt3(NamedTuple): + x: int + y: int + z: int diff --git a/maven/python-transpiler/src/test/resources/conformance/reduce_protocol.out b/maven/python-transpiler/src/test/resources/conformance/reduce_protocol.out new file mode 100644 index 00000000000..6d5e2d9f83d --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/reduce_protocol.out @@ -0,0 +1,34 @@ +True True {'a': 1, 'b': [2, 3]} +{'a': 1, 'b': [2, 3]} +True +P {} +P(1, [2, 3]) True +P(1, [2, 3]) False +P(1, [2, 3]) +reduce called +reduce called +reduce called +10 20 30 +setstate ['v'] +1 rebuilt +setstate ['v'] +1 rebuilt +setstate ['v'] +1 rebuilt +(None, {'x': 1, 'y': [1]}) +1 [1] +1 [1] +1 [1] +new 4 +new 4 +4 +new 5 +new 5 +5 +newkw 7 +newkw 7 +7 +a b True True +True +True +103 103 diff --git a/maven/python-transpiler/src/test/resources/conformance/reduce_protocol.py b/maven/python-transpiler/src/test/resources/conformance/reduce_protocol.py new file mode 100644 index 00000000000..4a1a4b392fb --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/reduce_protocol.py @@ -0,0 +1,99 @@ +import copy, pickle, copyreg + +class P: + def __init__(self, a, b): + self.a = a + self.b = b + def __repr__(self): + return "P(%r, %r)" % (self.a, self.b) + +p = P(1, [2, 3]) +r = p.__reduce_ex__(2) +print(r[0] is copyreg.__newobj__, r[1][0] is P, r[2]) +print(p.__getstate__()) +print(type(p.__reduce__()) is tuple) +q = copyreg.__newobj__(P) +print(type(q).__name__, q.__dict__) + +c = copy.copy(p) +print(c, c.b is p.b) +d = copy.deepcopy(p) +print(d, d.b is p.b) +print(pickle.loads(pickle.dumps(p))) + +class R: + def __init__(self, v): + self.v = v + def __reduce__(self): + print("reduce called") + return (R, (self.v * 10,)) +print(copy.copy(R(1)).v, copy.deepcopy(R(2)).v, pickle.loads(pickle.dumps(R(3))).v) + +class S: + def __init__(self): + self.v = 1 + self.cache = "big" + def __getstate__(self): + st = dict(self.__dict__) + del st['cache'] + return st + def __setstate__(self, st): + print("setstate", sorted(st)) + self.__dict__.update(st) + self.cache = "rebuilt" +for f in (copy.copy, copy.deepcopy, lambda o: pickle.loads(pickle.dumps(o))): + o = f(S()) + print(o.v, o.cache) + +class Sl: + __slots__ = ('x', 'y') + def __init__(self): + self.x = 1 + self.y = [1] +print(Sl().__getstate__()) +c = copy.copy(Sl()); print(c.x, c.y) +c = copy.deepcopy(Sl()); print(c.x, c.y) +c = pickle.loads(pickle.dumps(Sl())); print(c.x, c.y) + +class N: + def __new__(cls, n): + print("new", n) + o = object.__new__(cls) + o.n = n + return o + def __getnewargs__(self): + return (self.n,) +c = copy.copy(N(4)); print(c.n) +c = pickle.loads(pickle.dumps(N(5))); print(c.n) + +class NK: + def __new__(cls, *, k): + print("newkw", k) + o = object.__new__(cls) + o.k = k + return o + def __getnewargs_ex__(self): + return ((), {'k': self.k}) +c = copy.deepcopy(NK(k=7)); print(c.k) + +class Node: + def __init__(self, name): + self.name = name + self.peer = None +a = Node('a'); b = Node('b'); a.peer = b; b.peer = a +x = copy.deepcopy(a) +print(x.name, x.peer.name, x.peer.peer is x, x is not a) +y = pickle.loads(pickle.dumps(a)) +print(y.peer.peer is y) +lst = [a, a] +z = copy.deepcopy(lst) +print(z[0] is z[1]) + +class St: + def __init__(self): + self.v = 3 + def __getstate__(self): + return (self.v,) + def __setstate__(self, s): + self.v = s[0] + 100 +print(copy.copy(St()).v, pickle.loads(pickle.dumps(St())).v) From d2d600bdf13d4403ce02621709e7d7f297eda9b2 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 12:11:30 +0200 Subject: [PATCH 069/112] Python: a program that calls object.__reduce_ex__ without importing copy or pickle gets copyreg Co-Authored-By: Claude Opus 5.5 --- .../python/transpiler/api/PythonTranspiler.java | 15 +++++++++++++++ .../resources/conformance/reduce_no_import.out | 2 ++ .../resources/conformance/reduce_no_import.py | 7 +++++++ 3 files changed, 24 insertions(+) create mode 100644 maven/python-transpiler/src/test/resources/conformance/reduce_no_import.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/reduce_no_import.py diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java index 3e04fc87980..a86330dd51c 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java @@ -210,11 +210,15 @@ private static Set addStdlib(Map sources, Map work = new ArrayDeque(trees.values()); Map found = new LinkedHashMap(); Set opens = new HashSet(); + Set reduces = new HashSet(); for (Map.Entry e : sources.entrySet()) { Node.Module t = trees.get(e.getKey().replace('\\', '/')); if (t != null && mentionsOpen(e.getValue())) { opens.add(t); } + if (t != null && mentionsReduction(e.getValue())) { + reduces.add(t); + } } while (!work.isEmpty()) { Node.Module tree = work.pop(); @@ -231,6 +235,11 @@ private static Set addStdlib(Map sources, Map addStdlib(Map sources, Map Date: Wed, 28 Oct 2026 12:12:26 +0200 Subject: [PATCH 070/112] Python runtime: clear PMD findings Co-Authored-By: Claude Opus 5.5 --- .../src/main/java/com/codename1/python/runtime/PyImage.java | 5 ++--- .../src/main/java/com/codename1/python/runtime/PyServer.java | 2 +- .../com/codename1/python/transpiler/codegen/Emitter.java | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyImage.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyImage.java index e7bcd245704..1800cfdb976 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyImage.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyImage.java @@ -254,7 +254,7 @@ static PyModule module() { try { io.save(ci, bo, Py.toStr(a[1]), (float) Py.asDouble(a[2])); } catch (java.io.IOException e) { - throw Py.error(Py.OS_ERROR, String.valueOf(e.getMessage())); + throw Py.error(Py.OS_ERROR, String.valueOf(e.getMessage())); // NOPMD PreserveStackTrace - surfaced as a Python OSError } return new PyBytes(bo.toByteArray()); })); @@ -726,8 +726,7 @@ private static int sample(byte[] raw, int rowStart, int index, int depth) { return raw[rowStart + index * 2] & 0xff; } int bit = index * depth; - int v = (raw[rowStart + (bit >> 3)] & 0xff) >> (8 - depth - (bit & 7)) & ((1 << depth) - 1); - return v; + return (raw[rowStart + (bit >> 3)] & 0xff) >> (8 - depth - (bit & 7)) & ((1 << depth) - 1); } /// Unpacks unfiltered PNG rows into an image: gray to 1 band, RGB to 3, diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyServer.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyServer.java index 448e420a47d..c0da8315e32 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyServer.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyServer.java @@ -284,7 +284,7 @@ public static byte[] nextChunk(Response response) { } } } catch (java.io.UnsupportedEncodingException e) { - throw Py.error(Py.OS_ERROR, "UTF-8 is unsupported"); + throw Py.error(Py.OS_ERROR, "UTF-8 is unsupported"); // NOPMD PreserveStackTrace - surfaced as a Python OSError } finally { Gil.exit(); } diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Emitter.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Emitter.java index 99aa7ce3b76..7719b3bac02 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Emitter.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/Emitter.java @@ -2360,7 +2360,7 @@ private String joined(JoinedStr j) { return sb.append(")").toString(); } - private static final java.util.Set CLASS_FACTORIES = new java.util.HashSet(java.util.Arrays.asList( + private static final Set CLASS_FACTORIES = new HashSet(Arrays.asList( "namedtuple", "NamedTuple", "TypedDict", "make_dataclass", "Enum", "IntEnum", "StrEnum", "Flag", "IntFlag")); /// A call of a functional class factory that names its class after the From 2eeb9ec18b4fba4d121a0cbda9601171e120e708 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 12:43:08 +0200 Subject: [PATCH 071/112] Python backend: native e2e (--native), PIL codec out of the server build, larger VT stack for Python handlers e2e-test.sh --native builds the Flask and FastAPI samples through ParparVM and runs the same client checks, including concurrency and background tasks; parparvm-tests.yml runs it on the Linux backend leg. Two bugs the native build hit: - PyImage called com.codename1.ui directly, which a backend translation does not have: the C compile failed. The UI codec moves to PyImageNative behind a seam the client registry installs; build.sh leaves it out. - A FastAPI request overflowed the 64KB virtual-thread C stack in re's engine (SIGBUS, empty reply). HttpServer.requireVirtualThreadStack lets the generated BackendServer ask for 1MB, mapped lazily. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/parparvm-tests.yml | 24 ++++++ .../com/codename1/python/runtime/PyImage.java | 75 ++++++++--------- .../python/runtime/PyImageNative.java | 84 +++++++++++++++++++ .../transpiler/api/PythonTranspiler.java | 18 +++- scripts/python-backend/build.sh | 4 +- scripts/python-backend/e2e-test.sh | 13 ++- .../src/com/codename1/backend/HttpServer.java | 29 ++++++- 7 files changed, 199 insertions(+), 48 deletions(-) create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/PyImageNative.java diff --git a/.github/workflows/parparvm-tests.yml b/.github/workflows/parparvm-tests.yml index 52a64d4a332..c9be991d2e3 100644 --- a/.github/workflows/parparvm-tests.yml +++ b/.github/workflows/parparvm-tests.yml @@ -38,6 +38,10 @@ on: - 'maven/python-runtime/**' - 'maven/python-transpiler/**' - 'scripts/python-backend/sqlalchemy-realdb-test.sh' + - 'scripts/python-backend/build.sh' + - 'scripts/python-backend/e2e-test.sh' + - 'scripts/python-backend/e2e_client.py' + - 'scripts/python-backend/samples/**' - 'scripts/hellocodenameone/pom.xml' - 'scripts/hellocodenameone/backend/**' # The core classes the backend compiles as its own (the ORM and the entity @@ -107,6 +111,10 @@ on: - 'maven/python-runtime/**' - 'maven/python-transpiler/**' - 'scripts/python-backend/sqlalchemy-realdb-test.sh' + - 'scripts/python-backend/build.sh' + - 'scripts/python-backend/e2e-test.sh' + - 'scripts/python-backend/e2e_client.py' + - 'scripts/python-backend/samples/**' - 'scripts/hellocodenameone/pom.xml' - 'scripts/hellocodenameone/backend/**' # The core classes the backend compiles as its own (the ORM and the entity @@ -474,6 +482,22 @@ jobs: JAVA17_HOME: ${{ env.JDK_17_HOME }} CN1_SQLA_POSTGRES: postgres://cn1:cn1secret@127.0.0.1:5432/cn1?sslmode=disable + # The Flask and FastAPI samples built NATIVELY (ParparVM, clang) and + # driven over sockets by the same client as the JVM run. The only place + # Python handlers run on the backend's virtual threads, so the only run + # of the GIL's scheduler path. python-transpiler and python-runtime were + # installed by the step above; core, backend and build-engine by the + # contract install; JavaAPI explicitly, since the JVM tests may not. + - name: Python backend end to end, native + run: | + bash "$GITHUB_WORKSPACE/scripts/ci/retry.sh" mvn -B -f vm/pom.xml -pl JavaAPI install -DskipTests \ + -Dmaven.repo.local="$GITHUB_WORKSPACE/.m2-repo" + bash scripts/python-backend/e2e-test.sh --native "$GITHUB_WORKSPACE/.m2-repo" + env: + JAVA_HOME: ${{ env.JDK_8_HOME }} + JDK_8_HOME: ${{ env.JDK_8_HOME }} + JAVA17_HOME: ${{ env.JDK_17_HOME }} + - name: Run ParparVM Benchmark working-directory: vm # Each benchmark already creates its own mutator/collector threads. Run diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyImage.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyImage.java index 1800cfdb976..c5259246d37 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyImage.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyImage.java @@ -23,7 +23,6 @@ */ package com.codename1.python.runtime; -import java.io.ByteArrayOutputStream; /// The `_cn1image` module: pixel storage and the hot loops behind the /// bundled `PIL` package. An image is a [Img] handle holding interleaved @@ -38,6 +37,35 @@ private PyImage() { } /// One image: `w` by `h` pixels of `bands` interleaved bytes. + /// The platform image codec behind `native_decode`/`native_encode`: + /// Codename One's `Image` and `ImageIO`, installed by the generated + /// registry of a client build (`PyImageNative.install()`). It is a seam + /// rather than a direct call because a backend build has no UI package: + /// the backend translation takes every runtime class, and a reference + /// to `com.codename1.ui` from this one failed its C compile. There it + /// stays null and PIL uses its pure codecs. + interface Codec { + boolean available(); + + Img decode(byte[] data); + + byte[] encode(Img rgba, String format, float quality); + } + + private static Codec codec; + + static void setCodec(Codec c) { + codec = c; + } + + private static Codec requireCodec() { + Codec c = codec; + if (c == null) { + throw Py.error(Py.OS_ERROR, "this platform has no native image codec"); + } + return c; + } + static final class Img { final int w; final int h; @@ -220,44 +248,15 @@ static PyModule module() { } return new PyBytes(out); })); - m.setAttr("native_available", PyBuiltin.positional("native_available", a -> - com.codename1.ui.Display.isInitialized() ? Py.TRUE : Py.FALSE)); - m.setAttr("native_decode", PyBuiltin.positional("native_decode", a -> { - byte[] b = bytes(a[0]); - com.codename1.ui.Image ci = com.codename1.ui.Image.createImage(b, 0, b.length); - int w = ci.getWidth(); - int h = ci.getHeight(); - int[] argb = ci.getRGB(); - Img r = new Img(w, h, 4); - for (int p = 0; p < w * h; p++) { - int c = argb[p]; - r.d[p * 4] = (byte) (c >> 16); - r.d[p * 4 + 1] = (byte) (c >> 8); - r.d[p * 4 + 2] = (byte) c; - r.d[p * 4 + 3] = (byte) (c >>> 24); - } - return wrap(r); - })); - m.setAttr("native_encode", PyBuiltin.positional("native_encode", a -> { - Img s = convert(img(a[0]), 4); - int[] argb = new int[s.w * s.h]; - for (int p = 0; p < argb.length; p++) { - argb[p] = (s.d[p * 4 + 3] & 0xff) << 24 | (s.d[p * 4] & 0xff) << 16 - | (s.d[p * 4 + 1] & 0xff) << 8 | (s.d[p * 4 + 2] & 0xff); - } - com.codename1.ui.Image ci = com.codename1.ui.Image.createImage(argb, s.w, s.h); - com.codename1.ui.util.ImageIO io = com.codename1.ui.util.ImageIO.getImageIO(); - if (io == null) { - throw Py.error(Py.OS_ERROR, "this platform cannot encode " + Py.toStr(a[1])); - } - ByteArrayOutputStream bo = new ByteArrayOutputStream(); - try { - io.save(ci, bo, Py.toStr(a[1]), (float) Py.asDouble(a[2])); - } catch (java.io.IOException e) { - throw Py.error(Py.OS_ERROR, String.valueOf(e.getMessage())); // NOPMD PreserveStackTrace - surfaced as a Python OSError - } - return new PyBytes(bo.toByteArray()); + m.setAttr("native_available", PyBuiltin.positional("native_available", a -> { + Codec c = codec; + return c != null && c.available() ? Py.TRUE : Py.FALSE; })); + m.setAttr("native_decode", PyBuiltin.positional("native_decode", a -> + wrap(requireCodec().decode(bytes(a[0]))))); + m.setAttr("native_encode", PyBuiltin.positional("native_encode", a -> + new PyBytes(requireCodec().encode(convert(img(a[0]), 4), Py.toStr(a[1]), + (float) Py.asDouble(a[2]))))); return m; } diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyImageNative.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyImageNative.java new file mode 100644 index 00000000000..d8abab67efa --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyImageNative.java @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +/// PIL's native image codec over Codename One's `Image` and `ImageIO`. Kept +/// out of `PyImage` so that a backend build, which has no UI package, can +/// leave this class out; the generated registry of a client build installs +/// it. +public final class PyImageNative implements PyImage.Codec { + private PyImageNative() { + } + + /// Makes PIL decode and encode through the platform; idempotent. + public static void install() { + PyImage.setCodec(new PyImageNative()); + } + + @Override + public boolean available() { + return com.codename1.ui.Display.isInitialized(); + } + + @Override + public PyImage.Img decode(byte[] b) { + com.codename1.ui.Image ci = com.codename1.ui.Image.createImage(b, 0, b.length); + int w = ci.getWidth(); + int h = ci.getHeight(); + int[] argb = ci.getRGB(); + PyImage.Img r = new PyImage.Img(w, h, 4); + for (int p = 0; p < w * h; p++) { + int c = argb[p]; + r.d[p * 4] = (byte) (c >> 16); + r.d[p * 4 + 1] = (byte) (c >> 8); + r.d[p * 4 + 2] = (byte) c; + r.d[p * 4 + 3] = (byte) (c >>> 24); + } + return r; + } + + @Override + public byte[] encode(PyImage.Img s, String format, float quality) { + int[] argb = new int[s.w * s.h]; + for (int p = 0; p < argb.length; p++) { + argb[p] = (s.d[p * 4 + 3] & 0xff) << 24 | (s.d[p * 4] & 0xff) << 16 + | (s.d[p * 4 + 1] & 0xff) << 8 | (s.d[p * 4 + 2] & 0xff); + } + com.codename1.ui.Image ci = com.codename1.ui.Image.createImage(argb, s.w, s.h); + com.codename1.ui.util.ImageIO io = com.codename1.ui.util.ImageIO.getImageIO(); + if (io == null) { + throw Py.error(Py.OS_ERROR, "this platform cannot encode " + format); + } + ByteArrayOutputStream bo = new ByteArrayOutputStream(); + try { + io.save(ci, bo, format, quality); + } catch (IOException e) { + throw Py.error(Py.OS_ERROR, String.valueOf(e.getMessage())); // NOPMD PreserveStackTrace - surfaced as a Python OSError + } + return bo.toByteArray(); + } +} diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java index a86330dd51c..20e39c6c31a 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java @@ -136,11 +136,13 @@ public static Map transpile(Map sources, String Set stdlib = addStdlib(sources, trees); boolean server = false; boolean db = false; + boolean image = false; for (String text : sources.values()) { server |= text.contains("_cn1server"); db |= text.contains("_cn1db"); + image |= text.contains("_cn1image"); } - if (server || db) { + if (server || db || image) { // A web framework is bundled: serve it over the backend's HTTP // server when the build has one (a backend build). Elsewhere // serving raises, and the frameworks' test clients still work. @@ -152,6 +154,9 @@ public static Map transpile(Map sources, String boolean backend = cp.find("com/codename1/backend/HttpServer") != null; server &= backend; db = backend && (db || server); + // PIL's platform codec is Codename One's UI Image, which a + // backend build does not have (and leaves out of its classes). + image &= !backend; } catch (IOException e) { throw new IllegalStateException("cannot read the class path: " + e.getMessage(), e); } @@ -194,7 +199,7 @@ public static Map transpile(Map sources, String out.put(PACKAGE.replace('.', '/') + "/BackendDb.java", backendDb()); } out.put(PACKAGE.replace('.', '/') + "/PythonModules.java", registry(classes, mainModule, interop != null, server, db, - nativeModules)); + image, nativeModules)); return out; } @@ -555,6 +560,10 @@ private static String backendServer() { + " return true;\n" + " }\n" + " });\n" + // Transpiled Python frames are large and re recurses: the + // backend's 64KB virtual-thread stack overflowed on a route + // match. Mapped lazily, so this is address space, not memory. + + " HttpServer.requireVirtualThreadStack(1024 * 1024);\n" + " PyServer.install(new BackendServer());\n" + " }\n\n" + " @Override\n" @@ -901,7 +910,7 @@ static String withLineMap(String java, String pyFile) { } private static String registry(Map classes, String mainModule, boolean interop, boolean server, boolean db, - Map nativeModules) { + boolean image, Map nativeModules) { StringBuilder sb = new StringBuilder(); sb.append("// Generated by the Codename One Python transpiler.\n"); sb.append("package ").append(PACKAGE).append(";\n\n"); @@ -922,6 +931,9 @@ private static String registry(Map classes, String mainModule, b if (db) { sb.append(" BackendDb.install();\n"); } + if (image) { + sb.append(" com.codename1.python.runtime.PyImageNative.install();\n"); + } if (classes.containsKey("_cn1_http")) { sb.append(" com.codename1.python.runtime.Cn1Http.installDefault();\n"); } diff --git a/scripts/python-backend/build.sh b/scripts/python-backend/build.sh index 34c7ea96d0d..52313b1bc3d 100755 --- a/scripts/python-backend/build.sh +++ b/scripts/python-backend/build.sh @@ -68,7 +68,9 @@ public class PyBackendMain { } JAVA # The runtime, the one core package it uses, and the host file system shim. -(cd "$CLASSES" && unzip -qo "$RUNTIME" 'com/*' && unzip -qo "$CORE" 'com/codename1/util/*') +# PyImageNative is PIL's codec over the client UI, which a server lacks. +(cd "$CLASSES" && unzip -qo "$RUNTIME" 'com/*' -x 'com/codename1/python/runtime/PyImageNative*' \ + && unzip -qo "$CORE" 'com/codename1/util/*') "$J17/bin/javac" -nowarn --release 17 -d "$CLASSES" $(find "$REPO/scripts/python-bench/hostfs" -name '*.java') "$J17/bin/javac" -nowarn --release 17 -cp "$CLASSES:$BACKEND" -d "$CLASSES" $(find "$GEN" -name '*.java') # Line numbers and source file of the Python, for stack traces and debuggers. diff --git a/scripts/python-backend/e2e-test.sh b/scripts/python-backend/e2e-test.sh index f437509b4d6..9338be00af9 100755 --- a/scripts/python-backend/e2e-test.sh +++ b/scripts/python-backend/e2e-test.sh @@ -5,10 +5,17 @@ # drives it over real sockets with e2e_client.py -- HTTP routes, both kinds # of middleware, websockets and streamed responses. # -# e2e-test.sh [maven repo] +# e2e-test.sh [--native] [maven repo] +# +# --native builds each sample with build.sh's native path instead (ParparVM +# translation, clang) and drives the binary with the same checks. That is +# the only run of the GIL's virtual-thread scheduler, which the generated +# BackendServer installs and which exists only under ParparVM. # # Needs JAVA17_HOME, python3, and the inputs build.sh lists in its header. set -e +BUILD_MODE=--jvm +if [ "$1" = "--native" ]; then BUILD_MODE=; shift; fi HERE="$(cd "$(dirname "$0")" && pwd)" M2="${1:-$(cd "$HERE/../.." && pwd)/.m2-repo}" WORK="$(mktemp -d "${TMPDIR:-/tmp}/cn1-py-e2e.XXXXXX")" @@ -24,7 +31,7 @@ free_port() { run_sample() { local name="$1" kind="$2" - "$HERE/build.sh" --jvm "$HERE/samples/$name" "$WORK/$name" "$M2" + "$HERE/build.sh" $BUILD_MODE "$HERE/samples/$name" "$WORK/$name" "$M2" local port port="$(free_port)" "$WORK/$name" "$port" > "$WORK/$name.log" 2>&1 & @@ -47,4 +54,4 @@ run_sample() { run_sample fastapi-chat fastapi run_sample flask-todo flask -echo "python backend e2e: passed" +echo "python backend e2e${BUILD_MODE:+ ($BUILD_MODE)}${BUILD_MODE:- (native)}: passed" diff --git a/vm/backend/src/com/codename1/backend/HttpServer.java b/vm/backend/src/com/codename1/backend/HttpServer.java index a9267a61e93..443160ad84c 100644 --- a/vm/backend/src/com/codename1/backend/HttpServer.java +++ b/vm/backend/src/com/codename1/backend/HttpServer.java @@ -2143,7 +2143,30 @@ static boolean isSpoolable(String contentType, String contentEncoding) { /// they live in the virtual thread's own VM state, mapped lazily -- so this /// buys call depth rather than data. 64KB holds a few hundred nested Java /// frames, well past what an HTTP handler needs, and is mapped lazily too. - private static final int VT_STACK_BYTES = envIntAtLeast("CN1_HTTP_VT_STACK", 64 * 1024, 1); + /// + /// Not final: an application whose handlers are deeper than that raises it + /// with `requireVirtualThreadStack` before starting a server. Transpiled + /// Python is one: its frames are large and `re` recurses, and a FastAPI + /// request overflowed 64KB on its first route match -- a SIGBUS on the + /// connection's host with nothing but an empty reply to show for it. Plain + /// static rather than atomic: it is written before the server starts, and + /// every reader runs on a host thread that start created afterwards. + private static int vtStackBytes = envIntAtLeast("CN1_HTTP_VT_STACK", 64 * 1024, 1); + + /// Makes every virtual thread this process creates from now on have at + /// least `bytes` of C stack. Call it before `start`; it only ever raises + /// the size, so it composes with `CN1_HTTP_VT_STACK` and with other callers. + /// The stack is mapped lazily, so a generous size costs address space, not + /// memory. + /// + /// #### Parameters + /// + /// - `bytes`: the C stack a connection's handler needs + public static void requireVirtualThreadStack(int bytes) { + if (bytes > vtStackBytes) { + vtStackBytes = bytes; + } + } /// How often a virtual-thread host sweeps its deadlines, however busy it is. /// The same 250ms the idle poll waits, so a quiet host behaves exactly as @@ -3419,7 +3442,7 @@ private void drainTaskInbox(VtHost me) { synchronized (VIRTUAL_TASKS) { queued = (Runnable) VIRTUAL_TASKS.get(Long.valueOf(token)); } - long handle = VirtualThread.createTask(token, VT_STACK_BYTES); + long handle = VirtualThread.createTask(token, vtStackBytes); if (handle != 0 && queued != null) { // Kept so a task abandoned at shutdown can still be told. me.tasks.put(Long.valueOf(handle), queued); @@ -4385,7 +4408,7 @@ private void advance(VtHost me, int fd, long handle) { return; } if (handle == 0) { - handle = VirtualThread.create(fd, VT_STACK_BYTES); + handle = VirtualThread.create(fd, vtStackBytes); if (handle == 0) { // No stack. Serving it on this thread is not an option here: the // keep-alive wait is indefinite because it expects to park, so From b6f9578ad9addb6bd13ed862e3115e7d318fe369 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 13:39:07 +0200 Subject: [PATCH 072/112] Backend SQL: sslmode=verify-ca/verify-full and MySQL's ssl-mode/ssl-ca spellings verify-ca verifies the chain against sslrootcert without the host-name match, on both runtimes (JavaSE endpoint identification off; OpenSSL skips set1_host), which reaches a stock mysql:8.4 container through its generated ca.pem. require stays verifying, as documented. Co-Authored-By: Claude Opus 5.5 --- .../javase/com/codename1/backend/Tcp.java | 20 +++++- .../parparvm/com/codename1/backend/Tcp.java | 21 ++++++- vm/backend/native/cn1_backend_tlsclient.c | 13 +++- .../src/com/codename1/backend/Database.java | 61 ++++++++++++++----- .../src/com/codename1/backend/sql/MySql.java | 5 +- .../com/codename1/backend/sql/Postgres.java | 5 +- .../com/codename1/backend/sql/SqlLimits.java | 26 +++++++- 7 files changed, 123 insertions(+), 28 deletions(-) diff --git a/vm/backend/impl/javase/com/codename1/backend/Tcp.java b/vm/backend/impl/javase/com/codename1/backend/Tcp.java index cca7902b2a1..31633579ba9 100644 --- a/vm/backend/impl/javase/com/codename1/backend/Tcp.java +++ b/vm/backend/impl/javase/com/codename1/backend/Tcp.java @@ -125,6 +125,22 @@ private static int handshakeBudgetMillis() { /// As [#startTls(String)], verifying against the PEM bundle at `caFile` /// INSTEAD of the system trust store. See the translated twin for why. public void startTls(String host, String caFile) throws IOException { + startTls(host, caFile, true); + } + + /// As [#startTls(String,String)], with the name check optional. + /// + /// `verifyHost` false still verifies the chain against `caFile` and skips + /// only the match between the certificate and `host`: what + /// `sslmode=verify-ca` asks for. It is refused without a `caFile`, because + /// a public root vouches for names, and a chain to one without the name is + /// a certificate for anybody. + public void startTls(String host, String caFile, boolean verifyHost) + throws IOException { + if (!verifyHost && caFile == null) { + throw new IOException("Skipping the host name check needs a trust " + + "root of its own (sslrootcert)"); + } // BOTH STRINGS, and the name is the one that decides who the peer is // allowed to be. It crosses to a native as a C string, so a NUL inside it // ends the name there and OpenSSL verifies the certificate against the @@ -153,7 +169,9 @@ public void startTls(String host, String caFile) throws IOException { } SSLSocket upgraded = (SSLSocket) created; //NOPMD CloseResource - becomes this connection's socket in rebind() SSLParameters parameters = upgraded.getSSLParameters(); - parameters.setEndpointIdentificationAlgorithm("HTTPS"); + if (verifyHost) { + parameters.setEndpointIdentificationAlgorithm("HTTPS"); + } upgraded.setSSLParameters(parameters); // BOUNDED, for the reason the packaged arm's comment gives at its own // SSL_connect: connectTimeout was spent reaching the port, and a peer that diff --git a/vm/backend/impl/parparvm/com/codename1/backend/Tcp.java b/vm/backend/impl/parparvm/com/codename1/backend/Tcp.java index fb131536e98..1015eadecb2 100644 --- a/vm/backend/impl/parparvm/com/codename1/backend/Tcp.java +++ b/vm/backend/impl/parparvm/com/codename1/backend/Tcp.java @@ -85,6 +85,22 @@ public void startTls(String host) throws IOException { /// fails to load would verify against roots the caller deliberately did not /// choose, so that is an error rather than a fallback. public void startTls(String host, String caFile) throws IOException { + startTls(host, caFile, true); + } + + /// As [#startTls(String,String)], with the name check optional. + /// + /// `verifyHost` false still verifies the chain against `caFile` and skips + /// only the match between the certificate and `host`: what + /// `sslmode=verify-ca` asks for. It is refused without a `caFile`, because + /// a public root vouches for names, and a chain to one without the name is + /// a certificate for anybody. + public void startTls(String host, String caFile, boolean verifyHost) + throws IOException { + if (!verifyHost && caFile == null) { + throw new IOException("Skipping the host name check needs a trust " + + "root of its own (sslrootcert)"); + } // BOTH STRINGS, and the name is the one that decides who the peer is // allowed to be. It crosses to a native as a C string, so a NUL inside it // ends the name there and OpenSSL verifies the certificate against the @@ -116,7 +132,7 @@ public void startTls(String host, String caFile) throws IOException { long session; claim(claimed); try { - session = startTlsImpl(claimed[0], host, caFile); + session = startTlsImpl(claimed[0], host, caFile, verifyHost); } finally { release(); } @@ -365,7 +381,8 @@ private void checkOpen() throws IOException { private static native int setReadTimeoutImpl(long handle, int millis); private static native int writeImpl(long handle, byte[] buffer, int offset, int length); private static native int closeImpl(long handle); - private static native long startTlsImpl(long handle, String host, String caFile); + private static native long startTlsImpl(long handle, String host, String caFile, + boolean verifyHost); private static native String tlsErrorImpl(); private static native int tlsReadImpl(long session, byte[] buffer, int offset, int length); private static native int tlsWriteImpl(long session, byte[] buffer, int offset, int length); diff --git a/vm/backend/native/cn1_backend_tlsclient.c b/vm/backend/native/cn1_backend_tlsclient.c index 3f7fa92cc2f..3e69d864451 100644 --- a/vm/backend/native/cn1_backend_tlsclient.c +++ b/vm/backend/native/cn1_backend_tlsclient.c @@ -359,7 +359,7 @@ static int cn1IsIpLiteral(const char* host) { return inet_pton(AF_INET6, host, &v6) == 1; } -JAVA_LONG com_codename1_backend_Tcp_startTlsImpl___long_java_lang_String_java_lang_String_R_long(CODENAME_ONE_THREAD_STATE, JAVA_LONG handle, JAVA_OBJECT host, JAVA_OBJECT caFile) { +JAVA_LONG com_codename1_backend_Tcp_startTlsImpl___long_java_lang_String_java_lang_String_boolean_R_long(CODENAME_ONE_THREAD_STATE, JAVA_LONG handle, JAVA_OBJECT host, JAVA_OBJECT caFile, JAVA_BOOLEAN verifyHost) { SSL_CTX* ctx; SSL* ssl; char* h; @@ -419,7 +419,14 @@ JAVA_LONG com_codename1_backend_Tcp_startTlsImpl___long_java_lang_String_java_la * a host by address failed verification against a certificate that correctly * carried the IP -- after packaging only, since the Java SE arm checks both. * X509_VERIFY_PARAM_set1_ip_asc is the IP half of the same door. */ - if(cn1IsIpLiteral(h)) { + /* verifyHost is false only for sslmode=verify-ca, where the caller named a + * private trust root of its own (Database refuses verify-ca without one) and + * asked for the chain to be checked but not the name: a development + * container's generated certificate names no host anyone connects by. The + * chain is still verified -- SSL_VERIFY_PEER is set on the context. */ + if(!verifyHost) { + /* SNI above is still sent; only the name match is skipped. */ + } else if(cn1IsIpLiteral(h)) { if(X509_VERIFY_PARAM_set1_ip_asc(SSL_get0_param(ssl), h) != 1) { cn1ClientTlsRecordError("could not set the expected peer address"); SSL_free(ssl); @@ -590,7 +597,7 @@ void com_codename1_backend_Tcp_tlsCloseImpl___long(CODENAME_ONE_THREAD_STATE, JA #else /* CN1_BACKEND_NO_TLS */ -JAVA_LONG com_codename1_backend_Tcp_startTlsImpl___long_java_lang_String_java_lang_String_R_long(CODENAME_ONE_THREAD_STATE, JAVA_LONG handle, JAVA_OBJECT host, JAVA_OBJECT caFile) { +JAVA_LONG com_codename1_backend_Tcp_startTlsImpl___long_java_lang_String_java_lang_String_boolean_R_long(CODENAME_ONE_THREAD_STATE, JAVA_LONG handle, JAVA_OBJECT host, JAVA_OBJECT caFile, JAVA_BOOLEAN verifyHost) { return 0; } diff --git a/vm/backend/src/com/codename1/backend/Database.java b/vm/backend/src/com/codename1/backend/Database.java index 5f17413dddb..fad5a15249e 100644 --- a/vm/backend/src/com/codename1/backend/Database.java +++ b/vm/backend/src/com/codename1/backend/Database.java @@ -51,13 +51,23 @@ /// PostgreSQL without a branch. Parameters are always bound, never interpolated, /// on all three. /// -/// TLS has three settings, and none of them is "encrypted but unverified". -/// `sslmode=require` demands TLS whose certificate chains to a trusted root AND -/// carries the host's name; `sslmode=disable` is plaintext, deliberately; -/// `sslmode=prefer` uses TLS when the server offers it and fails loudly rather -/// than silently downgrading when verification does not hold. A managed instance -/// or a development container that presents a private CA is reached by naming it: -/// `sslrootcert=/path/to/ca.pem`. +/// TLS has five settings, and none of them is "encrypted but unverified". +/// `sslmode=require` (and its synonym `verify-full`) demands TLS whose +/// certificate chains to a trusted root AND carries the host's name; +/// `sslmode=verify-ca` demands a chain to the root named by `sslrootcert`, which +/// it requires, and does not check the name; `sslmode=disable` is plaintext, +/// deliberately; `sslmode=prefer` uses TLS when the server offers it and fails +/// loudly rather than silently downgrading when verification does not hold. A +/// managed instance or a development container that presents a private CA is +/// reached by naming it: `sslrootcert=/path/to/ca.pem`. MySQL's own spellings are +/// accepted too: `ssl-mode=DISABLED|PREFERRED|REQUIRED|VERIFY_CA|VERIFY_IDENTITY` +/// and `ssl-ca`. +/// +/// Unlike libpq and MySQL Connector, `require` here verifies. Those clients read +/// `require` as "encrypt, trust any certificate", which an on-path peer satisfies +/// with a certificate of its own; a stock `mysql:8.4` container, whose generated +/// certificate names no host, is reached with `sslmode=verify-ca` and its +/// `ca.pem` instead. /// /// SQLite goes through [Db], which is per-target: the engine is linked into /// the binary on the translated side and reached through a JDBC driver on the @@ -1305,18 +1315,15 @@ private static void applyQuery(Url out, String query) throws IOException { } String key = pair.substring(0, equals); String value = decode(pair.substring(equals + 1)); - if ("sslmode".equals(key) || "ssl".equals(key)) { - if (!"require".equals(value) && !"prefer".equals(value) - && !"disable".equals(value)) { - throw new IOException("sslmode must be require, prefer or " - + "disable, not '" + value + "'"); - } - out.sslMode = value; + if ("sslmode".equals(key) || "ssl".equals(key) + || "ssl-mode".equals(key) || "sslMode".equals(key)) { + out.sslMode = canonicalSslMode(value); } else if ("user".equals(key)) { out.user = value; } else if ("password".equals(key)) { out.password = value; - } else if ("sslrootcert".equals(key) || "sslca".equals(key)) { + } else if ("sslrootcert".equals(key) || "sslca".equals(key) + || "ssl-ca".equals(key)) { out.caFile = value; } else if ("connectTimeout".equals(key)) { try { @@ -1377,6 +1384,30 @@ private static String strip(String host) { return host; } + /// Reduces every accepted spelling of an sslmode to the three-plus-one values + /// the drivers implement: `disable`, `prefer`, `require` (verify-full) and + /// `verify-ca`. Anything else is refused rather than read as prefer, which + /// would accept plaintext. + static String canonicalSslMode(String value) throws IOException { + String v = value == null ? "" : value.trim(); + if ("disable".equals(v) || "DISABLED".equals(v) || "false".equals(v)) { + return "disable"; + } + if ("prefer".equals(v) || "PREFERRED".equals(v)) { + return "prefer"; + } + if ("require".equals(v) || "REQUIRED".equals(v) || "true".equals(v) + || "verify-full".equals(v) || "VERIFY_IDENTITY".equals(v)) { + return "require"; + } + if ("verify-ca".equals(v) || "VERIFY_CA".equals(v)) { + return "verify-ca"; + } + throw new IOException("sslmode must be disable, prefer, require, verify-ca " + + "or verify-full (or MySQL's DISABLED, PREFERRED, REQUIRED, " + + "VERIFY_CA, VERIFY_IDENTITY), not '" + value + "'"); + } + private static String decode(String value) throws IOException { if (value.indexOf('%') < 0) { return value; diff --git a/vm/backend/src/com/codename1/backend/sql/MySql.java b/vm/backend/src/com/codename1/backend/sql/MySql.java index b30a00ee23b..fcd56dccc6e 100644 --- a/vm/backend/src/com/codename1/backend/sql/MySql.java +++ b/vm/backend/src/com/codename1/backend/sql/MySql.java @@ -116,6 +116,7 @@ public static MySql connect(String host, int port, String database, String user, int socketTimeoutMillis) throws IOException { // BEFORE THE SOCKET, for the reason Postgres gives one engine over. SqlLimits.requireSslMode(sslMode); + SqlLimits.requireRootForVerifyCa(sslMode, caFile); Tcp connection = Tcp.connect(host, port <= 0 ? 3306 : port, timeoutMillis); // AFTER THE CONNECT, and before anything is said on it: the deadline // governs the conversation, which is where a peer can stall a worker. @@ -194,7 +195,7 @@ private void handshake(String host, String database, String user, String passwor boolean useTls = !"disable".equals(sslMode); if (useTls && (serverCapabilities & CLIENT_SSL) == 0) { - if ("require".equals(sslMode)) { + if (SqlLimits.demandsTls(sslMode)) { throw new IOException("The MySQL server at " + host + " does not offer TLS and sslmode=require"); } @@ -235,7 +236,7 @@ private void handshake(String host, String database, String user, String passwor } sendPacket(request.toByteArray()); try { - wire.getConnection().startTls(host, caFile); + wire.getConnection().startTls(host, caFile, SqlLimits.verifiesHost(sslMode)); } catch (IOException err) { // The SSLRequest packet has already gone out, so this connection // cannot go back to plaintext -- the server is waiting for a diff --git a/vm/backend/src/com/codename1/backend/sql/Postgres.java b/vm/backend/src/com/codename1/backend/sql/Postgres.java index 99297416ef8..fb157185d5a 100644 --- a/vm/backend/src/com/codename1/backend/sql/Postgres.java +++ b/vm/backend/src/com/codename1/backend/sql/Postgres.java @@ -102,6 +102,7 @@ public static Postgres connect(String host, int port, String database, String us // BEFORE THE SOCKET: a mode this code does not implement is a typo, and // reading it as prefer is how "require" spelled wrong accepts plaintext. SqlLimits.requireSslMode(sslMode); + SqlLimits.requireRootForVerifyCa(sslMode, caFile); Tcp connection = Tcp.connect(host, port <= 0 ? 5432 : port, timeoutMillis); // AFTER THE CONNECT, and before anything is said on it: the deadline // governs the conversation, which is where a peer can stall a worker. @@ -116,14 +117,14 @@ public static Postgres connect(String host, int port, String database, String us boolean secured = false; if (!plaintextByChoice) { boolean offered = requestTls(wire); - boolean required = "require".equals(sslMode); + boolean required = SqlLimits.demandsTls(sslMode); if (!offered && required) { throw new IOException("The server at " + host + " refused TLS and sslmode=require"); } if (offered) { try { - connection.startTls(host, caFile); + connection.startTls(host, caFile, SqlLimits.verifiesHost(sslMode)); secured = true; } catch (IOException err) { if (required) { diff --git a/vm/backend/src/com/codename1/backend/sql/SqlLimits.java b/vm/backend/src/com/codename1/backend/sql/SqlLimits.java index 0782d3328c7..f690043f382 100644 --- a/vm/backend/src/com/codename1/backend/sql/SqlLimits.java +++ b/vm/backend/src/com/codename1/backend/sql/SqlLimits.java @@ -84,10 +84,30 @@ static void requireSslMode(String sslMode) throws IOException { return; } if (!"require".equals(sslMode) && !"prefer".equals(sslMode) - && !"disable".equals(sslMode)) { + && !"disable".equals(sslMode) && !"verify-ca".equals(sslMode)) { throw new IOException("sslmode=" + sslMode + " is not one this runtime " - + "implements: use require, prefer or disable. It is refused " - + "rather than read as prefer, which would accept plaintext"); + + "implements: use require, verify-ca, prefer or disable. It is " + + "refused rather than read as prefer, which would accept plaintext"); + } + } + + /// Whether `sslMode` refuses a connection that cannot be encrypted. + static boolean demandsTls(String sslMode) { + return "require".equals(sslMode) || "verify-ca".equals(sslMode); + } + + /// Whether the certificate's name must match the host: every mode but + /// `verify-ca`. + static boolean verifiesHost(String sslMode) { + return !"verify-ca".equals(sslMode); + } + + /// Refuses `verify-ca` without its own trust root: skipping the name check + /// against the public roots would accept any certificate any CA issued. + static void requireRootForVerifyCa(String sslMode, String caFile) throws IOException { + if ("verify-ca".equals(sslMode) && caFile == null) { + throw new IOException("sslmode=verify-ca needs sslrootcert naming the " + + "CA (or server certificate) to trust"); } } } From 2fd7aafd3ce480377586e407e58f890e02b3901b Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 13:45:42 +0200 Subject: [PATCH 073/112] SQLAlchemy subset on MySQL: type_annotation_map, RETURNING refused at compile, concat/CAST names; realdb test resets MySQL The conformance programs give Mapped[str] a length through type_annotation_map (MySQL VARCHAR needs one, as in real SQLAlchemy) and take the inserted key from inserted_primary_key where the dialect has no RETURNING. orm and relations now pass on mysql:8.4 over sslmode=verify-ca. Co-Authored-By: Claude Opus 5.5 --- .../python/site/sqlalchemy/__init__.py | 29 ++++++++++++++++++- .../python/site/sqlalchemy/orm/__init__.py | 18 ++++++++---- .../resources/conformance/sqlalchemy_core.py | 9 ++++-- .../resources/conformance/sqlalchemy_orm.py | 1 + .../resources/conformance/sqlalchemy_query.py | 1 + .../conformance/sqlalchemy_relations.py | 1 + .../python-backend/sqlalchemy-realdb-test.sh | 11 +++++++ 7 files changed, 62 insertions(+), 8 deletions(-) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/__init__.py index 18acdce5d85..c5a5ebe61ea 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/__init__.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/__init__.py @@ -402,6 +402,9 @@ def __init__(self, left, op, right): self.type = Boolean() if op not in ("+", "-", "*", "/", "%") else left.type def _compile(self, ctx): + if self.op == "||" and getattr(ctx.dialect, "name", None) in ("mysql", "mariadb"): + # || is logical OR on MySQL; SQLAlchemy's MySQL compiler writes concat(). + return "concat(%s, %s)" % (self.left._compile(ctx), self.right._compile(ctx)) return "%s %s %s" % (self.left._compile(ctx), self.op, self.right._compile(ctx)) def _tables(self, acc): @@ -1480,7 +1483,9 @@ def __init__(self, expr, typ): def _compile(self, ctx): cc = ctx.columns_clause ctx.columns_clause = False - out = "CAST(%s AS %s)" % (self.expr._compile(ctx), self.type.ddl()) + cast_ddl = getattr(ctx.dialect, "cast_ddl", None) + typ = cast_ddl(self.type) if cast_ddl is not None else self.type.ddl() + out = "CAST(%s AS %s)" % (self.expr._compile(ctx), typ) ctx.columns_clause = cc return out @@ -1553,6 +1558,10 @@ def returning(self, *cols): def _returning_sql(self, ctx): if not self._returning: return "" + if not getattr(ctx.dialect, "insert_returning", True): + # As SQLAlchemy's MySQL compiler: MySQL has no RETURNING at all, and + # sending it anyway answers with a syntax error from the server. + raise exc.CompileError("RETURNING is not supported by this dialect's statement compiler.") q = ctx.qualify ctx.qualify = False s = " RETURNING " + ", ".join(c._compile(ctx) for c in self._returning) @@ -2125,6 +2134,24 @@ class MySQLDialect(_Dialect): autoincrement_suffix = " AUTO_INCREMENT" _quote_char = "`" + def cast_ddl(self, t): + # MySQL's CAST takes CHAR, SIGNED, DECIMAL, DATE...; not VARCHAR or + # INTEGER. The names SQLAlchemy's MySQL compiler renders. + if isinstance(t, (String, Text)): + n = getattr(t, "length", None) + return "CHAR" if n is None else "CHAR(%d)" % n + if isinstance(t, Integer): + return "SIGNED INTEGER" + if isinstance(t, Numeric): + return "DECIMAL" + if isinstance(t, DateTime): + return "DATETIME" + if isinstance(t, Date): + return "DATE" + if isinstance(t, LargeBinary): + return "BINARY" + return t.ddl() + def column_ddl(self, c, table): return _server_type(c.type, ( (Text, "TEXT"), (String, _varchar(self.name)), (BigInteger, "BIGINT"), diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/orm/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/orm/__init__.py index 2cba79b74c4..a60da5564f4 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/orm/__init__.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/orm/__init__.py @@ -102,9 +102,16 @@ def _analyze_str(s): return _name_of(s), False, False -def _sql_type(pytype): +def _sql_type(pytype, tmap=None): if pytype is None: return None + if tmap: + # DeclarativeBase.type_annotation_map: {str: String(100)} is how a model + # gives every Mapped[str] a length, which MySQL's VARCHAR demands. + for k in tmap: + if k is pytype or (isinstance(pytype, str) and getattr(k, "__name__", None) == pytype): + v = tmap[k] + return v() if isinstance(v, type) else v if isinstance(pytype, str): t = _PY_NAMES.get(pytype) else: @@ -117,13 +124,13 @@ def __init__(self, args, kw): self.args = args self.kw = kw - def _make(self, key, pytype, optional): + def _make(self, key, pytype, optional, tmap=None): args = list(self.args) name = key if args and isinstance(args[0], str): name = args.pop(0) if not any(_is_type(a) for a in args): - t = _sql_type(pytype) + t = _sql_type(pytype, tmap) if t is not None: args.insert(0, t) kw = dict(self.kw) @@ -611,6 +618,7 @@ def __new__(mcs, name, bases, ns, **kw): def _map_class(cls, ns, parent=None): ann = ns.get("__annotations__", {}) + tmap = getattr(cls, "type_annotation_map", None) attrs = [] rels = {} specs = [] @@ -628,14 +636,14 @@ def _map_class(cls, ns, parent=None): continue pyt, optional, is_list = _analyze(a) if isinstance(v, _MappedColumnSpec): - col = v._make(key, pyt, optional) + col = v._make(key, pyt, optional, tmap) specs.append((v, col)) elif isinstance(v, Column): col = v if col.name is None: col.name = col.key = key elif v is None and (isinstance(a, _MappedAnnotation) or (isinstance(a, str) and "Mapped" in a)): - t = _sql_type(pyt) + t = _sql_type(pyt, tmap) if t is None: raise exc.ArgumentError("Could not locate SQLAlchemy type for %s.%s" % (cls.__name__, key)) col = Column(key, t, nullable=optional) diff --git a/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_core.py b/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_core.py index 69d166f2432..edeadc7cc88 100644 --- a/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_core.py +++ b/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_core.py @@ -97,8 +97,13 @@ conn.rollback() print(conn.execute(select(func.count()).select_from(posts)).scalar()) print(conn.execute(select(posts.c.title).order_by(posts.c.likes.desc())).first()[0]) - r = conn.execute(insert(posts).values(title="r", likes=4).returning(posts.c.id, posts.c.likes)) - print(tuple(r.one())) + if engine.dialect.insert_returning: + r = conn.execute(insert(posts).values(title="r", likes=4).returning(posts.c.id, posts.c.likes)) + print(tuple(r.one())) + else: + # MySQL has no RETURNING; the generated key comes back on the result. + r = conn.execute(insert(posts).values(title="r", likes=4)) + print((r.inserted_primary_key[0], 4)) try: conn.execute(insert(users).values(id=1, name="dup")) except exc.IntegrityError: diff --git a/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_orm.py b/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_orm.py index 07082afdc31..ea3bd10ef91 100644 --- a/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_orm.py +++ b/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_orm.py @@ -5,6 +5,7 @@ class Base(DeclarativeBase): + type_annotation_map = {str: String(200)} pass diff --git a/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_query.py b/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_query.py index d4d0ef8bbb4..6e9ff611094 100644 --- a/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_query.py +++ b/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_query.py @@ -7,6 +7,7 @@ class Base(DeclarativeBase): + type_annotation_map = {str: String(200)} pass diff --git a/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_relations.py b/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_relations.py index 037c0daf4ef..d9b4ae189d2 100644 --- a/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_relations.py +++ b/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_relations.py @@ -6,6 +6,7 @@ class Base(DeclarativeBase): + type_annotation_map = {str: String(200)} pass diff --git a/scripts/python-backend/sqlalchemy-realdb-test.sh b/scripts/python-backend/sqlalchemy-realdb-test.sh index 8e818c4de78..d7e88cc852e 100755 --- a/scripts/python-backend/sqlalchemy-realdb-test.sh +++ b/scripts/python-backend/sqlalchemy-realdb-test.sh @@ -82,6 +82,17 @@ public class Reset { if ("postgresql".equals(db.dialect().getName())) { db.execute("DROP SCHEMA public CASCADE", new Object[0]); db.execute("CREATE SCHEMA public", new Object[0]); + } else if (db.dialect().getName().startsWith("m")) { + // MySQL and MariaDB: a database is the schema, and the URL's + // user owns it (MYSQL_USER is granted ALL on MYSQL_DATABASE). + String name = args[0]; + int q = name.indexOf('?'); + if (q >= 0) { + name = name.substring(0, q); + } + name = name.substring(name.lastIndexOf('/') + 1); + db.execute("DROP DATABASE IF EXISTS `" + name + "`", new Object[0]); + db.execute("CREATE DATABASE `" + name + "`", new Object[0]); } } finally { db.close(); From 7b10fa7fbfb6bf3d629821e08f219e0adfeb8ae6 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 13:53:12 +0200 Subject: [PATCH 074/112] Shared pure-Java image API (com.codename1.image) for devices and servers; Pillow runs on it RasterImage (non-premultiplied ARGB), RasterGraphics (antialiased paths, strokes with caps/joins, transform, clip, alpha, gradients, scaled images, a RasterFont seam for text), Resampler (Pillow's fixed-point convolution, byte for byte), PngCodec (every color type, palette, tRNS, 16-bit, Adam7) and JpegCodec (baseline + progressive decode, baseline encode). All marked @SharedWithBackend, so a backend can generate images; java.util.zip only. PIL's _cn1image now delegates resampling, premultiply and the PNG row helpers to the core classes, and JPEG open/save goes through JpegCodec on every target, servers included. New pil_jpeg conformance program, core RasterApiTest (checked against the JDK's ImageIO) and backend SharedRasterTest. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/parparvm-tests.yml | 2 + .../src/com/codename1/image/JpegCodec.java | 1020 ++++++++++++++++ .../src/com/codename1/image/PngCodec.java | 484 ++++++++ .../src/com/codename1/image/RasterFont.java | 70 ++ .../com/codename1/image/RasterGradient.java | 156 +++ .../com/codename1/image/RasterGraphics.java | 1083 +++++++++++++++++ .../src/com/codename1/image/RasterImage.java | 276 +++++ .../src/com/codename1/image/RasterPaint.java | 40 + .../src/com/codename1/image/RasterPath.java | 378 ++++++ .../src/com/codename1/image/Resampler.java | 285 +++++ .../src/com/codename1/image/package-info.java | 53 + .../developerguide/backend/SocialCard.java | 64 + docs/developer-guide/graphics.asciidoc | 15 + .../backend/image/SharedRasterTest.java | 72 ++ .../com/codename1/image/RasterApiTest.java | 394 ++++++ .../com/codename1/python/runtime/PyImage.java | 316 ++--- .../com/codename1/python/site/PIL/Image.py | 30 +- .../com/codename1/python/site/PIL/__init__.py | 9 +- .../test/resources/conformance/pil_jpeg.out | 10 + .../test/resources/conformance/pil_jpeg.py | 78 ++ scripts/python-backend/build.sh | 3 +- 21 files changed, 4583 insertions(+), 255 deletions(-) create mode 100644 CodenameOne/src/com/codename1/image/JpegCodec.java create mode 100644 CodenameOne/src/com/codename1/image/PngCodec.java create mode 100644 CodenameOne/src/com/codename1/image/RasterFont.java create mode 100644 CodenameOne/src/com/codename1/image/RasterGradient.java create mode 100644 CodenameOne/src/com/codename1/image/RasterGraphics.java create mode 100644 CodenameOne/src/com/codename1/image/RasterImage.java create mode 100644 CodenameOne/src/com/codename1/image/RasterPaint.java create mode 100644 CodenameOne/src/com/codename1/image/RasterPath.java create mode 100644 CodenameOne/src/com/codename1/image/Resampler.java create mode 100644 CodenameOne/src/com/codename1/image/package-info.java create mode 100644 docs/demos/backend/src/main/java/com/codenameone/developerguide/backend/SocialCard.java create mode 100644 maven/backend/src/test/java/com/codename1/backend/image/SharedRasterTest.java create mode 100644 maven/core-unittests/src/test/java/com/codename1/image/RasterApiTest.java create mode 100644 maven/python-transpiler/src/test/resources/conformance/pil_jpeg.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/pil_jpeg.py diff --git a/.github/workflows/parparvm-tests.yml b/.github/workflows/parparvm-tests.yml index c9be991d2e3..25d78d3d1bb 100644 --- a/.github/workflows/parparvm-tests.yml +++ b/.github/workflows/parparvm-tests.yml @@ -51,6 +51,7 @@ on: # marker; a class marked outside these directories needs its path here. - 'CodenameOne/src/com/codename1/impl/SharedWithBackend.java' - 'CodenameOne/src/com/codename1/impl/orm/**' + - 'CodenameOne/src/com/codename1/image/**' - 'CodenameOne/src/com/codename1/orm/session/**' - 'CodenameOne/src/com/codename1/orm/repository/**' - 'CodenameOne/src/com/codename1/annotations/db/**' @@ -124,6 +125,7 @@ on: # marker; a class marked outside these directories needs its path here. - 'CodenameOne/src/com/codename1/impl/SharedWithBackend.java' - 'CodenameOne/src/com/codename1/impl/orm/**' + - 'CodenameOne/src/com/codename1/image/**' - 'CodenameOne/src/com/codename1/orm/session/**' - 'CodenameOne/src/com/codename1/orm/repository/**' - 'CodenameOne/src/com/codename1/annotations/db/**' diff --git a/CodenameOne/src/com/codename1/image/JpegCodec.java b/CodenameOne/src/com/codename1/image/JpegCodec.java new file mode 100644 index 00000000000..2c6d79f7514 --- /dev/null +++ b/CodenameOne/src/com/codename1/image/JpegCodec.java @@ -0,0 +1,1020 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.image; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +/// Reads and writes JPEG files in pure Java. +/// +/// [#decode(byte[])] reads baseline and progressive Huffman-coded JPEG +/// (JFIF, Exif and Adobe files) with any chroma subsampling, restart +/// intervals, 8- and 16-bit quantization tables, gray, YCbCr and Adobe +/// CMYK/YCCK color. Arithmetic coding and 12-bit lossless JPEG are not +/// supported. Chroma is upsampled by replication, so a decode may differ from +/// libjpeg's by a few levels near sharp color edges. +/// +/// [#encode(RasterImage, float, boolean)] writes baseline JPEG with the +/// standard tables of the JPEG specification, 4:4:4 (no chroma subsampling), +/// scaled to a quality the way libjpeg scales them, so quality 0.75 here +/// gives the same tables as libjpeg's (and Pillow's) quality 75. Alpha is +/// dropped. +@com.codename1.impl.SharedWithBackend +public final class JpegCodec { + static final int[] ZIGZAG = { + 0, 1, 8, 16, 9, 2, 3, 10, 17, 24, 32, 25, 18, 11, 4, 5, 12, 19, 26, 33, 40, 48, 41, 34, 27, 20, 13, 6, 7, + 14, 21, 28, 35, 42, 49, 56, 57, 50, 43, 36, 29, 22, 15, 23, 30, 37, 44, 51, 58, 59, 52, 45, 38, 31, 39, 46, + 53, 60, 61, 54, 47, 55, 62, 63 + }; + + private static final double[] COS = new double[64]; + + static { + for (int x = 0; x < 8; x++) { + for (int u = 0; u < 8; u++) { + double cu = u == 0 ? 1.0 / Math.sqrt(2.0) : 1.0; + COS[x * 8 + u] = cu * Math.cos((2 * x + 1) * u * Math.PI / 16.0) / 2.0; + } + } + } + + private JpegCodec() { + } + + /// Returns true when `data` starts with a JPEG start-of-image marker. + /// + /// #### Parameters + /// + /// - `data`: file bytes + /// + /// #### Returns + /// + /// whether the data looks like a JPEG file + public static boolean isJpeg(byte[] data) { + return data != null && data.length > 3 && (data[0] & 0xff) == 0xff && (data[1] & 0xff) == 0xd8 + && (data[2] & 0xff) == 0xff; + } + + /// Returns the number of color components of a JPEG file: 1 for gray, + /// 3 for color, 4 for CMYK, or 0 when no frame header is found. + /// + /// #### Parameters + /// + /// - `data`: the file + /// + /// #### Returns + /// + /// the component count + public static int componentCount(byte[] data) { + int p = 2; + while (p + 4 <= data.length) { + if ((data[p] & 0xff) != 0xff) { + p++; + continue; + } + int m = data[p + 1] & 0xff; + if (m == 0xff) { + p++; + continue; + } + if (m == 0xd8 || m >= 0xd0 && m <= 0xd7 || m == 0x01) { + p += 2; + continue; + } + int len = (data[p + 2] & 0xff) << 8 | (data[p + 3] & 0xff); + if (m >= 0xc0 && m <= 0xcf && m != 0xc4 && m != 0xc8 && m != 0xcc) { + return p + 9 < data.length ? data[p + 9] & 0xff : 0; + } + p += 2 + len; + } + return 0; + } + + // ------------------------------------------------------------------ decoder + + private static final class Huffman { + final int[] maxcode = new int[18]; + final int[] valptr = new int[17]; + final int[] mincode = new int[17]; + final int[] vals; + + Huffman(byte[] counts, byte[] values) { + vals = new int[values.length]; + for (int i = 0; i < values.length; i++) { + vals[i] = values[i] & 0xff; + } + int code = 0; + int k = 0; + for (int l = 1; l <= 16; l++) { + int n = counts[l - 1] & 0xff; + valptr[l] = k; + mincode[l] = code; + code += n; + k += n; + maxcode[l] = n > 0 ? code - 1 : -1; + code <<= 1; + } + maxcode[17] = Integer.MAX_VALUE; + } + } + + private static final class Component { + int id; + int h; + int v; + int tq; + int blocksPerLine; + int blocksPerColumn; + int blocksPerLineForMcu; + int blocksPerColumnForMcu; + int[] coef; + int pred; + Huffman dc; + Huffman ac; + byte[] plane; + int planeWidth; + } + + /// The decoder's state: one instance per file. + private static final class Decoder { + final byte[] d; + int pos; + int bits; + int bitCount; + final int[][] quant = new int[4][]; + final Huffman[] dcTables = new Huffman[4]; + final Huffman[] acTables = new Huffman[4]; + Component[] comps = new Component[0]; + boolean progressive; + int width; + int height; + int maxH; + int maxV; + int mcusPerLine; + int mcusPerColumn; + int restart; + int adobe = -1; + int eobrun; + int acState; + int acNext; + + Decoder(byte[] d) { + this.d = d; + } + + int u16(int p) throws IOException { + if (p + 1 >= d.length) { + throw new IOException("JPEG file is truncated"); + } + return (d[p] & 0xff) << 8 | (d[p + 1] & 0xff); + } + + RasterImage run() throws IOException { + pos = 2; + boolean frame = false; + while (pos < d.length) { + if ((d[pos] & 0xff) != 0xff) { + pos++; + continue; + } + int m = d[pos + 1 < d.length ? pos + 1 : pos] & 0xff; + if (m == 0xff) { + pos++; + continue; + } + if (m == 0xd9) { + break; + } + if (m == 0xd8 || m >= 0xd0 && m <= 0xd7 || m == 0x01 || m == 0x00) { + pos += 2; + continue; + } + int len = u16(pos + 2); + int body = pos + 4; + int end = pos + 2 + len; + if (end > d.length) { + throw new IOException("JPEG file is truncated"); + } + switch (m) { + case 0xdb: + readQuant(body, end); + break; + case 0xc4: + readHuffman(body, end); + break; + case 0xdd: + restart = u16(body); + break; + case 0xee: + if (len >= 12 && d[body] == 'A' && d[body + 1] == 'd' && d[body + 2] == 'o') { + adobe = d[body + 11] & 0xff; + } + break; + case 0xc0: + case 0xc1: + case 0xc2: + progressive = m == 0xc2; + readFrame(body); + frame = true; + break; + case 0xda: + if (!frame) { + throw new IOException("JPEG scan before frame header"); + } + pos = readScan(body); + continue; + default: + if (m >= 0xc3 && m <= 0xcf && m != 0xc8 && m != 0xcc) { + throw new IOException("unsupported JPEG process (marker 0x" + Integer.toHexString(m) + ")"); + } + break; + } + pos = end; + } + if (!frame) { + throw new IOException("JPEG file has no frame"); + } + return output(); + } + + void readQuant(int p, int end) throws IOException { + while (p < end) { + int pq = (d[p] & 0xff) >> 4; + int tq = d[p] & 3; + p++; + int[] q = new int[64]; + for (int i = 0; i < 64; i++) { + if (pq == 0) { + q[ZIGZAG[i]] = d[p++] & 0xff; + } else { + q[ZIGZAG[i]] = u16(p); + p += 2; + } + } + quant[tq] = q; + } + } + + void readHuffman(int p, int end) { + while (p + 17 <= end) { + int tc = (d[p] & 0xff) >> 4; + int th = d[p] & 3; + byte[] counts = new byte[16]; + System.arraycopy(d, p + 1, counts, 0, 16); + int total = 0; + for (int i = 0; i < 16; i++) { + total += counts[i] & 0xff; + } + byte[] vals = new byte[total]; + System.arraycopy(d, p + 17, vals, 0, Math.min(total, d.length - p - 17)); + Huffman t = new Huffman(counts, vals); + if (tc == 0) { + dcTables[th] = t; + } else { + acTables[th] = t; + } + p += 17 + total; + } + } + + void readFrame(int p) throws IOException { + height = u16(p + 1); + width = u16(p + 3); + int n = d[p + 5] & 0xff; + if (width <= 0 || height <= 0 || n == 0 || n > 4) { + throw new IOException("unsupported JPEG frame " + width + "x" + height + "x" + n); + } + comps = new Component[n]; + maxH = 1; + maxV = 1; + for (int i = 0; i < n; i++) { + Component c = new Component(); + int q = p + 6 + i * 3; + c.id = d[q] & 0xff; + c.h = Math.max(1, (d[q + 1] & 0xff) >> 4); + c.v = Math.max(1, d[q + 1] & 15); + c.tq = d[q + 2] & 3; + maxH = Math.max(maxH, c.h); + maxV = Math.max(maxV, c.v); + comps[i] = c; + } + mcusPerLine = (width + 8 * maxH - 1) / (8 * maxH); + mcusPerColumn = (height + 8 * maxV - 1) / (8 * maxV); + for (int i = 0; i < n; i++) { + Component c = comps[i]; + c.blocksPerLine = ((width * c.h + maxH - 1) / maxH + 7) / 8; + c.blocksPerColumn = ((height * c.v + maxV - 1) / maxV + 7) / 8; + c.blocksPerLineForMcu = mcusPerLine * c.h; + c.blocksPerColumnForMcu = mcusPerColumn * c.v; + c.coef = new int[c.blocksPerLineForMcu * c.blocksPerColumnForMcu * 64]; + } + } + + int readBit() { + if (bitCount == 0) { + if (pos >= d.length) { + return 0; + } + int b = d[pos] & 0xff; + if (b == 0xff) { + int next = pos + 1 < d.length ? d[pos + 1] & 0xff : 0xd9; + if (next != 0) { + // A marker: the scan's data has ended; feed zeros. + return 0; + } + pos += 2; + } else { + pos++; + } + bits = b; + bitCount = 8; + } + bitCount--; + return bits >> bitCount & 1; + } + + int receive(int n) { + int v = 0; + for (int i = 0; i < n; i++) { + v = v << 1 | readBit(); + } + return v; + } + + int extend(int n) { + if (n == 0) { + return 0; + } + int v = receive(n); + return v >= 1 << (n - 1) ? v : v - (1 << n) + 1; + } + + int decode(Huffman t) throws IOException { + if (t == null) { + throw new IOException("JPEG scan uses an undefined Huffman table"); + } + int code = 0; + for (int l = 1; l <= 16; l++) { + code = code << 1 | readBit(); + if (code <= t.maxcode[l]) { + int idx = t.valptr[l] + code - t.mincode[l]; + return idx < t.vals.length ? t.vals[idx] : 0; + } + } + return 0; + } + + int readScan(int p) throws IOException { + int n = d[p] & 0xff; + Component[] sc = new Component[n]; + int q = p + 1; + for (int i = 0; i < n; i++) { + int id = d[q] & 0xff; + int tables = d[q + 1] & 0xff; + q += 2; + for (Component cand : comps) { + if (cand.id == id) { + sc[i] = cand; + } + } + if (sc[i] == null) { + throw new IOException("JPEG scan names an unknown component"); + } + sc[i].dc = dcTables[tables >> 4 & 3]; + sc[i].ac = acTables[tables & 3]; + } + int ss = d[q] & 0xff; + int se = d[q + 1] & 0xff; + int ah = (d[q + 2] & 0xff) >> 4; + int al = d[q + 2] & 15; + pos = q + 3; + bitCount = 0; + int total = n == 1 ? sc[0].blocksPerLine * sc[0].blocksPerColumn : mcusPerLine * mcusPerColumn; + int mcu = 0; + while (mcu < total) { + for (int i = 0; i < n; i++) { + sc[i].pred = 0; + } + eobrun = 0; + acState = 0; + int count = restart > 0 ? Math.min(restart, total - mcu) : total - mcu; + for (int i = 0; i < count; i++) { + if (n == 1) { + Component c = sc[0]; + int row = mcu / c.blocksPerLine; + int col = mcu % c.blocksPerLine; + block(c, 64 * (row * c.blocksPerLineForMcu + col), ss, se, ah, al); + } else { + int mr = mcu / mcusPerLine; + int mc = mcu % mcusPerLine; + for (int k = 0; k < n; k++) { + Component c = sc[k]; + for (int y = 0; y < c.v; y++) { + for (int x = 0; x < c.h; x++) { + int row = mr * c.v + y; + int col = mc * c.h + x; + block(c, 64 * (row * c.blocksPerLineForMcu + col), ss, se, ah, al); + } + } + } + } + mcu++; + } + bitCount = 0; + while (pos + 1 < d.length && !((d[pos] & 0xff) == 0xff && (d[pos + 1] & 0xff) != 0 + && (d[pos + 1] & 0xff) != 0xff)) { + pos++; + } + if (pos + 1 >= d.length) { + break; + } + int m = d[pos + 1] & 0xff; + if (m >= 0xd0 && m <= 0xd7) { + pos += 2; + } else { + break; + } + } + return pos; + } + + void block(Component c, int off, int ss, int se, int ah, int al) throws IOException { + int[] z = c.coef; + if (!progressive) { + int t = decode(c.dc); + c.pred += extend(t); + z[off] = c.pred; + int k = 1; + while (k < 64) { + int rs = decode(c.ac); + int s = rs & 15; + int r = rs >> 4; + if (s == 0) { + if (r < 15) { + break; + } + k += 16; + continue; + } + k += r; + if (k > 63) { + break; + } + z[off + ZIGZAG[k]] = extend(s); + k++; + } + return; + } + if (ss == 0) { + if (ah == 0) { + int t = decode(c.dc); + c.pred += extend(t) << al; + z[off] = c.pred; + } else if (readBit() != 0) { + z[off] |= 1 << al; + } + return; + } + if (ah == 0) { + acFirst(c, z, off, ss, se, al); + } else { + acSuccessive(c, z, off, ss, se, al); + } + } + + void acFirst(Component c, int[] z, int off, int ss, int se, int al) throws IOException { + if (eobrun > 0) { + eobrun--; + return; + } + int k = ss; + while (k <= se) { + int rs = decode(c.ac); + int s = rs & 15; + int r = rs >> 4; + if (s == 0) { + if (r < 15) { + eobrun = receive(r) + (1 << r) - 1; + break; + } + k += 16; + continue; + } + k += r; + if (k > 63) { + break; + } + z[off + ZIGZAG[k]] = extend(s) * (1 << al); + k++; + } + } + + void acSuccessive(Component c, int[] z, int off, int ss, int se, int al) throws IOException { + int k = ss; + int r = 0; + while (k <= se) { + int o = off + ZIGZAG[k]; + int sign = z[o] < 0 ? -1 : 1; + switch (acState) { + case 0: { + int rs = decode(c.ac); + int s = rs & 15; + r = rs >> 4; + if (s == 0) { + if (r < 15) { + eobrun = receive(r) + (1 << r); + acState = 4; + } else { + r = 16; + acState = 1; + } + } else { + acNext = extend(s); + acState = r != 0 ? 2 : 3; + } + continue; + } + case 1: + case 2: + if (z[o] != 0) { + z[o] += sign * (readBit() << al); + } else { + r--; + if (r == 0) { + acState = acState == 2 ? 3 : 0; + } + } + break; + case 3: + if (z[o] != 0) { + z[o] += sign * (readBit() << al); + } else { + z[o] = acNext << al; + acState = 0; + } + break; + default: + if (z[o] != 0) { + z[o] += sign * (readBit() << al); + } + break; + } + k++; + } + if (acState == 4) { + eobrun--; + if (eobrun == 0) { + acState = 0; + } + } + } + + RasterImage output() throws IOException { + double[] tmp = new double[64]; + int[] out = new int[64]; + for (Component c : comps) { + int[] q = quant[c.tq]; + if (q == null) { + throw new IOException("JPEG component uses an undefined quantization table"); + } + int pw = c.blocksPerLineForMcu * 8; + c.planeWidth = pw; + c.plane = new byte[pw * c.blocksPerColumnForMcu * 8]; + for (int br = 0; br < c.blocksPerColumnForMcu; br++) { + for (int bc = 0; bc < c.blocksPerLineForMcu; bc++) { + int off = 64 * (br * c.blocksPerLineForMcu + bc); + idct(c.coef, off, q, tmp, out); + for (int y = 0; y < 8; y++) { + int base = (br * 8 + y) * pw + bc * 8; + for (int x = 0; x < 8; x++) { + c.plane[base + x] = (byte) out[y * 8 + x]; + } + } + } + } + c.coef = null; + } + int[] px = new int[width * height]; + int n = comps.length; + boolean transform = n == 3 ? adobe != 0 : n == 4 && adobe == 2; + for (int y = 0; y < height; y++) { + for (int x = 0; x < width; x++) { + int c0 = sampleAt(comps[0], x, y); + int argb; + if (n == 1) { + argb = 0xff000000 | c0 << 16 | c0 << 8 | c0; + } else if (n == 2) { + argb = 0xff000000 | c0 << 16 | c0 << 8 | c0; + } else { + int c1 = sampleAt(comps[1], x, y); + int c2 = sampleAt(comps[2], x, y); + int r = c0; + int g = c1; + int b = c2; + if (transform) { + double yy = c0; + double cb = c1 - 128; + double cr = c2 - 128; + r = clamp((int) Math.floor(yy + 1.402 * cr + 0.5)); + g = clamp((int) Math.floor(yy - 0.344136 * cb - 0.714136 * cr + 0.5)); + b = clamp((int) Math.floor(yy + 1.772 * cb + 0.5)); + } + if (n == 4) { + int k = sampleAt(comps[3], x, y); + if (adobe >= 0) { + r = r * k / 255; + g = g * k / 255; + b = b * k / 255; + } else { + r = (255 - r) * (255 - k) / 255; + g = (255 - g) * (255 - k) / 255; + b = (255 - b) * (255 - k) / 255; + } + } + argb = 0xff000000 | r << 16 | g << 8 | b; + } + px[y * width + x] = argb; + } + } + return new RasterImage(width, height, px); + } + + int sampleAt(Component c, int x, int y) { + int sx = x * c.h / maxH; + int sy = y * c.v / maxV; + return c.plane[sy * c.planeWidth + sx] & 0xff; + } + } + + private static int clamp(int v) { + return v < 0 ? 0 : v > 255 ? 255 : v; + } + + private static void idct(int[] coef, int off, int[] q, double[] tmp, int[] out) { + for (int y = 0; y < 8; y++) { + for (int x = 0; x < 8; x++) { + double s = 0; + for (int u = 0; u < 8; u++) { + s += COS[x * 8 + u] * coef[off + y * 8 + u] * q[y * 8 + u]; + } + tmp[y * 8 + x] = s; + } + } + for (int x = 0; x < 8; x++) { + for (int y = 0; y < 8; y++) { + double s = 0; + for (int v = 0; v < 8; v++) { + s += COS[y * 8 + v] * tmp[v * 8 + x]; + } + out[y * 8 + x] = clamp((int) Math.floor(s + 128.5)); + } + } + } + + /// Decodes a JPEG file to an opaque ARGB image. + /// + /// #### Parameters + /// + /// - `data`: the file + /// + /// #### Returns + /// + /// the image + /// + /// #### Throws + /// + /// - `IOException`: when the data is not a JPEG file this codec can read + public static RasterImage decode(byte[] data) throws IOException { + if (!isJpeg(data)) { + throw new IOException("not a JPEG file"); + } + try { + return new Decoder(data).run(); + } catch (ArrayIndexOutOfBoundsException e) { + throw new IOException("corrupt JPEG file", e); + } + } + + // ------------------------------------------------------------------ encoder + + private static final int[] STD_LUMA_Q = { + 16, 11, 10, 16, 24, 40, 51, 61, 12, 12, 14, 19, 26, 58, 60, 55, 14, 13, 16, 24, 40, 57, 69, 56, + 14, 17, 22, 29, 51, 87, 80, 62, 18, 22, 37, 56, 68, 109, 103, 77, 24, 35, 55, 64, 81, 104, 113, 92, + 49, 64, 78, 87, 103, 121, 120, 101, 72, 92, 95, 98, 112, 100, 103, 99 + }; + private static final int[] STD_CHROMA_Q = { + 17, 18, 24, 47, 99, 99, 99, 99, 18, 21, 26, 66, 99, 99, 99, 99, 24, 26, 56, 99, 99, 99, 99, 99, + 47, 66, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, + 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99 + }; + private static final int[] DC_LUMA_BITS = {0, 1, 5, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0}; + private static final int[] DC_CHROMA_BITS = {0, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0}; + private static final int[] DC_VALS = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}; + private static final int[] AC_LUMA_BITS = {0, 2, 1, 3, 3, 2, 4, 3, 5, 5, 4, 4, 0, 0, 1, 0x7d}; + private static final int[] AC_LUMA_VALS = { + 0x01, 0x02, 0x03, 0x00, 0x04, 0x11, 0x05, 0x12, 0x21, 0x31, 0x41, 0x06, 0x13, 0x51, 0x61, 0x07, + 0x22, 0x71, 0x14, 0x32, 0x81, 0x91, 0xa1, 0x08, 0x23, 0x42, 0xb1, 0xc1, 0x15, 0x52, 0xd1, 0xf0, + 0x24, 0x33, 0x62, 0x72, 0x82, 0x09, 0x0a, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x25, 0x26, 0x27, 0x28, + 0x29, 0x2a, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, + 0x4a, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, + 0x6a, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, + 0x8a, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, + 0xa8, 0xa9, 0xaa, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xc2, 0xc3, 0xc4, 0xc5, + 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xe1, 0xe2, + 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, + 0xf9, 0xfa + }; + private static final int[] AC_CHROMA_BITS = {0, 2, 1, 2, 4, 4, 3, 4, 7, 5, 4, 4, 0, 1, 2, 0x77}; + private static final int[] AC_CHROMA_VALS = { + 0x00, 0x01, 0x02, 0x03, 0x11, 0x04, 0x05, 0x21, 0x31, 0x06, 0x12, 0x41, 0x51, 0x07, 0x61, 0x71, + 0x13, 0x22, 0x32, 0x81, 0x08, 0x14, 0x42, 0x91, 0xa1, 0xb1, 0xc1, 0x09, 0x23, 0x33, 0x52, 0xf0, + 0x15, 0x62, 0x72, 0xd1, 0x0a, 0x16, 0x24, 0x34, 0xe1, 0x25, 0xf1, 0x17, 0x18, 0x19, 0x1a, 0x26, + 0x27, 0x28, 0x29, 0x2a, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, + 0x49, 0x4a, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, + 0x69, 0x6a, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0xa2, 0xa3, 0xa4, 0xa5, + 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xc2, 0xc3, + 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, + 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, + 0xf9, 0xfa + }; + + /// Huffman codes by symbol: `codes[sym]` and `sizes[sym]`. + private static int[][] codeTable(int[] bits, int[] vals) { + int[] codes = new int[256]; + int[] sizes = new int[256]; + int code = 0; + int k = 0; + for (int l = 1; l <= 16; l++) { + for (int i = 0; i < bits[l - 1]; i++) { + codes[vals[k]] = code; + sizes[vals[k]] = l; + k++; + code++; + } + code <<= 1; + } + return new int[][] {codes, sizes}; + } + + private static final class BitWriter { + final ByteArrayOutputStream out; + int acc; + int count; + + BitWriter(ByteArrayOutputStream out) { + this.out = out; + } + + void write(int value, int n) { + for (int i = n - 1; i >= 0; i--) { + acc = acc << 1 | (value >> i & 1); + count++; + if (count == 8) { + out.write(acc); + if (acc == 0xff) { + out.write(0); + } + acc = 0; + count = 0; + } + } + } + + void flush() { + while (count != 0) { + write(1, 1); + } + } + } + + private static int[] scaledTable(int[] std, int quality) { + int scale = quality < 50 ? 5000 / quality : 200 - quality * 2; + int[] q = new int[64]; + for (int i = 0; i < 64; i++) { + int v = (std[i] * scale + 50) / 100; + q[i] = v < 1 ? 1 : v > 255 ? 255 : v; + } + return q; + } + + /// Encodes an image as baseline color JPEG. + /// + /// #### Parameters + /// + /// - `img`: the image; alpha is ignored + /// - `quality`: 0 to 1, as libjpeg's 0 to 100 divided by 100 + /// + /// #### Returns + /// + /// the JPEG file + public static byte[] encode(RasterImage img, float quality) { + return encode(img, quality, false); + } + + /// Encodes an image as baseline JPEG. + /// + /// #### Parameters + /// + /// - `img`: the image; alpha is ignored + /// - `quality`: 0 to 1, as libjpeg's 0 to 100 divided by 100 + /// - `grayscale`: true to write one gray component (the luma of each pixel) + /// instead of three + /// + /// #### Returns + /// + /// the JPEG file + public static byte[] encode(RasterImage img, float quality, boolean grayscale) { + int qi = (int) Math.floor(quality * 100 + 0.5); + qi = qi < 1 ? 1 : qi > 100 ? 100 : qi; + int[] ql = scaledTable(STD_LUMA_Q, qi); + int[] qc = scaledTable(STD_CHROMA_Q, qi); + int w = img.getWidth(); + int h = img.getHeight(); + int nc = grayscale ? 1 : 3; + ByteArrayOutputStream out = new ByteArrayOutputStream(w * h / 4 + 1024); + out.write(0xff); + out.write(0xd8); + byte[] jfif = {0x4a, 0x46, 0x49, 0x46, 0, 1, 1, 0, 0, 1, 0, 1, 0, 0}; + segment(out, 0xe0, jfif); + byte[] dqt = new byte[grayscale ? 65 : 130]; + dqt[0] = 0; + for (int i = 0; i < 64; i++) { + dqt[1 + i] = (byte) ql[ZIGZAG[i]]; + } + if (!grayscale) { + dqt[65] = 1; + for (int i = 0; i < 64; i++) { + dqt[66 + i] = (byte) qc[ZIGZAG[i]]; + } + } + segment(out, 0xdb, dqt); + byte[] sof = new byte[6 + nc * 3]; + sof[0] = 8; + sof[1] = (byte) (h >> 8); + sof[2] = (byte) h; + sof[3] = (byte) (w >> 8); + sof[4] = (byte) w; + sof[5] = (byte) nc; + for (int i = 0; i < nc; i++) { + sof[6 + i * 3] = (byte) (i + 1); + sof[7 + i * 3] = 0x11; + sof[8 + i * 3] = (byte) (i == 0 ? 0 : 1); + } + segment(out, 0xc0, sof); + ByteArrayOutputStream dht = new ByteArrayOutputStream(); + huffSegment(dht, 0x00, DC_LUMA_BITS, DC_VALS); + huffSegment(dht, 0x10, AC_LUMA_BITS, AC_LUMA_VALS); + if (!grayscale) { + huffSegment(dht, 0x01, DC_CHROMA_BITS, DC_VALS); + huffSegment(dht, 0x11, AC_CHROMA_BITS, AC_CHROMA_VALS); + } + segment(out, 0xc4, dht.toByteArray()); + byte[] sos = new byte[4 + nc * 2]; + sos[0] = (byte) nc; + for (int i = 0; i < nc; i++) { + sos[1 + i * 2] = (byte) (i + 1); + sos[2 + i * 2] = (byte) (i == 0 ? 0x00 : 0x11); + } + sos[2 + nc * 2] = 63; + segment(out, 0xda, sos); + int[][] dcl = codeTable(DC_LUMA_BITS, DC_VALS); + int[][] acl = codeTable(AC_LUMA_BITS, AC_LUMA_VALS); + int[][] dcc = codeTable(DC_CHROMA_BITS, DC_VALS); + int[][] acc = codeTable(AC_CHROMA_BITS, AC_CHROMA_VALS); + BitWriter bw = new BitWriter(out); + int[] px = img.getPixels(); + double[] yb = new double[64]; + double[] cbb = new double[64]; + double[] crb = new double[64]; + double[] tmp = new double[64]; + int[] pred = new int[3]; + for (int by = 0; by < h; by += 8) { + for (int bx = 0; bx < w; bx += 8) { + for (int y = 0; y < 8; y++) { + int sy = Math.min(by + y, h - 1); + for (int x = 0; x < 8; x++) { + int c = px[sy * w + Math.min(bx + x, w - 1)]; + int r = c >> 16 & 0xff; + int g = c >> 8 & 0xff; + int b = c & 0xff; + yb[y * 8 + x] = 0.299 * r + 0.587 * g + 0.114 * b - 128; + cbb[y * 8 + x] = -0.168736 * r - 0.331264 * g + 0.5 * b; + crb[y * 8 + x] = 0.5 * r - 0.418688 * g - 0.081312 * b; + } + } + pred[0] = encodeBlock(bw, yb, tmp, ql, pred[0], dcl, acl); + if (!grayscale) { + pred[1] = encodeBlock(bw, cbb, tmp, qc, pred[1], dcc, acc); + pred[2] = encodeBlock(bw, crb, tmp, qc, pred[2], dcc, acc); + } + } + } + bw.flush(); + out.write(0xff); + out.write(0xd9); + return out.toByteArray(); + } + + private static void segment(ByteArrayOutputStream out, int marker, byte[] body) { + out.write(0xff); + out.write(marker); + out.write((body.length + 2) >> 8); + out.write(body.length + 2); + out.write(body, 0, body.length); + } + + private static void huffSegment(ByteArrayOutputStream out, int tcth, int[] bits, int[] vals) { + out.write(tcth); + for (int i = 0; i < 16; i++) { + out.write(bits[i]); + } + for (int v : vals) { + out.write(v); + } + } + + private static int category(int v) { + int a = v < 0 ? -v : v; + int n = 0; + while (a != 0) { + n++; + a >>= 1; + } + return n; + } + + private static void emit(BitWriter bw, int v, int n) { + bw.write(v < 0 ? v + (1 << n) - 1 : v, n); + } + + private static int encodeBlock(BitWriter bw, double[] blk, double[] tmp, int[] q, int pred, int[][] dc, + int[][] ac) { + // Forward DCT, the transpose of the decoder's inverse. + for (int y = 0; y < 8; y++) { + for (int u = 0; u < 8; u++) { + double s = 0; + for (int x = 0; x < 8; x++) { + s += COS[x * 8 + u] * blk[y * 8 + x]; + } + tmp[y * 8 + u] = s; + } + } + int[] zz = new int[64]; + for (int u = 0; u < 8; u++) { + for (int v = 0; v < 8; v++) { + double s = 0; + for (int y = 0; y < 8; y++) { + s += COS[y * 8 + v] * tmp[y * 8 + u]; + } + int idx = v * 8 + u; + zz[idx] = (int) Math.floor(s / q[idx] + 0.5); + } + } + int diff = zz[0] - pred; + int cat = category(diff); + bw.write(dc[0][cat], dc[1][cat]); + emit(bw, diff, cat); + int run = 0; + for (int k = 1; k < 64; k++) { + int v = zz[ZIGZAG[k]]; + if (v == 0) { + run++; + continue; + } + while (run > 15) { + bw.write(ac[0][0xf0], ac[1][0xf0]); + run -= 16; + } + int s = category(v); + int sym = run << 4 | s; + bw.write(ac[0][sym], ac[1][sym]); + emit(bw, v, s); + run = 0; + } + if (run > 0) { + bw.write(ac[0][0], ac[1][0]); + } + return zz[0]; + } +} diff --git a/CodenameOne/src/com/codename1/image/PngCodec.java b/CodenameOne/src/com/codename1/image/PngCodec.java new file mode 100644 index 00000000000..e7a4e967d73 --- /dev/null +++ b/CodenameOne/src/com/codename1/image/PngCodec.java @@ -0,0 +1,484 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.image; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.zip.CRC32; +import java.util.zip.DataFormatException; +import java.util.zip.Deflater; +import java.util.zip.Inflater; + +/// Reads and writes PNG files in pure Java. +/// +/// [#decode(byte[])] accepts every PNG color type (gray, RGB, palette, gray +/// with alpha, RGBA), bit depths 1 to 16, `tRNS` transparency and Adam7 +/// interlacing. 16-bit samples are reduced to their high byte. [#encode(RasterImage)] +/// writes 8-bit RGBA, or RGB when every pixel is opaque. +/// +/// The lower-level [#unfilter(byte[], int, int, int, int)] and +/// [#expand(byte[], int, int, int, int)] keep the file's own sample layout +/// instead of converting to ARGB; the Python runtime's `PIL` uses them to +/// keep a palette image a palette image. +@com.codename1.impl.SharedWithBackend +public final class PngCodec { + private static final byte[] SIGNATURE = {(byte) 0x89, 'P', 'N', 'G', 13, 10, 26, 10}; + private static final int[][] ADAM7 = { + {0, 0, 8, 8}, {4, 0, 8, 8}, {0, 4, 4, 8}, {2, 0, 4, 4}, {0, 2, 2, 4}, {1, 0, 2, 2}, {0, 1, 1, 2} + }; + + private PngCodec() { + } + + /// Returns true when `data` starts with the PNG signature. + /// + /// #### Parameters + /// + /// - `data`: file bytes + /// + /// #### Returns + /// + /// whether the data is a PNG file + public static boolean isPng(byte[] data) { + if (data == null || data.length < 8) { + return false; + } + for (int i = 0; i < 8; i++) { + if (data[i] != SIGNATURE[i]) { + return false; + } + } + return true; + } + + private static int be32(byte[] d, int p) { + return (d[p] & 0xff) << 24 | (d[p + 1] & 0xff) << 16 | (d[p + 2] & 0xff) << 8 | (d[p + 3] & 0xff); + } + + /// Decodes a PNG file to a non-premultiplied ARGB image. + /// + /// #### Parameters + /// + /// - `data`: the file + /// + /// #### Returns + /// + /// the image + /// + /// #### Throws + /// + /// - `IOException`: when the data is not a PNG file this codec can read + public static RasterImage decode(byte[] data) throws IOException { + if (!isPng(data)) { + throw new IOException("not a PNG file"); + } + int pos = 8; + int w = 0; + int h = 0; + int depth = 0; + int ctype = -1; + int interlace = 0; + byte[] plte = null; + byte[] trns = null; + ByteArrayOutputStream idat = new ByteArrayOutputStream(); + while (pos + 8 <= data.length) { + int len = be32(data, pos); + if (len < 0 || pos + 12 + len > data.length) { + throw new IOException("truncated PNG chunk"); + } + int body = pos + 8; + String kind = new String(new char[] {(char) data[pos + 4], (char) data[pos + 5], + (char) data[pos + 6], (char) data[pos + 7]}); + if ("IHDR".equals(kind)) { + w = be32(data, body); + h = be32(data, body + 4); + depth = data[body + 8] & 0xff; + ctype = data[body + 9] & 0xff; + interlace = data[body + 12] & 0xff; + } else if ("PLTE".equals(kind)) { + plte = slice(data, body, len); + } else if ("tRNS".equals(kind)) { + trns = slice(data, body, len); + } else if ("IDAT".equals(kind)) { + idat.write(data, body, len); + } else if ("IEND".equals(kind)) { + break; + } + pos += 12 + len; + } + if (ctype < 0 || w <= 0 || h <= 0) { + throw new IOException("PNG file has no header"); + } + int channels = channels(ctype); + if (channels == 0 || (depth != 1 && depth != 2 && depth != 4 && depth != 8 && depth != 16)) { + throw new IOException("unsupported PNG type " + ctype + "/" + depth); + } + byte[] raw = inflate(idat.toByteArray()); + int bpp = Math.max(1, channels * depth / 8); + int[] argb = new int[w * h]; + if (interlace == 0) { + int row = (w * channels * depth + 7) / 8; + toArgb(expand(unfilter(raw, 0, row, h, bpp), w, h, ctype, depth), w, h, ctype, depth, + plte, trns, argb, 0, 0, 1, 1, w); + } else { + int off = 0; + for (int[] a : ADAM7) { + int pw = (w - a[0] + a[2] - 1) / a[2]; + int ph = (h - a[1] + a[3] - 1) / a[3]; + if (pw <= 0 || ph <= 0) { + continue; + } + int row = (pw * channels * depth + 7) / 8; + toArgb(expand(unfilter(raw, off, row, ph, bpp), pw, ph, ctype, depth), pw, ph, ctype, depth, + plte, trns, argb, a[0], a[1], a[2], a[3], w); + off += (row + 1) * ph; + } + } + return new RasterImage(w, h, argb); + } + + private static byte[] slice(byte[] d, int p, int n) { + byte[] r = new byte[n]; + System.arraycopy(d, p, r, 0, n); + return r; + } + + private static int channels(int ctype) { + switch (ctype) { + case 0: + case 3: + return 1; + case 2: + return 3; + case 4: + return 2; + case 6: + return 4; + default: + return 0; + } + } + + private static byte[] inflate(byte[] z) throws IOException { + Inflater inf = new Inflater(); + inf.setInput(z); + ByteArrayOutputStream out = new ByteArrayOutputStream(z.length * 4 + 64); + byte[] buf = new byte[8192]; + try { + while (!inf.finished()) { + int n = inf.inflate(buf); + if (n == 0 && (inf.needsInput() || inf.needsDictionary())) { + break; + } + out.write(buf, 0, n); + } + } catch (DataFormatException e) { + throw new IOException("corrupt PNG data: " + e.getMessage(), e); + } finally { + inf.end(); + } + return out.toByteArray(); + } + + private static void toArgb(byte[] s, int w, int h, int ctype, int depth, byte[] plte, byte[] trns, + int[] out, int x0, int y0, int dx, int dy, int stride) { + int bands = channels(ctype); + int tr = -1; + int tg = -1; + int tb = -1; + if (trns != null && ctype == 0 && trns.length >= 2) { + tr = scaleKey(((trns[0] & 0xff) << 8 | (trns[1] & 0xff)), depth); + } else if (trns != null && ctype == 2 && trns.length >= 6) { + tr = scaleKey(((trns[0] & 0xff) << 8 | (trns[1] & 0xff)), depth); + tg = scaleKey(((trns[2] & 0xff) << 8 | (trns[3] & 0xff)), depth); + tb = scaleKey(((trns[4] & 0xff) << 8 | (trns[5] & 0xff)), depth); + } + for (int y = 0; y < h; y++) { + for (int x = 0; x < w; x++) { + int o = (y * w + x) * bands; + int c; + switch (ctype) { + case 0: { + int g = s[o] & 0xff; + c = (g == tr ? 0 : 0xff000000) | g << 16 | g << 8 | g; + break; + } + case 2: { + int r = s[o] & 0xff; + int g = s[o + 1] & 0xff; + int b = s[o + 2] & 0xff; + c = (r == tr && g == tg && b == tb ? 0 : 0xff000000) | r << 16 | g << 8 | b; + break; + } + case 3: { + int i = s[o] & 0xff; + int a = trns != null && i < trns.length ? trns[i] & 0xff : 255; + int r = 0; + int g = 0; + int b = 0; + if (plte != null && i * 3 + 2 < plte.length) { + r = plte[i * 3] & 0xff; + g = plte[i * 3 + 1] & 0xff; + b = plte[i * 3 + 2] & 0xff; + } + c = a << 24 | r << 16 | g << 8 | b; + break; + } + case 4: { + int g = s[o] & 0xff; + c = (s[o + 1] & 0xff) << 24 | g << 16 | g << 8 | g; + break; + } + default: + c = (s[o + 3] & 0xff) << 24 | (s[o] & 0xff) << 16 | (s[o + 1] & 0xff) << 8 | (s[o + 2] & 0xff); + break; + } + out[(y0 + y * dy) * stride + x0 + x * dx] = c; + } + } + } + + /// Maps a tRNS key to the scale [#expand(byte[], int, int, int, int)] produces. + private static int scaleKey(int v, int depth) { + if (depth == 16) { + return v >> 8; + } + if (depth < 8) { + return (v & ((1 << depth) - 1)) * (255 / ((1 << depth) - 1)); + } + return v & 0xff; + } + + private static int paeth(int a, int b, int c) { + int p = a + b - c; + int pa = Math.abs(p - a); + int pb = Math.abs(p - b); + int pc = Math.abs(p - c); + if (pa <= pb && pa <= pc) { + return a; + } + return pb <= pc ? b : c; + } + + /// Reverses the five PNG row filters. + /// + /// #### Parameters + /// + /// - `data`: `h` rows, each a filter type byte followed by `row` bytes + /// - `offset`: where the first row starts in `data` + /// - `row`: bytes per row, without the filter byte + /// - `h`: number of rows + /// - `bpp`: the filter's byte distance, at least 1 + /// + /// #### Returns + /// + /// the `row * h` unfiltered bytes + /// + /// #### Throws + /// + /// - `IOException`: on truncated data or an unknown filter type + public static byte[] unfilter(byte[] data, int offset, int row, int h, int bpp) throws IOException { + if (data.length - offset < (row + 1) * h) { + throw new IOException("image file is truncated"); + } + byte[] out = new byte[row * h]; + for (int y = 0; y < h; y++) { + int ft = data[offset + y * (row + 1)] & 0xff; + int in = offset + y * (row + 1) + 1; + int o = y * row; + for (int x = 0; x < row; x++) { + int raw = data[in + x] & 0xff; + int a = x >= bpp ? out[o + x - bpp] & 0xff : 0; + int b = y > 0 ? out[o - row + x] & 0xff : 0; + int c = x >= bpp && y > 0 ? out[o - row + x - bpp] & 0xff : 0; + int v; + switch (ft) { + case 0: v = raw; break; + case 1: v = raw + a; break; + case 2: v = raw + b; break; + case 3: v = raw + ((a + b) >> 1); break; + case 4: v = raw + paeth(a, b, c); break; + default: throw new IOException("unknown PNG filter type " + ft); + } + out[o + x] = (byte) v; + } + } + return out; + } + + private static int sample(byte[] raw, int rowStart, int index, int depth) { + if (depth == 8) { + return raw[rowStart + index] & 0xff; + } + if (depth == 16) { + return raw[rowStart + index * 2] & 0xff; + } + int bit = index * depth; + return (raw[rowStart + (bit >> 3)] & 0xff) >> (8 - depth - (bit & 7)) & ((1 << depth) - 1); + } + + /// Unpacks unfiltered rows to one byte per sample: gray to 1 band, RGB to + /// 3, gray with alpha to 2, RGBA to 4 and palette to its 1-band indices. + /// Gray below 8 bits is scaled to 0..255, 16-bit samples keep their high + /// byte. + /// + /// #### Parameters + /// + /// - `raw`: unfiltered rows + /// - `w`: width + /// - `h`: height + /// - `ctype`: the PNG color type + /// - `depth`: the PNG bit depth + /// + /// #### Returns + /// + /// `w * h * bands` bytes + public static byte[] expand(byte[] raw, int w, int h, int ctype, int depth) { + int channels = channels(ctype); + int row = (w * channels * depth + 7) / 8; + byte[] r = new byte[w * h * channels]; + int scale = depth < 8 ? 255 / ((1 << depth) - 1) : 1; + for (int y = 0; y < h; y++) { + int rs = y * row; + for (int x = 0; x < w; x++) { + int o = (y * w + x) * channels; + if (ctype == 3) { + r[o] = (byte) sample(raw, rs, x, depth); + continue; + } + if (ctype == 4) { + r[o] = (byte) sample(raw, rs, x * 2, depth); + r[o + 1] = (byte) sample(raw, rs, x * 2 + 1, depth); + continue; + } + for (int b = 0; b < channels; b++) { + r[o + b] = (byte) (sample(raw, rs, x * channels + b, depth) * scale); + } + } + } + return r; + } + + /// Encodes an image as an 8-bit PNG: RGBA, or RGB when it is opaque. + /// + /// #### Parameters + /// + /// - `img`: the image + /// + /// #### Returns + /// + /// the PNG file + public static byte[] encode(RasterImage img) { + int w = img.getWidth(); + int h = img.getHeight(); + int[] px = img.getPixels(); + boolean opaque = true; + for (int i = 0, n = w * h; i < n && opaque; i++) { + opaque = px[i] >>> 24 == 255; + } + int bands = opaque ? 3 : 4; + byte[] rows = new byte[(w * bands + 1) * h]; + int o = 0; + for (int y = 0; y < h; y++) { + rows[o++] = 0; + for (int x = 0; x < w; x++) { + int c = px[y * w + x]; + rows[o++] = (byte) (c >> 16); + rows[o++] = (byte) (c >> 8); + rows[o++] = (byte) c; + if (!opaque) { + rows[o++] = (byte) (c >>> 24); + } + } + } + return encodeRows(w, h, 8, opaque ? 2 : 6, null, null, rows); + } + + /// Assembles a PNG file from rows that already carry their filter bytes. + /// + /// #### Parameters + /// + /// - `w`: width + /// - `h`: height + /// - `depth`: bit depth + /// - `ctype`: color type + /// - `plte`: the `PLTE` chunk body, or null + /// - `trns`: the `tRNS` chunk body, or null + /// - `rows`: `h` rows, each a filter type byte and its samples + /// + /// #### Returns + /// + /// the PNG file + public static byte[] encodeRows(int w, int h, int depth, int ctype, byte[] plte, byte[] trns, byte[] rows) { + ByteArrayOutputStream out = new ByteArrayOutputStream(rows.length / 2 + 128); + out.write(SIGNATURE, 0, 8); + byte[] ihdr = new byte[13]; + put32(ihdr, 0, w); + put32(ihdr, 4, h); + ihdr[8] = (byte) depth; + ihdr[9] = (byte) ctype; + chunk(out, "IHDR", ihdr); + if (plte != null) { + chunk(out, "PLTE", plte); + } + if (trns != null) { + chunk(out, "tRNS", trns); + } + Deflater def = new Deflater(6); + def.setInput(rows); + def.finish(); + ByteArrayOutputStream z = new ByteArrayOutputStream(rows.length / 2 + 64); + byte[] buf = new byte[8192]; + while (!def.finished()) { + int n = def.deflate(buf); + z.write(buf, 0, n); + } + def.end(); + chunk(out, "IDAT", z.toByteArray()); + chunk(out, "IEND", new byte[0]); + return out.toByteArray(); + } + + private static void put32(byte[] d, int p, int v) { + d[p] = (byte) (v >>> 24); + d[p + 1] = (byte) (v >>> 16); + d[p + 2] = (byte) (v >>> 8); + d[p + 3] = (byte) v; + } + + private static void chunk(ByteArrayOutputStream out, String kind, byte[] body) { + byte[] head = new byte[8]; + put32(head, 0, body.length); + for (int i = 0; i < 4; i++) { + head[4 + i] = (byte) kind.charAt(i); + } + out.write(head, 0, 8); + out.write(body, 0, body.length); + CRC32 crc = new CRC32(); + crc.update(head, 4, 4); + crc.update(body, 0, body.length); + byte[] c = new byte[4]; + put32(c, 0, (int) crc.getValue()); + out.write(c, 0, 4); + } +} diff --git a/CodenameOne/src/com/codename1/image/RasterFont.java b/CodenameOne/src/com/codename1/image/RasterFont.java new file mode 100644 index 00000000000..9870c023251 --- /dev/null +++ b/CodenameOne/src/com/codename1/image/RasterFont.java @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.image; + +/// The seam through which [RasterGraphics#drawString(String, double, double)] +/// draws text: a font turns a string into an outline, and the graphics fills +/// it like any other path. +/// +/// No implementation ships yet; a font engine plugs in here without any +/// change to the graphics. +@com.codename1.impl.SharedWithBackend +public interface RasterFont { + /// Returns the outline of `text` with its baseline starting at `(x, y)`. + /// + /// #### Parameters + /// + /// - `text`: the string to lay out + /// - `x`: left end of the baseline + /// - `y`: the baseline + /// + /// #### Returns + /// + /// the glyph outlines, filled with the non-zero rule + RasterPath getOutline(String text, double x, double y); + + /// Returns the advance width of `text`. + /// + /// #### Parameters + /// + /// - `text`: the string + /// + /// #### Returns + /// + /// the width in user-space units + double stringWidth(String text); + + /// Returns the distance from the top of the line to the baseline. + /// + /// #### Returns + /// + /// the ascent in user-space units + double getAscent(); + + /// Returns the distance from the baseline to the bottom of the line. + /// + /// #### Returns + /// + /// the descent in user-space units + double getDescent(); +} diff --git a/CodenameOne/src/com/codename1/image/RasterGradient.java b/CodenameOne/src/com/codename1/image/RasterGradient.java new file mode 100644 index 00000000000..e53a47d46d2 --- /dev/null +++ b/CodenameOne/src/com/codename1/image/RasterGradient.java @@ -0,0 +1,156 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.image; + +/// A linear or radial color gradient for [RasterGraphics#setPaint(RasterPaint)]. +/// +/// Colors are interpolated per channel, not premultiplied, between stops at +/// increasing fractions from 0 to 1. Before the first stop and after the last +/// one the end colors extend ("pad"). +@com.codename1.impl.SharedWithBackend +public final class RasterGradient implements RasterPaint { + private final boolean radial; + private final double x1; + private final double y1; + private final double x2; + private final double y2; + private final double radius; + private final float[] fractions; + private final int[] colors; + + private RasterGradient(boolean radial, double x1, double y1, double x2, double y2, double radius, + float[] fractions, int[] colors) { + if (fractions.length != colors.length || colors.length == 0) { + throw new IllegalArgumentException("one color per fraction is required"); + } + for (int i = 1; i < fractions.length; i++) { + if (fractions[i] < fractions[i - 1]) { + throw new IllegalArgumentException("fractions must not decrease"); + } + } + this.radial = radial; + this.x1 = x1; + this.y1 = y1; + this.x2 = x2; + this.y2 = y2; + this.radius = radius; + this.fractions = new float[fractions.length]; + System.arraycopy(fractions, 0, this.fractions, 0, fractions.length); + this.colors = new int[colors.length]; + System.arraycopy(colors, 0, this.colors, 0, colors.length); + } + + /// Creates a gradient along the line from `(x1, y1)` (fraction 0) to + /// `(x2, y2)` (fraction 1). + /// + /// #### Parameters + /// + /// - `x1`: start x + /// - `y1`: start y + /// - `x2`: end x + /// - `y2`: end y + /// - `fractions`: stop positions, 0 to 1, not decreasing + /// - `colors`: ARGB color of each stop + /// + /// #### Returns + /// + /// the gradient + public static RasterGradient linear(double x1, double y1, double x2, double y2, float[] fractions, + int[] colors) { + return new RasterGradient(false, x1, y1, x2, y2, 0, fractions, colors); + } + + /// Creates a circular gradient from the center (fraction 0) to the + /// circle of `radius` (fraction 1). + /// + /// #### Parameters + /// + /// - `cx`: center x + /// - `cy`: center y + /// - `radius`: radius, above zero + /// - `fractions`: stop positions, 0 to 1, not decreasing + /// - `colors`: ARGB color of each stop + /// + /// #### Returns + /// + /// the gradient + public static RasterGradient radial(double cx, double cy, double radius, float[] fractions, int[] colors) { + if (radius <= 0) { + throw new IllegalArgumentException("radius must be positive"); + } + return new RasterGradient(true, cx, cy, 0, 0, radius, fractions, colors); + } + + /// Returns the gradient's color at a point. + /// + /// #### Parameters + /// + /// - `x`: user-space x + /// - `y`: user-space y + /// + /// #### Returns + /// + /// the ARGB color + @Override + public int getColor(double x, double y) { + double t; + if (radial) { + double dx = x - x1; + double dy = y - y1; + t = Math.sqrt(dx * dx + dy * dy) / radius; + } else { + double vx = x2 - x1; + double vy = y2 - y1; + double len2 = vx * vx + vy * vy; + t = len2 == 0 ? 0 : ((x - x1) * vx + (y - y1) * vy) / len2; + } + return colorAt(t); + } + + private int colorAt(double t) { + int n = fractions.length; + if (t <= fractions[0]) { + return colors[0]; + } + if (t >= fractions[n - 1]) { + return colors[n - 1]; + } + int i = 1; + while (fractions[i] < t) { + i++; + } + double span = fractions[i] - fractions[i - 1]; + double f = span <= 0 ? 1 : (t - fractions[i - 1]) / span; + return lerp(colors[i - 1], colors[i], f); + } + + static int lerp(int a, int b, double f) { + int r = 0; + for (int s = 0; s < 32; s += 8) { + int ca = a >>> s & 0xff; + int cb = b >>> s & 0xff; + r |= ((int) (ca + (cb - ca) * f + 0.5) & 0xff) << s; + } + return r; + } +} diff --git a/CodenameOne/src/com/codename1/image/RasterGraphics.java b/CodenameOne/src/com/codename1/image/RasterGraphics.java new file mode 100644 index 00000000000..2f818f9af35 --- /dev/null +++ b/CodenameOne/src/com/codename1/image/RasterGraphics.java @@ -0,0 +1,1083 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.image; + +/// Draws antialiased shapes, gradients and images into a [RasterImage]. +/// +/// Everything is pure Java, so the same drawing code runs on a device, in +/// the simulator and on a Codename One server that generates images (charts, +/// thumbnails, social cards) with no display at all. +/// +/// - **Coordinates** are user space, mapped to pixels by an affine transform +/// ([#translate(double, double)], [#scale(double, double)], +/// [#rotate(double)], [#setTransform(double, double, double, double, double, double)]). +/// Pixel `(x, y)` covers the square from `(x, y)` to `(x + 1, y + 1)`, so a +/// one pixel wide line along `y = 10.5` covers row 10 exactly. +/// - **Painting** is source-over compositing of non-premultiplied colors, with +/// [#setColor(int)] or a [RasterPaint] such as [RasterGradient], scaled by +/// [#setAlpha(float)]. +/// - **Coverage** is computed on eight sub-scanlines per pixel with exact +/// horizontal area; [#setAntiAlias(boolean)] turns it off. +/// - **Clipping** is a rectangle in pixels: [#clipRect(double, double, double, double)] +/// intersects the bounds of a transformed rectangle with it. +/// - **Text** goes through the [RasterFont] seam set with [#setFont(RasterFont)]. +/// +/// A graphics object is not thread safe; create one per thread. +@com.codename1.impl.SharedWithBackend +public final class RasterGraphics { + /// Stroke ends stop exactly at the end points. + public static final int CAP_BUTT = 0; + /// Stroke ends are half circles. + public static final int CAP_ROUND = 1; + /// Stroke ends extend by half the line width. + public static final int CAP_SQUARE = 2; + /// Stroke corners extend to a sharp point, up to the miter limit. + public static final int JOIN_MITER = 0; + /// Stroke corners are rounded. + public static final int JOIN_ROUND = 1; + /// Stroke corners are cut flat. + public static final int JOIN_BEVEL = 2; + /// A point is inside when the path winds around it a non-zero number of times. + public static final int WIND_NON_ZERO = 0; + /// A point is inside when the path crosses a ray from it an odd number of times. + public static final int WIND_EVEN_ODD = 1; + + private static final int SUBSAMPLES = 8; + + private final RasterImage target; + private final int[] px; + private final int width; + private final int height; + private double ta = 1; + private double tb; + private double tc; + private double td = 1; + private double te; + private double tf; + private int color = 0xff000000; + private RasterPaint paint; + private float alpha = 1f; + private double strokeWidth = 1; + private int cap = CAP_BUTT; + private int join = JOIN_MITER; + private double miterLimit = 10; + private int windingRule = WIND_NON_ZERO; + private boolean antiAlias = true; + private int clipX0; + private int clipY0; + private int clipX1; + private int clipY1; + private RasterFont font; + + // Edge list of the shape being filled, in device space. + private double[] edges = new double[64]; + private int edgeCount; + + /// Creates a graphics that draws into `target`; also available as + /// [RasterImage#createGraphics()]. + /// + /// #### Parameters + /// + /// - `target`: the image to draw into + public RasterGraphics(RasterImage target) { + this.target = target; + this.px = target.getPixels(); + this.width = target.getWidth(); + this.height = target.getHeight(); + resetClip(); + } + + /// Returns the image this graphics draws into. + public RasterImage getTarget() { + return target; + } + + /// Sets a solid ARGB color and clears any paint. + /// + /// #### Parameters + /// + /// - `argb`: the color, not premultiplied + public void setColor(int argb) { + this.color = argb; + this.paint = null; + } + + /// Returns the solid color. + public int getColor() { + return color; + } + + /// Sets the paint used instead of the solid color; null returns to the color. + /// + /// #### Parameters + /// + /// - `paint`: a gradient or pattern + public void setPaint(RasterPaint paint) { + this.paint = paint; + } + + /// Returns the current paint, or null when drawing with the solid color. + public RasterPaint getPaint() { + return paint; + } + + /// Sets an opacity that multiplies everything drawn. + /// + /// #### Parameters + /// + /// - `alpha`: 0 (invisible) to 1 (opaque) + public void setAlpha(float alpha) { + this.alpha = alpha < 0 ? 0 : alpha > 1 ? 1 : alpha; + } + + /// Returns the opacity set by [#setAlpha(float)]. + public float getAlpha() { + return alpha; + } + + /// Sets the line width used by the draw methods, in user-space units. + /// + /// #### Parameters + /// + /// - `w`: the width + public void setStrokeWidth(double w) { + this.strokeWidth = w < 0 ? 0 : w; + } + + /// Returns the line width. + public double getStrokeWidth() { + return strokeWidth; + } + + /// Sets how open line ends are drawn. + /// + /// #### Parameters + /// + /// - `cap`: `CAP_BUTT`, `CAP_ROUND` or `CAP_SQUARE` + public void setLineCap(int cap) { + this.cap = cap; + } + + /// Sets how line corners are drawn. + /// + /// #### Parameters + /// + /// - `join`: `JOIN_MITER`, `JOIN_ROUND` or `JOIN_BEVEL` + public void setLineJoin(int join) { + this.join = join; + } + + /// Sets the longest a miter may be, as a multiple of half the line width; + /// sharper corners are beveled. + /// + /// #### Parameters + /// + /// - `limit`: at least 1 + public void setMiterLimit(double limit) { + this.miterLimit = Math.max(1, limit); + } + + /// Sets the rule deciding what is inside a self-intersecting path. + /// + /// #### Parameters + /// + /// - `rule`: `WIND_NON_ZERO` or `WIND_EVEN_ODD` + public void setWindingRule(int rule) { + this.windingRule = rule; + } + + /// Turns antialiasing on (the default) or off. + /// + /// #### Parameters + /// + /// - `aa`: true for smooth edges + public void setAntiAlias(boolean aa) { + this.antiAlias = aa; + } + + /// Sets the font [#drawString(String, double, double)] uses. + /// + /// #### Parameters + /// + /// - `font`: the font + public void setFont(RasterFont font) { + this.font = font; + } + + /// Returns the font, or null. + public RasterFont getFont() { + return font; + } + + // ------------------------------------------------------------- transform + + /// Moves the origin. + /// + /// #### Parameters + /// + /// - `dx`: horizontal offset + /// - `dy`: vertical offset + public void translate(double dx, double dy) { + te += ta * dx + tc * dy; + tf += tb * dx + td * dy; + } + + /// Scales user space. + /// + /// #### Parameters + /// + /// - `sx`: horizontal factor + /// - `sy`: vertical factor + public void scale(double sx, double sy) { + ta *= sx; + tb *= sx; + tc *= sy; + td *= sy; + } + + /// Rotates user space about the origin. + /// + /// #### Parameters + /// + /// - `radians`: the angle; positive turns x toward y (clockwise on screen) + public void rotate(double radians) { + double c = Math.cos(radians); + double s = Math.sin(radians); + double na = ta * c + tc * s; + double nb = tb * c + td * s; + double nc = -ta * s + tc * c; + double nd = -tb * s + td * c; + ta = na; + tb = nb; + tc = nc; + td = nd; + } + + /// Replaces the transform: device `x = a * x + c * y + e`, + /// `y = b * x + d * y + f`. + /// + /// #### Parameters + /// + /// - `a`: x scale + /// - `b`: y shear + /// - `c`: x shear + /// - `d`: y scale + /// - `e`: x translation + /// - `f`: y translation + public void setTransform(double a, double b, double c, double d, double e, double f) { + ta = a; + tb = b; + tc = c; + td = d; + te = e; + tf = f; + } + + /// Returns the transform as `{a, b, c, d, e, f}`. + public double[] getTransform() { + return new double[] {ta, tb, tc, td, te, tf}; + } + + /// Restores the identity transform. + public void resetTransform() { + setTransform(1, 0, 0, 1, 0, 0); + } + + // ------------------------------------------------------------------ clip + + /// Removes the clip, so the whole image can be drawn. + public void resetClip() { + clipX0 = 0; + clipY0 = 0; + clipX1 = width; + clipY1 = height; + } + + /// Intersects the clip with the pixel bounds of a transformed rectangle. + /// + /// #### Parameters + /// + /// - `x`: left + /// - `y`: top + /// - `w`: width + /// - `h`: height + public void clipRect(double x, double y, double w, double h) { + double[] xs = {x, x + w, x + w, x}; + double[] ys = {y, y, y + h, y + h}; + double minX = Double.MAX_VALUE; + double minY = Double.MAX_VALUE; + double maxX = -Double.MAX_VALUE; + double maxY = -Double.MAX_VALUE; + for (int i = 0; i < 4; i++) { + double dx = ta * xs[i] + tc * ys[i] + te; + double dy = tb * xs[i] + td * ys[i] + tf; + minX = Math.min(minX, dx); + minY = Math.min(minY, dy); + maxX = Math.max(maxX, dx); + maxY = Math.max(maxY, dy); + } + clipX0 = Math.max(clipX0, (int) Math.floor(minX + 0.5)); + clipY0 = Math.max(clipY0, (int) Math.floor(minY + 0.5)); + clipX1 = Math.min(clipX1, (int) Math.floor(maxX + 0.5)); + clipY1 = Math.min(clipY1, (int) Math.floor(maxY + 0.5)); + } + + /// Replaces the clip with the pixel bounds of a transformed rectangle. + /// + /// #### Parameters + /// + /// - `x`: left + /// - `y`: top + /// - `w`: width + /// - `h`: height + public void setClip(double x, double y, double w, double h) { + resetClip(); + clipRect(x, y, w, h); + } + + /// Returns the clip as pixel bounds `{x0, y0, x1, y1}`, right and bottom + /// exclusive. + public int[] getClipBounds() { + return new int[] {clipX0, clipY0, clipX1, clipY1}; + } + + // ---------------------------------------------------------- convenience + + /// Replaces every pixel inside the clip with a color, without blending. + /// + /// #### Parameters + /// + /// - `argb`: the color + public void clear(int argb) { + for (int y = clipY0; y < clipY1; y++) { + for (int x = clipX0; x < clipX1; x++) { + px[y * width + x] = argb; + } + } + } + + /// Fills a rectangle. + /// + /// #### Parameters + /// + /// - `x`: left + /// - `y`: top + /// - `w`: width + /// - `h`: height + public void fillRect(double x, double y, double w, double h) { + fill(RasterPath.rect(x, y, w, h)); + } + + /// Strokes a rectangle's outline. + /// + /// #### Parameters + /// + /// - `x`: left + /// - `y`: top + /// - `w`: width + /// - `h`: height + public void drawRect(double x, double y, double w, double h) { + draw(RasterPath.rect(x, y, w, h)); + } + + /// Fills a rectangle with elliptical corners. + /// + /// #### Parameters + /// + /// - `x`: left + /// - `y`: top + /// - `w`: width + /// - `h`: height + /// - `arcW`: horizontal corner diameter + /// - `arcH`: vertical corner diameter + public void fillRoundRect(double x, double y, double w, double h, double arcW, double arcH) { + fill(RasterPath.roundRect(x, y, w, h, arcW, arcH)); + } + + /// Strokes a rectangle with elliptical corners. + /// + /// #### Parameters + /// + /// - `x`: left + /// - `y`: top + /// - `w`: width + /// - `h`: height + /// - `arcW`: horizontal corner diameter + /// - `arcH`: vertical corner diameter + public void drawRoundRect(double x, double y, double w, double h, double arcW, double arcH) { + draw(RasterPath.roundRect(x, y, w, h, arcW, arcH)); + } + + /// Fills the ellipse inscribed in a rectangle. + /// + /// #### Parameters + /// + /// - `x`: left + /// - `y`: top + /// - `w`: width + /// - `h`: height + public void fillOval(double x, double y, double w, double h) { + fill(RasterPath.ellipse(x, y, w, h)); + } + + /// Strokes the ellipse inscribed in a rectangle. + /// + /// #### Parameters + /// + /// - `x`: left + /// - `y`: top + /// - `w`: width + /// - `h`: height + public void drawOval(double x, double y, double w, double h) { + draw(RasterPath.ellipse(x, y, w, h)); + } + + /// Fills a pie slice of the ellipse inscribed in a rectangle. + /// + /// #### Parameters + /// + /// - `x`: left + /// - `y`: top + /// - `w`: width + /// - `h`: height + /// - `startDeg`: start angle, degrees counterclockwise from three o'clock + /// - `extentDeg`: extent in degrees + public void fillArc(double x, double y, double w, double h, double startDeg, double extentDeg) { + fill(RasterPath.arc(x, y, w, h, startDeg, extentDeg, RasterPath.ARC_PIE)); + } + + /// Strokes an open arc of the ellipse inscribed in a rectangle. + /// + /// #### Parameters + /// + /// - `x`: left + /// - `y`: top + /// - `w`: width + /// - `h`: height + /// - `startDeg`: start angle, degrees counterclockwise from three o'clock + /// - `extentDeg`: extent in degrees + public void drawArc(double x, double y, double w, double h, double startDeg, double extentDeg) { + draw(RasterPath.arc(x, y, w, h, startDeg, extentDeg, RasterPath.ARC_OPEN)); + } + + /// Strokes a straight line. + /// + /// #### Parameters + /// + /// - `x1`: start x + /// - `y1`: start y + /// - `x2`: end x + /// - `y2`: end y + public void drawLine(double x1, double y1, double x2, double y2) { + draw(new RasterPath().moveTo(x1, y1).lineTo(x2, y2)); + } + + /// Fills a polygon. + /// + /// #### Parameters + /// + /// - `xs`: vertex x values + /// - `ys`: vertex y values + /// - `n`: vertex count + public void fillPolygon(double[] xs, double[] ys, int n) { + fill(RasterPath.polygon(xs, ys, n)); + } + + /// Strokes a closed polygon. + /// + /// #### Parameters + /// + /// - `xs`: vertex x values + /// - `ys`: vertex y values + /// - `n`: vertex count + public void drawPolygon(double[] xs, double[] ys, int n) { + draw(RasterPath.polygon(xs, ys, n)); + } + + /// Strokes an open chain of lines. + /// + /// #### Parameters + /// + /// - `xs`: vertex x values + /// - `ys`: vertex y values + /// - `n`: vertex count + public void drawPolyline(double[] xs, double[] ys, int n) { + draw(RasterPath.polyline(xs, ys, n)); + } + + /// Draws text with the font set by [#setFont(RasterFont)]. + /// + /// #### Parameters + /// + /// - `text`: the string + /// - `x`: left end of the baseline + /// - `y`: the baseline + public void drawString(String text, double x, double y) { + if (font == null) { + throw new IllegalStateException("no RasterFont set"); + } + int rule = windingRule; + windingRule = WIND_NON_ZERO; + fill(font.getOutline(text, x, y)); + windingRule = rule; + } + + /// Draws an image at its own size. + /// + /// #### Parameters + /// + /// - `img`: the image + /// - `x`: left + /// - `y`: top + public void drawImage(RasterImage img, double x, double y) { + drawImage(img, x, y, img.getWidth(), img.getHeight()); + } + + /// Draws an image scaled to a rectangle, sampled bilinearly through the + /// transform. For large reductions scale it first with + /// [RasterImage#scale(int, int, int)], which filters every source pixel. + /// + /// #### Parameters + /// + /// - `img`: the image + /// - `x`: left + /// - `y`: top + /// - `w`: width + /// - `h`: height + public void drawImage(RasterImage img, double x, double y, double w, double h) { + if (w == 0 || h == 0) { + return; + } + RasterPaint saved = paint; + paint = new ImagePaint(img, x, y, w / img.getWidth(), h / img.getHeight()); + fill(RasterPath.rect(x, y, w, h)); + paint = saved; + } + + /// Samples an image placed at `(x, y)` with a scale, bilinearly. + private static final class ImagePaint implements RasterPaint { + private final RasterImage img; + private final double x; + private final double y; + private final double sx; + private final double sy; + + ImagePaint(RasterImage img, double x, double y, double sx, double sy) { + this.img = img; + this.x = x; + this.y = y; + this.sx = sx; + this.sy = sy; + } + + @Override + public int getColor(double ux, double uy) { + int w = img.getWidth(); + int h = img.getHeight(); + int[] p = img.getPixels(); + double fx = (ux - x) / sx - 0.5; + double fy = (uy - y) / sy - 0.5; + int x0 = (int) Math.floor(fx); + int y0 = (int) Math.floor(fy); + double ax = fx - x0; + double ay = fy - y0; + int x1 = clampI(x0 + 1, w); + int y1 = clampI(y0 + 1, h); + x0 = clampI(x0, w); + y0 = clampI(y0, h); + int top = RasterGradient.lerp(p[y0 * w + x0], p[y0 * w + x1], ax); + int bottom = RasterGradient.lerp(p[y1 * w + x0], p[y1 * w + x1], ax); + return RasterGradient.lerp(top, bottom, ay); + } + } + + private static int clampI(int v, int n) { + return v < 0 ? 0 : v >= n ? n - 1 : v; + } + + // ------------------------------------------------------------- filling + + /// Fills a path with the current color or paint and winding rule. + /// + /// #### Parameters + /// + /// - `path`: the shape + public void fill(RasterPath path) { + edgeCount = 0; + double[][] polys = flatten(path); + for (double[] poly : polys) { + addPolygon(poly, poly.length / 2, false); + } + rasterize(windingRule); + } + + /// Strokes a path with the current line width, caps and joins. + /// + /// #### Parameters + /// + /// - `path`: the shape + public void draw(RasterPath path) { + edgeCount = 0; + double hw = strokeWidth / 2; + if (hw <= 0) { + return; + } + double[][] polys = flatten(path); + boolean[] closed = closedFlags(path, polys.length); + for (int i = 0; i < polys.length; i++) { + strokePolyline(polys[i], closed[i], hw); + } + rasterize(WIND_NON_ZERO); + } + + private static boolean[] closedFlags(RasterPath path, int count) { + boolean[] r = new boolean[count]; + int sub = -1; + for (int i = 0; i < path.opCount; i++) { + if (path.ops[i] == RasterPath.MOVE) { + sub++; + } else if (path.ops[i] == RasterPath.CLOSE && sub >= 0 && sub < count) { + r[sub] = true; + } + } + return r; + } + + /// How many line segments a curve with this device-space control + /// polygon length needs. + private int segments(double[] k, int from, int n) { + double len = 0; + for (int i = 0; i + 1 < n; i++) { + double x0 = ta * k[from + i * 2] + tc * k[from + i * 2 + 1]; + double y0 = tb * k[from + i * 2] + td * k[from + i * 2 + 1]; + double x1 = ta * k[from + i * 2 + 2] + tc * k[from + i * 2 + 3]; + double y1 = tb * k[from + i * 2 + 2] + td * k[from + i * 2 + 3]; + len += Math.sqrt((x1 - x0) * (x1 - x0) + (y1 - y0) * (y1 - y0)); + } + int s = (int) Math.ceil(Math.sqrt(len) * 1.5); + return s < 2 ? 2 : s > 200 ? 200 : s; + } + + /// Flattens a path to user-space polylines, one per subpath. + private double[][] flatten(RasterPath path) { + java.util.ArrayList list = new java.util.ArrayList(); + double[] cur = new double[32]; + int n = 0; + double[] k = path.coords; + int c = 0; + double lx = 0; + double ly = 0; + double[] seg = new double[8]; + for (int i = 0; i < path.opCount; i++) { + byte o = path.ops[i]; + if (o == RasterPath.MOVE) { + if (n > 0) { + list.add(trim(cur, n)); + } + n = 0; + lx = k[c]; + ly = k[c + 1]; + c += 2; + cur = push(cur, n, lx, ly); + n += 2; + } else if (o == RasterPath.LINE) { + lx = k[c]; + ly = k[c + 1]; + c += 2; + cur = push(cur, n, lx, ly); + n += 2; + } else if (o == RasterPath.QUAD || o == RasterPath.CUBIC) { + int pts = o == RasterPath.QUAD ? 3 : 4; + seg[0] = lx; + seg[1] = ly; + System.arraycopy(k, c, seg, 2, (pts - 1) * 2); + c += (pts - 1) * 2; + int steps = segments(seg, 0, pts); + for (int s = 1; s <= steps; s++) { + double t = (double) s / steps; + double u = 1 - t; + double x; + double y; + if (pts == 3) { + x = u * u * seg[0] + 2 * u * t * seg[2] + t * t * seg[4]; + y = u * u * seg[1] + 2 * u * t * seg[3] + t * t * seg[5]; + } else { + x = u * u * u * seg[0] + 3 * u * u * t * seg[2] + 3 * u * t * t * seg[4] + t * t * t * seg[6]; + y = u * u * u * seg[1] + 3 * u * u * t * seg[3] + 3 * u * t * t * seg[5] + t * t * t * seg[7]; + } + cur = push(cur, n, x, y); + n += 2; + } + lx = seg[(pts - 1) * 2]; + ly = seg[(pts - 1) * 2 + 1]; + } else if (n > 0) { + // CLOSE: the next segment starts back at the subpath's start. + lx = cur[0]; + ly = cur[1]; + } + } + if (n > 0) { + list.add(trim(cur, n)); + } + double[][] r = new double[list.size()][]; + for (int i = 0; i < r.length; i++) { + r[i] = list.get(i); + } + return r; + } + + private static double[] push(double[] a, int n, double x, double y) { + double[] r = a; + if (n + 2 > r.length) { + r = new double[r.length * 2]; + System.arraycopy(a, 0, r, 0, n); + } + r[n] = x; + r[n + 1] = y; + return r; + } + + private static double[] trim(double[] a, int n) { + double[] r = new double[n]; + System.arraycopy(a, 0, r, 0, n); + return r; + } + + /// Adds a closed user-space polygon to the edge list, transformed. + /// With `orient` the polygon is first made counterclockwise, so the + /// pieces of a stroke union under the non-zero rule. + private void addPolygon(double[] p, int count, boolean orient) { + if (count < 2) { + return; + } + boolean reverse = false; + if (orient) { + double area = 0; + for (int i = 0; i < count; i++) { + int j = (i + 1) % count; + area += p[i * 2] * p[j * 2 + 1] - p[j * 2] * p[i * 2 + 1]; + } + double det = ta * td - tb * tc; + reverse = area * det < 0; + } + for (int i = 0; i < count; i++) { + int a = reverse ? count - 1 - i : i; + int b = reverse ? (a - 1 + count) % count : (a + 1) % count; + double x0 = ta * p[a * 2] + tc * p[a * 2 + 1] + te; + double y0 = tb * p[a * 2] + td * p[a * 2 + 1] + tf; + double x1 = ta * p[b * 2] + tc * p[b * 2 + 1] + te; + double y1 = tb * p[b * 2] + td * p[b * 2 + 1] + tf; + if (y0 == y1) { + continue; + } + if (edgeCount * 5 + 5 > edges.length) { + double[] e = new double[edges.length * 2]; + System.arraycopy(edges, 0, e, 0, edgeCount * 5); + edges = e; + } + int o = edgeCount * 5; + if (y0 < y1) { + edges[o] = x0; + edges[o + 1] = y0; + edges[o + 2] = x1; + edges[o + 3] = y1; + edges[o + 4] = 1; + } else { + edges[o] = x1; + edges[o + 1] = y1; + edges[o + 2] = x0; + edges[o + 3] = y0; + edges[o + 4] = -1; + } + edgeCount++; + } + } + + // ------------------------------------------------------------- stroking + + private void quad(double ax, double ay, double bx, double by, double cx, double cy, double dx, double dy) { + addPolygon(new double[] {ax, ay, bx, by, cx, cy, dx, dy}, 4, true); + } + + private void circle(double x, double y, double r) { + double scale = Math.sqrt(Math.abs(ta * td - tb * tc)); + int n = (int) Math.ceil(Math.sqrt(r * scale) * 6); + n = n < 8 ? 8 : n > 128 ? 128 : n; + double[] p = new double[n * 2]; + for (int i = 0; i < n; i++) { + double a = 2 * Math.PI * i / n; + p[i * 2] = x + r * Math.cos(a); + p[i * 2 + 1] = y + r * Math.sin(a); + } + addPolygon(p, n, true); + } + + private void strokePolyline(double[] p, boolean closed, double hw) { + // Drop repeated points so every segment has a direction. + double[] q = new double[p.length]; + int n = 0; + for (int i = 0; i < p.length; i += 2) { + if (n == 0 || p[i] != q[n * 2 - 2] || p[i + 1] != q[n * 2 - 1]) { + q[n * 2] = p[i]; + q[n * 2 + 1] = p[i + 1]; + n++; + } + } + if (closed && n > 1 && q[0] == q[n * 2 - 2] && q[1] == q[n * 2 - 1]) { + n--; + } + if (n == 1) { + if (cap == CAP_ROUND) { + circle(q[0], q[1], hw); + } else if (cap == CAP_SQUARE) { + quad(q[0] - hw, q[1] - hw, q[0] + hw, q[1] - hw, q[0] + hw, q[1] + hw, q[0] - hw, q[1] + hw); + } + return; + } + int segs = closed ? n : n - 1; + for (int i = 0; i < segs; i++) { + int j = (i + 1) % n; + double x0 = q[i * 2]; + double y0 = q[i * 2 + 1]; + double x1 = q[j * 2]; + double y1 = q[j * 2 + 1]; + double dx = x1 - x0; + double dy = y1 - y0; + double len = Math.sqrt(dx * dx + dy * dy); + double nx = -dy / len * hw; + double ny = dx / len * hw; + quad(x0 + nx, y0 + ny, x1 + nx, y1 + ny, x1 - nx, y1 - ny, x0 - nx, y0 - ny); + } + int first = closed ? 0 : 1; + int last = closed ? n - 1 : n - 2; + for (int i = first; i <= last; i++) { + int a = (i - 1 + n) % n; + int b = (i + 1) % n; + joinAt(q[a * 2], q[a * 2 + 1], q[i * 2], q[i * 2 + 1], q[b * 2], q[b * 2 + 1], hw); + } + if (!closed) { + capAt(q[2], q[3], q[0], q[1], hw); + capAt(q[n * 2 - 4], q[n * 2 - 3], q[n * 2 - 2], q[n * 2 - 1], hw); + } + } + + /// Caps the end `(x, y)` of a segment arriving from `(fx, fy)`. + private void capAt(double fx, double fy, double x, double y, double hw) { + if (cap == CAP_ROUND) { + circle(x, y, hw); + } else if (cap == CAP_SQUARE) { + double dx = x - fx; + double dy = y - fy; + double len = Math.sqrt(dx * dx + dy * dy); + double ux = dx / len * hw; + double uy = dy / len * hw; + quad(x - uy, y + ux, x + ux - uy, y + uy + ux, x + ux + uy, y + uy - ux, x + uy, y - ux); + } + } + + private void joinAt(double ax, double ay, double x, double y, double bx, double by, double hw) { + if (join == JOIN_ROUND) { + circle(x, y, hw); + return; + } + double d1x = x - ax; + double d1y = y - ay; + double d2x = bx - x; + double d2y = by - y; + double l1 = Math.sqrt(d1x * d1x + d1y * d1y); + double l2 = Math.sqrt(d2x * d2x + d2y * d2y); + d1x /= l1; + d1y /= l1; + d2x /= l2; + d2y /= l2; + double cross = d1x * d2y - d1y * d2x; + if (Math.abs(cross) < 1e-12 && d1x * d2x + d1y * d2y > 0) { + return; + } + double side = cross > 0 ? -1 : 1; + double n1x = -d1y * side * hw; + double n1y = d1x * side * hw; + double n2x = -d2y * side * hw; + double n2y = d2x * side * hw; + if (join == JOIN_MITER) { + double mx = n1x + n2x; + double my = n1y + n2y; + double ml = Math.sqrt(mx * mx + my * my); + if (ml > 1e-12) { + double cosHalf = (mx * n1x + my * n1y) / (ml * hw); + if (cosHalf > 1e-9 && 1 / cosHalf <= miterLimit) { + double len = hw / cosHalf; + double tx = x + mx / ml * len; + double ty = y + my / ml * len; + quad(x, y, x + n1x, y + n1y, tx, ty, x + n2x, y + n2y); + return; + } + } + } + addPolygon(new double[] {x, y, x + n1x, y + n1y, x + n2x, y + n2y}, 3, true); + } + + // ---------------------------------------------------------- rasterizing + + private void rasterize(int rule) { + if (edgeCount == 0 || clipX0 >= clipX1 || clipY0 >= clipY1) { + return; + } + double minY = Double.MAX_VALUE; + double maxY = -Double.MAX_VALUE; + double minX = Double.MAX_VALUE; + double maxX = -Double.MAX_VALUE; + for (int i = 0; i < edgeCount; i++) { + int o = i * 5; + minY = Math.min(minY, edges[o + 1]); + maxY = Math.max(maxY, edges[o + 3]); + minX = Math.min(minX, Math.min(edges[o], edges[o + 2])); + maxX = Math.max(maxX, Math.max(edges[o], edges[o + 2])); + } + int y0 = Math.max(clipY0, (int) Math.floor(minY)); + int y1 = Math.min(clipY1, (int) Math.ceil(maxY)); + int x0 = Math.max(clipX0, (int) Math.floor(minX)); + int x1 = Math.min(clipX1, (int) Math.ceil(maxX) + 1); + if (y0 >= y1 || x0 >= x1) { + return; + } + int sub = antiAlias ? SUBSAMPLES : 1; + double weight = 1.0 / sub; + double[] cov = new double[x1 - x0 + 1]; + double[] xs = new double[16]; + int[] dirs = new int[16]; + double inv = ta * td - tb * tc; + boolean invertible = Math.abs(inv) > 1e-300; + for (int y = y0; y < y1; y++) { + boolean any = false; + for (int s = 0; s < sub; s++) { + double sy = y + (s + 0.5) / sub; + int n = 0; + for (int i = 0; i < edgeCount; i++) { + int o = i * 5; + if (sy < edges[o + 1] || sy >= edges[o + 3]) { + continue; + } + double x = edges[o] + (sy - edges[o + 1]) * (edges[o + 2] - edges[o]) / (edges[o + 3] - edges[o + 1]); + if (n == xs.length) { + double[] nx = new double[n * 2]; + System.arraycopy(xs, 0, nx, 0, n); + xs = nx; + int[] nd = new int[n * 2]; + System.arraycopy(dirs, 0, nd, 0, n); + dirs = nd; + } + int k = n; + while (k > 0 && xs[k - 1] > x) { + xs[k] = xs[k - 1]; + dirs[k] = dirs[k - 1]; + k--; + } + xs[k] = x; + dirs[k] = (int) edges[o + 4]; + n++; + } + int wind = 0; + for (int i = 0; i + 1 < n; i++) { + wind += dirs[i]; + boolean inside = rule == WIND_EVEN_ODD ? ((i + 1) & 1) == 1 : wind != 0; + if (inside) { + if (antiAlias) { + span(cov, x0, x1, xs[i], xs[i + 1], weight); + } else { + span(cov, x0, x1, Math.floor(xs[i] + 0.5), Math.floor(xs[i + 1] + 0.5), weight); + } + any = true; + } + } + } + if (!any) { + continue; + } + for (int x = x0; x < x1; x++) { + double c = cov[x - x0]; + cov[x - x0] = 0; + if (c <= 0) { + continue; + } + if (c > 1) { + c = 1; + } + int src; + if (paint == null) { + src = color; + } else if (invertible) { + double dx = x + 0.5 - te; + double dy = y + 0.5 - tf; + double ux = (td * dx - tc * dy) / inv; + double uy = (-tb * dx + ta * dy) / inv; + src = paint.getColor(ux, uy); + } else { + continue; + } + blend(y * width + x, src, c * alpha); + } + } + } + + private static void span(double[] cov, int x0, int x1, double a, double b, double w) { + double xa = Math.max(a, x0); + double xb = Math.min(b, x1); + if (xb <= xa) { + return; + } + int ia = (int) Math.floor(xa); + int ib = (int) Math.floor(xb); + if (ia == ib) { + cov[ia - x0] += (xb - xa) * w; + return; + } + cov[ia - x0] += (ia + 1 - xa) * w; + for (int i = ia + 1; i < ib; i++) { + cov[i - x0] += w; + } + if (ib < x1) { + cov[ib - x0] += (xb - ib) * w; + } + } + + private void blend(int idx, int src, double cover) { + double sa = (src >>> 24) / 255.0 * cover; + if (sa <= 0) { + return; + } + int dst = px[idx]; + double da = (dst >>> 24) / 255.0; + double oa = sa + da * (1 - sa); + if (oa <= 0) { + px[idx] = 0; + return; + } + double fd = da * (1 - sa); + int r = (int) (((src >> 16 & 0xff) * sa + (dst >> 16 & 0xff) * fd) / oa + 0.5); + int g = (int) (((src >> 8 & 0xff) * sa + (dst >> 8 & 0xff) * fd) / oa + 0.5); + int b = (int) (((src & 0xff) * sa + (dst & 0xff) * fd) / oa + 0.5); + int a = (int) (oa * 255 + 0.5); + px[idx] = (a > 255 ? 255 : a) << 24 | (r > 255 ? 255 : r) << 16 | (g > 255 ? 255 : g) << 8 | (b > 255 ? 255 : b); + } +} diff --git a/CodenameOne/src/com/codename1/image/RasterImage.java b/CodenameOne/src/com/codename1/image/RasterImage.java new file mode 100644 index 00000000000..107c9d33b7c --- /dev/null +++ b/CodenameOne/src/com/codename1/image/RasterImage.java @@ -0,0 +1,276 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.image; + +/// A mutable image held in memory as 32-bit ARGB pixels. +/// +/// Pixels are **not** premultiplied: `0x80FF0000` is half transparent pure +/// red. They are stored row by row with no padding, so pixel `(x, y)` is +/// `getPixels()[y * getWidth() + x]`. +/// +/// Unlike `com.codename1.ui.Image` this class needs no display and no +/// platform: it runs on a device, in the simulator and on a Codename One +/// server alike. Draw into it with [RasterGraphics], read and write files with +/// [PngCodec] and [JpegCodec], and hand the pixels to the UI with +/// `Image.createImage(raster.getPixels(), w, h)` when you need one. +@com.codename1.impl.SharedWithBackend +public final class RasterImage { + private final int width; + private final int height; + private final int[] pixels; + + /// Creates a fully transparent image. + /// + /// #### Parameters + /// + /// - `width`: width in pixels, at least 1 + /// - `height`: height in pixels, at least 1 + public RasterImage(int width, int height) { + this(width, height, new int[checkSize(width, height)]); + } + + /// Wraps an existing ARGB array without copying it. + /// + /// #### Parameters + /// + /// - `width`: width in pixels + /// - `height`: height in pixels + /// - `argb`: at least `width * height` non-premultiplied ARGB pixels + public RasterImage(int width, int height, int[] argb) { + if (argb.length < checkSize(width, height)) { + throw new IllegalArgumentException("pixel array too small"); + } + this.width = width; + this.height = height; + this.pixels = argb; + } + + private static int checkSize(int w, int h) { + if (w <= 0 || h <= 0 || (long) w * h > Integer.MAX_VALUE) { + throw new IllegalArgumentException("bad image size " + w + "x" + h); + } + return w * h; + } + + /// Returns the width in pixels. + public int getWidth() { + return width; + } + + /// Returns the height in pixels. + public int getHeight() { + return height; + } + + /// Returns the live pixel array; writes to it change the image. + public int[] getPixels() { + return pixels; + } + + /// Returns the ARGB value at `(x, y)`. + /// + /// #### Parameters + /// + /// - `x`: column + /// - `y`: row + /// + /// #### Returns + /// + /// the pixel; an `ArrayIndexOutOfBoundsException` or + /// `IllegalArgumentException` for a point outside the image + public int getRGB(int x, int y) { + return pixels[index(x, y)]; + } + + /// Sets the ARGB value at `(x, y)`, replacing what was there. + /// + /// #### Parameters + /// + /// - `x`: column + /// - `y`: row + /// - `argb`: the new pixel + public void setRGB(int x, int y, int argb) { + pixels[index(x, y)] = argb; + } + + private int index(int x, int y) { + if (x < 0 || y < 0 || x >= width || y >= height) { + throw new IllegalArgumentException("pixel (" + x + ", " + y + ") outside " + width + "x" + height); + } + return y * width + x; + } + + /// Fills the whole image with one ARGB value. + /// + /// #### Parameters + /// + /// - `argb`: the pixel value + public void fill(int argb) { + for (int i = 0, n = width * height; i < n; i++) { + pixels[i] = argb; + } + } + + /// Returns a copy of this image. + public RasterImage copy() { + return subImage(0, 0, width, height); + } + + /// Returns a copy of a rectangle of this image. + /// + /// #### Parameters + /// + /// - `x`: left edge + /// - `y`: top edge + /// - `w`: width + /// - `h`: height; the rectangle must lie inside the image + /// + /// #### Returns + /// + /// a new image + public RasterImage subImage(int x, int y, int w, int h) { + if (x < 0 || y < 0 || w <= 0 || h <= 0 || x + w > width || y + h > height) { + throw new IllegalArgumentException("rectangle outside the image"); + } + int[] out = new int[w * h]; + for (int r = 0; r < h; r++) { + System.arraycopy(pixels, (y + r) * width + x, out, r * w, w); + } + return new RasterImage(w, h, out); + } + + /// Returns this image resized with one of the [Resampler] filters. + /// + /// Color is filtered premultiplied, so transparent pixels do not darken + /// the edges of what is next to them. The convolution filters give the + /// same bytes as Pillow's `resize`. + /// + /// #### Parameters + /// + /// - `w`: new width + /// - `h`: new height + /// - `filter`: `Resampler.NEAREST`, `BILINEAR`, `BICUBIC`, `LANCZOS`, + /// `BOX` or `HAMMING` + /// + /// #### Returns + /// + /// a new image + public RasterImage scale(int w, int h, int filter) { + checkSize(w, h); + byte[] rgba = Resampler.premultiply(toRGBA(), 4); + byte[] out = Resampler.resample(rgba, width, height, 4, w, h, new double[] {0, 0, width, height}, filter); + return fromRGBA(w, h, Resampler.unpremultiply(out, 4)); + } + + /// Returns the pixels as `R, G, B, A` bytes, row by row. + public byte[] toRGBA() { + int n = width * height; + byte[] d = new byte[n * 4]; + for (int p = 0; p < n; p++) { + int c = pixels[p]; + d[p * 4] = (byte) (c >> 16); + d[p * 4 + 1] = (byte) (c >> 8); + d[p * 4 + 2] = (byte) c; + d[p * 4 + 3] = (byte) (c >>> 24); + } + return d; + } + + /// Builds an image from `R, G, B, A` bytes, row by row. + /// + /// #### Parameters + /// + /// - `w`: width + /// - `h`: height + /// - `rgba`: `w * h * 4` bytes, not premultiplied + /// + /// #### Returns + /// + /// a new image + public static RasterImage fromRGBA(int w, int h, byte[] rgba) { + int n = checkSize(w, h); + int[] px = new int[n]; + for (int p = 0; p < n; p++) { + px[p] = (rgba[p * 4 + 3] & 0xff) << 24 | (rgba[p * 4] & 0xff) << 16 + | (rgba[p * 4 + 1] & 0xff) << 8 | (rgba[p * 4 + 2] & 0xff); + } + return new RasterImage(w, h, px); + } + + /// Returns this image turned 90 degrees clockwise. + public RasterImage rotate90() { + int[] out = new int[width * height]; + for (int y = 0; y < height; y++) { + for (int x = 0; x < width; x++) { + out[x * height + (height - 1 - y)] = pixels[y * width + x]; + } + } + return new RasterImage(height, width, out); + } + + /// Returns this image turned 180 degrees. + public RasterImage rotate180() { + int n = width * height; + int[] out = new int[n]; + for (int i = 0; i < n; i++) { + out[n - 1 - i] = pixels[i]; + } + return new RasterImage(width, height, out); + } + + /// Returns this image turned 90 degrees counterclockwise. + public RasterImage rotate270() { + int[] out = new int[width * height]; + for (int y = 0; y < height; y++) { + for (int x = 0; x < width; x++) { + out[(width - 1 - x) * height + y] = pixels[y * width + x]; + } + } + return new RasterImage(height, width, out); + } + + /// Returns this image mirrored left to right. + public RasterImage flipHorizontal() { + int[] out = new int[width * height]; + for (int y = 0; y < height; y++) { + for (int x = 0; x < width; x++) { + out[y * width + width - 1 - x] = pixels[y * width + x]; + } + } + return new RasterImage(width, height, out); + } + + /// Returns this image mirrored top to bottom. + public RasterImage flipVertical() { + int[] out = new int[width * height]; + for (int y = 0; y < height; y++) { + System.arraycopy(pixels, y * width, out, (height - 1 - y) * width, width); + } + return new RasterImage(width, height, out); + } + + /// Returns a [RasterGraphics] that draws into this image. + public RasterGraphics createGraphics() { + return new RasterGraphics(this); + } +} diff --git a/CodenameOne/src/com/codename1/image/RasterPaint.java b/CodenameOne/src/com/codename1/image/RasterPaint.java new file mode 100644 index 00000000000..9aca433cf77 --- /dev/null +++ b/CodenameOne/src/com/codename1/image/RasterPaint.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.image; + +/// What [RasterGraphics] fills with when it is not a single color: a +/// gradient ([RasterGradient]) or your own pattern. +@com.codename1.impl.SharedWithBackend +public interface RasterPaint { + /// Returns the non-premultiplied ARGB color at a point. + /// + /// #### Parameters + /// + /// - `x`: horizontal position in user space (before the graphics transform) + /// - `y`: vertical position in user space + /// + /// #### Returns + /// + /// the color at that point + int getColor(double x, double y); +} diff --git a/CodenameOne/src/com/codename1/image/RasterPath.java b/CodenameOne/src/com/codename1/image/RasterPath.java new file mode 100644 index 00000000000..d9d60df8e89 --- /dev/null +++ b/CodenameOne/src/com/codename1/image/RasterPath.java @@ -0,0 +1,378 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.image; + +/// A shape made of straight lines and quadratic and cubic Bezier curves, for +/// [RasterGraphics#fill(RasterPath)] and [RasterGraphics#draw(RasterPath)]. +/// +/// A path is a list of subpaths, each started by [#moveTo(double, double)]. +/// Filling closes every subpath implicitly; stroking draws a subpath closed +/// only after [#closePath()]. +/// +/// The static factories build the common shapes. Angles are in degrees, +/// counterclockwise from three o'clock, as on screen with `y` pointing down +/// (so a positive extent turns toward the top). +@com.codename1.impl.SharedWithBackend +public final class RasterPath { + static final byte MOVE = 0; + static final byte LINE = 1; + static final byte QUAD = 2; + static final byte CUBIC = 3; + static final byte CLOSE = 4; + + /// An arc drawn alone, both ends open. + public static final int ARC_OPEN = 0; + /// An arc closed by the straight line between its ends. + public static final int ARC_CHORD = 1; + /// An arc closed through the center, a pie slice. + public static final int ARC_PIE = 2; + + private static final double KAPPA = 0.5522847498307936; + + byte[] ops = new byte[16]; + int opCount; + double[] coords = new double[32]; + int coordCount; + private boolean open; + + private void op(byte o, int n) { + if (opCount == ops.length) { + byte[] b = new byte[ops.length * 2]; + System.arraycopy(ops, 0, b, 0, opCount); + ops = b; + } + if (coordCount + n > coords.length) { + double[] c = new double[Math.max(coords.length * 2, coordCount + n)]; + System.arraycopy(coords, 0, c, 0, coordCount); + coords = c; + } + ops[opCount++] = o; + } + + private void ensureOpen() { + if (!open) { + throw new IllegalStateException("moveTo must start a subpath"); + } + } + + /// Starts a new subpath at a point. + /// + /// #### Parameters + /// + /// - `x`: x + /// - `y`: y + /// + /// #### Returns + /// + /// this path + public RasterPath moveTo(double x, double y) { + op(MOVE, 2); + coords[coordCount++] = x; + coords[coordCount++] = y; + open = true; + return this; + } + + /// Adds a straight line to a point. + /// + /// #### Parameters + /// + /// - `x`: x + /// - `y`: y + /// + /// #### Returns + /// + /// this path + public RasterPath lineTo(double x, double y) { + ensureOpen(); + op(LINE, 2); + coords[coordCount++] = x; + coords[coordCount++] = y; + return this; + } + + /// Adds a quadratic Bezier curve. + /// + /// #### Parameters + /// + /// - `cx`: control point x + /// - `cy`: control point y + /// - `x`: end x + /// - `y`: end y + /// + /// #### Returns + /// + /// this path + public RasterPath quadTo(double cx, double cy, double x, double y) { + ensureOpen(); + op(QUAD, 4); + coords[coordCount++] = cx; + coords[coordCount++] = cy; + coords[coordCount++] = x; + coords[coordCount++] = y; + return this; + } + + /// Adds a cubic Bezier curve. + /// + /// #### Parameters + /// + /// - `c1x`: first control point x + /// - `c1y`: first control point y + /// - `c2x`: second control point x + /// - `c2y`: second control point y + /// - `x`: end x + /// - `y`: end y + /// + /// #### Returns + /// + /// this path + public RasterPath curveTo(double c1x, double c1y, double c2x, double c2y, double x, double y) { + ensureOpen(); + op(CUBIC, 6); + coords[coordCount++] = c1x; + coords[coordCount++] = c1y; + coords[coordCount++] = c2x; + coords[coordCount++] = c2y; + coords[coordCount++] = x; + coords[coordCount++] = y; + return this; + } + + /// Closes the current subpath with a line back to its start. + /// + /// #### Returns + /// + /// this path + public RasterPath closePath() { + if (open) { + op(CLOSE, 0); + open = false; + } + return this; + } + + /// Returns true when the path has no segments. + public boolean isEmpty() { + return opCount == 0; + } + + /// Creates a closed polygon. + /// + /// #### Parameters + /// + /// - `xs`: x of each vertex + /// - `ys`: y of each vertex + /// - `n`: number of vertices to use + /// + /// #### Returns + /// + /// the path + public static RasterPath polygon(double[] xs, double[] ys, int n) { + RasterPath p = polyline(xs, ys, n); + return p.closePath(); + } + + /// Creates an open chain of lines. + /// + /// #### Parameters + /// + /// - `xs`: x of each vertex + /// - `ys`: y of each vertex + /// - `n`: number of vertices to use + /// + /// #### Returns + /// + /// the path + public static RasterPath polyline(double[] xs, double[] ys, int n) { + RasterPath p = new RasterPath(); + for (int i = 0; i < n; i++) { + if (i == 0) { + p.moveTo(xs[i], ys[i]); + } else { + p.lineTo(xs[i], ys[i]); + } + } + return p; + } + + /// Creates a rectangle. + /// + /// #### Parameters + /// + /// - `x`: left + /// - `y`: top + /// - `w`: width + /// - `h`: height + /// + /// #### Returns + /// + /// the path + public static RasterPath rect(double x, double y, double w, double h) { + return new RasterPath().moveTo(x, y).lineTo(x + w, y).lineTo(x + w, y + h).lineTo(x, y + h).closePath(); + } + + /// Creates a rectangle with elliptical corners. + /// + /// #### Parameters + /// + /// - `x`: left + /// - `y`: top + /// - `w`: width + /// - `h`: height + /// - `arcW`: horizontal diameter of a corner + /// - `arcH`: vertical diameter of a corner + /// + /// #### Returns + /// + /// the path + public static RasterPath roundRect(double x, double y, double w, double h, double arcW, double arcH) { + double rx = Math.min(Math.abs(arcW) / 2, Math.abs(w) / 2); + double ry = Math.min(Math.abs(arcH) / 2, Math.abs(h) / 2); + if (rx <= 0 || ry <= 0) { + return rect(x, y, w, h); + } + double kx = rx * KAPPA; + double ky = ry * KAPPA; + RasterPath p = new RasterPath(); + p.moveTo(x + rx, y).lineTo(x + w - rx, y); + p.curveTo(x + w - rx + kx, y, x + w, y + ry - ky, x + w, y + ry); + p.lineTo(x + w, y + h - ry); + p.curveTo(x + w, y + h - ry + ky, x + w - rx + kx, y + h, x + w - rx, y + h); + p.lineTo(x + rx, y + h); + p.curveTo(x + rx - kx, y + h, x, y + h - ry + ky, x, y + h - ry); + p.lineTo(x, y + ry); + p.curveTo(x, y + ry - ky, x + rx - kx, y, x + rx, y); + return p.closePath(); + } + + /// Creates the ellipse inscribed in a rectangle. + /// + /// #### Parameters + /// + /// - `x`: left + /// - `y`: top + /// - `w`: width + /// - `h`: height + /// + /// #### Returns + /// + /// the path + public static RasterPath ellipse(double x, double y, double w, double h) { + RasterPath p = arc(x, y, w, h, 0, 360, ARC_OPEN); + return p.closePath(); + } + + /// Creates an arc of the ellipse inscribed in a rectangle. + /// + /// #### Parameters + /// + /// - `x`: left of the ellipse's bounds + /// - `y`: top of the ellipse's bounds + /// - `w`: width of the ellipse + /// - `h`: height of the ellipse + /// - `startDeg`: start angle in degrees + /// - `extentDeg`: angular extent in degrees, negative for clockwise + /// - `type`: `ARC_OPEN`, `ARC_CHORD` or `ARC_PIE` + /// + /// #### Returns + /// + /// the path + public static RasterPath arc(double x, double y, double w, double h, double startDeg, double extentDeg, + int type) { + double cx = x + w / 2; + double cy = y + h / 2; + double rx = w / 2; + double ry = h / 2; + double ext = Math.max(-360, Math.min(360, extentDeg)); + int n = (int) Math.ceil(Math.abs(ext) / 90.0); + if (n == 0) { + n = 1; + } + double step = Math.toRadians(ext) / n; + double a = Math.toRadians(startDeg); + RasterPath p = new RasterPath(); + double sx = cx + rx * Math.cos(a); + double sy = cy - ry * Math.sin(a); + if (type == ARC_PIE) { + p.moveTo(cx, cy).lineTo(sx, sy); + } else { + p.moveTo(sx, sy); + } + double k = 4.0 / 3.0 * Math.tan(step / 4); + for (int i = 0; i < n; i++) { + double a1 = a + step; + double c0 = Math.cos(a); + double s0 = Math.sin(a); + double c1 = Math.cos(a1); + double s1 = Math.sin(a1); + p.curveTo(cx + rx * (c0 - k * s0), cy - ry * (s0 + k * c0), + cx + rx * (c1 + k * s1), cy - ry * (s1 - k * c1), + cx + rx * c1, cy - ry * s1); + a = a1; + } + if (type == ARC_PIE || type == ARC_CHORD) { + p.closePath(); + } + return p; + } + + /// Appends every subpath of another path to this one. + /// + /// #### Parameters + /// + /// - `other`: the path to append + /// + /// #### Returns + /// + /// this path + public RasterPath append(RasterPath other) { + int c = 0; + for (int i = 0; i < other.opCount; i++) { + byte o = other.ops[i]; + double[] k = other.coords; + switch (o) { + case MOVE: + moveTo(k[c], k[c + 1]); + c += 2; + break; + case LINE: + lineTo(k[c], k[c + 1]); + c += 2; + break; + case QUAD: + quadTo(k[c], k[c + 1], k[c + 2], k[c + 3]); + c += 4; + break; + case CUBIC: + curveTo(k[c], k[c + 1], k[c + 2], k[c + 3], k[c + 4], k[c + 5]); + c += 6; + break; + default: + closePath(); + break; + } + } + return this; + } +} diff --git a/CodenameOne/src/com/codename1/image/Resampler.java b/CodenameOne/src/com/codename1/image/Resampler.java new file mode 100644 index 00000000000..e6f400c2f0a --- /dev/null +++ b/CodenameOne/src/com/codename1/image/Resampler.java @@ -0,0 +1,285 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.image; + +/// Resampling of interleaved 8-bit band data, the two-pass fixed-point +/// convolution of Pillow's `Resample.c`. +/// +/// The results match Pillow byte for byte, which is why the Python runtime's +/// `PIL.Image.resize` delegates here. Pixels are `bands` consecutive bytes +/// (`L`, `LA`, `RGB`, `RGBA`, ...), rows packed with no padding. Images with +/// alpha should be premultiplied first ([#premultiply(byte[], int)]) and +/// unpremultiplied afterwards, as Pillow does, so transparent pixels do not +/// bleed their color into their neighbours. +/// +/// [RasterImage#scale(int, int, int)] wraps this for ARGB images. +@com.codename1.impl.SharedWithBackend +public final class Resampler { + /// Nearest neighbour: each output pixel takes the input pixel under its center. + public static final int NEAREST = 0; + /// Lanczos, a windowed sinc with a support of three pixels. + public static final int LANCZOS = 1; + /// Linear interpolation (a triangle filter). + public static final int BILINEAR = 2; + /// Cubic interpolation, Pillow's `a = -0.5` kernel. + public static final int BICUBIC = 3; + /// Box filter: every input pixel under an output pixel weighs the same. + public static final int BOX = 4; + /// Hamming-windowed sinc with a support of one pixel. + public static final int HAMMING = 5; + + private static final int PRECISION_BITS = 22; + + private Resampler() { + } + + private static double support(int filter) { + switch (filter) { + case BOX: + return 0.5; + case BICUBIC: + return 2.0; + case LANCZOS: + return 3.0; + default: + return 1.0; + } + } + + private static double sinc(double x) { + if (x == 0.0) { + return 1.0; + } + double px = x * Math.PI; + return Math.sin(px) / px; + } + + private static double kernel(int filter, double x) { + double t = x < 0 ? -x : x; + switch (filter) { + case BOX: + return x > -0.5 && x <= 0.5 ? 1.0 : 0.0; + case HAMMING: + if (t == 0.0) { + return 1.0; + } + if (t >= 1.0) { + return 0.0; + } + t = t * Math.PI; + return Math.sin(t) / t * (0.54 + 0.46 * Math.cos(t)); + case BICUBIC: + if (t < 1.0) { + return (1.5 * t - 2.5) * t * t + 1; + } + if (t < 2.0) { + return (((t - 5) * t + 8) * t - 4) * -0.5; + } + return 0.0; + case LANCZOS: + return x > -3.0 && x < 3.0 ? sinc(x) * sinc(x / 3.0) : 0.0; + default: + return t < 1.0 ? 1.0 - t : 0.0; + } + } + + private static int coeffs(int inSize, double in0, double in1, int outSize, int[] bounds, int[][] out, + int filter) { + double scale = (in1 - in0) / outSize; + double filterscale = scale < 1.0 ? 1.0 : scale; + double support = support(filter) * filterscale; + int ksize = (int) Math.ceil(support) * 2 + 1; + int[] kk = new int[outSize * ksize]; + double[] k = new double[ksize]; + for (int xx = 0; xx < outSize; xx++) { + double center = in0 + (xx + 0.5) * scale; + double ww = 0.0; + double ss = 1.0 / filterscale; + int xmin = (int) (center - support + 0.5); + if (xmin < 0) { + xmin = 0; + } + int xmax = (int) (center + support + 0.5); + if (xmax > inSize) { + xmax = inSize; + } + xmax -= xmin; + for (int x = 0; x < xmax; x++) { + double w = kernel(filter, (x + xmin - center + 0.5) * ss); + k[x] = w; + ww += w; + } + for (int x = 0; x < xmax; x++) { + double v = ww != 0.0 ? k[x] / ww : k[x]; + kk[xx * ksize + x] = v < 0 ? (int) (-0.5 + v * (1 << PRECISION_BITS)) + : (int) (0.5 + v * (1 << PRECISION_BITS)); + } + bounds[xx * 2] = xmin; + bounds[xx * 2 + 1] = xmax; + } + out[0] = kk; + return ksize; + } + + private static byte clip8(int v) { + int s = v >> PRECISION_BITS; + return (byte) (s < 0 ? 0 : s > 255 ? 255 : s); + } + + /// Resamples the region `box` (`x0, y0, x1, y1`, in source pixels, may be + /// fractional) of a `w` x `h` image to `nw` x `nh`. + /// + /// #### Parameters + /// + /// - `src`: the source pixels, `w * h * bands` bytes + /// - `w`: source width + /// - `h`: source height + /// - `bands`: bytes per pixel + /// - `nw`: output width + /// - `nh`: output height + /// - `box`: the source region, four values + /// - `filter`: one of the filter constants of this class + /// + /// #### Returns + /// + /// a new array of `nw * nh * bands` bytes + public static byte[] resample(byte[] src, int w, int h, int bands, int nw, int nh, double[] box, int filter) { + if (filter == NEAREST) { + return nearest(src, w, h, bands, nw, nh, box); + } + byte[] cur = src; + int cw = w; + boolean horizontal = nw != w || box[0] != 0 || box[2] != nw; + boolean vertical = nh != h || box[1] != 0 || box[3] != nh; + if (horizontal) { + int[] bounds = new int[nw * 2]; + int[][] kh = new int[1][]; + int ks = coeffs(cw, box[0], box[2], nw, bounds, kh, filter); + byte[] r = new byte[nw * h * bands]; + for (int y = 0; y < h; y++) { + for (int x = 0; x < nw; x++) { + int xmin = bounds[x * 2]; + int n = bounds[x * 2 + 1]; + for (int b = 0; b < bands; b++) { + int ss = 1 << (PRECISION_BITS - 1); + for (int j = 0; j < n; j++) { + ss += (cur[(y * cw + xmin + j) * bands + b] & 0xff) * kh[0][x * ks + j]; + } + r[(y * nw + x) * bands + b] = clip8(ss); + } + } + } + cur = r; + cw = nw; + } + if (vertical) { + int[] bounds = new int[nh * 2]; + int[][] kv = new int[1][]; + int ks = coeffs(h, box[1], box[3], nh, bounds, kv, filter); + byte[] r = new byte[cw * nh * bands]; + for (int y = 0; y < nh; y++) { + int ymin = bounds[y * 2]; + int n = bounds[y * 2 + 1]; + for (int x = 0; x < cw; x++) { + for (int b = 0; b < bands; b++) { + int ss = 1 << (PRECISION_BITS - 1); + for (int j = 0; j < n; j++) { + ss += (cur[((ymin + j) * cw + x) * bands + b] & 0xff) * kv[0][y * ks + j]; + } + r[(y * cw + x) * bands + b] = clip8(ss); + } + } + } + cur = r; + } + if (cur == src) { //NOPMD CompareObjectsWithEquals - identity is meant + cur = new byte[src.length]; + System.arraycopy(src, 0, cur, 0, src.length); + } + return cur; + } + + private static byte[] nearest(byte[] src, int w, int h, int bands, int nw, int nh, double[] box) { + byte[] r = new byte[nw * nh * bands]; + double sx = (box[2] - box[0]) / nw; + double sy = (box[3] - box[1]) / nh; + for (int y = 0; y < nh; y++) { + int iy = (int) Math.floor(box[1] + (y + 0.5) * sy); + iy = iy < 0 ? 0 : iy >= h ? h - 1 : iy; + for (int x = 0; x < nw; x++) { + int ix = (int) Math.floor(box[0] + (x + 0.5) * sx); + ix = ix < 0 ? 0 : ix >= w ? w - 1 : ix; + System.arraycopy(src, (iy * w + ix) * bands, r, (y * nw + x) * bands, bands); + } + } + return r; + } + + /// Premultiplies the color bands by the last (alpha) band, Pillow's + /// rounding (`RGBA` to `RGBa`). + /// + /// #### Parameters + /// + /// - `d`: interleaved pixels whose last band is alpha + /// - `bands`: bytes per pixel + /// + /// #### Returns + /// + /// a new premultiplied array + public static byte[] premultiply(byte[] d, int bands) { + byte[] r = new byte[d.length]; + System.arraycopy(d, 0, r, 0, d.length); + for (int q = 0; q + bands <= d.length; q += bands) { + int alpha = d[q + bands - 1] & 0xff; + for (int b = 0; b < bands - 1; b++) { + int t = (d[q + b] & 0xff) * alpha + 128; + r[q + b] = (byte) (((t >> 8) + t) >> 8); + } + } + return r; + } + + /// Reverses [#premultiply(byte[], int)] in place, as Pillow's `rgbA2rgba`. + /// + /// #### Parameters + /// + /// - `d`: interleaved premultiplied pixels whose last band is alpha + /// - `bands`: bytes per pixel + /// + /// #### Returns + /// + /// `d` + public static byte[] unpremultiply(byte[] d, int bands) { + for (int q = 0; q + bands <= d.length; q += bands) { + int alpha = d[q + bands - 1] & 0xff; + if (alpha == 0 || alpha == 255) { + continue; + } + for (int b = 0; b < bands - 1; b++) { + int v = (255 * (d[q + b] & 0xff)) / alpha; + d[q + b] = (byte) (v > 255 ? 255 : v); + } + } + return d; + } +} diff --git a/CodenameOne/src/com/codename1/image/package-info.java b/CodenameOne/src/com/codename1/image/package-info.java new file mode 100644 index 00000000000..4dd9ef4839b --- /dev/null +++ b/CodenameOne/src/com/codename1/image/package-info.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ + +/// Pure-Java images: pixels, drawing, resampling and PNG/JPEG codecs that +/// need no display, so the same code runs on a device, in the simulator and +/// on a Codename One server. +/// +/// - [RasterImage] holds non-premultiplied ARGB pixels and rotates, flips, +/// crops and resizes them. +/// - [RasterGraphics] draws antialiased lines, rectangles, ellipses, arcs, +/// polygons and Bezier paths ([RasterPath]), with transforms, clipping, +/// alpha, gradients ([RasterGradient]) and images. Text goes through the +/// [RasterFont] seam. +/// - [Resampler] is Pillow's resampling convolution, byte for byte. +/// - [PngCodec] and [JpegCodec] read and write files. +/// +/// A server builds a social card like this: +/// +/// ```java +/// RasterImage card = new RasterImage(1200, 630); +/// RasterGraphics g = card.createGraphics(); +/// g.setPaint(RasterGradient.linear(0, 0, 0, 630, +/// new float[] {0, 1}, new int[] {0xff1e3a8a, 0xff0ea5e9})); +/// g.fillRect(0, 0, 1200, 630); +/// g.setPaint(null); +/// g.setColor(0xccffffff); +/// g.fillRoundRect(60, 60, 1080, 510, 48, 48); +/// byte[] png = PngCodec.encode(card); +/// ``` +/// +/// Every class is shared with the backend runtime, and depends on nothing +/// but the subset of the JDK every Codename One target provides. +package com.codename1.image; diff --git a/docs/demos/backend/src/main/java/com/codenameone/developerguide/backend/SocialCard.java b/docs/demos/backend/src/main/java/com/codenameone/developerguide/backend/SocialCard.java new file mode 100644 index 00000000000..101fbbf76e7 --- /dev/null +++ b/docs/demos/backend/src/main/java/com/codenameone/developerguide/backend/SocialCard.java @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codenameone.developerguide.backend; + +import com.codename1.image.JpegCodec; +import com.codename1.image.PngCodec; +import com.codename1.image.RasterGradient; +import com.codename1.image.RasterGraphics; +import com.codename1.image.RasterImage; +import com.codename1.image.Resampler; + +import java.io.IOException; + +/// Draws a sharing card and a thumbnail of an upload on the server. +public final class SocialCard { + private SocialCard() { + } + + // tag::raster-image-card[] + public static byte[] card(int accent) { + RasterImage card = new RasterImage(1200, 630); + RasterGraphics g = card.createGraphics(); + g.setPaint(RasterGradient.linear(0, 0, 0, 630, + new float[] {0, 1}, new int[] {0xff0f172a, accent})); + g.fillRect(0, 0, 1200, 630); + g.setPaint(null); + g.setColor(0xccffffff); + g.fillRoundRect(60, 60, 1080, 510, 48, 48); + g.setColor(accent); + g.setStrokeWidth(12); + g.setLineCap(RasterGraphics.CAP_ROUND); + g.drawLine(120, 500, 1080, 500); + return PngCodec.encode(card); + } + + public static byte[] thumbnail(byte[] upload) throws IOException { + RasterImage src = JpegCodec.isJpeg(upload) + ? JpegCodec.decode(upload) : PngCodec.decode(upload); + int w = 320; + int h = Math.max(1, src.getHeight() * w / src.getWidth()); + return JpegCodec.encode(src.scale(w, h, Resampler.LANCZOS), 0.85f); + } + // end::raster-image-card[] +} diff --git a/docs/developer-guide/graphics.asciidoc b/docs/developer-guide/graphics.asciidoc index 7e6f9127fe8..8bcc100745f 100644 --- a/docs/developer-guide/graphics.asciidoc +++ b/docs/developer-guide/graphics.asciidoc @@ -1081,3 +1081,18 @@ The following classes are available for different kinds of visualisations: | `XYMultipleSeriesDataset` / `XYMultipleSeriesRenderer` | Extends `LineChart` with date-aware labelling on the X axis for time series data. |=== + +=== Images without a display + +`com.codename1.image` draws and encodes images in pure Java, with no display and no platform code. The same calls work on a device, in the simulator and in a Codename One backend, which makes it the tool for server-generated sharing cards, thumbnails of uploads and chart exports. + +* `RasterImage` holds non-premultiplied ARGB pixels, and crops, rotates, flips and resizes them. Its resize filters (`NEAREST`, `BILINEAR`, `BICUBIC`, `LANCZOS`, `BOX` and `HAMMING`) produce the same bytes as Pillow. +* `RasterGraphics` draws antialiased lines, rectangles, rounded rectangles, ellipses, arcs, polygons and Bezier paths, with stroke width, caps and joins, an affine transform, a clip rectangle, alpha, linear and radial gradients and scaled images. +* `PngCodec` reads every PNG variant and writes 8-bit PNG; `JpegCodec` reads baseline and progressive JPEG and writes baseline JPEG. + +[source,java] +---- +include::../demos/backend/src/main/java/com/codenameone/developerguide/backend/SocialCard.java[tag=raster-image-card,indent=0] +---- + +To show the result in the UI, pass the pixels to `Image.createImage(raster.getPixels(), width, height)`. Text isn't drawn yet: `RasterGraphics.drawString` renders through a `RasterFont`, and Codename One doesn't ship one. diff --git a/maven/backend/src/test/java/com/codename1/backend/image/SharedRasterTest.java b/maven/backend/src/test/java/com/codename1/backend/image/SharedRasterTest.java new file mode 100644 index 00000000000..0abcc7c905c --- /dev/null +++ b/maven/backend/src/test/java/com/codename1/backend/image/SharedRasterTest.java @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.image; + +import com.codename1.image.JpegCodec; +import com.codename1.image.PngCodec; +import com.codename1.image.RasterGradient; +import com.codename1.image.RasterGraphics; +import com.codename1.image.RasterImage; +import com.codename1.image.Resampler; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.io.IOException; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/// The core's `com.codename1.image` classes are shared with the backend: a +/// server draws, resizes and encodes images with the same code a device runs. +class SharedRasterTest { + + @Test + @DisplayName("a server draws a card and writes it as PNG and JPEG") + void serverGeneratesImages() throws IOException { + RasterImage card = new RasterImage(120, 63); + RasterGraphics g = card.createGraphics(); + g.setPaint(RasterGradient.linear(0, 0, 0, 63, new float[] {0, 1}, new int[] {0xff1e3a8a, 0xff0ea5e9})); + g.fillRect(0, 0, 120, 63); + g.setPaint(null); + g.setColor(0xccffffff); + g.fillRoundRect(6, 6, 108, 51, 10, 10); + g.setColor(0xff000000); + g.setStrokeWidth(2); + g.drawOval(40, 12, 40, 40); + + byte[] png = PngCodec.encode(card); + assertArrayEquals(card.getPixels(), PngCodec.decode(png).getPixels()); + + byte[] jpg = JpegCodec.encode(card, 0.9f); + RasterImage back = JpegCodec.decode(jpg); + assertEquals(120, back.getWidth()); + int a = card.getRGB(60, 3); + int b = back.getRGB(60, 3); + assertTrue(Math.abs((a & 0xff) - (b & 0xff)) < 12, "JPEG keeps the gradient's color"); + + RasterImage thumb = card.scale(30, 16, Resampler.LANCZOS); + assertEquals(30, thumb.getWidth()); + assertEquals(16, thumb.getHeight()); + } +} diff --git a/maven/core-unittests/src/test/java/com/codename1/image/RasterApiTest.java b/maven/core-unittests/src/test/java/com/codename1/image/RasterApiTest.java new file mode 100644 index 00000000000..ed705568c7f --- /dev/null +++ b/maven/core-unittests/src/test/java/com/codename1/image/RasterApiTest.java @@ -0,0 +1,394 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.image; + +import org.junit.jupiter.api.Test; + +import javax.imageio.IIOImage; +import javax.imageio.ImageIO; +import javax.imageio.ImageWriteParam; +import javax.imageio.ImageWriter; +import javax.imageio.stream.MemoryCacheImageOutputStream; +import java.awt.image.BufferedImage; +import java.awt.image.IndexColorModel; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.Random; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/// The pure-Java image API, checked against the JDK's own codecs where +/// there is one to compare with. +class RasterApiTest { + + private static RasterImage noise(int w, int h, boolean alpha, long seed) { + Random r = new Random(seed); + RasterImage img = new RasterImage(w, h); + int[] p = img.getPixels(); + for (int i = 0; i < p.length; i++) { + p[i] = r.nextInt() | (alpha ? 0 : 0xff000000); + } + return img; + } + + private static RasterImage smooth(int w, int h) { + RasterImage img = new RasterImage(w, h); + for (int y = 0; y < h; y++) { + for (int x = 0; x < w; x++) { + int r = x * 255 / (w - 1); + int g = y * 255 / (h - 1); + int b = (x + y) * 255 / (w + h - 2); + img.setRGB(x, y, 0xff000000 | r << 16 | g << 8 | b); + } + } + return img; + } + + private static BufferedImage toAwt(RasterImage img) { + BufferedImage bi = new BufferedImage(img.getWidth(), img.getHeight(), BufferedImage.TYPE_INT_RGB); + bi.setRGB(0, 0, img.getWidth(), img.getHeight(), img.getPixels(), 0, img.getWidth()); + return bi; + } + + /// Mean and max per-channel difference, ignoring alpha. + private static double[] diff(RasterImage a, BufferedImage b) { + double sum = 0; + int max = 0; + for (int y = 0; y < a.getHeight(); y++) { + for (int x = 0; x < a.getWidth(); x++) { + int p = a.getRGB(x, y); + int q = b.getRGB(x, y); + for (int s = 0; s < 24; s += 8) { + int d = Math.abs((p >> s & 0xff) - (q >> s & 0xff)); + sum += d; + max = Math.max(max, d); + } + } + } + return new double[] {sum / (a.getWidth() * a.getHeight() * 3.0), max}; + } + + private static byte[] jpegWithImageIO(BufferedImage bi, boolean progressive) throws IOException { + ImageWriter w = ImageIO.getImageWritersByFormatName("jpeg").next(); + ImageWriteParam p = w.getDefaultWriteParam(); + p.setCompressionMode(ImageWriteParam.MODE_EXPLICIT); + p.setCompressionQuality(0.95f); + if (progressive) { + p.setProgressiveMode(ImageWriteParam.MODE_DEFAULT); + } + ByteArrayOutputStream bo = new ByteArrayOutputStream(); + MemoryCacheImageOutputStream out = new MemoryCacheImageOutputStream(bo); + w.setOutput(out); + w.write(null, new IIOImage(bi, null, null), p); + out.close(); + w.dispose(); + return bo.toByteArray(); + } + + @Test + void pngRoundTripKeepsEveryPixel() throws IOException { + RasterImage a = noise(37, 23, true, 1); + assertArrayEquals(a.getPixels(), PngCodec.decode(PngCodec.encode(a)).getPixels()); + RasterImage b = noise(19, 41, false, 2); + byte[] png = PngCodec.encode(b); + assertEquals(2, png[8 + 8 + 9], "opaque images are written as RGB"); + assertArrayEquals(b.getPixels(), PngCodec.decode(png).getPixels()); + } + + @Test + void pngWrittenByTheJdkDecodes() throws IOException { + RasterImage src = noise(29, 17, false, 3); + ByteArrayOutputStream bo = new ByteArrayOutputStream(); + ImageIO.write(toAwt(src), "png", bo); + assertArrayEquals(src.getPixels(), PngCodec.decode(bo.toByteArray()).getPixels()); + + byte[] r = {0, (byte) 255, 0, 10}; + byte[] g = {0, 0, (byte) 255, 20}; + byte[] b = {0, 0, 0, 30}; + byte[] a = {(byte) 255, (byte) 255, (byte) 255, 0}; + BufferedImage pal = new BufferedImage(6, 3, BufferedImage.TYPE_BYTE_INDEXED, new IndexColorModel(2, 4, r, g, b, a)); + for (int y = 0; y < 3; y++) { + for (int x = 0; x < 6; x++) { + pal.getRaster().setSample(x, y, 0, (x + y) % 4); + } + } + bo.reset(); + ImageIO.write(pal, "png", bo); + RasterImage d = PngCodec.decode(bo.toByteArray()); + for (int y = 0; y < 3; y++) { + for (int x = 0; x < 6; x++) { + int i = (x + y) % 4; + int expected = i == 3 ? 0x000a141e : pal.getRGB(x, y); + assertEquals(expected, d.getRGB(x, y), "palette pixel " + x + "," + y); + } + } + + BufferedImage gray = new BufferedImage(9, 4, BufferedImage.TYPE_USHORT_GRAY); + for (int y = 0; y < 4; y++) { + for (int x = 0; x < 9; x++) { + gray.getRaster().setSample(x, y, 0, x * 7000 + y * 300); + } + } + bo.reset(); + ImageIO.write(gray, "png", bo); + RasterImage g16 = PngCodec.decode(bo.toByteArray()); + for (int y = 0; y < 4; y++) { + for (int x = 0; x < 9; x++) { + int v = (x * 7000 + y * 300) >> 8; + assertEquals(0xff000000 | v << 16 | v << 8 | v, g16.getRGB(x, y)); + } + } + } + + @Test + void adam7InterlacedPngDecodes() throws IOException { + int w = 11; + int h = 7; + RasterImage src = noise(w, h, false, 4); + int[][] passes = {{0, 0, 8, 8}, {4, 0, 8, 8}, {0, 4, 4, 8}, {2, 0, 4, 4}, {0, 2, 2, 4}, {1, 0, 2, 2}, {0, 1, 1, 2}}; + ByteArrayOutputStream rows = new ByteArrayOutputStream(); + for (int[] p : passes) { + for (int y = p[1]; y < h; y += p[3]) { + if (p[0] >= w) { + break; + } + rows.write(0); + for (int x = p[0]; x < w; x += p[2]) { + int c = src.getRGB(x, y); + rows.write(c >> 16 & 0xff); + rows.write(c >> 8 & 0xff); + rows.write(c & 0xff); + } + } + } + byte[] png = PngCodec.encodeRows(w, h, 8, 2, null, null, rows.toByteArray()); + png[8 + 8 + 12] = 1; + assertArrayEquals(src.getPixels(), PngCodec.decode(png).getPixels()); + } + + @Test + void jpegFromTheJdkDecodesBaselineAndProgressive() throws IOException { + RasterImage src = smooth(70, 45); + BufferedImage bi = toAwt(src); + for (boolean progressive : new boolean[] {false, true}) { + byte[] jpg = jpegWithImageIO(bi, progressive); + RasterImage mine = JpegCodec.decode(jpg); + BufferedImage theirs = ImageIO.read(new ByteArrayInputStream(jpg)); + double[] d = diff(mine, theirs); + assertTrue(d[0] < 2.0 && d[1] <= 24, "progressive=" + progressive + " mean " + d[0] + " max " + d[1]); + assertEquals(3, JpegCodec.componentCount(jpg)); + } + } + + @Test + void jpegWeWriteIsReadByTheJdk() throws IOException { + RasterImage src = smooth(53, 31); + byte[] jpg = JpegCodec.encode(src, 0.9f); + BufferedImage theirs = ImageIO.read(new ByteArrayInputStream(jpg)); + assertEquals(53, theirs.getWidth()); + double[] d = diff(src, theirs); + assertTrue(d[0] < 3.0, "mean " + d[0]); + double[] own = diff(JpegCodec.decode(jpg), theirs); + assertTrue(own[1] <= 2, "our decoder agrees with the JDK's on our file: " + own[1]); + + byte[] grayJpg = JpegCodec.encode(src, 0.75f, true); + assertEquals(1, JpegCodec.componentCount(grayJpg)); + int c = JpegCodec.decode(grayJpg).getRGB(10, 10); + assertEquals(c >> 16 & 0xff, c & 0xff); + assertThrows(IOException.class, () -> JpegCodec.decode(new byte[] {1, 2, 3, 4})); + } + + @Test + void resizeKeepsSolidColorAndDoesNotBleedTransparency() { + int[] filters = {Resampler.NEAREST, Resampler.BILINEAR, Resampler.BICUBIC, Resampler.LANCZOS, + Resampler.BOX, Resampler.HAMMING}; + RasterImage solid = new RasterImage(9, 7); + solid.fill(0xff336699); + RasterImage half = new RasterImage(8, 8); + for (int y = 0; y < 8; y++) { + for (int x = 0; x < 8; x++) { + half.setRGB(x, y, x < 4 ? 0xffff0000 : 0x0000ff00); + } + } + for (int f : filters) { + RasterImage s = solid.scale(4, 13, f); + for (int p : s.getPixels()) { + assertEquals(0xff336699, p, "filter " + f); + } + RasterImage h = half.scale(5, 5, f); + for (int p : h.getPixels()) { + if (p >>> 24 != 0) { + assertEquals(0, p >> 8 & 0xff, "no green from transparent pixels, filter " + f); + } + } + } + } + + @Test + void rotateFlipAndSubImage() { + RasterImage a = noise(3, 2, true, 5); + RasterImage r = a.rotate90(); + assertEquals(2, r.getWidth()); + assertEquals(a.getRGB(0, 0), r.getRGB(1, 0)); + assertEquals(a.getRGB(0, 1), r.getRGB(0, 0)); + assertArrayEquals(a.getPixels(), r.rotate270().getPixels()); + assertArrayEquals(a.rotate180().getPixels(), a.flipHorizontal().flipVertical().getPixels()); + assertEquals(a.getRGB(2, 1), a.subImage(1, 1, 2, 1).getRGB(1, 0)); + assertThrows(IllegalArgumentException.class, () -> a.getRGB(3, 0)); + } + + @Test + void fillsAreExactOnPixelEdgesAndAntialiasedBetween() { + RasterImage img = new RasterImage(20, 20); + RasterGraphics g = img.createGraphics(); + g.setColor(0xff0000ff); + g.fillRect(2, 2, 3, 3); + assertEquals(0xff0000ff, img.getRGB(2, 2)); + assertEquals(0xff0000ff, img.getRGB(4, 4)); + assertEquals(0, img.getRGB(5, 4)); + assertEquals(0, img.getRGB(1, 2)); + g.fillRect(10.5, 10, 1, 1); + int a = img.getRGB(10, 10) >>> 24; + assertTrue(a > 120 && a < 136, "half covered pixel alpha " + a); + + g.setColor(0xffff0000); + g.drawLine(0, 15.5, 20, 15.5); + assertEquals(0xffff0000, img.getRGB(7, 15)); + assertEquals(0, img.getRGB(7, 14)); + assertEquals(0, img.getRGB(7, 16)); + } + + @Test + void ovalsStrokesAndWindingRules() { + RasterImage img = new RasterImage(40, 40); + RasterGraphics g = img.createGraphics(); + g.setColor(0xff00ff00); + g.fillOval(0, 0, 40, 40); + assertEquals(0xff00ff00, img.getRGB(20, 20)); + assertEquals(0, img.getRGB(1, 1)); + + RasterImage ring = new RasterImage(30, 30); + RasterGraphics rg = ring.createGraphics(); + RasterPath p = RasterPath.rect(0, 0, 30, 30).append(RasterPath.rect(10, 10, 10, 10)); + rg.setColor(0xff000000); + rg.fill(p); + assertEquals(0xff000000, ring.getRGB(15, 15), "non-zero: same winding fills the hole"); + ring.fill(0); + rg.setWindingRule(RasterGraphics.WIND_EVEN_ODD); + rg.fill(p); + assertEquals(0, ring.getRGB(15, 15), "even-odd leaves the hole"); + assertEquals(0xff000000, ring.getRGB(5, 5)); + + RasterImage s = new RasterImage(30, 30); + RasterGraphics sg = s.createGraphics(); + sg.setColor(0xff000000); + sg.setStrokeWidth(4); + sg.setLineJoin(RasterGraphics.JOIN_MITER); + sg.drawRect(5, 5, 20, 20); + assertEquals(0xff000000, s.getRGB(3, 3), "miter fills the corner"); + assertEquals(0xff000000, s.getRGB(15, 6)); + assertEquals(0, s.getRGB(15, 15)); + s.fill(0); + sg.setLineJoin(RasterGraphics.JOIN_BEVEL); + sg.drawRect(5, 5, 20, 20); + assertEquals(0, s.getRGB(3, 3), "bevel cuts the corner"); + s.fill(0); + sg.setLineCap(RasterGraphics.CAP_SQUARE); + sg.drawLine(10, 15, 20, 15); + assertEquals(0xff000000, s.getRGB(8, 15)); + s.fill(0); + sg.setLineCap(RasterGraphics.CAP_BUTT); + sg.drawLine(10, 15, 20, 15); + assertEquals(0, s.getRGB(8, 15)); + sg.setStrokeWidth(2); + sg.draw(new RasterPath().moveTo(2, 2).quadTo(15, 30, 28, 2)); + sg.draw(new RasterPath().moveTo(2, 28).curveTo(10, 0, 20, 0, 28, 28)); + sg.drawArc(5, 5, 20, 20, 0, 180); + } + + @Test + void transformClipAlphaGradientAndImages() { + RasterImage img = new RasterImage(20, 20); + RasterGraphics g = img.createGraphics(); + g.translate(10, 0); + g.scale(2, 2); + g.setColor(0xffffffff); + g.fillRect(0, 0, 2, 2); + assertEquals(0xffffffff, img.getRGB(13, 3)); + assertEquals(0, img.getRGB(9, 3)); + g.resetTransform(); + + g.setClip(0, 0, 5, 5); + g.setColor(0xff123456); + g.fillRect(0, 0, 20, 20); + assertEquals(0xff123456, img.getRGB(4, 4)); + assertEquals(0, img.getRGB(5, 5)); + g.resetClip(); + + img.fill(0xff000000); + g.setAlpha(0.5f); + g.setColor(0xffffffff); + g.fillRect(0, 0, 20, 20); + int v = img.getRGB(10, 10) & 0xff; + assertTrue(v >= 127 && v <= 128, "half white over black: " + v); + g.setAlpha(1); + + g.setPaint(RasterGradient.linear(0, 0, 20, 0, new float[] {0, 1}, new int[] {0xff000000, 0xffffffff})); + g.fillRect(0, 0, 20, 20); + assertTrue((img.getRGB(1, 5) & 0xff) < (img.getRGB(18, 5) & 0xff)); + g.setPaint(RasterGradient.radial(10, 10, 10, new float[] {0, 1}, new int[] {0xffff0000, 0xff0000ff})); + g.fillRect(0, 0, 20, 20); + assertTrue((img.getRGB(10, 10) >> 16 & 0xff) > 200); + g.setPaint(null); + + RasterImage tile = new RasterImage(2, 2); + tile.fill(0xff00ff00); + g.drawImage(tile, 4, 4, 8, 8); + assertEquals(0xff00ff00, img.getRGB(7, 7)); + assertThrows(IllegalStateException.class, () -> g.drawString("x", 0, 0)); + g.setFont(new RasterFont() { + public RasterPath getOutline(String text, double x, double y) { + return RasterPath.rect(x, y - 5, 5 * text.length(), 5); + } + + public double stringWidth(String text) { + return 5 * text.length(); + } + + public double getAscent() { + return 5; + } + + public double getDescent() { + return 0; + } + }); + g.setColor(0xffabcdef); + g.drawString("ab", 0, 19); + assertEquals(0xffabcdef, img.getRGB(8, 16)); + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyImage.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyImage.java index c5259246d37..99f955ed68e 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyImage.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyImage.java @@ -23,6 +23,13 @@ */ package com.codename1.python.runtime; +import com.codename1.image.JpegCodec; +import com.codename1.image.PngCodec; +import com.codename1.image.RasterImage; +import com.codename1.image.Resampler; + +import java.io.IOException; + /// The `_cn1image` module: pixel storage and the hot loops behind the /// bundled `PIL` package. An image is a [Img] handle holding interleaved @@ -30,9 +37,6 @@ /// Pillow's `tobytes()`. Every arithmetic here follows Pillow's own fixed /// point code so results are byte identical with it. final class PyImage { - /// Pillow's resampling precision, `32 - 8 - 2`. - private static final int PRECISION_BITS = 22; - private PyImage() { } @@ -236,9 +240,12 @@ static PyModule module() { return wrap(transpose(u, 5)); })); m.setAttr("png_unfilter", PyBuiltin.positional("png_unfilter", a -> - new PyBytes(unfilter(bytes(a[0]), i(a[1]), i(a[2]), i(a[3]))))); + new PyBytes(pngUnfilter(bytes(a[0]), i(a[1]), i(a[2]), i(a[3]))))); m.setAttr("png_expand", PyBuiltin.positional("png_expand", a -> - wrap(expand(bytes(a[0]), i(a[1]), i(a[2]), i(a[3]), i(a[4]))))); + wrap(pngExpand(bytes(a[0]), i(a[1]), i(a[2]), i(a[3]), i(a[4]))))); + m.setAttr("jpeg_decode", PyBuiltin.positional("jpeg_decode", a -> wrap(jpegDecode(bytes(a[0]))))); + m.setAttr("jpeg_encode", PyBuiltin.positional("jpeg_encode", a -> + new PyBytes(jpegEncode(img(a[0]), (float) Py.asDouble(a[1]))))); m.setAttr("png_rows", PyBuiltin.positional("png_rows", a -> { Img s = img(a[0]); int row = s.w * s.bands; @@ -395,183 +402,77 @@ private static Img affine(Img s, int w, int h, double[] a, byte[] fill) { static final int FILTER_BOX = 4; static final int FILTER_HAMMING = 5; - private static double support(int filter) { - switch (filter) { - case FILTER_BOX: - return 0.5; - case FILTER_BICUBIC: - return 2.0; - case FILTER_LANCZOS: - return 3.0; - default: - return 1.0; - } + /// Pillow's two-pass resampling, shared with every Java program through + /// `com.codename1.image.Resampler` (same fixed point, same bytes). + private static Img resample(Img s, int w, int h, double[] box, int filter) { + Img r = new Img(w, h, s.bands); + byte[] d = Resampler.resample(s.d, s.w, s.h, s.bands, w, h, box, filter); + System.arraycopy(d, 0, r.d, 0, d.length); + return r; } - private static double sinc(double x) { - if (x == 0.0) { - return 1.0; - } - double px = x * Math.PI; - return Math.sin(px) / px; - } - - /// Pillow's filter kernels (`Resample.c`), evaluated at signed `x`. - private static double kernel(int filter, double x) { - double t = x < 0 ? -x : x; - switch (filter) { - case FILTER_BOX: - return x > -0.5 && x <= 0.5 ? 1.0 : 0.0; - case FILTER_HAMMING: - if (t == 0.0) { - return 1.0; - } - if (t >= 1.0) { - return 0.0; - } - t = t * Math.PI; - return Math.sin(t) / t * (0.54 + 0.46 * Math.cos(t)); - case FILTER_BICUBIC: - if (t < 1.0) { - return (1.5 * t - 2.5) * t * t + 1; - } - if (t < 2.0) { - return (((t - 5) * t + 8) * t - 4) * -0.5; - } - return 0.0; - case FILTER_LANCZOS: - return x > -3.0 && x < 3.0 ? sinc(x) * sinc(x / 3.0) : 0.0; - default: - return t < 1.0 ? 1.0 - t : 0.0; - } + /// RGBA to RGBa (or LA to La): Pillow's `MULDIV255` premultiply. + private static Img premultiply(Img s) { + Img r = new Img(s.w, s.h, s.bands); + byte[] d = Resampler.premultiply(s.d, s.bands); + System.arraycopy(d, 0, r.d, 0, d.length); + return r; } - /// Filter coefficients for one axis, as Pillow's `precompute_coeffs` - /// and `normalize_coeffs_8bpc` make them. Returns `ksize`; `bounds` - /// receives `xmin, count` per output position. - private static int coeffs(int inSize, double in0, double in1, int outSize, int[] bounds, int[][] out, - int filter) { - double scale = (in1 - in0) / outSize; - double filterscale = scale < 1.0 ? 1.0 : scale; - double support = support(filter) * filterscale; - int ksize = (int) Math.ceil(support) * 2 + 1; - int[] kk = new int[outSize * ksize]; - double[] k = new double[ksize]; - for (int xx = 0; xx < outSize; xx++) { - double center = in0 + (xx + 0.5) * scale; - double ww = 0.0; - double ss = 1.0 / filterscale; - int xmin = (int) (center - support + 0.5); - if (xmin < 0) { - xmin = 0; - } - int xmax = (int) (center + support + 0.5); - if (xmax > inSize) { - xmax = inSize; - } - xmax -= xmin; - for (int x = 0; x < xmax; x++) { - double w = kernel(filter, (x + xmin - center + 0.5) * ss); - k[x] = w; - ww += w; - } - for (int x = 0; x < xmax; x++) { - double v = ww != 0.0 ? k[x] / ww : k[x]; - kk[xx * ksize + x] = v < 0 ? (int) (-0.5 + v * (1 << PRECISION_BITS)) - : (int) (0.5 + v * (1 << PRECISION_BITS)); - } - bounds[xx * 2] = xmin; - bounds[xx * 2 + 1] = xmax; - } - out[0] = kk; - return ksize; + /// RGBa to RGBA (or La to LA), as Pillow's `rgbA2rgba`; in place. + private static Img unpremultiply(Img s) { + Resampler.unpremultiply(s.d, s.bands); + return s; } - private static byte clip8(int v) { - int s = v >> PRECISION_BITS; - return (byte) (s < 0 ? 0 : s > 255 ? 255 : s); + private static Img fromBands(int w, int h, int bands, byte[] d) { + Img r = new Img(w, h, bands); + System.arraycopy(d, 0, r.d, 0, r.d.length); + return r; } - private static Img resample(Img s, int w, int h, double[] box, int filter) { - Img cur = s; - boolean horizontal = w != s.w || box[0] != 0 || box[2] != w; - boolean vertical = h != s.h || box[1] != 0 || box[3] != h; - if (horizontal) { - int[] bounds = new int[w * 2]; - int[][] kh = new int[1][]; - int ks = coeffs(cur.w, box[0], box[2], w, bounds, kh, filter); - Img r = new Img(w, cur.h, cur.bands); - for (int y = 0; y < cur.h; y++) { - for (int x = 0; x < w; x++) { - int xmin = bounds[x * 2]; - int n = bounds[x * 2 + 1]; - for (int b = 0; b < cur.bands; b++) { - int ss = 1 << (PRECISION_BITS - 1); - for (int j = 0; j < n; j++) { - ss += (cur.d[(y * cur.w + xmin + j) * cur.bands + b] & 0xff) * kh[0][x * ks + j]; - } - r.d[(y * w + x) * cur.bands + b] = clip8(ss); - } - } - } - cur = r; - } - if (vertical) { - int[] bounds = new int[h * 2]; - int[][] kv = new int[1][]; - int ks = coeffs(cur.h, box[1], box[3], h, bounds, kv, filter); - Img r = new Img(cur.w, h, cur.bands); - for (int y = 0; y < h; y++) { - int ymin = bounds[y * 2]; - int n = bounds[y * 2 + 1]; - for (int x = 0; x < cur.w; x++) { - for (int b = 0; b < cur.bands; b++) { - int ss = 1 << (PRECISION_BITS - 1); - for (int j = 0; j < n; j++) { - ss += (cur.d[((ymin + j) * cur.w + x) * cur.bands + b] & 0xff) * kv[0][y * ks + j]; - } - r.d[(y * cur.w + x) * cur.bands + b] = clip8(ss); - } - } - } - cur = r; - } - if (cur == s) { //NOPMD CompareObjectsWithEquals - identity is meant - cur = crop(s, 0, 0, s.w, s.h); - } - return cur; + private static RuntimeException osError(IOException e) { + return Py.error(Py.OS_ERROR, String.valueOf(e.getMessage())); } - /// RGBA to RGBa (or LA to La): Pillow's `MULDIV255` premultiply. - private static Img premultiply(Img s) { - Img r = crop(s, 0, 0, s.w, s.h); - int n = s.bands; - for (int p = 0, c = s.w * s.h; p < c; p++) { - int q = p * n; - int alpha = s.d[q + n - 1] & 0xff; - for (int b = 0; b < n - 1; b++) { - int t = (s.d[q + b] & 0xff) * alpha + 128; - r.d[q + b] = (byte) (((t >> 8) + t) >> 8); + /// Decodes a JPEG file with the shared pure-Java codec, to "L" (one band) + /// for a gray file and "RGB" (three) otherwise, as Pillow opens them. + private static Img jpegDecode(byte[] data) { + RasterImage ri; + try { + ri = JpegCodec.decode(data); + } catch (IOException e) { + throw osError(e); // NOPMD PreserveStackTrace - surfaced as a Python OSError + } + int bands = JpegCodec.componentCount(data) == 1 ? 1 : 3; + int[] px = ri.getPixels(); + Img r = new Img(ri.getWidth(), ri.getHeight(), bands); + for (int p = 0; p < px.length; p++) { + int c = px[p]; + if (bands == 1) { + r.d[p] = (byte) c; + } else { + r.d[p * 3] = (byte) (c >> 16); + r.d[p * 3 + 1] = (byte) (c >> 8); + r.d[p * 3 + 2] = (byte) c; } } return r; } - /// RGBa to RGBA (or La to LA), as Pillow's `rgbA2rgba`. - private static Img unpremultiply(Img s) { - int n = s.bands; - for (int p = 0, c = s.w * s.h; p < c; p++) { - int q = p * n; - int alpha = s.d[q + n - 1] & 0xff; - if (alpha == 0 || alpha == 255) { - continue; - } - for (int b = 0; b < n - 1; b++) { - int v = (255 * (s.d[q + b] & 0xff)) / alpha; - s.d[q + b] = (byte) (v > 255 ? 255 : v); - } + /// Encodes an "L" or "RGB" image as baseline JPEG. + private static byte[] jpegEncode(Img s, float quality) { + if (s.bands != 1 && s.bands != 3) { + throw Py.error(Py.OS_ERROR, "cannot write this mode as JPEG"); } - return s; + int[] px = new int[s.w * s.h]; + for (int p = 0; p < px.length; p++) { + int r = s.d[p * s.bands] & 0xff; + int g = s.bands == 1 ? r : s.d[p * 3 + 1] & 0xff; + int b = s.bands == 1 ? r : s.d[p * 3 + 2] & 0xff; + px[p] = 0xff000000 | r << 16 | g << 8 | b; + } + return JpegCodec.encode(new RasterImage(s.w, s.h, px), quality, s.bands == 1); } /// Maps a 1-band index image through a 256 entry RGBA table. @@ -675,85 +576,16 @@ private static Img blurRows(Img s, int r) { return o; } - private static int paeth(int a, int b, int c) { - int p = a + b - c; - int pa = Math.abs(p - a); - int pb = Math.abs(p - b); - int pc = Math.abs(p - c); - if (pa <= pb && pa <= pc) { - return a; + private static byte[] pngUnfilter(byte[] data, int row, int h, int bpp) { + try { + return PngCodec.unfilter(data, 0, row, h, bpp); + } catch (IOException e) { + throw osError(e); // NOPMD PreserveStackTrace - surfaced as a Python OSError } - return pb <= pc ? b : c; } - /// Reverses the five PNG row filters; `data` holds `h` rows of a filter - /// byte followed by `row` bytes, and `bpp` is the filter's byte distance. - private static byte[] unfilter(byte[] data, int row, int h, int bpp) { - if (data.length < (row + 1) * h) { - throw Py.error(Py.OS_ERROR, "image file is truncated"); - } - byte[] out = new byte[row * h]; - for (int y = 0; y < h; y++) { - int ft = data[y * (row + 1)] & 0xff; - int in = y * (row + 1) + 1; - int o = y * row; - for (int x = 0; x < row; x++) { - int raw = data[in + x] & 0xff; - int a = x >= bpp ? out[o + x - bpp] & 0xff : 0; - int b = y > 0 ? out[o - row + x] & 0xff : 0; - int c = x >= bpp && y > 0 ? out[o - row + x - bpp] & 0xff : 0; - int v; - switch (ft) { - case 0: v = raw; break; - case 1: v = raw + a; break; - case 2: v = raw + b; break; - case 3: v = raw + ((a + b) >> 1); break; - case 4: v = raw + paeth(a, b, c); break; - default: throw Py.error(Py.OS_ERROR, "unknown PNG filter type " + ft); - } - out[o + x] = (byte) v; - } - } - return out; - } - - private static int sample(byte[] raw, int rowStart, int index, int depth) { - if (depth == 8) { - return raw[rowStart + index] & 0xff; - } - if (depth == 16) { - return raw[rowStart + index * 2] & 0xff; - } - int bit = index * depth; - return (raw[rowStart + (bit >> 3)] & 0xff) >> (8 - depth - (bit & 7)) & ((1 << depth) - 1); - } - - /// Unpacks unfiltered PNG rows into an image: gray to 1 band, RGB to 3, - /// gray+alpha to 2, RGBA to 4 and palette to its 1-band indices. - private static Img expand(byte[] raw, int w, int h, int ctype, int depth) { - int channels = ctype == 0 || ctype == 3 ? 1 : ctype == 2 ? 3 : ctype == 4 ? 2 : 4; - int row = (w * channels * depth + 7) / 8; + private static Img pngExpand(byte[] raw, int w, int h, int ctype, int depth) { int bands = ctype == 0 || ctype == 3 ? 1 : ctype == 2 ? 3 : ctype == 4 ? 2 : 4; - Img r = new Img(w, h, bands); - for (int y = 0; y < h; y++) { - int rs = y * row; - for (int x = 0; x < w; x++) { - int o = (y * w + x) * bands; - if (ctype == 3) { - r.d[o] = (byte) sample(raw, rs, x, depth); - continue; - } - int scale = depth < 8 ? 255 / ((1 << depth) - 1) : 1; - if (ctype == 4) { - r.d[o] = (byte) sample(raw, rs, x * 2, depth); - r.d[o + 1] = (byte) sample(raw, rs, x * 2 + 1, depth); - continue; - } - for (int b = 0; b < channels; b++) { - r.d[o + b] = (byte) (sample(raw, rs, x * channels + b, depth) * scale); - } - } - } - return r; + return fromBands(w, h, bands, PngCodec.expand(raw, w, h, ctype, depth)); } } diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/Image.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/Image.py index ce5a9a765ea..6a36a0308dc 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/Image.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/Image.py @@ -444,6 +444,8 @@ def save(self, fp, format=None, **params): format = format.upper() if format == "PNG": data = _png_encode(self) + elif format in ("JPEG", "JPG"): + data = _jpeg_encode(self, params.get("quality", 75)) else: data = _native_encode(self, format, params.get("quality", 75)) if filename is not None: @@ -496,10 +498,14 @@ def open(fp, mode="r", formats=None): im = _png_decode(data) im.format = "PNG" return im - kind = None if data[:3] == b"\xff\xd8\xff": - kind = "JPEG" - elif data[:4] == b"GIF8": + handle = _cn1image.jpeg_decode(data) + mode = "L" if len(_cn1image.getpixel(handle, 0, 0)) == 1 else "RGB" + im = _wrap(mode, _cn1image.size(handle), handle) + im.format = "JPEG" + return im + kind = None + if data[:4] == b"GIF8": kind = "GIF" elif data[:2] == b"BM": kind = "BMP" @@ -508,11 +514,11 @@ def open(fp, mode="r", formats=None): if kind is None: raise UnidentifiedImageError("cannot identify image file %r" % (fp if isinstance(fp, str) else fp,)) if not _cn1image.native_available(): - raise OSError(kind + " needs a running Codename One display to decode; only PNG is built in") + raise OSError(kind + " needs a running Codename One display to decode; only PNG and JPEG are built in") handle = _cn1image.native_decode(data) im = _wrap("RGBA", _cn1image.size(handle), handle) im.format = kind - if kind in ("JPEG", "BMP"): + if kind == "BMP": im = im.convert("RGB") im.format = kind return im @@ -520,12 +526,20 @@ def open(fp, mode="r", formats=None): def _native_encode(im, format, quality): if not _cn1image.native_available(): - raise OSError(format + " needs a running Codename One display to encode; only PNG is built in") - if format == "JPEG" and im.mode == "RGBA": - raise OSError("cannot write mode RGBA as JPEG") + raise OSError(format + " needs a running Codename One display to encode; only PNG and JPEG are built in") return _cn1image.native_encode(im._im, format.lower(), quality / 100.0) +def _jpeg_encode(im, quality): + # The shared pure-Java codec (com.codename1.image.JpegCodec): baseline, + # libjpeg's quality scaling, so it works on servers as well as devices. + if im.mode == "1": + im = im.convert("L") + if im.mode not in ("L", "RGB"): + raise OSError("cannot write mode %s as JPEG" % im.mode) + return _cn1image.jpeg_encode(im._im, int(quality) / 100.0) + + def _chunk(kind, body): return struct.pack(">I", len(body)) + kind + body + struct.pack(">I", zlib.crc32(kind + body) & 0xffffffff) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/__init__.py index 9e652241f9f..69ef6aa8344 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/__init__.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/__init__.py @@ -1,12 +1,13 @@ # Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. # Licensed under the GNU General Public License version 2 with the Classpath # exception, as the rest of the Codename One Python runtime. -"""A Pillow subset: images in "L", "RGB" and "RGBA" with PNG built in. +"""A Pillow subset: images in "L", "RGB" and "RGBA" with PNG and JPEG built in. Pixels live in the runtime's `_cn1image` module, which also does every loop -over them. PNG is read and written here over zlib, so it works on servers as -well as devices; JPEG and the other formats go through Codename One's own -image codecs and need a running display. +over them. Resampling and the PNG and JPEG codecs are Codename One's pure-Java +`com.codename1.image` package, so they work on servers as well as devices; +GIF, BMP and WEBP go through the platform's image codecs and need a running +display. """ __version__ = "11.0.0" diff --git a/maven/python-transpiler/src/test/resources/conformance/pil_jpeg.out b/maven/python-transpiler/src/test/resources/conformance/pil_jpeg.out new file mode 100644 index 00000000000..1e5b65b960f --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/pil_jpeg.out @@ -0,0 +1,10 @@ +True True +JPEG RGB (40, 24) +rgb close True True +JPEG L (33, 17) +gray close True True +quality orders size True +JPEG RGB (40, 24) +re-encode (40, 24) +RGBA OSError cannot write mode RGBA as JPEG +P OSError cannot write mode P as JPEG diff --git a/maven/python-transpiler/src/test/resources/conformance/pil_jpeg.py b/maven/python-transpiler/src/test/resources/conformance/pil_jpeg.py new file mode 100644 index 00000000000..3561a39eaed --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/pil_jpeg.py @@ -0,0 +1,78 @@ +# Pillow JPEG: round trips through BytesIO and a file path, RGB and L, the +# quality setting, and the modes JPEG cannot hold. JPEG is lossy and codecs +# round differently, so pixels are compared within a tolerance. +import io +import os + +from PIL import Image + + +def gradient(mode, w, h): + im = Image.new(mode, (w, h)) + for y in range(h): + for x in range(w): + if mode == "L": + im.putpixel((x, y), (x * 255 // (w - 1) + y * 2) % 256) + else: + im.putpixel((x, y), (x * 255 // (w - 1), y * 255 // (h - 1), (x + y) * 4 % 256)) + return im + + +def error(a, b): + total = 0 + worst = 0 + n = 0 + for y in range(a.height): + for x in range(a.width): + p = a.getpixel((x, y)) + q = b.getpixel((x, y)) + if not isinstance(p, tuple): + p, q = (p,), (q,) + for u, v in zip(p, q): + d = abs(u - v) + total += d + worst = max(worst, d) + n += 1 + return total / n, worst + + +rgb = gradient("RGB", 40, 24) +buf = io.BytesIO() +rgb.save(buf, format="JPEG", quality=92) +data = buf.getvalue() +print(data[:3] == b"\xff\xd8\xff", data[-2:] == b"\xff\xd9") +back = Image.open(io.BytesIO(data)) +print(back.format, back.mode, back.size) +mean, worst = error(rgb, back) +print("rgb close", mean < 4, worst < 60) + +gray = gradient("L", 33, 17) +buf = io.BytesIO() +gray.save(buf, "JPEG") +g = Image.open(io.BytesIO(buf.getvalue())) +print(g.format, g.mode, g.size) +mean, worst = error(gray, g) +print("gray close", mean < 4, worst < 40) + +low = io.BytesIO() +high = io.BytesIO() +rgb.save(low, "JPEG", quality=20) +rgb.save(high, "JPEG", quality=95) +print("quality orders size", len(low.getvalue()) < len(high.getvalue())) + +path = "pil_jpeg_conformance_tmp.jpg" +rgb.save(path) +with Image.open(path) as f: + print(f.format, f.mode, f.size) +os.remove(path) + +again = io.BytesIO() +back.save(again, "JPEG") +print("re-encode", Image.open(io.BytesIO(again.getvalue())).size) + +for mode in ("RGBA", "P"): + try: + Image.new(mode, (2, 2)).save(io.BytesIO(), "JPEG") + print(mode, "saved") + except OSError as e: + print(mode, "OSError", e) diff --git a/scripts/python-backend/build.sh b/scripts/python-backend/build.sh index 52313b1bc3d..10ac4265a7f 100755 --- a/scripts/python-backend/build.sh +++ b/scripts/python-backend/build.sh @@ -68,7 +68,8 @@ public class PyBackendMain { } JAVA # The runtime, the one core package it uses, and the host file system shim. -# PyImageNative is PIL's codec over the client UI, which a server lacks. +# PyImageNative is PIL's GIF/BMP/WEBP codec over the client UI, which a server +# lacks; PNG and JPEG are com.codename1.image, which the backend jar carries. (cd "$CLASSES" && unzip -qo "$RUNTIME" 'com/*' -x 'com/codename1/python/runtime/PyImageNative*' \ && unzip -qo "$CORE" 'com/codename1/util/*') "$J17/bin/javac" -nowarn --release 17 -d "$CLASSES" $(find "$REPO/scripts/python-bench/hostfs" -name '*.java') From 1316815d1b6ad6fb1ce1e83da6bfae92e94c5711 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 13:53:55 +0200 Subject: [PATCH 075/112] Core: drop a redundant qualified name Co-Authored-By: Claude Opus 5.5 --- CodenameOne/src/com/codename1/impl/AbstractDBCursor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CodenameOne/src/com/codename1/impl/AbstractDBCursor.java b/CodenameOne/src/com/codename1/impl/AbstractDBCursor.java index da639cbe588..57bec15951a 100644 --- a/CodenameOne/src/com/codename1/impl/AbstractDBCursor.java +++ b/CodenameOne/src/com/codename1/impl/AbstractDBCursor.java @@ -448,7 +448,7 @@ protected boolean legacyWasNullBeforeAnyRead() { /// `Database.COLUMN_UNKNOWN` (the default) where the engine cannot say. Only called on an /// open cursor positioned on a row, with a valid index. protected int columnType(int index) throws IOException { - return com.codename1.db.Database.COLUMN_UNKNOWN; + return Database.COLUMN_UNKNOWN; } @Override From fc2572059a7e19df81a515b88d8058ff2fdf8302 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 14:04:25 +0200 Subject: [PATCH 076/112] Python SQLAlchemy: Session compiles for its bind's dialect; typed DECIMAL/date columns from server databases Session._run executed statements without pushing the engine's dialect, so every ORM statement compiled as SQLite (CAST AS VARCHAR, || on MySQL). It was not a getattr bug in the runtime; getattr_method and sqlalchemy_dialect_compile hold that down. The PostgreSQL and MySQL drivers now record each column's kind on QueryRows, exposed as Database.columnTypes(List); the generated BackendDb passes it through PySql.Result.types and _cn1db.run, and _cn1dbapi turns DECIMAL/NUMERIC into decimal.Decimal and DATE/DATETIME/TIMESTAMP/TIME into datetime values, raw text() queries included. The realdb SQLAlchemy step also runs against the job's MySQL service. All four programs pass on mysql:8.4 (sslmode=verify-ca) and PostgreSQL 16. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/parparvm-tests.yml | 7 ++- .../com/codename1/python/runtime/PySql.java | 5 ++ .../codename1/python/runtime/StdModules.java | 13 ++++- .../transpiler/api/PythonTranspiler.java | 3 + .../python/site/sqlalchemy/_cn1dbapi.py | 36 +++++++++++- .../python/site/sqlalchemy/orm/__init__.py | 10 +++- .../resources/conformance/getattr_method.out | 15 +++++ .../resources/conformance/getattr_method.py | 55 +++++++++++++++++++ .../sqlalchemy_dialect_compile.out | 3 + .../conformance/sqlalchemy_dialect_compile.py | 9 +++ .../src/com/codename1/backend/Database.java | 18 ++++++ .../src/com/codename1/backend/sql/MySql.java | 29 +++++++++- .../com/codename1/backend/sql/Postgres.java | 24 ++++++++ .../com/codename1/backend/sql/QueryRows.java | 11 ++++ 14 files changed, 231 insertions(+), 7 deletions(-) create mode 100644 maven/python-transpiler/src/test/resources/conformance/getattr_method.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/getattr_method.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/sqlalchemy_dialect_compile.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/sqlalchemy_dialect_compile.py diff --git a/.github/workflows/parparvm-tests.yml b/.github/workflows/parparvm-tests.yml index 25d78d3d1bb..bdcd4dbe7be 100644 --- a/.github/workflows/parparvm-tests.yml +++ b/.github/workflows/parparvm-tests.yml @@ -471,9 +471,11 @@ jobs: # The SQLAlchemy conformance programs against the PostgreSQL service # above: the suite in python-transpiler runs them on SQLite only, so this # is the one place the postgresql dialect and the _cn1db bridge meet a - # real server. PostgreSQL only; the MySQL family is run by hand. + # real server. PostgreSQL and the job's MySQL service; MariaDB is run by + # hand. Last of the steps using MySQL: each program drops and recreates + # the cn1 database. # python-runtime compiles at release 17, hence JDK 17 for the install. - - name: Python SQLAlchemy on PostgreSQL + - name: Python SQLAlchemy on PostgreSQL and MySQL run: | (cd maven && bash "$GITHUB_WORKSPACE/scripts/ci/retry.sh" mvn -B -pl python-runtime,python-transpiler \ install -DskipTests -Dspotbugs.skip=true -Dpmd.skip=true -Dcheckstyle.skip=true \ @@ -483,6 +485,7 @@ jobs: JAVA_HOME: ${{ env.JDK_17_HOME }} JAVA17_HOME: ${{ env.JDK_17_HOME }} CN1_SQLA_POSTGRES: postgres://cn1:cn1secret@127.0.0.1:5432/cn1?sslmode=disable + CN1_SQLA_MYSQL: mysql://cn1:cn1secret@127.0.0.1:3306/cn1?sslmode=disable # The Flask and FastAPI samples built NATIVELY (ParparVM, clang) and # driven over sockets by the same client as the JVM run. The only place diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PySql.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PySql.java index 1f2fe47e013..2d0e7431f7e 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PySql.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PySql.java @@ -81,6 +81,11 @@ public static final class Result { public final List rows = new ArrayList(); public long changes = -1; public long lastRowId = -1; + /// Per column, the kind of value a server database sent as text that + /// the DB-API layer turns into a Python type -- "decimal", "date", + /// "datetime", "datetimetz" or "time" -- or null; null altogether when + /// the engine reports none (SQLite). + public String[] types; } /// The value a SQLite engine that cannot report storage classes most diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java index 494919a1625..c4a03dcdd8d 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/StdModules.java @@ -459,8 +459,19 @@ private static PyModule sql(String moduleName, boolean server) { } rows[i] = new PyTuple(t); } + if (!server) { + // sqlite3 (_cn1sql): SQLite reports no column types. + return new PyTuple(new Object[] {new PyTuple(cols), new PyList(rows), PyInt.of(r.changes), + PyInt.of(r.lastRowId)}); + } + // _cn1db adds each column's kind (PySql.Result.types) for _cn1dbapi. + Object[] kinds = new Object[cols.length]; + for (int i = 0; i < kinds.length; i++) { + kinds[i] = r.types != null && i < r.types.length && r.types[i] != null + ? Py.str(r.types[i]) : Py.NONE; + } return new PyTuple(new Object[] {new PyTuple(cols), new PyList(rows), PyInt.of(r.changes), - PyInt.of(r.lastRowId)}); + PyInt.of(r.lastRowId), new PyTuple(kinds)}); })); sqlControl(m, "begin", 0); sqlControl(m, "commit", 1); diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java index 20e39c6c31a..f154faf4bc4 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java @@ -499,6 +499,9 @@ private static String backendDb() { + " }\n" + " r.rows.add(values);\n" + " }\n" + + " // DECIMAL, DATE and the like arrive as text; their kinds let\n" + + " // _cn1dbapi hand back decimal.Decimal and datetime values.\n" + + " r.types = Database.columnTypes(rows);\n" + " return r;\n" + " }\n\n" + " @Override\n" diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/_cn1dbapi.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/_cn1dbapi.py index 6d3d867de6a..d716d1abecb 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/_cn1dbapi.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/_cn1dbapi.py @@ -9,6 +9,8 @@ import sqlite3 as _sqlite3 import _cn1db +import datetime as _dt +import decimal as _decimal paramstyle = "qmark" @@ -74,6 +76,36 @@ def _named(sql, params): return "".join(out), tuple(values) +def _datetime(text): + # PostgreSQL writes a timestamptz offset as +HH when it has no minutes. + if len(text) > 3 and text[-3] in "+-" and text[-2:].isdigit(): + text += ":00" + return _dt.datetime.fromisoformat(text) + + +# A server database sends DECIMAL/NUMERIC and the date and time types as text; +# the backend reports each column's kind so they come back as the Python types +# psycopg2 and mysqlclient return. +_PARSERS = { + "decimal": _decimal.Decimal, + "date": _dt.date.fromisoformat, + "datetime": _datetime, + "datetimetz": _datetime, + "time": _dt.time.fromisoformat, +} + + +def _typed(rows, kinds): + parsers = [_PARSERS.get(k) if k is not None else None for k in kinds] + if not any(p is not None for p in parsers): + return list(rows) + out = [] + for row in rows: + out.append(tuple(p(v) if p is not None and isinstance(v, str) else v + for p, v in zip(parsers, row))) + return out + + class Cursor: def __init__(self, conn): self.connection = conn @@ -96,10 +128,10 @@ def execute(self, sql, params=()): _cn1db.begin(c._handle) c._in_tx = True try: - cols, rows, changes, rowid = _cn1db.run(c._handle, sql, tuple(params or ()), query) + cols, rows, changes, rowid, kinds = _cn1db.run(c._handle, sql, tuple(params or ()), query) except OSError as e: raise _error(e) - self._rows = list(rows) + self._rows = _typed(rows, kinds) if query else list(rows) self._at = 0 self.description = tuple((n, None, None, None, None, None, None) for n in cols) if query else None self.rowcount = len(self._rows) if query else changes diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/orm/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/orm/__init__.py index a60da5564f4..94d2e668005 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/orm/__init__.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/orm/__init__.py @@ -19,6 +19,7 @@ import datetime as _dt import typing as _typing +import sqlalchemy as _sa from sqlalchemy import (Column, ForeignKey, Integer, String, Float, Boolean, DateTime, Date, LargeBinary, MetaData, Table, TypeEngine, select, exc, _is_type, _to_type) @@ -1174,7 +1175,14 @@ def _eager_rels(self, stmt): return out def _run(self, statement, params): - res = statement._execute_on(self.connection(), params, self) + conn = self.connection() + # Compile for the bind's dialect, as Connection.execute does: without it a + # Session statement compiled as SQLite (CAST AS VARCHAR, || on MySQL). + _sa._active.append(conn.engine.dialect) + try: + res = statement._execute_on(conn, params, self) + finally: + _sa._active.pop() eager = self._eager_rels(statement) if not eager: return res diff --git a/maven/python-transpiler/src/test/resources/conformance/getattr_method.out b/maven/python-transpiler/src/test/resources/conformance/getattr_method.out new file mode 100644 index 00000000000..c27c4348823 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/getattr_method.out @@ -0,0 +1,15 @@ +Base True - +False common +dflt False +Child False special:1 +False common +dflt True +GrandChild False special:1 +False common +dflt True +False special:2 +False special:3 +False +fallback +special:x +special:x diff --git a/maven/python-transpiler/src/test/resources/conformance/getattr_method.py b/maven/python-transpiler/src/test/resources/conformance/getattr_method.py new file mode 100644 index 00000000000..63312cfa242 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/getattr_method.py @@ -0,0 +1,55 @@ +class Base: + name = "base" + + def common(self): + return "common" + + +class Child(Base): + name = "child" + + def special(self, x): + return "special:" + str(x) + + +class GrandChild(Child): + pass + + +class Holder: + def __init__(self, d): + self.dialect = d + + +class Ctx: + def __init__(self): + self.dialect = Base() + + def use(self, d): + self.dialect = d + return self + + +def compile_with(ctx): + fn = getattr(ctx.dialect, "special", None) + return fn("x") if fn is not None else "fallback" + + +for obj in (Base(), Child(), GrandChild()): + m = getattr(obj, "special", None) + print(type(obj).__name__, m is None, m(1) if m is not None else "-") + c = getattr(obj, "common", None) + print(c is None, c()) + print(getattr(obj, "missing", "dflt"), hasattr(obj, "special")) + +h = Holder(Child()) +f = getattr(h.dialect, "special", None) +print(f is None, f(2) if f is not None else "-") +h.dialect = GrandChild() +f = getattr(h.dialect, "special", None) +print(f is None, f(3) if f is not None else "-") +print(getattr(Child, "special", None) is None) +ctx = Ctx() +print(compile_with(ctx)) +print(compile_with(ctx.use(Child()))) +print(compile_with(ctx.use(GrandChild()))) diff --git a/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_dialect_compile.out b/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_dialect_compile.out new file mode 100644 index 00000000000..87e2fafee75 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_dialect_compile.out @@ -0,0 +1,3 @@ +mysql True True False False +postgresql False False True True +sqlite False False True True diff --git a/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_dialect_compile.py b/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_dialect_compile.py new file mode 100644 index 00000000000..c11b8189488 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/sqlalchemy_dialect_compile.py @@ -0,0 +1,9 @@ +from sqlalchemy import Column, Integer, String, MetaData, Table, cast, select +from sqlalchemy.dialects import mysql, postgresql, sqlite + +md = MetaData() +t = Table("t", md, Column("id", Integer, primary_key=True), Column("age", Integer)) +stmt = select(cast(t.c.age, String) + "y") +for d in (mysql.dialect(), postgresql.dialect(), sqlite.dialect()): + sql = str(stmt.compile(dialect=d)) + print(d.name, "concat(" in sql, "AS CHAR)" in sql, "AS VARCHAR)" in sql, "||" in sql) diff --git a/vm/backend/src/com/codename1/backend/Database.java b/vm/backend/src/com/codename1/backend/Database.java index fad5a15249e..e3e4d402a55 100644 --- a/vm/backend/src/com/codename1/backend/Database.java +++ b/vm/backend/src/com/codename1/backend/Database.java @@ -33,6 +33,7 @@ import com.codename1.backend.sql.Dialect; import com.codename1.backend.sql.MySql; import com.codename1.backend.sql.Postgres; +import com.codename1.backend.sql.QueryRows; /// One database API over SQLite, PostgreSQL and MySQL, chosen by URL. /// @@ -324,6 +325,23 @@ public List query(String sql, Object[] params) throws IOException { } } + /// The kind of each column of a result `query` returned, in the order the + /// statement projects them, for the columns whose values come back as text + /// that still has to be parsed: `"decimal"` (an exact DECIMAL or NUMERIC, + /// kept as its digits), `"date"`, `"datetime"` (no zone), `"datetimetz"` + /// (PostgreSQL's timestamp with time zone, which carries its offset) and + /// `"time"`. Every other column is null, because its value is already the + /// Long, Double, String or byte\[\] it is. + /// + /// Answers null for rows this class did not produce, and for SQLite, which + /// has no column types to report. + public static String[] columnTypes(List rows) { + if (rows instanceof QueryRows) { + return ((QueryRows) rows).getColumnTypes(); + } + return null; + } + private List queryUntraced(String sql, Object[] params) throws IOException { try { params = portableParameters(params); diff --git a/vm/backend/src/com/codename1/backend/sql/MySql.java b/vm/backend/src/com/codename1/backend/sql/MySql.java index fcd56dccc6e..9b3537ea71c 100644 --- a/vm/backend/src/com/codename1/backend/sql/MySql.java +++ b/vm/backend/src/com/codename1/backend/sql/MySql.java @@ -634,6 +634,13 @@ private long executePrepared(int statementId, Object[] params, Column[] columns, } } readPacket(); // EOF ending the definitions + if (rows != null) { + String[] kinds = new String[resultColumns]; + for (int iter = 0 ; iter < resultColumns ; iter++) { + kinds[iter] = kind(resultDefinitions[iter].type); + } + rows.setColumnTypes(kinds); + } while (true) { Packet packet = readPacket(); int marker = packet.body[0] & 0xff; @@ -655,7 +662,27 @@ private long executePrepared(int statementId, Object[] params, Column[] columns, } } - /// A binary row: a 0x00 marker, a null bitmap offset by two bits, then each + /// The portable column kind (`Database.columnTypes(List)`) of a binary + /// protocol type: only the kinds whose values arrive as text a caller must + /// parse. + private static String kind(int type) { + switch (type) { + case 0x00: // DECIMAL + case 0xf6: // NEWDECIMAL + return "decimal"; + case 0x0a: // DATE + return "date"; + case 0x0c: // DATETIME + case 0x07: // TIMESTAMP + return "datetime"; + case 0x0b: // TIME + return "time"; + default: + return null; + } + } + + /// A binary row: a 0x00 marker, a null bitmap offset by two bits, then each /// non-null value in its column's binary form. private static Map decodeBinaryRow(byte[] body, Column[] columns) throws IOException { Reader reader = new Reader(body); diff --git a/vm/backend/src/com/codename1/backend/sql/Postgres.java b/vm/backend/src/com/codename1/backend/sql/Postgres.java index fb157185d5a..98a003a3768 100644 --- a/vm/backend/src/com/codename1/backend/sql/Postgres.java +++ b/vm/backend/src/com/codename1/backend/sql/Postgres.java @@ -620,6 +620,11 @@ private Result collect(String sql) throws IOException { types[iter] = intAt(message.body, at + 6); at += 18; // + type size (2), modifier (4), format (2) } + String[] kinds = new String[columns]; + for (int iter = 0 ; iter < columns ; iter++) { + kinds[iter] = kind(types[iter]); + } + result.rows.setColumnTypes(kinds); break; } case DATA_ROW: { @@ -731,6 +736,25 @@ private static byte[] encodeParameter(Object value) { return Wire.utf8(String.valueOf(value)); } + /// The portable column kind (`Database.columnTypes(List)`) of a type OID: + /// only the kinds whose values arrive as text a caller must parse. + private static String kind(int typeOid) { + switch (typeOid) { + case 1700: + return "decimal"; + case 1082: + return "date"; + case 1114: + return "datetime"; + case 1184: + return "datetimetz"; + case 1083: + return "time"; + default: + return null; + } + } + /// Maps a text-format value to the same Java types the SQLite path returns. /// The OIDs are the stable built-in ones from pg_type; a type this does not /// know stays a String, which is what the server sent. diff --git a/vm/backend/src/com/codename1/backend/sql/QueryRows.java b/vm/backend/src/com/codename1/backend/sql/QueryRows.java index 7372d77a4c3..43ccccf820d 100644 --- a/vm/backend/src/com/codename1/backend/sql/QueryRows.java +++ b/vm/backend/src/com/codename1/backend/sql/QueryRows.java @@ -30,6 +30,7 @@ public final class QueryRows extends ArrayList { private static final long serialVersionUID = 1L; private int columnCount; + private String[] columnTypes; public QueryRows(int columnCount) { this.columnCount = columnCount; @@ -42,4 +43,14 @@ public int getColumnCount() { public void setColumnCount(int columnCount) { this.columnCount = columnCount; } + + /// The portable kind of each column, in projection order, or null when the + /// driver did not record them. See `Database.columnTypes(List)`. + public String[] getColumnTypes() { + return columnTypes == null ? null : (String[]) columnTypes.clone(); + } + + public void setColumnTypes(String[] columnTypes) { + this.columnTypes = columnTypes == null ? null : (String[]) columnTypes.clone(); + } } From e1ff7909c8ef046131218ca201e64d1a3cc8e6a1 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 14:05:52 +0200 Subject: [PATCH 077/112] python-runtime: exclude PySql.Result.types from UWF (written by the generated BackendDb) Co-Authored-By: Claude Opus 5.5 --- maven/python-runtime/spotbugs-exclude.xml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/maven/python-runtime/spotbugs-exclude.xml b/maven/python-runtime/spotbugs-exclude.xml index 93b4c09145e..a5106fbbcba 100644 --- a/maven/python-runtime/spotbugs-exclude.xml +++ b/maven/python-runtime/spotbugs-exclude.xml @@ -103,4 +103,12 @@ + + + + + + From ea628bbc3324f2421def647d39ced7af9c961c44 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 14:13:57 +0200 Subject: [PATCH 078/112] Backend: @VirtualThreadStack and the python.virtualThreadStack build hint The annotation is found across the class index like @OpenTelemetry, parsed at build time (bytes, KB or MB; a bad value fails naming the class), and the largest of several wins. The generated entry point raises the stack before the builder. Python backends take the size from the build hint (Maven, Gradle, and -vtstack / CN1_PY_VT_STACK in scripts/python-backend/build.sh), replacing the hard-coded 1MB, which stays the default. Co-Authored-By: Claude Opus 5.5 --- .../codename1/maven/PythonProjectBuilder.java | 29 +++++- .../RestControllerAnnotationProcessor.java | 50 +++++++++++ ...RestControllerAnnotationProcessorTest.java | 61 +++++++++++++ .../build/shared/BuildHintsGeneral.java | 10 +++ .../java/com/codename1/gradle/AppSupport.java | 8 +- .../gradle/tasks/TranspilePythonTask.java | 9 +- .../codename1/maven/TranspilePythonMojo.java | 4 +- .../codename1/python/transpiler/api/Main.java | 23 ++++- .../transpiler/api/PythonTranspiler.java | 62 ++++++++++++- .../transpiler/VirtualThreadStackTest.java | 89 +++++++++++++++++++ scripts/python-backend/build.sh | 5 +- .../annotations/VirtualThreadStack.java | 60 +++++++++++++ 12 files changed, 399 insertions(+), 11 deletions(-) create mode 100644 maven/python-transpiler/src/test/java/com/codename1/python/transpiler/VirtualThreadStackTest.java create mode 100644 vm/backend/src/com/codename1/backend/annotations/VirtualThreadStack.java diff --git a/maven/build-engine/src/main/java/com/codename1/maven/PythonProjectBuilder.java b/maven/build-engine/src/main/java/com/codename1/maven/PythonProjectBuilder.java index d1f684d7313..650ecb27af7 100644 --- a/maven/build-engine/src/main/java/com/codename1/maven/PythonProjectBuilder.java +++ b/maven/build-engine/src/main/java/com/codename1/maven/PythonProjectBuilder.java @@ -78,6 +78,20 @@ public PythonProjectBuilder(File sourceDir, File outputDir, String mainModule, L /// `java-runtime` supplies the device's `java.*`. The project's own Java /// sources are not /// compiled yet when Python is transpiled, so they cannot be imported. + /// The build hint naming the C stack a served request's virtual thread + /// gets, such as `2MB`; see [#virtualThreadStack(String)]. + public static final String VIRTUAL_THREAD_STACK_HINT = "codename1.arg.python.virtualThreadStack"; + + private String virtualThreadStack; + + /// The value of [#VIRTUAL_THREAD_STACK_HINT]: bytes, or a number with a + /// `KB` or `MB` suffix. Null or blank keeps the transpiler's 1MB default; + /// anything else that is not a size fails [#run()]. + public PythonProjectBuilder virtualThreadStack(String size) { + this.virtualThreadStack = size; + return this; + } + public PythonProjectBuilder classPath(java.util.List entries) { this.classPath = entries; return this; @@ -140,7 +154,7 @@ public boolean run() throws BuildException { } Map out; try { - out = PythonTranspiler.transpile(sources, entry, classPath, nativeModules); + out = PythonTranspiler.transpile(sources, entry, classPath, nativeModules, stackBytes()); } catch (PythonSyntaxError e) { throw new BuildException("src/main/python/" + e.file + ":" + e.line + ":" + e.col + ": " + e.reason, e); } catch (IllegalArgumentException e) { @@ -222,4 +236,17 @@ private static void listFiles(File dir, List into) { } } } + + private int stackBytes() throws BuildException { + if (virtualThreadStack == null || virtualThreadStack.trim().length() == 0) { + return PythonTranspiler.DEFAULT_VIRTUAL_THREAD_STACK; + } + int bytes = PythonTranspiler.parseStackSize(virtualThreadStack); + if (bytes <= 0) { + throw new BuildException(VIRTUAL_THREAD_STACK_HINT + "=" + virtualThreadStack + + " is not a stack size. Write a positive number of bytes, or a number followed by " + + "KB or MB, such as 2MB."); + } + return bytes; + } } diff --git a/maven/build-engine/src/main/java/com/codename1/maven/processors/RestControllerAnnotationProcessor.java b/maven/build-engine/src/main/java/com/codename1/maven/processors/RestControllerAnnotationProcessor.java index a7fcd73a662..718debf6294 100644 --- a/maven/build-engine/src/main/java/com/codename1/maven/processors/RestControllerAnnotationProcessor.java +++ b/maven/build-engine/src/main/java/com/codename1/maven/processors/RestControllerAnnotationProcessor.java @@ -91,6 +91,7 @@ public final class RestControllerAnnotationProcessor extends AbstractAnnotationP private static final String WEBSOCKET_MAPPING = PKG + "WebSocketMapping;"; private static final String WEBSOCKET_INTERFACE = "com/codename1/backend/WebSocket"; static final String OPEN_TELEMETRY = PKG + "OpenTelemetry;"; + static final String VIRTUAL_THREAD_STACK = PKG + "VirtualThreadStack;"; /// The application.properties key that enables tracing without touching the /// source, read at build time from the module directory -- where Config reads @@ -232,6 +233,10 @@ public void setDevTools(boolean devTools) { /// and MCP endpoints. Settled in [#finish] beside [#telemetry]. BackendSettings settings; + /// The C stack `@VirtualThreadStack` asks for, in bytes, or 0 when no class + /// asks. Settled in [#finish] beside [#telemetry]. + int virtualThreadStack; + private final Map routeShapes = new LinkedHashMap(); /** Which controller claimed each shape, so a clash names the other one. */ @@ -1536,6 +1541,7 @@ public void finish(ProcessorContext ctx) throws ProcessingException { return; } resolveTelemetry(ctx); + resolveVirtualThreadStack(ctx); if (ctx.hasErrors()) { return; } @@ -3185,6 +3191,12 @@ String generateBootstrap(String packageName) { // runtime's internal access, so none of them is public API. sb.append(" com.codename1.impl.backend.BackendAccess cn1Access =\n") .append(" com.codename1.impl.backend.BackendAccess.get();\n"); + if (virtualThreadStack > 0) { + // Before the builder exists, so before any server can start: the + // runtime reads the size when it creates a connection's thread. + sb.append(" com.codename1.backend.HttpServer.requireVirtualThreadStack(") + .append(virtualThreadStack).append(");\n"); + } sb.append(" com.codename1.backend.Backend.Builder cn1Builder = " + "com.codename1.backend.Backend.builder()"); if (telemetry) { @@ -3351,6 +3363,44 @@ void resolveTelemetry(ProcessorContext ctx) { } } + /// The C stack `@VirtualThreadStack` asks for, across the whole class index + /// the way [#resolveTelemetry] looks for `@OpenTelemetry`. + /// + /// Several classes may carry it and the LARGEST size wins rather than the + /// build failing: the runtime call only ever raises the size, so a module + /// that combines two libraries each asking for its own floor gets what both + /// need. A value that is not a size is a build error naming the class. + void resolveVirtualThreadStack(ProcessorContext ctx) { + virtualThreadStack = 0; + for (AnnotatedClass cls : ctx.getClassIndex().values()) { + AnnotationValues vts = cls.getClassAnnotation(VIRTUAL_THREAD_STACK); + if (vts == null) { + continue; + } + if (!BuildHintAnnotationProcessor.hasBackingSource(cls, ctx.getCompileSourceRoots(), + ctx.getSourceEncoding())) { + continue; + } + String raw = vts.getStringOrDefault("value", ""); + int bytes = parseStackSize(raw); + if (bytes <= 0) { + ctx.error(cls, "@VirtualThreadStack(\"" + raw + "\") on " + cls.getBinaryName() + + " is not a stack size. Write a positive number of bytes, or a number " + + "followed by KB or MB, such as \"512KB\" or \"1MB\"."); + return; + } + if (bytes > virtualThreadStack) { + virtualThreadStack = bytes; + } + } + } + + /// Parses a size the way the Python build hint does, so the annotation and + /// the hint accept exactly the same spellings. + static int parseStackSize(String raw) { + return com.codename1.python.transpiler.api.PythonTranspiler.parseStackSize(raw); + } + /// `cn1.otel.enabled` from the module's application properties, the files /// Config reads at run time. Only a literal truth value counts: a /// `${...}` reference cannot be resolved at build time, and building the diff --git a/maven/build-engine/src/test/java/com/codename1/maven/processors/RestControllerAnnotationProcessorTest.java b/maven/build-engine/src/test/java/com/codename1/maven/processors/RestControllerAnnotationProcessorTest.java index 152cb7bea1f..f5bb2d57698 100644 --- a/maven/build-engine/src/test/java/com/codename1/maven/processors/RestControllerAnnotationProcessorTest.java +++ b/maven/build-engine/src/test/java/com/codename1/maven/processors/RestControllerAnnotationProcessorTest.java @@ -308,6 +308,67 @@ public void aBlankServiceNameIsNoServiceName() throws Exception { .indexOf(".tracing(new com.codename1.backend.otel.OtlpTracer(null))") >= 0); } + /// Resolves `@VirtualThreadStack` the way the build does and returns the + /// processor, with the context's errors left for the caller to read. + private RestControllerAnnotationProcessor resolveStack(String source, ProcessorContext[] out) + throws Exception { + File classes = compile(source); + File root = sourceRootWith("Notes.java", source); + ProcessorContext ctx = new ProcessorContext(classes, tmp.newFolder(), + ClassScanner.scan(classes), new SystemStreamLog(), tmp.newFolder(), + new Properties(), null, Collections.singletonList(root.getAbsolutePath()), + "UTF-8", Collections.emptyList()); + RestControllerAnnotationProcessor proc = new RestControllerAnnotationProcessor(); + proc.resolveVirtualThreadStack(ctx); + out[0] = ctx; + return proc; + } + + @Test + public void virtualThreadStackRaisesTheStackInTheEntryPoint() throws Exception { + ProcessorContext[] ctx = new ProcessorContext[1]; + RestControllerAnnotationProcessor proc = resolveStack(CONTROLLER_SOURCE.replace( + "@RestController\n", "@RestController\n@VirtualThreadStack(\"2mb\")\n"), ctx); + assertFalse(String.valueOf(ctx[0].getErrors()), ctx[0].hasErrors()); + assertEquals(2 * 1024 * 1024, proc.virtualThreadStack); + String boot = proc.generateBootstrap("com.example"); + int call = boot.indexOf( + "com.codename1.backend.HttpServer.requireVirtualThreadStack(2097152);"); + assertTrue(boot, call >= 0); + assertTrue("raised before the builder can start a server", + call < boot.indexOf("cn1Builder = ")); + } + + @Test + public void noVirtualThreadStackLeavesTheRuntimeDefault() throws Exception { + ProcessorContext[] ctx = new ProcessorContext[1]; + RestControllerAnnotationProcessor proc = resolveStack(CONTROLLER_SOURCE, ctx); + assertEquals(0, proc.virtualThreadStack); + assertTrue(proc.generateBootstrap("com.example") + .indexOf("requireVirtualThreadStack") < 0); + } + + @Test + public void aBadVirtualThreadStackFailsTheBuildNamingTheClass() throws Exception { + ProcessorContext[] ctx = new ProcessorContext[1]; + resolveStack(CONTROLLER_SOURCE.replace( + "@RestController\n", "@RestController\n@VirtualThreadStack(\"lots\")\n"), ctx); + assertTrue(ctx[0].hasErrors()); + assertTrue(String.valueOf(ctx[0].getErrors()), + String.valueOf(ctx[0].getErrors()).indexOf("com.example.Notes") >= 0); + } + + @Test + public void stackSizesParse() { + assertEquals(4096, RestControllerAnnotationProcessor.parseStackSize("4096")); + assertEquals(512 * 1024, RestControllerAnnotationProcessor.parseStackSize(" 512 KB ")); + assertEquals(1024 * 1024, RestControllerAnnotationProcessor.parseStackSize("1MB")); + assertEquals(-1, RestControllerAnnotationProcessor.parseStackSize("0")); + assertEquals(-1, RestControllerAnnotationProcessor.parseStackSize("-1MB")); + assertEquals(-1, RestControllerAnnotationProcessor.parseStackSize("4096MB")); + assertEquals(-1, RestControllerAnnotationProcessor.parseStackSize("1GB")); + } + @Test public void anUntracedBuildNamesRoutesButNeverLinksTheTracer() throws Exception { // The same controller without the annotation. The OTLP tracer is not diff --git a/maven/build-hint-catalog/src/main/java/com/codename1/build/shared/BuildHintsGeneral.java b/maven/build-hint-catalog/src/main/java/com/codename1/build/shared/BuildHintsGeneral.java index e19b9ccbf17..8641c01f61b 100644 --- a/maven/build-hint-catalog/src/main/java/com/codename1/build/shared/BuildHintsGeneral.java +++ b/maven/build-hint-catalog/src/main/java/com/codename1/build/shared/BuildHintsGeneral.java @@ -307,6 +307,16 @@ static void register(List h) { .type(HintType.STRING) .platform("general")); + h.add(new Hint("python.virtualThreadStack") + .group(HintGroup.GENERAL) + .type(HintType.STRING) + .def("1MB") + .platform("general") + .doc("The C stack each request's virtual thread gets in a Python backend: bytes, " + + "or a number followed by `KB` or `MB`, such as `2MB`. Raise it when a " + + "handler recurses deeply. `CN1_HTTP_VT_STACK` can still raise it at " + + "start-up; the larger of the two applies.")); + h.add(new Hint("watchMain") .group(HintGroup.GENERAL) .type(HintType.STRING) diff --git a/maven/codenameone-gradle-plugin/src/main/java/com/codename1/gradle/AppSupport.java b/maven/codenameone-gradle-plugin/src/main/java/com/codename1/gradle/AppSupport.java index fdcdfc30f17..70dc77266ea 100644 --- a/maven/codenameone-gradle-plugin/src/main/java/com/codename1/gradle/AppSupport.java +++ b/maven/codenameone-gradle-plugin/src/main/java/com/codename1/gradle/AppSupport.java @@ -161,8 +161,11 @@ static void apply(final Project project, final ProjectLayout layout, final Coden addFramework(project, "implementation", ext.getVersion(), com.codename1.maven.PythonProjectBuilder.RUNTIME_ARTIFACT); } - final String pythonMain = AppSettings.read(layout.settingsFile()) + final java.util.Properties pythonSettings = AppSettings.read(layout.settingsFile()); + final String pythonMain = pythonSettings .getProperty(com.codename1.maven.PythonProjectImporter.MAIN_MODULE_SETTING); + final String pythonStack = pythonSettings + .getProperty(com.codename1.maven.PythonProjectBuilder.VIRTUAL_THREAD_STACK_HINT); TaskProvider python = project.getTasks().register( "transpilePython", com.codename1.gradle.tasks.TranspilePythonTask.class, t -> { common(t, project, layout, ext, userProperties); @@ -172,6 +175,9 @@ static void apply(final Project project, final ProjectLayout layout, final Coden if (pythonMain != null) { t.getMainModule().set(pythonMain); } + if (pythonStack != null) { + t.getVirtualThreadStack().set(pythonStack); + } t.getJavaClassPath().from(project.getConfigurations().getByName("compileClasspath")); t.getOutputDirectory().set(new File(layout.buildDir(), "generated/sources/cn1-python")); }); diff --git a/maven/codenameone-gradle-plugin/src/main/java/com/codename1/gradle/tasks/TranspilePythonTask.java b/maven/codenameone-gradle-plugin/src/main/java/com/codename1/gradle/tasks/TranspilePythonTask.java index 96b81f5cd44..7ae3be19db5 100644 --- a/maven/codenameone-gradle-plugin/src/main/java/com/codename1/gradle/tasks/TranspilePythonTask.java +++ b/maven/codenameone-gradle-plugin/src/main/java/com/codename1/gradle/tasks/TranspilePythonTask.java @@ -64,6 +64,12 @@ public abstract class TranspilePythonTask extends Cn1Task { @Classpath public abstract ConfigurableFileCollection getJavaClassPath(); + /// The `codename1.arg.python.virtualThreadStack` build hint, or unset for + /// the transpiler's default. + @Input + @Optional + public abstract Property getVirtualThreadStack(); + @OutputDirectory public abstract DirectoryProperty getOutputDirectory(); @@ -72,7 +78,8 @@ public void transpile() { try { new PythonProjectBuilder(getPythonDirectory().get().getAsFile(), getOutputDirectory().get().getAsFile(), getMainModule().getOrNull(), log()) - .classPath(new java.util.ArrayList(getJavaClassPath().getFiles())).run(); + .classPath(new java.util.ArrayList(getJavaClassPath().getFiles())) + .virtualThreadStack(getVirtualThreadStack().getOrNull()).run(); } catch (BuildException ex) { throw new GradleException(ex.getMessage(), ex.getCause() == null ? ex : ex.getCause()); } diff --git a/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/TranspilePythonMojo.java b/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/TranspilePythonMojo.java index 06fb18645c4..2a8f46f119e 100644 --- a/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/TranspilePythonMojo.java +++ b/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/TranspilePythonMojo.java @@ -69,7 +69,9 @@ protected void executeImpl() throws MojoExecutionException, MojoFailureException entry = properties.getProperty(PythonProjectImporter.MAIN_MODULE_SETTING); } PythonProjectBuilder builder = new PythonProjectBuilder(pythonSourceDir, pythonOutputDir, entry, - MavenLog.of(getLog())).classPath(compileClassPath()); + MavenLog.of(getLog())).classPath(compileClassPath()) + .virtualThreadStack(properties == null ? null + : properties.getProperty(PythonProjectBuilder.VIRTUAL_THREAD_STACK_HINT)); if (PythonProjectBuilder.isPythonProject(pythonSourceDir)) { checkJdk(); checkRuntimeDependency(); diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/Main.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/Main.java index 401b4b8ad9b..3217148b305 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/Main.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/Main.java @@ -36,7 +36,11 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -/// Command line: `Main [-cp ] [main module]`. +/// Command line: `Main [-cp ] [-vtstack ] [main module]`. +/// +/// `-vtstack` is the C stack a served request's virtual thread gets, as the +/// `python.virtualThreadStack` build hint takes it: bytes, or a number with a +/// `KB` or `MB` suffix. 1MB when not given. /// /// Transpiles every `.py` under the source root and writes the Java sources /// under the output root. Exit status is 0 on success, 1 when a source does @@ -58,8 +62,20 @@ public static void main(String[] args) throws IOException { System.arraycopy(args, 2, rest, 0, rest.length); args = rest; } + int stack = PythonTranspiler.DEFAULT_VIRTUAL_THREAD_STACK; + if (args.length >= 2 && "-vtstack".equals(args[0])) { + stack = PythonTranspiler.parseStackSize(args[1]); + if (stack <= 0) { + System.err.println("-vtstack " + args[1] + " is not a stack size (bytes, or a number with KB or MB)"); + System.exit(2); + return; + } + String[] rest = new String[args.length - 2]; + System.arraycopy(args, 2, rest, 0, rest.length); + args = rest; + } if (args.length < 2 || args.length > 3) { - System.err.println("usage: Main [-cp ] [main module]"); + System.err.println("usage: Main [-cp ] [-vtstack ] [main module]"); System.exit(2); return; } @@ -75,7 +91,8 @@ public static void main(String[] args) throws IOException { } Map out; try { - out = PythonTranspiler.transpile(sources, args.length == 3 ? args[2] : null, classPath); + out = PythonTranspiler.transpile(sources, args.length == 3 ? args[2] : null, classPath, + java.util.Collections.emptyMap(), stack); } catch (PythonSyntaxError e) { System.err.println(e.getMessage()); System.exit(1); diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java index f154faf4bc4..2892fe6d3cb 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java @@ -111,6 +111,62 @@ public static Map transpile(Map sources, String /// installs each before any module runs public static Map transpile(Map sources, String mainModule, List classPath, Map nativeModules) { + return transpile(sources, mainModule, classPath, nativeModules, DEFAULT_VIRTUAL_THREAD_STACK); + } + + /// The C stack a served request's virtual thread gets unless the build asks + /// for another: transpiled Python frames are large and `re` recurses, and + /// the backend's own 64KB default overflowed on a FastAPI route match. + public static final int DEFAULT_VIRTUAL_THREAD_STACK = 1024 * 1024; + + /// Parses a stack size: `1048576`, `1024KB` or `1MB` (suffix case ignored, + /// blanks around the number allowed). -1 for anything else, for zero, or + /// for a size that does not fit an int. + /// + /// @param raw the size as written + /// @return the size in bytes, or -1 + public static int parseStackSize(String raw) { + String s = raw == null ? "" : raw.trim(); + long unit = 1; + if (s.length() > 2) { + String suffix = s.substring(s.length() - 2); + if ("KB".equalsIgnoreCase(suffix)) { + unit = 1024; + } else if ("MB".equalsIgnoreCase(suffix)) { + unit = 1024 * 1024; + } + if (unit > 1) { + s = s.substring(0, s.length() - 2).trim(); + } + } + if (s.length() == 0 || s.length() > 10) { + return -1; + } + long n = 0; + for (int i = 0; i < s.length(); i++) { + char c = s.charAt(i); + if (c < '0' || c > '9') { + return -1; + } + n = n * 10 + (c - '0'); + } + long bytes = n * unit; + return bytes <= 0 || bytes > Integer.MAX_VALUE ? -1 : (int) bytes; + } + + /// @param sources relative `.py` path to source text + /// @param mainModule the module run as `__main__`, or null for a library + /// @param classPath as for [#transpile(Map, String, List)] + /// @param nativeModules as for [#transpile(Map, String, List, Map)] + /// @param virtualThreadStack the bytes of C stack the generated server + /// asks each request's virtual thread to have (the build hint + /// `python.virtualThreadStack`); the backend only ever raises its + /// size, so `CN1_HTTP_VT_STACK` can still ask for more at start-up + public static Map transpile(Map sources, String mainModule, + List classPath, Map nativeModules, int virtualThreadStack) { + if (virtualThreadStack <= 0) { + throw new IllegalArgumentException("virtual thread stack must be positive: " + virtualThreadStack); + } Map out = new LinkedHashMap(); Map classes = new LinkedHashMap(); Map trees = new LinkedHashMap(); @@ -193,7 +249,7 @@ public static Map transpile(Map sources, String out.put(PACKAGE.replace('.', '/') + "/JavaInterop.java", interop); } if (server) { - out.put(PACKAGE.replace('.', '/') + "/BackendServer.java", backendServer()); + out.put(PACKAGE.replace('.', '/') + "/BackendServer.java", backendServer(virtualThreadStack)); } if (db) { out.put(PACKAGE.replace('.', '/') + "/BackendDb.java", backendDb()); @@ -530,7 +586,7 @@ private static String backendDb() { /// slow streamed response -- sleeping between chunks, or draining to a /// slow client -- leaves the other workers free. Background tasks run /// once the response has been handed back or its last chunk written. - private static String backendServer() { + private static String backendServer(int virtualThreadStack) { return "// Generated by the Codename One Python transpiler.\n" + "package " + PACKAGE + ";\n\n" + "import com.codename1.backend.HttpServer;\n" @@ -566,7 +622,7 @@ private static String backendServer() { // Transpiled Python frames are large and re recurses: the // backend's 64KB virtual-thread stack overflowed on a route // match. Mapped lazily, so this is address space, not memory. - + " HttpServer.requireVirtualThreadStack(1024 * 1024);\n" + + " HttpServer.requireVirtualThreadStack(" + virtualThreadStack + ");\n" + " PyServer.install(new BackendServer());\n" + " }\n\n" + " @Override\n" diff --git a/maven/python-transpiler/src/test/java/com/codename1/python/transpiler/VirtualThreadStackTest.java b/maven/python-transpiler/src/test/java/com/codename1/python/transpiler/VirtualThreadStackTest.java new file mode 100644 index 00000000000..638675c503a --- /dev/null +++ b/maven/python-transpiler/src/test/java/com/codename1/python/transpiler/VirtualThreadStackTest.java @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.transpiler; + +import com.codename1.python.transpiler.api.PythonTranspiler; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collections; +import java.util.Map; +import java.util.TreeMap; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/// The `python.virtualThreadStack` build hint reaches the generated server. +class VirtualThreadStackTest { + @TempDir + Path tmp; + + /// A served program, transpiled against a class path that has the + /// backend's `HttpServer` -- the one thing that makes it a backend build. + private String backendServer(int stack) throws Exception { + Path cls = tmp.resolve("com/codename1/backend/HttpServer.class"); + if (!Files.exists(cls)) { + // A stand-in class: only its presence is read. + Path src = tmp.resolve("src/com/codename1/backend/HttpServer.java"); + Files.createDirectories(src.getParent()); + Files.write(src, "package com.codename1.backend; public class HttpServer { }" + .getBytes(java.nio.charset.StandardCharsets.US_ASCII)); + assertEquals(0, javax.tools.ToolProvider.getSystemJavaCompiler().run(null, null, null, + "-d", tmp.toString(), src.toString())); + } + Map sources = new TreeMap(); + sources.put("main.py", "import _cn1server\n"); + Map out = PythonTranspiler.transpile(sources, "main", + Collections.singletonList(tmp.toFile()), Collections.emptyMap(), stack); + String server = out.get("com/codename1/generated/python/BackendServer.java"); + assertTrue(server != null, String.valueOf(out.keySet())); + return server; + } + + @Test + void theDefaultIsOneMegabyte() throws Exception { + assertTrue(backendServer(PythonTranspiler.DEFAULT_VIRTUAL_THREAD_STACK) + .contains("HttpServer.requireVirtualThreadStack(1048576);")); + } + + @Test + void theHintSetsTheStack() throws Exception { + String server = backendServer(PythonTranspiler.parseStackSize("4MB")); + assertTrue(server.contains("HttpServer.requireVirtualThreadStack(4194304);"), server); + } + + @Test + void sizesParse() { + assertEquals(4096, PythonTranspiler.parseStackSize("4096")); + assertEquals(512 * 1024, PythonTranspiler.parseStackSize(" 512 kb ")); + assertEquals(2 * 1024 * 1024, PythonTranspiler.parseStackSize("2MB")); + assertEquals(-1, PythonTranspiler.parseStackSize("")); + assertEquals(-1, PythonTranspiler.parseStackSize("0KB")); + assertEquals(-1, PythonTranspiler.parseStackSize("1.5MB")); + assertEquals(-1, PythonTranspiler.parseStackSize("4096MB")); + assertEquals(-1, PythonTranspiler.parseStackSize(new File("x").getName())); + } +} diff --git a/scripts/python-backend/build.sh b/scripts/python-backend/build.sh index 10ac4265a7f..e46f005d6cc 100755 --- a/scripts/python-backend/build.sh +++ b/scripts/python-backend/build.sh @@ -52,8 +52,11 @@ GEN="$W/gen" CLASSES="$W/classes" DEMO="$W/demo" mkdir -p "$GEN" "$CLASSES" "$DEMO" +# CN1_PY_VT_STACK: the C stack a request's virtual thread gets, as the +# codename1.arg.python.virtualThreadStack build hint takes it (2MB, 512KB, +# bytes). Unset keeps the transpiler's 1MB. "$J17/bin/java" -cp "$TRANSPILER" com.codename1.python.transpiler.api.Main -cp "$JAVAAPI:$CORE:$BACKEND" \ - "$APP/src" "$GEN" main + ${CN1_PY_VT_STACK:+-vtstack "$CN1_PY_VT_STACK"} "$APP/src" "$GEN" main [ -f "$GEN/com/codename1/generated/python/BackendServer.java" ] || { echo "the application does not serve anything (no Flask/FastAPI run): nothing to build" >&2; exit 1; } mkdir -p "$GEN/com/pybackend" diff --git a/vm/backend/src/com/codename1/backend/annotations/VirtualThreadStack.java b/vm/backend/src/com/codename1/backend/annotations/VirtualThreadStack.java new file mode 100644 index 00000000000..70309425a69 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/annotations/VirtualThreadStack.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.annotations; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/// Gives every connection's virtual thread at least this much C stack. +/// +/// A virtual thread's C stack holds native call depth, not Java data, and the +/// default of 64KB is far more than an ordinary handler reaches. A handler that +/// recurses deeply -- a recursive descent parser, a deep template, a regular +/// expression engine -- can still run out, and an overflow there kills the +/// connection's host thread with nothing but an empty reply to show for it. +/// Put this on any class in the backend module to raise the floor: +/// +/// ```java +/// @VirtualThreadStack("1MB") +/// @RestController +/// public class ReportController { ... } +/// ``` +/// +/// The size is a number of bytes, optionally followed by `KB` or `MB` (case +/// does not matter; `1KB` is 1024 bytes). It is read at build time, so a value +/// that is not a size fails the build and names the class it sits on. When +/// several classes carry the annotation the largest size wins, the same way +/// `HttpServer.requireVirtualThreadStack` only ever raises the size. +/// +/// The `CN1_HTTP_VT_STACK` environment variable still applies at start-up, and +/// the larger of the two is used. The stack is mapped lazily, so a generous +/// size costs address space rather than memory. +@Retention(RetentionPolicy.CLASS) +@Target(ElementType.TYPE) +public @interface VirtualThreadStack { + /// The minimum C stack per virtual thread: bytes, or a number with a `KB` + /// or `MB` suffix, such as `"512KB"` or `"1MB"`. + String value(); +} From 98c30a67d37169b4067bc22a93716880cfe23a8b Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 14:16:06 +0200 Subject: [PATCH 079/112] Docs: database TLS modes, typed columns, virtual-thread stack size Backend-Data: sslmode/sslrootcert (verify-ca verifies the chain only, require/verify-full the chain and host) and MySQL's ssl-mode/ssl-ca spellings; Database.columnTypes for DECIMAL/NUMERIC/date columns. Backend-Operations: @VirtualThreadStack and CN1_HTTP_VT_STACK. Python-Interop: the python.virtualThreadStack build hint. Snippets live in docs/demos and are included by tag. Co-Authored-By: Claude Opus 5.5 --- .../backend/DatabaseSnippets.java | 10 +++++ .../developerguide/backend/DeepReports.java | 42 +++++++++++++++++++ .../developer-guide/backend-web.properties | 7 ++++ .../snippets/developer-guide/backend-web.sh | 4 ++ .../developer-guide/python-backend.properties | 5 +++ docs/developer-guide/Backend-Data.asciidoc | 41 ++++++++++++++++++ .../Backend-Operations.asciidoc | 24 +++++++++++ docs/developer-guide/Python-Interop.asciidoc | 15 +++++++ 8 files changed, 148 insertions(+) create mode 100644 docs/demos/backend/src/main/java/com/codenameone/developerguide/backend/DeepReports.java create mode 100644 docs/demos/common/src/main/snippets/developer-guide/python-backend.properties diff --git a/docs/demos/backend/src/main/java/com/codenameone/developerguide/backend/DatabaseSnippets.java b/docs/demos/backend/src/main/java/com/codenameone/developerguide/backend/DatabaseSnippets.java index cbb5faafc66..9e8cf0522a2 100644 --- a/docs/demos/backend/src/main/java/com/codenameone/developerguide/backend/DatabaseSnippets.java +++ b/docs/demos/backend/src/main/java/com/codenameone/developerguide/backend/DatabaseSnippets.java @@ -44,6 +44,16 @@ public static List open(DataSource db) throws IOException { return rows; } + public static String[] types(DataSource db) throws IOException { +// tag::backend-database-column-types[] +List rows = db.query("SELECT total, placed FROM orders", new Object[0]); +String[] types = Database.columnTypes(rows); +// types[0] is "decimal": the value is a String of digits, such as "19.90" +// types[1] is "date", "datetime" or "datetimetz", depending on the column +// end::backend-database-column-types[] + return types; + } + public static long insert(DataSource db) throws IOException { // tag::backend-database-insert[] long id = db.insert("INSERT INTO note (body) VALUES (?)", diff --git a/docs/demos/backend/src/main/java/com/codenameone/developerguide/backend/DeepReports.java b/docs/demos/backend/src/main/java/com/codenameone/developerguide/backend/DeepReports.java new file mode 100644 index 00000000000..a416ff75f81 --- /dev/null +++ b/docs/demos/backend/src/main/java/com/codenameone/developerguide/backend/DeepReports.java @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codenameone.developerguide.backend; + +import com.codename1.backend.annotations.GetMapping; +import com.codename1.backend.annotations.RestController; +import com.codename1.backend.annotations.VirtualThreadStack; + +// tag::backend-virtual-thread-stack[] +@VirtualThreadStack("1MB") +@RestController +public class DeepReports { + @GetMapping("/reports/depth") + public String depth() { + return String.valueOf(nest(200)); + } + + private static int nest(int n) { + return n == 0 ? 0 : 1 + nest(n - 1); + } +} +// end::backend-virtual-thread-stack[] diff --git a/docs/demos/common/src/main/snippets/developer-guide/backend-web.properties b/docs/demos/common/src/main/snippets/developer-guide/backend-web.properties index 4f4ce443bd4..1aa3131dbe3 100644 --- a/docs/demos/common/src/main/snippets/developer-guide/backend-web.properties +++ b/docs/demos/common/src/main/snippets/developer-guide/backend-web.properties @@ -5,3 +5,10 @@ cn1.cors.allowedOrigins=https://app.example.com cn1.cors.allowCredentials=true cn1.cors.exposedHeaders=X-Request-Id // end::backend-web-properties-001[] + +// tag::backend-database-tls-001[] +# PostgreSQL behind a private CA: the chain is verified, the host name is not +cn1.datasource.url=postgres://app:secret@db.internal:5432/app?sslmode=verify-ca&sslrootcert=/etc/ssl/db-ca.pem +# MySQL's own spellings work too: chain and host name both verified +cn1.datasource.url=mysql://app:secret@db.internal/app?ssl-mode=VERIFY_IDENTITY&ssl-ca=/etc/ssl/db-ca.pem +// end::backend-database-tls-001[] diff --git a/docs/demos/common/src/main/snippets/developer-guide/backend-web.sh b/docs/demos/common/src/main/snippets/developer-guide/backend-web.sh index 1c30456e877..229ab5a6cf0 100644 --- a/docs/demos/common/src/main/snippets/developer-guide/backend-web.sh +++ b/docs/demos/common/src/main/snippets/developer-guide/backend-web.sh @@ -9,3 +9,7 @@ mvn -pl backend -am -Dcodename1.platform=backend cn1:backend <2> ./gradlew :backend:backendWebApp <1> ./gradlew :backend:runBackend <2> // end::backend-web-hosting-gradle[] + +// tag::backend-vt-stack-sh-001[] +CN1_HTTP_VT_STACK=1048576 ./server +// end::backend-vt-stack-sh-001[] diff --git a/docs/demos/common/src/main/snippets/developer-guide/python-backend.properties b/docs/demos/common/src/main/snippets/developer-guide/python-backend.properties new file mode 100644 index 00000000000..b7cf144fb8c --- /dev/null +++ b/docs/demos/common/src/main/snippets/developer-guide/python-backend.properties @@ -0,0 +1,5 @@ +// Generated from docs/developer-guide source blocks. Edit the guide snippets here, not inline. + +// tag::python-backend-properties-001[] +codename1.arg.python.virtualThreadStack=2MB +// end::python-backend-properties-001[] diff --git a/docs/developer-guide/Backend-Data.asciidoc b/docs/developer-guide/Backend-Data.asciidoc index d15eab7a32c..2449ad09433 100644 --- a/docs/developer-guide/Backend-Data.asciidoc +++ b/docs/developer-guide/Backend-Data.asciidoc @@ -32,6 +32,47 @@ families spell that collation differently. Which one is used comes from the server rather than from the URL scheme, so a `mysql://` URL pointed at MariaDB is still correct. +==== Encrypted connections + +TLS is set on the URL. `sslmode=require` and its synonym `verify-full` demand +TLS whose certificate chains to a trusted root and names the host the URL +connects to. `sslmode=verify-ca` with `sslrootcert=` verifies the chain +up to that CA and doesn't check the host name, which is what a managed instance +or a container with a generated certificate needs. `sslmode=prefer` uses TLS when +the server offers it, and `sslmode=disable` is plaintext: + +[source,properties] +---- +include::../demos/common/src/main/snippets/developer-guide/backend-web.properties[tag=backend-database-tls-001,indent=0] +---- + +MySQL's own spellings are accepted on any URL: `ssl-mode=DISABLED`, `PREFERRED`, +`REQUIRED`, `VERIFY_CA` or `VERIFY_IDENTITY`, and `ssl-ca` for the CA file. An +unknown mode is refused at start-up rather than read as `prefer`. + +Unlike libpq and MySQL Connector/J, `require` verifies here. Those clients read +it to mean encrypt but trust any certificate, which an on-path peer satisfies with +a certificate of its own. There is no setting that encrypts without verifying. + +==== Typed columns + +PostgreSQL and MySQL send DECIMAL, NUMERIC, date and time values as text, and +they come back as the `String` the server sent, so an exact amount keeps every +digit. `Database.columnTypes` says which columns those are, so code that maps +rows to objects can parse each value by its kind: + +[source,java,role=cn1-no-try] +---- +include::../demos/backend/src/main/java/com/codenameone/developerguide/backend/DatabaseSnippets.java[tag=backend-database-column-types,indent=0] +---- + +The kinds are `decimal`, `date`, `datetime` (no zone), `datetimetz` +(PostgreSQL's timestamp with time zone) and `time`. Every other column is null, +because its value is already a `Long`, `Double`, `String` or `byte[]`. SQLite has +no column types to report, so the call answers null there. + +==== Portable statements + Statements are written once, in one portable form: `?` for every parameter, and plain unquoted names. PostgreSQL binds `$1` rather than `?`, and that difference stops inside `execute` and `query` rather than at every call site. SQL already diff --git a/docs/developer-guide/Backend-Operations.asciidoc b/docs/developer-guide/Backend-Operations.asciidoc index 98d19f31a23..95044bfaa83 100644 --- a/docs/developer-guide/Backend-Operations.asciidoc +++ b/docs/developer-guide/Backend-Operations.asciidoc @@ -171,6 +171,30 @@ no TLS, and what it does need is exactly what a translated binary is good at. core, that many concurrent slow calls of that kind occupy every host while other connections wait. A server whose handlers spend their time in SQLite should size for that, or run the thread pool with `CN1_HTTP_POLL_MODE=0`. +* Each virtual thread has 64KB of C stack. That holds a few hundred nested Java + frames, well past what a handler usually needs, but deep recursion -- a + recursive parser, a template nested many levels -- can run out, and an overflow ends + the connection's host thread with an empty reply. Put `@VirtualThreadStack` on + any class in the backend module to raise it; the size is read at build time, + in bytes or with a `KB` or `MB` suffix, and when several classes ask, the + largest wins: ++ +[source,java] +---- +include::../demos/backend/src/main/java/com/codenameone/developerguide/backend/DeepReports.java[tag=backend-virtual-thread-stack,indent=0] +---- ++ +The `CN1_HTTP_VT_STACK` environment variable sets it at start-up instead, in +bytes, and the larger of the two applies. Memory backs a page of the stack only +once it's touched, so a generous size costs address space rather than memory: ++ +[source,bash] +---- +include::../demos/common/src/main/snippets/developer-guide/backend-web.sh[tag=backend-vt-stack-sh-001,indent=0] +---- ++ +A Python backend starts at 1MB and takes its size from a build hint instead; +see <>. * TLS runs on the thread pool, whatever the poll mode says. The TLS layer can't park a read yet, and a blocking read on a virtual thread holds its host for the duration, so one idle TLS client per core would occupy every one of them. The diff --git a/docs/developer-guide/Python-Interop.asciidoc b/docs/developer-guide/Python-Interop.asciidoc index 62080144d2c..9e94754d0d1 100644 --- a/docs/developer-guide/Python-Interop.asciidoc +++ b/docs/developer-guide/Python-Interop.asciidoc @@ -357,6 +357,21 @@ column types, so on a server database its numbers and dates come back as the text the server sent. SQLAlchemy's own drivers, such as `psycopg`, aren't used: the URL `postgresql+psycopg://` selects the backend's driver. +==== Stack size + +Every request runs on a virtual thread with 1MB of C stack, because transpiled +Python frames are large and `re` recurses. A handler that recurses deeper than +that raises it with a build hint in `codenameone_settings.properties`, in bytes +or with a `KB` or `MB` suffix: + +[source,properties] +---- +include::../demos/common/src/main/snippets/developer-guide/python-backend.properties[tag=python-backend-properties-001,indent=0] +---- + +`CN1_HTTP_VT_STACK` can still raise it when the server starts, and the larger +of the two applies. + === Compared with CPython, PyPy and GraalPy The same programs, run on one Apple silicon Mac with From abeed314a66a8fea7274a800a72fb966a3a0c670 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 14:22:01 +0200 Subject: [PATCH 080/112] TrueTypeFont: pure-Java TrueType fonts for com.codename1.image, embedded Aileron default; Pillow truetype/load_default on it TrueTypeFont (SharedWithBackend) reads head/hhea/maxp/cmap 4+12/loca/glyf (composites)/hmtx/kern 0/name, emits unhinted quadratic outlines into RasterPath with a bounded glyph cache, and refuses CFF fonts with an IOException. Aileron Regular (CC0, the font Pillow's load_default() embeds) ships as DefaultFontData with its legal code and a NOTICE entry. The Python runtime's ImageFont gains FreeTypeFont, truetype() and a scalable load_default(size); ImageDraw text/multiline_text/textbbox/textlength render through RasterGraphics with Pillow's basic-layout metrics and anchors (FreeTypeMetrics emulates FreeType's 26.6 rounding). pil_truetype documents the tolerance against CPython+Pillow; the flask sample renders a badge PNG/JPEG. Co-Authored-By: Claude Opus 5.5 --- .../com/codename1/image/DefaultFontData.java | 244 ++++ .../src/com/codename1/image/PngCodec.java | 2 +- .../src/com/codename1/image/RasterFont.java | 5 +- .../src/com/codename1/image/TrueTypeFont.java | 1110 +++++++++++++++++ .../src/com/codename1/image/package-info.java | 3 +- CodenameOne/third-party/aileron/LICENSE.txt | 121 ++ NOTICE | 11 + .../com/codename1/image/TrueTypeFontTest.java | 224 ++++ .../python/runtime/FreeTypeMetrics.java | 240 ++++ .../com/codename1/python/runtime/PyImage.java | 53 + .../codename1/python/site/PIL/ImageDraw.py | 127 +- .../codename1/python/site/PIL/ImageFont.py | 117 +- .../resources/conformance/pil_truetype.out | 35 + .../resources/conformance/pil_truetype.py | 126 ++ scripts/python-backend/e2e_client.py | 11 + .../samples/flask-todo/src/main.py | 29 + 16 files changed, 2406 insertions(+), 52 deletions(-) create mode 100644 CodenameOne/src/com/codename1/image/DefaultFontData.java create mode 100644 CodenameOne/src/com/codename1/image/TrueTypeFont.java create mode 100644 CodenameOne/third-party/aileron/LICENSE.txt create mode 100644 maven/core-unittests/src/test/java/com/codename1/image/TrueTypeFontTest.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/FreeTypeMetrics.java create mode 100644 maven/python-transpiler/src/test/resources/conformance/pil_truetype.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/pil_truetype.py diff --git a/CodenameOne/src/com/codename1/image/DefaultFontData.java b/CodenameOne/src/com/codename1/image/DefaultFontData.java new file mode 100644 index 00000000000..876d07afadd --- /dev/null +++ b/CodenameOne/src/com/codename1/image/DefaultFontData.java @@ -0,0 +1,244 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.image; + +import java.io.IOException; +import java.util.zip.Inflater; + +/// Aileron Regular, embedded for [TrueTypeFont#createDefault(double)]. +/// +/// The font is the subset Pillow embeds for `ImageFont.load_default()` +/// (12676 bytes): Aileron by Sora Sagano (dotcolon.net), released into the +/// public domain under CC0 1.0 ("No Rights Reserved", as its `name` table +/// says). The license is kept in `CodenameOne/third-party/aileron` and the +/// root `NOTICE`. The bytes are zlib-compressed and base64-encoded here +/// because classes are what the backend shares with the core, not +/// resources. It is data, not code: replace it whole, never edit it. +@com.codename1.impl.SharedWithBackend +final class DefaultFontData { + private static final int SIZE = 12676; + + private static final String DATA = "" + + "eNq1ewlgVEW2aFV1ujvpdDrp9JI96aSz70lvSSeEkHQ6JCELWTqQjYTsIRshINtTQHggIIqijsoT" + + "RR+ODPIBwXFBXnREEZfnY4AZGVRcB+f7GQYZ5Edz7z9V93YSwHHk/f871L11q05Vnf2cqntBGCHk" + + "i9YgCRp2OmvLt0oeykcoowdaQ4sdRU70a/RnhOIvw3N0cVVlzb8GbapEKOE6QvLB4pq6gsV/WzEO" + + "z1ro/7yyJi2z59jadxHCy+G5tX2gbfjruVUZCEl+DW17u9uWDPM8rITiL0K/Z3f/iq7YopW+CMlg" + + "zbDrPZ1tHZoFwe9A349QrD3QILtIDsJYM12/Z2B0+Rdq/Wp4/iOUD/uH2tte+vPB/QhJa6H/LwNt" + + "y4fRq8gJffPg2TDYNtCZNB7eCUOzAIfdw0NLRn1X+pYjFHceYBSI0o6DQ0tUuyMX+Ob+HXlLKF7o" + + "w+BdH9H7Z4qwp/g+PlVyUGKg+CKChB+Mk5zkg+H6EvT3SQ6ymab/RlnLKCpEUvZM2Cjali9CSPA4" + + "vh96PclmQrEtE+64BWXiPAognZpNPn3qurL6CpjF8AOREMDhhOQk+hrQe5At408OUcph9qsiqkAt" + + "Am6hKygb1yAXcSAnFAv5EiWQbTDPAZSEc1AHlHCSjuqgrwqd5z/D/jDLeRSD0/gfoC0ByjoouVBS" + + "oERCyYaSBCUTSjKUDApPx0K9m84DxYG/Q0ZJLbKSu1AYWYI6yKsojRyHexkUG5S34PkcrO8JdQOy" + + "kTa4z0QdkoVw/wDKZ9DvgvvHcF8E9xJkJh0oiuxFLWQQGSQvoWCyGAWRIqQhlSgVm9BCijOOAxjA" + + "AfiQiZMBfw2KJu2olKiBzm4oMSgcRyEdq89FpegSykPf8N8RBdS/R05JPcBCOymFfjoO4HApKsXv" + + "okCcjbSkChUQgrBEjTD+nB8nIF38AdLiNODpOGqFO+VRucQJ66ag5TgRhTJ8kkErTiIDtGXDOlYo" + + "evwUskLRk37g990Aezf0/U++j4TAOgqkwxNIRzVGLKGitj0DT5g9e6BNcA9HftCiQnei59Bv0EF0" + + "GP0WvWoIMoQZYg12gyMy6gcC1ke1g/YfEPsDDaHQn836Mc/zX/Av8y/xh/lD/AH++QtzL1ReKL5g" + + "uZD8aaV08S1afssPy9AkECai3t8AACR4SGVyTy+Ft9JH5eun9tdodfqAwKDgkNCw8AhDZJQxOiY2" + + "Lj4hMSk5JTUtPSPTZLZYbVnZ9pzcGXkz82cVFIJXKp5dUlo2p7yismpudU1tnat+3vyGxqbmlgWt" + + "beif/jrE+8LFXeze+ZNQ7T0D7uqi/t4+eh9Gg0MjS1D3P18CxYp/eagarUMnsQJX4CY8gjfhPfgo" + + "PonP4ov4GlESAyklm8geclaikkRJOiQ7Ja9ITksuSn708PNweuz2OOoxLo2VVkv7pGukB6QnpWel" + + "4zKtLFo2R9Yj2yw7LPtcHiyvlXfJ75fvlo/Jz8uveCo8Qz0TPe2eFZ5Nnn2euz1PeY57JXrN81rt" + + "9YbXXxREEa/IV7QoHlYcVZz3Vnrneru8t3of8j7nfUmpVMYqZytblU8q9yn/4qPwmePT5bPe50mf" + + "/T5jPud8LvpcUylVUapSVZOqX7VStVP1pupHt5vCV8lR0D0Z8kJKiCYoQx2plmgiNQFqHOmN1Rp8" + + "dXwvt+/EXm4brt17Yi85OuHAWxDPneYxN3EG8TiZI4gHbcnmr6CXwYdJkDdCNqPERDL1WllUrPnJ" + + "CmVXZVZ0dFb0YRgyFk2rAO/CX+Fc8inAg4/UGONMAVD2OBUN66Dgr8bHxwXvS2MCB/OGoAioyU06" + + "o9xoY8ViYsUkZ0WXBJ34V2Gdi9PsaY1QGrqm1Xc1hjfiXVuz/gS/LPeNWaWF30NUEiXMHouSEYrR" + + "yuQ6oyUVx5nCcYA6FZutNotJB3VjbBzcsFYfYFFhjS4PW8xEtXievXXEo0meZ2lZ0FNnr2tXNHk3" + + "DjRxO1PMqabHTfYjc7u9KsplffVWhydGwY3PdMCjfFHVkOqSsRN7Zlj9vvLM4r67JrObW4TQkcBf" + + "Ia+CTOTARx0KQ3EI+Yuc1BixySNKptPqM60Ws7vVOtmyYlNV1aaqh+rxWD3XZDca7cavWctBVidH" + + "XQ+4XA9MPEYOZZrS08zmtHTuC9YULD4yfufD+oNkDEWhVOC3zqiWGaNiLcCFVCDYasoM0MUa2YoB" + + "+hwMnJFq9aZMm8UYJVdhMtja1Hb3NtfsnOJN/QXWrEJHuSPPPznXxT1QuyA7OTdBW12O1zq593S1" + + "3fkdvqq5OfMGFUXehfaCLAkeHn67eIbXj0RexLxNEv4KbWD6wbRjlaAVgifq4A/gAJCZF/SZZH4B" + + "Riqmj+1Ll9qz8/Mlyre49Lfqvvj2CwYbzh9A/+WGDccgXqPVL5wCZtuXSpQUru4t/CGDrUPV+E/4" + + "T+CJEbYFyAPkcfI4W5wtIM6EUx7X7Fqc2d6euXiX5nFFHo6fkZOwOebRR2M2J+TMeAHGVqE6sKUY" + + "qv3YCBoEqgncMeGrR+xH4B+OOHky64MPGP78Z/wmtBW8mQdYCmiyaWvD3IrmhlM0tQEfz6Mr2CbQ" + + "bTFdOXUK8VnQHgP29e9ohFqLVZR8CbOqEcGi6Lw/8J9jM/mQjg0wakzZeT/khZNLgh1RvQoBuTJ7" + + "8zf4GaxQEvAVTtWKM7kPyNhEpoM4Bf6ug8vDYHOecFcb1XKLzWrCD7uOPWCMJ4cmKkj94sZGPcDl" + + "wsQqmDMI6IjURVo01ChAPXyx0ZaHTZl6HbD7ZVzNZYwXWGfl3OmHXVixuqdzoLTF6cxAvKMuLbvs" + + "VwrFosX91YWLm9naKYCnHuZMhIdwDBMYo9KwKSAcpgMVTwKVo3eqg9NXwVtnlTzQ85EOFpDXF8ya" + + "+4xanV9baMsulwtLLhguxEP+jQXNiz9UKJzO4ry7nLNnmmcVFSgUC/pbK+3zGI8i4RINdCuoJ4yh" + + "Uoy0YJPaBEbgi0l0vYvbi12u+ufDQg/s378/CX/EFeCyzNjvKN7g/0gQjA2DB4aSgLfeja7cqAEW" + + "yW1WbBq6Y3vnGS12keA7iqoritrSqtPxJu5YRFI6fntN98Da4wrF2vryisaFeLvzcmqYIL8kmF8D" + + "fAlAxqkV1NQi9RHYGAcrSGxWjZs3YJ84tW/V5r6rj4zOH/XETSSwe6cizmKfVVNSllddhU+t7R24" + + "BzuP/66zTqFYxn2bl8x1NJdVLGysKWljcsiERYOBHpoX6LARR1pIMPfrJlzGrSWH4ieWkX4n85/J" + + "/N9IIOClB81NnpLZlMtgqMZOOinJdBR3ZJfd39W/udKenrPi5IqZbasLC1Y7NpSUQYpQVoEHNM25" + + "vd1dfbnN/poqW11nZ52taqLIbHM4bOYJrjTPXl5un1EGvMlgvJ/kjZohIHJfIxexuYE3MTfxpYTy" + + "CviSD/whY9wrIl+wNi+ZMorroHyh/GGyYLYIOa+c6onbD/vfYJXCP2Jgd26Z20YJs31hrHKaZ6de" + + "QBjY01Bd0eIe13XuayoLmsFeh3yYysIH62IsMfg6VUVuCG/HB/bscb7jeIfiVYVexVfxq4KNg8Qi" + + "1VDwVa4JP0MLvjuLR9SXOGC+M+J8OqklBVvwGZiLqneS4x0HTCjonJG/gk8DXwNBtoLOgZOXG/Ow" + + "TTR0GX1wE4E9Fg72phiISxqZaI3Ps2QXezRIVpkFRox1V7X3pthiUgrT88qSzTNLilpHJg652QJr" + + "WflfSfTkV5BXF8BaEpCS23JUREdjjNUWIEgyj5hE9Y9zy9aUGYFtYFuCeAGWrJzpEZtZ8vBw/uqO" + + "vCdX5f+rJfDuB+97e/ayByu2vbB67nxvXao1u3ZVc26ol9rm5RsTb3c64mbi/+Kuq32ju1rWb8+Y" + + "f2/NlkcS4jLmH9nTurvX/tJDd66bnRBXXtv1aET6jLNxhbmDJZ1OwBvMk9wBdiIHrwE+A8zcROO0" + + "BJ/g5jZdH73UzJ34z46rV6nfxHa2G5bQPJbMgTF+kNXEuW1GLkvCaoivQFqMDkIqEA9WzVxqXGwS" + + "hiZ8rLi4a06U9ZN3syP3chs3hIa09fnHlJjyhiMC73tIj4f8KovmLVRKySGPgO1DZ77w8G1foCQT" + + "56V5tkFfjwnBt6eBrdwHMg1hEhVdh4xxeCY2gXwFjuKld93nDS4qanfP6hWdu6MIuNGH1j645zAZ" + + "W9OnUOx5+q61T+9RKDo3nNi5610qP0rTLqBJSaPBDRQJUYHSAA9427/seGy10fL+KYuuobt9fkj2" + + "HSNZZL/frx/a8YqvB3DpUd/7li/eqJJOnBNjPcybBvNCXI3RgULrhD+Sxlnw69xDuJx7wekkh5yX" + + "nZcn4WMAnsZ7CgnxK1L3Ci7kXnPh10n/ZSd3EqBv5AXNLE1qo2VKnW5kjNVixPd9Pycibykw4IHp" + + "jKkdeopisDzfSfkwyZmlTZO4bBVwp3GU4mKCK9nq4ta7XHg1U4pU7hQ5xL2JcyfpRa+wPBpB3DG9" + + "4nLRgwGMbGCLF6GqoXNpBTmJLg5URQ190uG+1gECqMn/ZVSBYU5XRlRjz1BbQsLW9doZ+Nkpfo4K" + + "OHlhowSSXcAJm3AOjlvxvwoAq7pO3MbtwnHFF2CK7XhomgykggxMasp9F2X7xA5xTokW+rUCnWab" + + "TC4xSmLj9DpKcQiWaF3EmLGyf7UthLi2vfcuo7v9xbKRbdxO3PVkU9lh7t+g7RHSO4XjGpjPB6FI" + + "IxZmwbEWtYmsWc+9KnGtx0WSWtKycgv3LAzbvWIrns98SBrLncdA9hqa41AnYYilmTJ1JFSWf8Lm" + + "ffu49/ad++PKlX9cScaO737qrbee2s3l7Ny46fHHN20U9bgY1laB15uux0aqyVazW4131C7orIjI" + + "WeM6mxPc1BRo2bTJQvZ7d7vqehUe3NfAuSO60zopd13E6z3yGuDlDzt+FKMPkNN8QNQ1imWMEBoB" + + "RTJjRmFGlS0iMSL+sX37ANuJ0ysZrrO9snKsXnJA9vjx3UeObNzJMBbxnQf4stgHc8M2RbA1inCA" + + "gL8+BpTE7UhIojJqQZDSp8Evcp3r85yIqianpYJ7fH26JnNGZ1Dy1iObUrJHR+WY+wP4EUVjTX61" + + "OjqfSPOz+vw8QDZm4HE80BLD/AdNvcXtgDvw042T4LrjWPzH9u5liiZZ2Sx7SeHcpUMbu0Y2ezf5" + + "NjhmVJc3dSxaT8aGmwuKrWab3T+k0zV/cLDDUZpnySvUBcyrql8k6EMU0NjBfCzk1kAY5GWk45rr" + + "Gikg/RM7aCaCUQvg1cH4INoHYEJVRq0FzCB4gI1Av+fay2s9iYvI5j02T0ZATOWJ2qYlS5q0ifgF" + + "ri7J31FV5fBPwr8R1oWoR1YKOVAk7GhDwEzWgUd//3QTqN19ZDGFoWeJB5nfg6hl9MYCHOj8wbHF" + + "O3ePvLhw+5OjOx9laXPNxPNwf5w0TjxDuuhY8JRkSLBF2GwbNWyoREMUJ99sv/blwt8ea788jqO5" + + "87gJYvPXOJh7nnuKjgPtJnOn8nMBs2bX1faxV9qwH3cZ++Hd3EFcSWFhF0eyBX8ItmuhWYEOk+yJ" + + "v+OnuFX4G67N6Swgic6iiY8EmhfC3koK+yW5sPuD/bYaS3t7n5Yosya0WZKmm/YZUqAXmyc3GiAr" + + "2G8dF8YHwP4FZGU6/jSMr8oi32axfdIfYJ/UQ3kaQ2k2afCBe+91HnAc+AOezz2Lt3NDQv7J78Vv" + + "8J+zNWgW8wY3jO93sX0OO9N8UfCRdG/4YlZdFXEMCnlENPjJDyG/CqMaGjmZIeQw7adpON0yBOjD" + + "sDYgLtM6A8fqtHilV1h6XVnr/Y9FEXlrr1eDbGD+fpVE8eWM4ktzsiOrihsbVcnaucO1LVsj4t6e" + + "UWquKLb6r20S1isFvSuCfaoahU/l5eCZmV+dlpHDgnpcu3jNQ4PGEo3R5YrPnF3SVFNe0tJGDI8s" + + "37JD42cuIZ9yn2QnctrBzpYVnYvCFrSw/esV/D3QEyja21RQspkmg3VAyzJlQ2h/QX1NvqXYp0Gz" + + "srFv3f3E0DPvzr6ugYHc7IbmLVvWPsLwDWe5shvfm4OeWkJ54k6TSRCg2eDG+VnAdRbgTD6diKzL" + + "ThSRxvqJOoYrRVqYH08AvjqWYWjlkcwbTOEKy8RB3hgZC1DcGll1vqXMq8F3bX3/hm2Di5Z/lV8n" + + "x4k+OFCvaZ83M6uxZdvGux7bXtlQ8zyVu44e+pMrsNdAGmAo1U3qDEwWM9sIBuhin5UMDLiysyXy" + + "ktyI8B6yXBdpLuDM+L2C5JiQ7JnSYo3Ig1TggQH8StpUNJ3cZdqAAzcwxXITU7DH+tHzalyPpRWF" + + "FXUS/FM88ueGS8OWrPwfCq/M3LryoEivSYaVcFFT/MKoFBBSkC+Z32KRzgxuExAKw4ICAQaEeiyP" + + "HHtRrcsVmT23biHeEJLPncXRGSGZOZiM3PUN+ZL7/YzkWkobfQOxD/wzPT+adnKkNrWkGgyphoWQ" + + "T4wZaHWilfjDfuQ7PhU9y+B1U/ABwBOaX1v01G/OF0ZKvBeH+pSC1xTGb5sYiwnpIE65SRUO9ohE" + + "WjxBt7yZ5TEL0JjkGvzg65ku1/eD349fsD0HOh6Pvce5iwDv5H9EnwA87KI0zE/bdCyCgN/eiUmo" + + "MSg6MZVg0LeXI9XB/h55HtFp/mZS6l5L4gcyjJ7iGwjohopJx9gXwK4Sb6nLQ9kyKlxcUbGlORV+" + + "abGljlq8KyqbO4pDC6Pnn+XG3XfiX54bYU1xRBSjSdoUsJ7+hvVkTEy6KTF5DTvqbpFS/yiIyL+R" + + "Schtg8lsD6hmeQpNU6ZlKf+O0caNPNr40hMNDU80EMOuy7t2XeZeWLagdenS1gXM5+SCzzHc4nNE" + + "mm/H53DfkbN1N/ocil8usw+3j1D/rDn8pI/wn0jEQT+j8ziPhNP8DlOrlbmlhYP1quJ6lyvdSMLl" + + "s6SuesiN/WuKA2FMIPiV5wEnyHI00/zgtLRDPz3r+LK+3atB05yUkpXbXD8wzzWobND0ptizHdXz" + + "O4ihoaw9KTom3i+osmB2Ra2rJz4mNtVfPzt/dg2TtZa/jrXkHmqTN+uleKynO4glsaaIxEyTBy4u" + + "drXj97j/DFHF6mUzpckZKjO2FKxaVcDmKgBZE7C0gGnedioxMVFWEkIVprJNARoqy7c45gL3LjXk" + + "JZbbo7JxHncqJjrVtmTNN8j9JhL/CPPJWL5BQz/+8Z3edxbs3kVHgd9Jwb8X4STeAHdLXuL9XNey" + + "ZV3Pta8aXbRsFR2Dj3EFMC4fj3GF+DUa38fhchbG0n0LBFg5zUwCbCYcfLRjy+b2Y7/t2rChHau4" + + "K2fOYCWWvf8+W49PxT/AGDVdj/pTcB5A3UzMUPzhXOT8/AgfH/8Eq1b++7Yn/g0Wfh2ithMnmjQk" + + "hUvFpyjfYexBoroxV8EHuXZs5j7H67gPHI4CvMlRxK0SeJEPuQo9u6fnTkwdAtiJI1UJOVWOHIjC" + + "VG/ZXtrCNtQsFcAqZaLR2yc1ISHVx9uYqPR2OJTe1dXeSofDm5wODNJXNBqSLFKpLcnQWKEPCsxK" + + "ywmsmxNV5JnlWRQ1py4wJ01Yv5XPQo9JDMJ5q9r0mN1+QkL9WiTkQH8V8NLo3Gi5jy7jJo/GKFqx" + + "rFkvIPxXAYWfQA6vnFw66x/jR3EqR5/ia7iD7RNNWqZs4A2NZuoI9fQMIWswJDllVliG+447NqgD" + + "jXPohZ5NRPOfSKLIReabEug7R70feCe/aANlqAEwjnFb381JCMaOjz/mjn5M5Nzll1/Gfi9zx6u6" + + "fZv859nzZtkSs2VN/l2upiXLyMUL3IsXLuDSPf+x9d5jx+7d2jivpLn+4fmW+DLXnd2L7gT/sxxV" + + "46v4CdhvUg2UBxjjbCZ2xaX33FNyzz3fsSt+Yo/4oz41FJ/CKhbHpp3JWSmWmVZza5/D0ed40BwZ" + + "aY4kYwV9BQV9CmOW0SiePVWx92DesJov5Zpu8vxeQzPPmUeEI3x2hNaYdfhw1oGJv2dRXo8SB8bu" + + "PNRm2pc1w52H0r3Dt2gvXsZO20VsZtvj4+3xeFk8vdN1s4HOI246Y+RGW5xJyq7YmxJYgk3sdn2S" + + "TJjXir9Cn0HM9IA1gStGa0t5RTP+6+Vx6NND3+tiHz1RfJ32kTPQR+g4vIX1eYkj6XGPMHyvew4F" + + "gAIszIMXTcLSmQImp9vrnpPBSuhZKLGw9xK+1Me5303cfBcON7+ddh2JnnZQOvkOg8m+lPFElLwo" + + "c0HaAm8p37wFmJjp/LrOIDDfB5e1glySMDbNpG/E2OcVQJsWn8DnyQ7AV0utUxC1VGe0yY1xdMMS" + + "YDtXmlmKT2SSJHMizixLSiI7Ep5+OgG/u23zPfe+C3tuIQ89CnlooBAdp7JQ4f2YTA6RFx91vUEz" + + "UfXbEeELXXIJUU1cgShBc1FVTMjD0mLP6HRdpJ7Ro+N/xF/CfLBjiZkebNgxwY15Ll1BrcVPT2ZH" + + "4XEtSpr0pqVJ1C8027uC1R4zhTxJJ3PBAgWTS2ogjRLeceuxavJlvwbFi3UMPJkj1gnUO8S6BPa4" + + "u8W6B7S/L9alKBe2mEJdhnzxPrHuibzxabHuBfXrYt0bBRCzWFdCfTX9CsLDi71zR2IdIxXKFesE" + + "6rViXYLMqF+se0D7a2JdCvpyRqzLUDjeKtY9gcajYt0L6hfEujdKJR5iXQn1OagCDYGtVqNe1I16" + + "0Chawp464d4JWr0Mrh10D10xZKju7e4ZXWKo7lzSObKsswMaZ8GofgY3hMDsZ/X2d44MQYWO70ZL" + + "oa8N+lB1Z/fS/jaoZMBMGSgdZaIcVIfKUD2snnPTLCk3j85IzUjPzKHfDuWIC6RMTviPEHCxtiXQ" + + "S9sNNyxchWqgJR3+prf2AOQoamfwy8QR6VBscM1BA4DKIpiTwnRBaz/MvBDGpSILK1nIBE/ZsHDn" + + "yJLeoUGDgHRVjSE9Xaj2DI22Dw0ug470VFt6zkDbos6h0a7U/t6FmamWVEuWKTMb/TNO3EJ+DcCN" + + "QJ8Bam0A2QajhqB5aKTNUNPW3TY49MtgqNxH0TCyw34wDd3B/lJB7m6W9DN8UqF0QgvqGR0dtqel" + + "3XHHHakdlKz+ocHUwc7R/3fzTH59w99N38P9xG+U2SsBTfYAG5CBVXqCnitAv5WQVavAF/tB/PYH" + + "69aCt9KDZw5EQWDHISgUhUFWHwH8iIT9rxH2TzFgf3HgBRJQIkqCnCUFMExj6pEJYjWDeK2gBlkg" + + "XjuoQi6aAXvMmSgfpFWACpEDFSEnKkazUQnk9WXgQcpBqytBzeaC/GrAgutAHevRPDQfNaBG1ISa" + + "UQtagFpBElvQc+hu9CC6F/0O7UUvot+iw+gI+g06hv4DHUVb0Rh6XfBMPP3CBDO/Qb+aovyhtNPv" + + "mqiaExrxxC/1EHBhDigpcTjn1CJVf9voINDtwbjJs349khTOqjWg8PLKObCfq64sZ7s6sRextRh3" + + "+3u722CGofZ+GivErwAxW0kG/JWxmj9wyIO1z2HfeBHooe/GQ8URlezaLdZ/CiJtGkQaO1HE7ItF" + + "GVBLaasEGgnzzBh47KbRCRycTmPoJI0C9m4q9bdQKVLXPrxkGKkWdY4MitS59xd0ZZVIqz8SvgUL" + + "Bx3AsKoU/hRMFtGgBwKEQL0SShb/ZzZDNCKyzbRdGi09CfQEwbrX0W3++EP8S/y3/CX+PXTbP/7N" + + "Xwh3mmffjfJ/g4sf/9ovGHHppufvfjFS/pNj/vdtUnNBKLfNhSv81Z/pvShG6P+Pv9vgjgD/9U3P" + + "n0O5zGrX/uEYUbP4D37hGh/wf2H3qyxXuV2KPoKyEsp89nQ/v/8fwF39v+AaUMy/yF+GXSTU+Tf5" + + "T1nr7/jr/FV+/FbKb0ePbnPE7/g3+N/D/RSU738W8gSUb/57lAsz8xO3MeLaba5w/oanF//bkvnx" + + "n9rTL/9RP1kFsYBm1bD/gxgtE3s86RdoLGP2n8zSgyB2+LBRN1ts4C1fmuvET9ADWZ+WxQsaOwnk" + + "BWoaBX7xTwrrqSFiy34hRVN4yEVKpn8JL2eUeAF+NEOh9PndtFogi+xCzAmk+0lGgR7+iFgQ4CMB" + + "erQs0/H4WW9LT8kwxK5oluHQ/EbIbWhe485oaD5TBNkLzVlovlIHucpUprJkki507pM/HL/5/xj8" + + "H8OKwa8="; + + private DefaultFontData() { + } + + /// Returns a fresh copy of the font file. + static byte[] aileron() throws IOException { + byte[] z = decode(DATA); + Inflater inf = new Inflater(); + try { + inf.setInput(z); + byte[] out = new byte[SIZE]; + int n = 0; + while (n < SIZE && !inf.finished()) { + int r = inf.inflate(out, n, SIZE - n); + if (r == 0 && (inf.needsInput() || inf.needsDictionary())) { + break; + } + n += r; + } + if (n != SIZE) { + throw new IOException("embedded font is truncated"); + } + return out; + } catch (java.util.zip.DataFormatException e) { + IOException x = new IOException("embedded font is corrupt"); + x.initCause(e); + throw x; + } finally { + inf.end(); + } + } + + private static byte[] decode(String s) { + int n = s.length(); + byte[] out = new byte[n / 4 * 3]; + int o = 0; + int acc = 0; + int bits = 0; + for (int i = 0; i < n; i++) { + int c = s.charAt(i); + int v; + if (c >= 'A' && c <= 'Z') { + v = c - 'A'; + } else if (c >= 'a' && c <= 'z') { + v = c - 'a' + 26; + } else if (c >= '0' && c <= '9') { + v = c - '0' + 52; + } else if (c == '+') { + v = 62; + } else if (c == '/') { + v = 63; + } else { + continue; + } + acc = (acc << 6) | v; + bits += 6; + if (bits >= 8) { + bits -= 8; + out[o++] = (byte) (acc >> bits); + } + } + if (o == out.length) { + return out; + } + byte[] r = new byte[o]; + System.arraycopy(out, 0, r, 0, o); + return r; + } +} diff --git a/CodenameOne/src/com/codename1/image/PngCodec.java b/CodenameOne/src/com/codename1/image/PngCodec.java index e7a4e967d73..185f5ddfffc 100644 --- a/CodenameOne/src/com/codename1/image/PngCodec.java +++ b/CodenameOne/src/com/codename1/image/PngCodec.java @@ -108,7 +108,7 @@ public static RasterImage decode(byte[] data) throws IOException { } int body = pos + 8; String kind = new String(new char[] {(char) data[pos + 4], (char) data[pos + 5], - (char) data[pos + 6], (char) data[pos + 7]}); + (char) data[pos + 6], (char) data[pos + 7]}); if ("IHDR".equals(kind)) { w = be32(data, body); h = be32(data, body + 4); diff --git a/CodenameOne/src/com/codename1/image/RasterFont.java b/CodenameOne/src/com/codename1/image/RasterFont.java index 9870c023251..ec698d61aa4 100644 --- a/CodenameOne/src/com/codename1/image/RasterFont.java +++ b/CodenameOne/src/com/codename1/image/RasterFont.java @@ -26,8 +26,9 @@ /// draws text: a font turns a string into an outline, and the graphics fills /// it like any other path. /// -/// No implementation ships yet; a font engine plugs in here without any -/// change to the graphics. +/// [TrueTypeFont] is the implementation that ships: TrueType fonts read from +/// bytes, with an embedded default. Another font engine plugs in here +/// without any change to the graphics. @com.codename1.impl.SharedWithBackend public interface RasterFont { /// Returns the outline of `text` with its baseline starting at `(x, y)`. diff --git a/CodenameOne/src/com/codename1/image/TrueTypeFont.java b/CodenameOne/src/com/codename1/image/TrueTypeFont.java new file mode 100644 index 00000000000..e3fc515afc4 --- /dev/null +++ b/CodenameOne/src/com/codename1/image/TrueTypeFont.java @@ -0,0 +1,1110 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.image; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +/// A scalable font read from the bytes of a TrueType file (`.ttf`, an +/// OpenType `.otf` with TrueType outlines, or one face of a `.ttc` +/// collection), drawn by [RasterGraphics#drawString(String, double, double)]. +/// +/// The font is pure Java and needs no platform font engine, so it renders the +/// same text on a device and on a server. It reads the `head`, `hhea`, +/// `maxp`, `cmap` (formats 4 and 12), `loca`, `glyf` (composite glyphs +/// included), `hmtx`, `kern` (format 0) and `name` tables. Glyphs are not +/// hinted: outlines are scaled exactly, as quadratic curves, and the +/// rasterizer's antialiasing smooths them. OpenType layout (`GSUB`, `GPOS`) +/// is not applied, so ligatures and `GPOS` kerning are absent, and fonts +/// with PostScript (`CFF`) outlines are refused with an `IOException`. +/// +/// ```java +/// TrueTypeFont font = TrueTypeFont.createDefault(48); +/// RasterGraphics g = card.createGraphics(); +/// g.setFont(font); +/// g.setColor(0xff202020); +/// g.drawString("Hello", 40, 40 + font.getAscent()); +/// ``` +/// +/// [#createDefault(double)] is Aileron Regular, a font embedded in the +/// library, so text can be drawn without shipping a font file. +/// +/// An instance keeps a bounded cache of glyph outlines and is not safe to use +/// from two threads at once; [#deriveFont(double)] is cheap and gives each +/// thread its own. +@com.codename1.impl.SharedWithBackend +public final class TrueTypeFont implements RasterFont { + private static final int CACHE_LIMIT = 512; + private static final int MAX_COMPONENT_DEPTH = 8; + + private static final int OP_MOVE = 0; + private static final int OP_LINE = 1; + private static final int OP_QUAD = 2; + private static final int OP_CLOSE = 3; + + private final Face face; + private final double size; + private final double scale; + private final Map cache = new HashMap(); + + /// The parsed tables, shared by every size of one face. + private static final class Face { + byte[] d; + int unitsPerEm; + int locaLong; + int numGlyphs; + int ascender; + int descender; + int lineGap; + int numHMetrics; + int hmtx; + int loca; + int locaLen; + int glyf; + int glyfLen; + int cmap12 = -1; + int cmap4 = -1; + boolean symbol; + int kernPairs; + int kernStart; + String family = ""; + String style = ""; + } + + /// One glyph in font units: path commands and its control box. + private static final class Glyph { + float[] cmds = new float[0]; + int len; + float xMin; + float yMin; + float xMax; + float yMax; + boolean empty = true; + + void add(float v) { + if (len == cmds.length) { + float[] n = new float[Math.max(16, len * 2)]; + System.arraycopy(cmds, 0, n, 0, len); + cmds = n; + } + cmds[len++] = v; + } + + void point(int op, float x, float y) { + add(op); + add(x); + add(y); + include(x, y); + } + + void include(float x, float y) { + if (empty) { + xMin = x; + xMax = x; + yMin = y; + yMax = y; + empty = false; + } else { + xMin = Math.min(xMin, x); + xMax = Math.max(xMax, x); + yMin = Math.min(yMin, y); + yMax = Math.max(yMax, y); + } + } + } + + private TrueTypeFont(Face face, double size) { + if (Double.isNaN(size) || size <= 0) { + throw new IllegalArgumentException("font size must be positive: " + size); + } + this.face = face; + this.size = size; + this.scale = size / face.unitsPerEm; + } + + /// Reads a font from the bytes of a TrueType or OpenType file. + /// + /// #### Parameters + /// + /// - `data`: the file's contents; the array is kept, not copied + /// - `size`: the size in pixels, the height of one em + /// + /// #### Returns + /// + /// the font + /// + /// #### Throws + /// + /// - `IOException`: if the data is not a font this class can read + public static TrueTypeFont load(byte[] data, double size) throws IOException { + return load(data, 0, size); + } + + /// Reads one face of a font file, which for a `.ttc` collection may be + /// any of its faces. + /// + /// #### Parameters + /// + /// - `data`: the file's contents; the array is kept, not copied + /// - `index`: the face in a collection, 0 for a single font + /// - `size`: the size in pixels, the height of one em + /// + /// #### Returns + /// + /// the font + /// + /// #### Throws + /// + /// - `IOException`: if the data is not a font this class can read + public static TrueTypeFont load(byte[] data, int index, double size) throws IOException { + return new TrueTypeFont(parse(data, index), size); + } + + /// Returns Aileron Regular, the scalable font embedded in the library, at + /// `size` pixels. It is the font Pillow's `ImageFont.load_default()` + /// returns, and it covers printable ASCII. + /// + /// #### Parameters + /// + /// - `size`: the size in pixels + /// + /// #### Returns + /// + /// the font + public static TrueTypeFont createDefault(double size) { + try { + return load(DefaultFontData.aileron(), size); + } catch (IOException e) { + throw corrupt(e); + } + } + + private static IllegalStateException corrupt(IOException e) { + IllegalStateException x = new IllegalStateException("embedded font is corrupt: " + e.getMessage()); + x.initCause(e); + return x; + } + + /// Returns the bytes of the embedded Aileron Regular font file, a new + /// copy on every call, for code that wants the file itself. + /// + /// #### Returns + /// + /// the `.ttf` file + public static byte[] getDefaultFontData() { + try { + return DefaultFontData.aileron(); + } catch (IOException e) { + throw corrupt(e); + } + } + + /// Returns this face at another size. The parsed tables are shared; the + /// glyph cache is not. + /// + /// #### Parameters + /// + /// - `newSize`: the size in pixels + /// + /// #### Returns + /// + /// the font at `newSize` + public TrueTypeFont deriveFont(double newSize) { + return new TrueTypeFont(face, newSize); + } + + /// Returns the size in pixels: the height of one em. + /// + /// #### Returns + /// + /// the size + public double getSize() { + return size; + } + + /// Returns the family name from the `name` table, such as `Aileron`. + /// + /// #### Returns + /// + /// the family, empty if the font names none + public String getFamilyName() { + return face.family; + } + + /// Returns the style name from the `name` table, such as `Regular`. + /// + /// #### Returns + /// + /// the style, empty if the font names none + public String getStyleName() { + return face.style; + } + + /// Returns the font units per em: glyph coordinates are scaled by + /// `getSize() / getUnitsPerEm()`. + /// + /// #### Returns + /// + /// the units per em + public int getUnitsPerEm() { + return face.unitsPerEm; + } + + /// Returns the number of glyphs in the font. + /// + /// #### Returns + /// + /// the glyph count + public int getGlyphCount() { + return face.numGlyphs; + } + + /// Returns the ascender of the `hhea` table, in font units. + /// + /// #### Returns + /// + /// the ascender, positive above the baseline + public int getAscenderUnits() { + return face.ascender; + } + + /// Returns the descender of the `hhea` table, in font units. + /// + /// #### Returns + /// + /// the descender, negative below the baseline + public int getDescenderUnits() { + return face.descender; + } + + /// Returns the distance from the baseline to the top of the line, in pixels. + /// + /// #### Returns + /// + /// the ascent + @Override + public double getAscent() { + return face.ascender * scale; + } + + /// Returns the distance from the baseline to the bottom of the line, in pixels. + /// + /// #### Returns + /// + /// the descent, positive + @Override + public double getDescent() { + return -face.descender * scale; + } + + /// Returns the extra space the font asks for between lines, in pixels. + /// + /// #### Returns + /// + /// the line gap + public double getLineGap() { + return face.lineGap * scale; + } + + /// Returns the glyph that shows a character, from the `cmap` table. + /// + /// #### Parameters + /// + /// - `codePoint`: a Unicode code point + /// + /// #### Returns + /// + /// the glyph index, 0 (the missing glyph) when the font has none + public int glyphIndex(int codePoint) { + Face f = face; + int g = 0; + if (f.cmap12 >= 0) { + g = lookup12(f, codePoint); + } + if (g == 0 && f.cmap4 >= 0 && codePoint >= 0 && codePoint <= 0xffff) { + g = lookup4(f, codePoint); + if (g == 0 && f.symbol && codePoint < 0x100) { + g = lookup4(f, codePoint | 0xf000); + } + } + return g < f.numGlyphs ? g : 0; + } + + /// Returns whether the font has a glyph for a character. + /// + /// #### Parameters + /// + /// - `codePoint`: a Unicode code point + /// + /// #### Returns + /// + /// true if the character maps to a glyph other than the missing glyph + public boolean canDisplay(int codePoint) { + return glyphIndex(codePoint) != 0; + } + + /// Returns a glyph's advance width from the `hmtx` table, in font units. + /// + /// #### Parameters + /// + /// - `glyph`: the glyph index + /// + /// #### Returns + /// + /// the advance + public int glyphAdvance(int glyph) { + Face f = face; + int n = f.numHMetrics; + if (n <= 0 || glyph < 0) { + return 0; + } + int i = glyph < n ? glyph : n - 1; + return u16(f.d, f.hmtx + i * 4); + } + + /// Returns the kerning between two glyphs from the `kern` table, in font + /// units, to add to the advance of the first. + /// + /// #### Parameters + /// + /// - `left`: the first glyph index + /// - `right`: the glyph that follows it + /// + /// #### Returns + /// + /// the adjustment, 0 when the pair is not kerned + public int kerning(int left, int right) { + Face f = face; + int lo = 0; + int hi = f.kernPairs - 1; + long key = ((long) left << 16) | right; + while (lo <= hi) { + int mid = (lo + hi) >>> 1; + int p = f.kernStart + mid * 6; + long k = ((long) u16(f.d, p) << 16) | u16(f.d, p + 2); + if (k < key) { + lo = mid + 1; + } else if (k > key) { + hi = mid - 1; + } else { + return s16(f.d, p + 4); + } + } + return 0; + } + + /// Returns a glyph's control box -- the bounds of every point of its + /// outline, on and off the curve -- in font units, y up. + /// + /// #### Parameters + /// + /// - `glyph`: the glyph index + /// + /// #### Returns + /// + /// `{xMin, yMin, xMax, yMax}`, or null for a glyph with no outline + public double[] glyphBounds(int glyph) { + Glyph g = glyph(glyph); + if (g.empty) { + return null; + } + return new double[] {g.xMin, g.yMin, g.xMax, g.yMax}; + } + + /// Appends a glyph's outline to a path, scaled to this size, with its + /// origin on the baseline at `(x, y)`. + /// + /// #### Parameters + /// + /// - `path`: the path to append to + /// - `glyph`: the glyph index + /// - `x`: the pen position + /// - `y`: the baseline + public void appendGlyph(RasterPath path, int glyph, double x, double y) { + Glyph g = glyph(glyph); + float[] c = g.cmds; + double s = scale; + int i = 0; + while (i < g.len) { + int op = (int) c[i]; + if (op == OP_MOVE) { + path.moveTo(x + c[i + 1] * s, y - c[i + 2] * s); + i += 3; + } else if (op == OP_LINE) { + path.lineTo(x + c[i + 1] * s, y - c[i + 2] * s); + i += 3; + } else if (op == OP_QUAD) { + path.quadTo(x + c[i + 1] * s, y - c[i + 2] * s, x + c[i + 3] * s, y - c[i + 4] * s); + i += 5; + } else { + path.closePath(); + i++; + } + } + } + + /// Returns the outline of `text` with its baseline starting at `(x, y)`, + /// glyph after glyph by their advances and `kern` table kerning. + /// + /// #### Parameters + /// + /// - `text`: the string + /// - `x`: left end of the baseline + /// - `y`: the baseline + /// + /// #### Returns + /// + /// the outline, filled with the non-zero rule + @Override + public RasterPath getOutline(String text, double x, double y) { + RasterPath p = new RasterPath(); + layout(text, p, x, y); + return p; + } + + /// Returns the advance width of `text`: the sum of its glyph advances and + /// `kern` table kerning, unrounded. + /// + /// #### Parameters + /// + /// - `text`: the string + /// + /// #### Returns + /// + /// the width in pixels + @Override + public double stringWidth(String text) { + return layout(text, null, 0, 0) * scale; + } + + /// Lays out `text`, appending its glyphs to `path` when it is not null, + /// and returns the total advance in font units. + private int layout(String text, RasterPath path, double x, double y) { + int pen = 0; + int prev = -1; + int n = text.length(); + int i = 0; + while (i < n) { + int cp = text.charAt(i++); + if (cp >= 0xd800 && cp < 0xdc00 && i < n) { + int lo = text.charAt(i); + if (lo >= 0xdc00 && lo < 0xe000) { + cp = ((cp - 0xd800) << 10) + (lo - 0xdc00) + 0x10000; + i++; + } + } + int g = glyphIndex(cp); + if (prev >= 0) { + pen += kerning(prev, g); + } + if (path != null) { + appendGlyph(path, g, x + pen * scale, y); + } + pen += glyphAdvance(g); + prev = g; + } + return pen; + } + + private Glyph glyph(int index) { + Integer key = Integer.valueOf(index); + Glyph g = cache.get(key); + if (g == null) { + g = new Glyph(); + if (index >= 0 && index < face.numGlyphs) { + decode(face, index, g, new double[] {1, 0, 0, 1, 0, 0}, 0); + } + if (cache.size() >= CACHE_LIMIT) { + cache.clear(); + } + cache.put(key, g); + } + return g; + } + + /// Decodes glyph `index` into `out`, transformed by `m = {a, b, c, d, e, f}`: + /// `x' = a x + c y + e`, `y' = b x + d y + f`. + private static void decode(Face f, int index, Glyph out, double[] m, int depth) { + byte[] data = f.d; + int start; + int end; + if (f.locaLong != 0) { + if (index * 4 + 8 > f.locaLen) { + return; + } + start = (int) u32(data, f.loca + index * 4); + end = (int) u32(data, f.loca + index * 4 + 4); + } else { + if (index * 2 + 4 > f.locaLen) { + return; + } + start = u16(data, f.loca + index * 2) * 2; + end = u16(data, f.loca + index * 2 + 2) * 2; + } + if (end - start < 10 || start < 0 || end > f.glyfLen) { + return; + } + int p = f.glyf + start; + int limit = f.glyf + end; + int contours = s16(data, p); + if (contours >= 0) { + simple(data, p, limit, contours, out, m); + } else if (depth < MAX_COMPONENT_DEPTH) { + composite(f, data, p + 10, limit, out, m, depth); + } + } + + private static void simple(byte[] data, int p, int limit, int contours, Glyph out, double[] m) { + int q = p + 10; + if (contours == 0 || q + contours * 2 + 2 > limit) { + return; + } + int[] ends = new int[contours]; + for (int i = 0; i < contours; i++) { + ends[i] = u16(data, q); + q += 2; + } + int count = ends[contours - 1] + 1; + q += 2 + u16(data, q); + byte[] flags = new byte[count]; + int i = 0; + while (i < count) { + if (q >= limit) { + return; + } + int fl = data[q++] & 0xff; + flags[i++] = (byte) fl; + if ((fl & 8) != 0) { + if (q >= limit) { + return; + } + int rep = data[q++] & 0xff; + while (rep > 0 && i < count) { + flags[i++] = (byte) fl; + rep--; + } + } + } + int[] xs = new int[count]; + int[] ys = new int[count]; + q = coords(data, q, limit, flags, xs, 2, 16); + if (q < 0 || coords(data, q, limit, flags, ys, 4, 32) < 0) { + return; + } + float[] px = new float[count]; + float[] py = new float[count]; + for (i = 0; i < count; i++) { + px[i] = (float) (m[0] * xs[i] + m[2] * ys[i] + m[4]); + py[i] = (float) (m[1] * xs[i] + m[3] * ys[i] + m[5]); + } + int s = 0; + for (int k = 0; k < contours; k++) { + int en = ends[k]; + if (en < s || en >= count) { + return; + } + contour(out, px, py, flags, s, en); + s = en + 1; + } + } + + private static int coords(byte[] data, int start, int limit, byte[] flags, int[] v, int shortBit, int sameBit) { + int q = start; + int cur = 0; + for (int i = 0; i < flags.length; i++) { + int fl = flags[i]; + if ((fl & shortBit) != 0) { + if (q >= limit) { + return -1; + } + int dv = data[q++] & 0xff; + cur += (fl & sameBit) != 0 ? dv : -dv; + } else if ((fl & sameBit) == 0) { + if (q + 2 > limit) { + return -1; + } + cur += s16(data, q); + q += 2; + } + v[i] = cur; + } + return q; + } + + /// Emits one contour as lines and quadratic curves, inserting the + /// on-curve point implied between two consecutive off-curve points. + private static void contour(Glyph out, float[] px, float[] py, byte[] flags, int s, int e) { + int n = e - s + 1; + int first = -1; + for (int i = 0; i < n; i++) { + if ((flags[s + i] & 1) != 0) { + first = i; + break; + } + } + float sx; + float sy; + int begin; + int steps; + if (first >= 0) { + sx = px[s + first]; + sy = py[s + first]; + begin = first + 1; + steps = n - 1; + } else { + sx = (px[s] + px[e]) / 2; + sy = (py[s] + py[e]) / 2; + begin = 0; + steps = n; + } + out.point(OP_MOVE, sx, sy); + boolean pending = false; + float cx = 0; + float cy = 0; + for (int k = 0; k < steps; k++) { + int i = s + (begin + k) % n; + float x = px[i]; + float y = py[i]; + if ((flags[i] & 1) != 0) { + if (pending) { + quad(out, cx, cy, x, y); + pending = false; + } else { + out.point(OP_LINE, x, y); + } + } else { + if (pending) { + quad(out, cx, cy, (cx + x) / 2, (cy + y) / 2); + } + cx = x; + cy = y; + pending = true; + } + } + if (pending) { + quad(out, cx, cy, sx, sy); + } + out.add(OP_CLOSE); + } + + private static void quad(Glyph out, float cx, float cy, float x, float y) { + out.add(OP_QUAD); + out.add(cx); + out.add(cy); + out.add(x); + out.add(y); + out.include(cx, cy); + out.include(x, y); + } + + private static void composite(Face f, byte[] data, int start, int limit, Glyph out, double[] m, int depth) { + int q = start; + while (q + 4 <= limit) { + int flags = u16(data, q); + int child = u16(data, q + 2); + q += 4; + double dx; + double dy; + int argLen = (flags & 1) != 0 ? 4 : 2; + if (q + argLen > limit) { + return; + } + if (argLen == 4) { + dx = s16(data, q); + dy = s16(data, q + 2); + } else { + dx = data[q]; + dy = data[q + 1]; + } + q += argLen; + if ((flags & 2) == 0) { + // Point-matched placement (anchor points instead of an + // offset) is rare outside old CJK fonts; the component is + // placed at the origin rather than refused. + dx = 0; + dy = 0; + } + double ca = 1; + double cb = 0; + double cc = 0; + double cd = 1; + if ((flags & 8) != 0) { + if (q + 2 > limit) { + return; + } + ca = f2dot14(data, q); + cd = ca; + q += 2; + } else if ((flags & 0x40) != 0) { + if (q + 4 > limit) { + return; + } + ca = f2dot14(data, q); + cd = f2dot14(data, q + 2); + q += 4; + } else if ((flags & 0x80) != 0) { + if (q + 8 > limit) { + return; + } + ca = f2dot14(data, q); + cb = f2dot14(data, q + 2); + cc = f2dot14(data, q + 4); + cd = f2dot14(data, q + 6); + q += 8; + } + // parent(child(p)), child(p) = (ca x + cc y + dx, cb x + cd y + dy) + double[] n = new double[] { + m[0] * ca + m[2] * cb, + m[1] * ca + m[3] * cb, + m[0] * cc + m[2] * cd, + m[1] * cc + m[3] * cd, + m[0] * dx + m[2] * dy + m[4], + m[1] * dx + m[3] * dy + m[5] + }; + decode(f, child, out, n, depth + 1); + if ((flags & 0x20) == 0) { + return; + } + } + } + + private static double f2dot14(byte[] data, int p) { + return s16(data, p) / 16384.0; + } + + private static int lookup12(Face f, int cp) { + byte[] d = f.d; + int t = f.cmap12; + int lo = 0; + int hi = (int) u32(d, t + 12) - 1; + while (lo <= hi) { + int mid = (lo + hi) >>> 1; + int g = t + 16 + mid * 12; + long first = u32(d, g); + long last = u32(d, g + 4); + if (cp < first) { + hi = mid - 1; + } else if (cp > last) { + lo = mid + 1; + } else { + return (int) (u32(d, g + 8) + cp - first); + } + } + return 0; + } + + private static int lookup4(Face f, int cp) { + byte[] d = f.d; + int t = f.cmap4; + int segX2 = u16(d, t + 6); + int ends = t + 14; + int starts = ends + segX2 + 2; + int deltas = starts + segX2; + int offsets = deltas + segX2; + if (offsets + segX2 > d.length) { + return 0; + } + int lo = 0; + int hi = segX2 / 2 - 1; + int seg = -1; + while (lo <= hi) { + int mid = (lo + hi) >>> 1; + if (cp > u16(d, ends + mid * 2)) { + lo = mid + 1; + } else if (cp < u16(d, starts + mid * 2)) { + hi = mid - 1; + } else { + seg = mid; + lo = hi + 1; + } + } + if (seg < 0) { + return 0; + } + int startCode = u16(d, starts + seg * 2); + int delta = u16(d, deltas + seg * 2); + int ro = u16(d, offsets + seg * 2); + if (ro == 0) { + return (cp + delta) & 0xffff; + } + int gp = offsets + seg * 2 + ro + (cp - startCode) * 2; + if (gp + 2 > d.length) { + return 0; + } + int g = u16(d, gp); + return g == 0 ? 0 : (g + delta) & 0xffff; + } + + private static Face parse(byte[] d, int index) throws IOException { + if (d == null || d.length < 12) { + throw new IOException("not a font: too short"); + } + int base = 0; + long tag = u32(d, 0); + if (tag == 0x74746366L) { + // 'ttcf': a collection of faces + check(d, 0, 12); + int n = (int) u32(d, 8); + if (index < 0 || index >= n) { + throw new IOException("font index " + index + " out of range; the collection has " + n); + } + check(d, 12 + index * 4, 4); + base = (int) u32(d, 12 + index * 4); + check(d, base, 12); + tag = u32(d, base); + } else if (index != 0) { + throw new IOException("font index " + index + " out of range; the file has one face"); + } + if (tag == 0x4f54544fL) { + // 'OTTO': OpenType with CFF outlines + throw new IOException("fonts with PostScript (CFF) outlines are not supported;" + + " use a TrueType-flavoured font (.ttf)"); + } + if (tag != 0x00010000L && tag != 0x74727565L) { + throw new IOException("not a TrueType font"); + } + int numTables = u16(d, base + 4); + check(d, base + 12, numTables * 16); + Face f = new Face(); + f.d = d; + int[] tables = new int[8]; + int[] lengths = new int[8]; + for (int i = 0; i < tables.length; i++) { + tables[i] = -1; + } + for (int i = 0; i < numTables; i++) { + int r = base + 12 + i * 16; + int which = tableSlot(u32(d, r)); + if (which >= 0) { + int off = (int) u32(d, r + 8); + int len = (int) u32(d, r + 12); + check(d, off, len); + tables[which] = off; + lengths[which] = len; + } + } + int head = tables[0]; + int hhea = tables[1]; + int maxp = tables[2]; + int cmap = tables[3]; + f.hmtx = tables[4]; + f.loca = tables[5]; + f.locaLen = lengths[5]; + f.glyf = tables[6]; + f.glyfLen = lengths[6]; + if (head < 0 || hhea < 0 || maxp < 0 || cmap < 0 || f.hmtx < 0 || f.loca < 0 || f.glyf < 0) { + throw new IOException("not a TrueType font: a required table is missing"); + } + check(d, head, 54); + check(d, hhea, 36); + check(d, maxp, 6); + f.unitsPerEm = u16(d, head + 18); + if (f.unitsPerEm < 16 || f.unitsPerEm > 16384) { + throw new IOException("bad unitsPerEm " + f.unitsPerEm); + } + f.locaLong = s16(d, head + 50); + f.ascender = s16(d, hhea + 4); + f.descender = s16(d, hhea + 6); + f.lineGap = s16(d, hhea + 8); + f.numHMetrics = u16(d, hhea + 34); + f.numGlyphs = u16(d, maxp + 4); + if (f.numHMetrics * 4 > lengths[4]) { + throw new IOException("hmtx table is too short"); + } + parseCmap(f, cmap); + if (f.cmap12 < 0 && f.cmap4 < 0) { + throw new IOException("the font has no Unicode cmap (format 4 or 12)"); + } + if (tables[7] >= 0) { + parseKern(f, tables[7], lengths[7]); + } + parseName(f, d, base, numTables); + return f; + } + + private static int tableSlot(long tag) { + if (tag == 0x68656164L) { + return 0; // head + } + if (tag == 0x68686561L) { + return 1; // hhea + } + if (tag == 0x6d617870L) { + return 2; // maxp + } + if (tag == 0x636d6170L) { + return 3; // cmap + } + if (tag == 0x686d7478L) { + return 4; // hmtx + } + if (tag == 0x6c6f6361L) { + return 5; // loca + } + if (tag == 0x676c7966L) { + return 6; // glyf + } + if (tag == 0x6b65726eL) { + return 7; // kern + } + return -1; + } + + private static void parseCmap(Face f, int cmap) throws IOException { + byte[] d = f.d; + check(d, cmap, 4); + int n = u16(d, cmap + 2); + check(d, cmap + 4, n * 8); + int rank4 = 0; + for (int i = 0; i < n; i++) { + int r = cmap + 4 + i * 8; + int platform = u16(d, r); + int encoding = u16(d, r + 2); + int sub = cmap + (int) u32(d, r + 4); + check(d, sub, 2); + int format = u16(d, sub); + boolean unicode = platform == 0 || (platform == 3 && (encoding == 1 || encoding == 10)); + if (format == 12 && unicode && f.cmap12 < 0) { + check(d, sub, 16); + check(d, sub + 16, (int) u32(d, sub + 12) * 12); + f.cmap12 = sub; + } else if (format == 4) { + check(d, sub, 14); + check(d, sub, u16(d, sub + 2)); + int rank = unicode ? 2 : (platform == 3 && encoding == 0 ? 1 : 0); + if (rank > rank4) { + rank4 = rank; + f.cmap4 = sub; + } + } + } + f.symbol = rank4 == 1; + } + + private static void parseKern(Face f, int kern, int len) { + byte[] d = f.d; + if (len < 4 || u16(d, kern) != 0) { + // Apple's version 1 table, or an empty one: no kerning + return; + } + int n = u16(d, kern + 2); + int p = kern + 4; + int end = kern + len; + for (int i = 0; i < n && p + 6 <= end; i++) { + int subLen = u16(d, p + 2); + int coverage = u16(d, p + 4); + boolean horizontal = (coverage & 1) != 0; + boolean plain = (coverage & 6) == 0; + if ((coverage >> 8) == 0 && horizontal && plain && p + 14 <= end) { + f.kernPairs = Math.min(u16(d, p + 6), (end - (p + 14)) / 6); + f.kernStart = p + 14; + return; + } + if (subLen < 6) { + return; + } + p += subLen; + } + } + + private static void parseName(Face f, byte[] d, int base, int numTables) { + int name = -1; + int len = 0; + for (int i = 0; i < numTables; i++) { + int r = base + 12 + i * 16; + if (u32(d, r) == 0x6e616d65L) { + name = (int) u32(d, r + 8); + len = (int) u32(d, r + 12); + } + } + if (name < 0 || len < 6 || name > d.length - len) { + return; + } + int count = u16(d, name + 2); + int strings = name + u16(d, name + 4); + int end = name + len; + String[] best = new String[3]; + int[] rank = new int[3]; + for (int i = 0; i < count && name + 6 + i * 12 + 12 <= end; i++) { + int r = name + 6 + i * 12; + int platform = u16(d, r); + int encoding = u16(d, r + 2); + int id = u16(d, r + 6); + int l = u16(d, r + 8); + int o = strings + u16(d, r + 10); + if ((id != 1 && id != 2) || o + l > end) { + continue; + } + int score; + if (platform == 3 && (encoding == 0 || encoding == 1 || encoding == 10)) { + score = u16(d, r + 4) == 0x409 ? 4 : 3; + } else if (platform == 0) { + score = 2; + } else if (platform == 1 && encoding == 0) { + score = 1; + } else { + continue; + } + if (score <= rank[id]) { + continue; + } + StringBuilder sb = new StringBuilder(l); + if (score == 1) { + for (int k = 0; k < l; k++) { + sb.append((char) (d[o + k] & 0xff)); + } + } else { + for (int k = 0; k + 1 < l; k += 2) { + sb.append((char) u16(d, o + k)); + } + } + rank[id] = score; + best[id] = sb.toString(); + } + if (best[1] != null) { + f.family = best[1]; + } + if (best[2] != null) { + f.style = best[2]; + } + } + + private static void check(byte[] d, int off, int len) throws IOException { + if (off < 0 || len < 0 || off > d.length - len) { + throw new IOException("truncated font data"); + } + } + + private static int u16(byte[] d, int p) { + return ((d[p] & 0xff) << 8) | (d[p + 1] & 0xff); + } + + private static int s16(byte[] d, int p) { + return (short) u16(d, p); + } + + private static long u32(byte[] d, int p) { + return ((long) u16(d, p) << 16) | u16(d, p + 2); + } +} diff --git a/CodenameOne/src/com/codename1/image/package-info.java b/CodenameOne/src/com/codename1/image/package-info.java index 4dd9ef4839b..1c16cc1921c 100644 --- a/CodenameOne/src/com/codename1/image/package-info.java +++ b/CodenameOne/src/com/codename1/image/package-info.java @@ -30,7 +30,8 @@ /// - [RasterGraphics] draws antialiased lines, rectangles, ellipses, arcs, /// polygons and Bezier paths ([RasterPath]), with transforms, clipping, /// alpha, gradients ([RasterGradient]) and images. Text goes through the -/// [RasterFont] seam. +/// [RasterFont] seam; [TrueTypeFont] reads TrueType fonts and embeds a +/// default one, so text needs no font file. /// - [Resampler] is Pillow's resampling convolution, byte for byte. /// - [PngCodec] and [JpegCodec] read and write files. /// diff --git a/CodenameOne/third-party/aileron/LICENSE.txt b/CodenameOne/third-party/aileron/LICENSE.txt new file mode 100644 index 00000000000..0e259d42c99 --- /dev/null +++ b/CodenameOne/third-party/aileron/LICENSE.txt @@ -0,0 +1,121 @@ +Creative Commons Legal Code + +CC0 1.0 Universal + + CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE + LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN + ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS + INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES + REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS + PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM + THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED + HEREUNDER. + +Statement of Purpose + +The laws of most jurisdictions throughout the world automatically confer +exclusive Copyright and Related Rights (defined below) upon the creator +and subsequent owner(s) (each and all, an "owner") of an original work of +authorship and/or a database (each, a "Work"). + +Certain owners wish to permanently relinquish those rights to a Work for +the purpose of contributing to a commons of creative, cultural and +scientific works ("Commons") that the public can reliably and without fear +of later claims of infringement build upon, modify, incorporate in other +works, reuse and redistribute as freely as possible in any form whatsoever +and for any purposes, including without limitation commercial purposes. +These owners may contribute to the Commons to promote the ideal of a free +culture and the further production of creative, cultural and scientific +works, or to gain reputation or greater distribution for their Work in +part through the use and efforts of others. + +For these and/or other purposes and motivations, and without any +expectation of additional consideration or compensation, the person +associating CC0 with a Work (the "Affirmer"), to the extent that he or she +is an owner of Copyright and Related Rights in the Work, voluntarily +elects to apply CC0 to the Work and publicly distribute the Work under its +terms, with knowledge of his or her Copyright and Related Rights in the +Work and the meaning and intended legal effect of CC0 on those rights. + +1. Copyright and Related Rights. A Work made available under CC0 may be +protected by copyright and related or neighboring rights ("Copyright and +Related Rights"). Copyright and Related Rights include, but are not +limited to, the following: + + i. the right to reproduce, adapt, distribute, perform, display, + communicate, and translate a Work; + ii. moral rights retained by the original author(s) and/or performer(s); +iii. publicity and privacy rights pertaining to a person's image or + likeness depicted in a Work; + iv. rights protecting against unfair competition in regards to a Work, + subject to the limitations in paragraph 4(a), below; + v. rights protecting the extraction, dissemination, use and reuse of data + in a Work; + vi. database rights (such as those arising under Directive 96/9/EC of the + European Parliament and of the Council of 11 March 1996 on the legal + protection of databases, and under any national implementation + thereof, including any amended or successor version of such + directive); and +vii. other similar, equivalent or corresponding rights throughout the + world based on applicable law or treaty, and any national + implementations thereof. + +2. Waiver. To the greatest extent permitted by, but not in contravention +of, applicable law, Affirmer hereby overtly, fully, permanently, +irrevocably and unconditionally waives, abandons, and surrenders all of +Affirmer's Copyright and Related Rights and associated claims and causes +of action, whether now known or unknown (including existing as well as +future claims and causes of action), in the Work (i) in all territories +worldwide, (ii) for the maximum duration provided by applicable law or +treaty (including future time extensions), (iii) in any current or future +medium and for any number of copies, and (iv) for any purpose whatsoever, +including without limitation commercial, advertising or promotional +purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each +member of the public at large and to the detriment of Affirmer's heirs and +successors, fully intending that such Waiver shall not be subject to +revocation, rescission, cancellation, termination, or any other legal or +equitable action to disrupt the quiet enjoyment of the Work by the public +as contemplated by Affirmer's express Statement of Purpose. + +3. Public License Fallback. Should any part of the Waiver for any reason +be judged legally invalid or ineffective under applicable law, then the +Waiver shall be preserved to the maximum extent permitted taking into +account Affirmer's express Statement of Purpose. In addition, to the +extent the Waiver is so judged Affirmer hereby grants to each affected +person a royalty-free, non transferable, non sublicensable, non exclusive, +irrevocable and unconditional license to exercise Affirmer's Copyright and +Related Rights in the Work (i) in all territories worldwide, (ii) for the +maximum duration provided by applicable law or treaty (including future +time extensions), (iii) in any current or future medium and for any number +of copies, and (iv) for any purpose whatsoever, including without +limitation commercial, advertising or promotional purposes (the +"License"). The License shall be deemed effective as of the date CC0 was +applied by Affirmer to the Work. Should any part of the License for any +reason be judged legally invalid or ineffective under applicable law, such +partial invalidity or ineffectiveness shall not invalidate the remainder +of the License, and in such case Affirmer hereby affirms that he or she +will not (i) exercise any of his or her remaining Copyright and Related +Rights in the Work or (ii) assert any associated claims and causes of +action with respect to the Work, in either case contrary to Affirmer's +express Statement of Purpose. + +4. Limitations and Disclaimers. + + a. No trademark or patent rights held by Affirmer are waived, abandoned, + surrendered, licensed or otherwise affected by this document. + b. Affirmer offers the Work as-is and makes no representations or + warranties of any kind concerning the Work, express, implied, + statutory or otherwise, including without limitation warranties of + title, merchantability, fitness for a particular purpose, non + infringement, or the absence of latent or other defects, accuracy, or + the present or absence of errors, whether or not discoverable, all to + the greatest extent permissible under applicable law. + c. Affirmer disclaims responsibility for clearing rights of other persons + that may apply to the Work or any use thereof, including without + limitation any person's Copyright and Related Rights in the Work. + Further, Affirmer disclaims responsibility for obtaining any necessary + consents, permissions or other rights required for any use of the + Work. + d. Affirmer understands and acknowledges that Creative Commons is not a + party to this document and has no duty or obligation with respect to + this CC0 or use of the Work. diff --git a/NOTICE b/NOTICE index 25cfa1a1698..39753e86dfb 100644 --- a/NOTICE +++ b/NOTICE @@ -255,3 +255,14 @@ Pillow's notice: Copyright (c) 1997-2003 by Secret Labs AB Copyright (c) 1996-2003 by Fredrik Lundh Licensed under the MIT-CMU (HPND) license; the full text is in the file. + +================================================================================ +Aileron (font) +================================================================================ + +CodenameOne/src/com/codename1/image/DefaultFontData.java embeds Aileron +Regular by Sora Sagano (https://dotcolon.net/fonts/aileron), the subset +Pillow embeds for ImageFont.load_default(). It is the default font of +com.codename1.image.TrueTypeFont and of the Python runtime's PIL.ImageFont. +Aileron is dedicated to the public domain under CC0 1.0 Universal; the legal +code is kept in CodenameOne/third-party/aileron/LICENSE.txt. diff --git a/maven/core-unittests/src/test/java/com/codename1/image/TrueTypeFontTest.java b/maven/core-unittests/src/test/java/com/codename1/image/TrueTypeFontTest.java new file mode 100644 index 00000000000..abcd624db58 --- /dev/null +++ b/maven/core-unittests/src/test/java/com/codename1/image/TrueTypeFontTest.java @@ -0,0 +1,224 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.image; + +import org.junit.jupiter.api.Test; + +import java.awt.Font; +import java.awt.Graphics2D; +import java.awt.RenderingHints; +import java.awt.font.FontRenderContext; +import java.awt.font.GlyphVector; +import java.awt.image.BufferedImage; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/// [TrueTypeFont] checked against the JDK's font engine: advances, glyph +/// outlines (composite glyphs included) and rendered coverage. +class TrueTypeFontTest { + private static final String ROBOTO = "Ports/JavaSE/src/com/codename1/impl/javase/Roboto-Regular.ttf"; + + private static byte[] roboto() throws IOException { + Path p = Paths.get(ROBOTO); + for (int i = 0; i < 4 && !Files.exists(p); i++) { + p = Paths.get("..").resolve(p); + } + return Files.readAllBytes(p); + } + + private static Font awt(byte[] data, float size) throws Exception { + return Font.createFont(Font.TRUETYPE_FONT, new ByteArrayInputStream(data)).deriveFont(size); + } + + private static final FontRenderContext FRC = new FontRenderContext(null, true, true); + + @Test + void defaultFontIsAileron() { + TrueTypeFont f = TrueTypeFont.createDefault(20); + assertEquals("Aileron", f.getFamilyName()); + assertEquals("Regular", f.getStyleName()); + assertEquals(1000, f.getUnitsPerEm()); + assertTrue(f.canDisplay('A') && f.canDisplay('~')); + assertTrue(f.getAscent() > 0 && f.getDescent() > 0); + assertEquals(40, f.deriveFont(40).getSize(), 0); + assertEquals(2 * f.stringWidth("Hello"), f.deriveFont(40).stringWidth("Hello"), 1e-9); + assertEquals(TrueTypeFont.getDefaultFontData().length, TrueTypeFont.getDefaultFontData().length); + } + + @Test + void advancesMatchTheJdk() throws Exception { + for (byte[] data : new byte[][] {roboto(), TrueTypeFont.getDefaultFontData()}) { + TrueTypeFont f = TrueTypeFont.load(data, 32); + Font j = awt(data, 32); + for (String s : new String[] {"Hello, World!", "The quick brown fox", "0123456789", "caf\u00e9"}) { + double jw = j.getStringBounds(s, FRC).getWidth(); + assertEquals(jw, f.stringWidth(s), 0.05, s); + } + } + } + + @Test + void outlinesMatchTheJdk() throws Exception { + byte[] data = roboto(); + TrueTypeFont f = TrueTypeFont.load(data, 48); + Font j = awt(data, 48); + // \u00e9 and \u00c5 are composite glyphs in Roboto + for (String s : new String[] {"g", "Q", "\u00e9", "\u00c5", "%&@"}) { + int w = 160; + int h = 80; + BufferedImage ref = new BufferedImage(w, h, BufferedImage.TYPE_INT_ARGB); + Graphics2D g2 = ref.createGraphics(); + g2.setRenderingHint(RenderingHints.KEY_ANTIALIASING, RenderingHints.VALUE_ANTIALIAS_ON); + g2.setRenderingHint(RenderingHints.KEY_FRACTIONALMETRICS, RenderingHints.VALUE_FRACTIONALMETRICS_ON); + g2.setColor(java.awt.Color.WHITE); + GlyphVector gv = j.createGlyphVector(FRC, s); + g2.fill(gv.getOutline(10, 60)); + g2.dispose(); + RasterImage ours = new RasterImage(w, h); + RasterGraphics g = ours.createGraphics(); + g.setColor(0xffffffff); + g.setFont(f); + g.drawString(s, 10, 60); + double a = 0; + double b = 0; + double diff = 0; + for (int y = 0; y < h; y++) { + for (int x = 0; x < w; x++) { + int ra = ref.getRGB(x, y) >>> 24; + int oa = ours.getRGB(x, y) >>> 24; + a += ra; + b += oa; + diff += Math.abs(ra - oa); + } + } + assertTrue(a > 0, s); + assertEquals(a, b, a * 0.03, s + " coverage"); + assertTrue(diff < a * 0.12, s + " differs in " + diff / a + " of its ink"); + } + double[] e = f.glyphBounds(f.glyphIndex(0xe9)); + assertNotNull(e); + assertTrue(e[3] > f.glyphBounds(f.glyphIndex('e'))[3], "the accent sits above the e"); + } + + @Test + void kernTableFormatZero() throws Exception { + byte[] base = TrueTypeFont.getDefaultFontData(); + TrueTypeFont plain = TrueTypeFont.load(base, 1000); + int a = plain.glyphIndex('A'); + int v = plain.glyphIndex('V'); + assertEquals(0, plain.kerning(a, v)); + // a kern table: version 0, one horizontal format 0 subtable, one pair + byte[] kern = new byte[4 + 14 + 6]; + put16(kern, 2, 1); + put16(kern, 6, 14 + 6); + put16(kern, 8, 1); + put16(kern, 10, 1); + put16(kern, 18, a); + put16(kern, 20, v); + put16(kern, 22, -80); + TrueTypeFont kerned = TrueTypeFont.load(withTable(base, "kern", kern), 1000); + assertEquals(-80, kerned.kerning(a, v)); + assertEquals(0, kerned.kerning(v, a)); + assertEquals(plain.stringWidth("AV") - 80, kerned.stringWidth("AV"), 1e-9); + assertEquals(plain.stringWidth("VA"), kerned.stringWidth("VA"), 1e-9); + } + + @Test + void refusesWhatItCannotRead() { + assertThrows(IOException.class, () -> TrueTypeFont.load(new byte[] {1, 2, 3}, 12)); + byte[] otto = new byte[64]; + otto[0] = 'O'; + otto[1] = 'T'; + otto[2] = 'T'; + otto[3] = 'O'; + IOException e = assertThrows(IOException.class, () -> TrueTypeFont.load(otto, 12)); + assertTrue(e.getMessage().contains("CFF")); + byte[] cut = TrueTypeFont.getDefaultFontData(); + byte[] half = new byte[cut.length / 2]; + System.arraycopy(cut, 0, half, 0, half.length); + assertThrows(IOException.class, () -> TrueTypeFont.load(half, 12)); + assertThrows(IllegalArgumentException.class, () -> TrueTypeFont.createDefault(0)); + } + + @Test + void cacheStaysBounded() throws Exception { + TrueTypeFont f = TrueTypeFont.load(roboto(), 12); + RasterPath p = new RasterPath(); + for (int g = 0; g < f.getGlyphCount(); g++) { + f.appendGlyph(p, g, 0, 0); + } + assertTrue(f.getGlyphCount() > 512); + assertTrue(f.stringWidth("still works") > 0); + } + + private static void put16(byte[] d, int p, int v) { + d[p] = (byte) (v >> 8); + d[p + 1] = (byte) v; + } + + private static int u16(byte[] d, int p) { + return ((d[p] & 0xff) << 8) | (d[p + 1] & 0xff); + } + + private static int u32(byte[] d, int p) { + return (u16(d, p) << 16) | u16(d, p + 2); + } + + private static void put32(byte[] d, int p, int v) { + put16(d, p, v >>> 16); + put16(d, p + 2, v); + } + + /// Returns `font` with one more table, appended after the others. + private static byte[] withTable(byte[] font, String tag, byte[] table) { + int n = u16(font, 4); + int shift = 16; + byte[] out = new byte[font.length + shift + table.length + 3 & ~3]; + System.arraycopy(font, 0, out, 0, 12); + put16(out, 4, n + 1); + for (int i = 0; i < n; i++) { + System.arraycopy(font, 12 + i * 16, out, 12 + i * 16, 16); + put32(out, 12 + i * 16 + 8, u32(font, 12 + i * 16 + 8) + shift); + } + int r = 12 + n * 16; + for (int i = 0; i < 4; i++) { + out[r + i] = (byte) tag.charAt(i); + } + int dataStart = 12 + n * 16; + System.arraycopy(font, dataStart, out, dataStart + shift, font.length - dataStart); + int at = font.length + shift + 3 & ~3; + out = java.util.Arrays.copyOf(out, at + table.length); + put32(out, r + 8, at); + put32(out, r + 12, table.length); + System.arraycopy(table, 0, out, at, table.length); + return out; + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/FreeTypeMetrics.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/FreeTypeMetrics.java new file mode 100644 index 00000000000..cc69c880b97 --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/FreeTypeMetrics.java @@ -0,0 +1,240 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +import com.codename1.image.RasterGraphics; +import com.codename1.image.RasterImage; +import com.codename1.image.RasterPath; +import com.codename1.image.TrueTypeFont; + +/// Pillow's basic text layout over a [TrueTypeFont], in FreeType's 26.6 +/// fixed point, so `getlength`, `getbbox`, `getmetrics` and the anchors give +/// the integers Pillow gives. +/// +/// Pillow loads glyphs with `FT_LOAD_DEFAULT`, so FreeType rounds every +/// advance to a whole pixel, the ascender up and the descender down. Glyph +/// boxes come from the control box of the scaled outline, floored and ceiled +/// to pixels (`FT_GLYPH_BBOX_PIXELS`). Outlines are not hinted here; a font +/// with TrueType instructions can therefore differ from FreeType by a pixel +/// at the edge of a box, never in an advance. +/// +/// Kerning follows Pillow's `text_layout_fallback`, which adds the pixel +/// count of a `kern` pair to an advance kept in 64ths of a pixel -- a pair +/// moves the pen by a 64th per pixel of kerning. That is what Pillow does, +/// so it is what this does. +final class FreeTypeMetrics { + final TrueTypeFont font; + private final long scale; + final long ascender; + final long descender; + + FreeTypeMetrics(TrueTypeFont font) { + this.font = font; + long size26 = (long) (font.getSize() * 64); + int upem = font.getUnitsPerEm(); + this.scale = ((size26 << 16) + upem / 2) / upem; + this.ascender = ceil(mulFix(font.getAscenderUnits(), scale)); + this.descender = floor(mulFix(font.getDescenderUnits(), scale)); + } + + static long mulFix(long a, long b) { + long p = a * b; + return p >= 0 ? (p + 0x8000) >> 16 : -((-p + 0x8000) >> 16); + } + + static long floor(long v) { + return v & -64; + } + + static long ceil(long v) { + return (v + 63) & -64; + } + + static long round(long v) { + return (v + 32) & -64; + } + + /// Pillow's `PIXEL`: a 26.6 value rounded to whole pixels. + static int pixel(long v) { + return (int) (((v + 32) & -64) >> 6); + } + + /// Text layout: glyph indices and advances in 26.6. + static final class Layout { + final int[] glyphs; + final long[] advances; + int count; + long position; + + Layout(int capacity) { + glyphs = new int[capacity]; + advances = new long[capacity]; + } + } + + /// Returns whether `anchor` is one of Pillow's horizontal anchors (null + /// meaning the default, `la`). + static boolean validAnchor(String anchor) { + return anchor == null || (anchor.length() == 2 && "lmr".indexOf(anchor.charAt(0)) >= 0 + && "atmsbd".indexOf(anchor.charAt(1)) >= 0); + } + + Layout layout(String text) { + int n = text.length(); + Layout l = new Layout(n); + int last = 0; + int i = 0; + while (i < n) { + int cp = text.charAt(i++); + if (cp >= 0xd800 && cp < 0xdc00 && i < n) { + int lo = text.charAt(i); + if (lo >= 0xdc00 && lo < 0xe000) { + cp = ((cp - 0xd800) << 10) + (lo - 0xdc00) + 0x10000; + i++; + } + } + int g = font.glyphIndex(cp); + int k = l.count; + if (last != 0 && g != 0) { + long delta = round(mulFix(font.kerning(last, g), scale)); + l.advances[k - 1] += pixel(delta); + } + l.glyphs[k] = g; + l.advances[k] = round(mulFix(font.glyphAdvance(g), scale)); + l.count++; + last = g; + } + for (int k = 0; k < l.count; k++) { + l.position += l.advances[k]; + } + return l; + } + + long length(String text) { + return layout(text).position; + } + + /// Returns `{width, height, xOffset, yOffset, xMin, yMax}` as Pillow's + /// `bounding_box_and_anchors` computes them for horizontal text; the + /// anchor must pass [#validAnchor(String)]. + int[] box(Layout l, String anchor) { + long position = 0; + int xMin = 0; + int xMax = 0; + int yMin = 0; + int yMax = 0; + for (int i = 0; i < l.count; i++) { + int px = pixel(position); + position += l.advances[i]; + int advanced = pixel(position); + if (advanced > xMax) { + xMax = advanced; + } + double[] b = font.glyphBounds(l.glyphs[i]); + if (b == null) { + continue; + } + int bx0 = (int) (floor(scaled(b[0])) >> 6) + px; + int by0 = (int) (floor(scaled(b[1])) >> 6); + int bx1 = (int) (ceil(scaled(b[2])) >> 6) + px; + int by1 = (int) (ceil(scaled(b[3])) >> 6); + xMax = Math.max(xMax, bx1); + xMin = Math.min(xMin, bx0); + yMax = Math.max(yMax, by1); + yMin = Math.min(yMin, by0); + } + String a = anchor == null ? "la" : anchor; + int xAnchor; + int yAnchor; + switch (a.charAt(0)) { + case 'l': + xAnchor = 0; + break; + case 'm': + xAnchor = l.count == 0 ? 0 : pixel(l.position / 2); + break; + default: + xAnchor = l.count == 0 ? 0 : pixel(l.position); + break; + } + switch (a.charAt(1)) { + case 'a': + yAnchor = pixel(ascender); + break; + case 't': + yAnchor = yMax; + break; + case 'm': + yAnchor = pixel((ascender + descender) / 2); + break; + case 's': + yAnchor = 0; + break; + case 'b': + yAnchor = yMin; + break; + default: + yAnchor = pixel(descender); + break; + } + if (l.count == 0) { + xAnchor = 0; + yAnchor = 0; + } + return new int[] {xMax - xMin, yMax - yMin, xMin - xAnchor, yAnchor - yMax, xMin, yMax}; + } + + private long scaled(double units) { + return mulFix(Math.round(units), scale); + } + + /// Renders the layout as an 8-bit coverage mask of `box`'s size, the + /// glyph origins at Pillow's pixel pen positions. + byte[] mask(Layout l, int[] box, boolean mono) { + int w = box[0]; + int h = box[1]; + byte[] out = new byte[w * h]; + if (w <= 0 || h <= 0) { + return out; + } + RasterImage img = new RasterImage(w, h); + RasterGraphics g = img.createGraphics(); + g.setAntiAlias(!mono); + g.setColor(0xffffffff); + RasterPath path = new RasterPath(); + // the mask's left edge is xMin, its top edge yMax above the baseline + long position = 0; + for (int i = 0; i < l.count; i++) { + int px = pixel(position); + position += l.advances[i]; + font.appendGlyph(path, l.glyphs[i], px - box[4], box[5]); + } + g.fill(path); + int[] px = img.getPixels(); + for (int i = 0; i < out.length; i++) { + int a = px[i] >>> 24; + out[i] = (byte) (mono ? (a >= 128 ? 255 : 0) : a); + } + return out; + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyImage.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyImage.java index 99f955ed68e..47d5509cbbc 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyImage.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyImage.java @@ -27,6 +27,7 @@ import com.codename1.image.PngCodec; import com.codename1.image.RasterImage; import com.codename1.image.Resampler; +import com.codename1.image.TrueTypeFont; import java.io.IOException; @@ -255,6 +256,43 @@ static PyModule module() { } return new PyBytes(out); })); + m.setAttr("ft_load", PyBuiltin.positional("ft_load", a -> { + try { + return new PyJavaObject(new FreeTypeMetrics( + TrueTypeFont.load(bytes(a[0]), i(a[1]), Py.asDouble(a[2])))); + } catch (IOException e) { + throw osError(e); + } + })); + m.setAttr("ft_default_data", PyBuiltin.positional("ft_default_data", a -> + new PyBytes(TrueTypeFont.getDefaultFontData()))); + m.setAttr("ft_variant", PyBuiltin.positional("ft_variant", a -> + new PyJavaObject(new FreeTypeMetrics(ft(a[0]).font.deriveFont(Py.asDouble(a[1])))))); + m.setAttr("ft_name", PyBuiltin.positional("ft_name", a -> { + TrueTypeFont f = ft(a[0]).font; + return new PyTuple(new Object[] {new PyStr(f.getFamilyName()), new PyStr(f.getStyleName())}); + })); + m.setAttr("ft_metrics", PyBuiltin.positional("ft_metrics", a -> { + FreeTypeMetrics f = ft(a[0]); + return new PyTuple(new Object[] {PyInt.of(FreeTypeMetrics.pixel(f.ascender)), + PyInt.of(-FreeTypeMetrics.pixel(f.descender))}); + })); + m.setAttr("ft_length", PyBuiltin.positional("ft_length", a -> + PyInt.of(ft(a[0]).length(Py.toStr(a[1]))))); + m.setAttr("ft_bbox", PyBuiltin.positional("ft_bbox", a -> { + FreeTypeMetrics f = ft(a[0]); + int[] b = ftBox(f, f.layout(Py.toStr(a[1])), a[2]); + return new PyTuple(new Object[] {PyInt.of(b[0]), PyInt.of(b[1]), PyInt.of(b[2]), PyInt.of(b[3])}); + })); + m.setAttr("ft_mask", PyBuiltin.positional("ft_mask", a -> { + FreeTypeMetrics f = ft(a[0]); + FreeTypeMetrics.Layout l = f.layout(Py.toStr(a[1])); + int[] b = ftBox(f, l, a[2]); + Img r = new Img(Math.max(0, b[0]), Math.max(0, b[1]), 1); + byte[] cov = f.mask(l, b, Py.isTrue(a[3])); + System.arraycopy(cov, 0, r.d, 0, Math.min(cov.length, r.d.length)); + return new PyTuple(new Object[] {wrap(r), PyInt.of(b[2]), PyInt.of(b[3])}); + })); m.setAttr("native_available", PyBuiltin.positional("native_available", a -> { Codec c = codec; return c != null && c.available() ? Py.TRUE : Py.FALSE; @@ -267,6 +305,21 @@ static PyModule module() { return m; } + private static FreeTypeMetrics ft(Object o) { + if (o instanceof PyJavaObject && ((PyJavaObject) o).value instanceof FreeTypeMetrics) { + return (FreeTypeMetrics) ((PyJavaObject) o).value; + } + throw Py.typeError("not a TrueType font"); + } + + private static int[] ftBox(FreeTypeMetrics f, FreeTypeMetrics.Layout l, Object anchor) { + String a = Py.NONE.equals(anchor) ? null : Py.toStr(anchor); + if (!FreeTypeMetrics.validAnchor(a)) { + throw Py.error(Py.VALUE_ERROR, "bad anchor specified: " + a); + } + return f.box(l, a); + } + private static Img crop(Img s, int x0, int y0, int x1, int y1) { Img r = new Img(Math.max(0, x1 - x0), Math.max(0, y1 - y0), s.bands); for (int y = 0; y < r.h; y++) { diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/ImageDraw.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/ImageDraw.py index 98e8b75b67c..15e68e9a01b 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/ImageDraw.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/ImageDraw.py @@ -36,6 +36,8 @@ def __init__(self, im, mode=None): self.mode = im.mode self.ink = _Image._color(im.mode, -1) self.fill = False + self.font = None + self.fontmode = "1" if im.mode in ("1", "P", "I", "F") else "L" def _ink(self, color): return None if color is None else _Image._color(self.mode, color) @@ -169,84 +171,129 @@ def polygon(self, xy, fill=None, outline=None, width=1): a, b = q[i], q[(i + 1) % len(q)] self._segment(a[0], a[1], b[0], b[1], ink, max(1, width)) - def _font(self, font): - if font is None: + def getfont(self): + if self.font is None: from PIL import ImageFont - font = ImageFont.load_default() - return font + self.font = ImageFont.load_default() + return self.font - def _spacing(self, font, spacing): - return self.textbbox((0, 0), "A", font)[3] + spacing + def _font(self, font, font_size=None): + if font is not None: + return font + if font_size is not None: + from PIL import ImageFont + return ImageFont.load_default(font_size) + return self.getfont() + + def _spacing(self, font, spacing, stroke_width=0): + return self.textbbox((0, 0), "A", font, stroke_width=stroke_width)[3] + stroke_width + spacing + + @staticmethod + def _scalable(font): + return hasattr(font, "getmask2") - def text(self, xy, text, fill=None, font=None, anchor=None, spacing=4, align="left", *args, **kwargs): - """Draws `text` with a bitmap font: each line's glyph mask is filled - with the ink at the integer position, as Pillow's basic layout does.""" - font = self._font(font) + def text(self, xy, text, fill=None, font=None, anchor=None, spacing=4, align="left", + direction=None, features=None, language=None, stroke_width=0, stroke_fill=None, + embedded_color=False, *args, **kwargs): + """Draws `text`. A TrueType font's coverage mask is filled with the + ink at the anchor's integer position, as Pillow does; the bitmap + font's glyph mask likewise, at the top-left corner.""" + font = self._font(font, kwargs.get("font_size")) if isinstance(text, bytes): text = text.decode("latin-1") if "\n" in text: - return self.multiline_text(xy, text, fill, font, anchor, spacing, align) + return self.multiline_text(xy, text, fill, font, anchor, spacing, align, + direction, features, language, stroke_width) ink = self._ink(fill) if ink is None: ink = self.ink - mask = font.getmask(text, self.mode) + x, y = int(xy[0]), int(xy[1]) + if self._scalable(font): + mask, offset = font.getmask2(text, self.fontmode, direction, features, language, + anchor=anchor) + x += offset[0] + y += offset[1] + else: + mask = font.getmask(text, self.mode) if mask.width == 0 or mask.height == 0: return - x, y = int(xy[0]), int(xy[1]) self.im.paste(ink, (x, y, x + mask.width, y + mask.height), mask) - def multiline_text(self, xy, text, fill=None, font=None, anchor=None, spacing=4, align="left", - *args, **kwargs): - font = self._font(font) + def _lines(self, xy, text, font, anchor, spacing, align, stroke_width=0): + if anchor is None: + anchor = "la" + elif len(anchor) != 2: + raise ValueError("anchor must be a 2 character string") + elif anchor[1] in "tb": + raise ValueError("anchor not supported for multiline text") lines = text.split("\n") - line_spacing = self._spacing(font, spacing) + line_spacing = self._spacing(font, spacing, stroke_width) widths = [self.textlength(line, font) for line in lines] widest = max(widths) top = xy[1] + if anchor[1] == "m": + top -= (len(lines) - 1) * line_spacing / 2.0 + elif anchor[1] == "d": + top -= (len(lines) - 1) * line_spacing + out = [] for line, w in zip(lines, widths): left = xy[0] + diff = widest - w + if anchor[0] == "m": + left -= diff / 2.0 + elif anchor[0] == "r": + left -= diff if align == "center": - left += (widest - w) / 2.0 + left += diff / 2.0 elif align == "right": - left += widest - w + left += diff elif align != "left": raise ValueError('align must be "left", "center" or "right"') - self.text((left, top), line, fill, font) + out.append(((left, top), line)) top += line_spacing + return out, anchor + + def multiline_text(self, xy, text, fill=None, font=None, anchor=None, spacing=4, align="left", + direction=None, features=None, language=None, stroke_width=0, *args, **kwargs): + font = self._font(font, kwargs.get("font_size")) + lines, anchor = self._lines(xy, text, font, anchor, spacing, align, stroke_width) + for pos, line in lines: + self.text(pos, line, fill, font, anchor if self._scalable(font) else None) - def textlength(self, text, font=None, *args, **kwargs): + def textlength(self, text, font=None, direction=None, features=None, language=None, + embedded_color=False, *args, **kwargs): if "\n" in text: raise ValueError("can't measure length of multiline text") - return self._font(font).getlength(text) + font = self._font(font, kwargs.get("font_size")) + if self._scalable(font): + return font.getlength(text, self.fontmode, direction, features, language) + return font.getlength(text) - def textbbox(self, xy, text, font=None, anchor=None, spacing=4, align="left", *args, **kwargs): - font = self._font(font) + def textbbox(self, xy, text, font=None, anchor=None, spacing=4, align="left", direction=None, + features=None, language=None, stroke_width=0, embedded_color=False, *args, **kwargs): + font = self._font(font, kwargs.get("font_size")) if "\n" in text: - return self.multiline_textbbox(xy, text, font, anchor, spacing, align) - b = font.getbbox(text) + return self.multiline_textbbox(xy, text, font, anchor, spacing, align, direction, + features, language, stroke_width) + if self._scalable(font): + b = font.getbbox(text, self.fontmode, direction, features, language, stroke_width, anchor) + else: + b = font.getbbox(text) return (b[0] + xy[0], b[1] + xy[1], b[2] + xy[0], b[3] + xy[1]) def multiline_textbbox(self, xy, text, font=None, anchor=None, spacing=4, align="left", + direction=None, features=None, language=None, stroke_width=0, *args, **kwargs): - font = self._font(font) - lines = text.split("\n") - line_spacing = self._spacing(font, spacing) - widths = [self.textlength(line, font) for line in lines] - widest = max(widths) + font = self._font(font, kwargs.get("font_size")) + lines, anchor = self._lines(xy, text, font, anchor, spacing, align, stroke_width) bbox = None - top = xy[1] - for line, w in zip(lines, widths): - left = xy[0] - if align == "center": - left += (widest - w) / 2.0 - elif align == "right": - left += widest - w - b = self.textbbox((left, top), line, font) + for pos, line in lines: + b = self.textbbox(pos, line, font, anchor if self._scalable(font) else None, + stroke_width=stroke_width) if bbox is None: bbox = list(b) else: bbox = [min(bbox[0], b[0]), min(bbox[1], b[1]), max(bbox[2], b[2]), max(bbox[3], b[3])] - top += line_spacing return tuple(bbox) diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/ImageFont.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/ImageFont.py index 36995620cad..e49c66715d0 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/ImageFont.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/ImageFont.py @@ -34,19 +34,27 @@ # renderer of Pillow's ImageFont, with Pillow's embedded default font # (courB08, the PIL font data and its glyph sheet) kept byte for byte. # -"""PIL raster fonts. Only Pillow's built-in bitmap font is available. +"""PIL fonts: Pillow's built-in bitmap font and scalable TrueType fonts. -`load_default()` returns that bitmap font: it is what Pillow returns when it -is built without FreeType. A Pillow with FreeType returns a scalable font -instead (Aileron), which this runtime does not have, so text drawn with -`load_default()` matches `ImageFont.load_default_imagefont()` in CPython. -`truetype()` raises, since no TrueType rasterizer is built in. +`truetype()` reads a TrueType font (a `.ttf`, an `.otf` with TrueType +outlines, or a face of a `.ttc`) with Codename One's pure-Java font engine, +`com.codename1.image.TrueTypeFont`. `load_default()` returns Aileron Regular, +the scalable font a Pillow with FreeType returns, embedded in that engine; +`load_default_imagefont()` returns the bitmap font. + +Layout is Pillow's basic layout (`Layout.BASIC`) whatever `layout_engine` +asks for: one glyph per character, advances rounded to whole pixels, kerning +from the `kern` table only, no shaping. Glyphs are not hinted, while FreeType +hints Aileron with its auto-hinter, so an advance or a box edge can differ +from Pillow's by a pixel at small sizes; `getmetrics()` and the anchor +arithmetic are Pillow's own. PostScript-flavoured (CFF) fonts raise OSError. """ import base64 import io import struct +import _cn1image from PIL import Image @@ -113,12 +121,105 @@ def load_default_imagefont(): return f +class Layout: + BASIC = 0 + RAQM = 1 + + +MAX_STRING_LENGTH = 1000000 + + +def _text(text): + if isinstance(text, bytes): + return text.decode("latin-1") + return text + + +class FreeTypeFont: + """A scalable TrueType font at one size.""" + + def __init__(self, font=None, size=10, index=0, encoding="", layout_engine=None): + if size <= 0: + raise ValueError("font size must be greater than 0, not " + str(size)) + self.path = font + self.size = size + self.index = index + self.encoding = encoding + self.layout_engine = Layout.BASIC + if isinstance(font, (str, bytes)) or hasattr(font, "__fspath__"): + with open(font, "rb") as f: + data = f.read() + else: + data = font.read() + self.font = _cn1image.ft_load(data, index, float(size)) + + def __getstate__(self): + return [self.path, self.size, self.index, self.encoding, self.layout_engine] + + def _check(self, direction, features, language): + if direction is not None or features is not None or language is not None: + raise KeyError("setting text direction, language or font features is not " + "supported without libraqm") + + def getname(self): + return _cn1image.ft_name(self.font) + + def getmetrics(self): + return _cn1image.ft_metrics(self.font) + + def getlength(self, text, mode="", direction=None, features=None, language=None): + self._check(direction, features, language) + return _cn1image.ft_length(self.font, _text(text)) / 64 + + def getbbox(self, text, mode="", direction=None, features=None, language=None, + stroke_width=0, anchor=None): + self._check(direction, features, language) + w, h, x, y = _cn1image.ft_bbox(self.font, _text(text), anchor) + left, top = x - stroke_width, y - stroke_width + return left, top, left + w + 2 * stroke_width, top + h + 2 * stroke_width + + def getmask2(self, text, mode="", direction=None, features=None, language=None, + stroke_width=0, anchor=None, ink=0, start=None, *args, **kwargs): + """Returns `(mask, offset)`: an "L" (or "1") coverage image of the + text and where its top-left corner lies relative to the anchor.""" + self._check(direction, features, language) + handle, x, y = _cn1image.ft_mask(self.font, _text(text), anchor, mode == "1") + w, h = _cn1image.size(handle) + return Image._wrap("L", (w, h), handle), (x, y) + + def getmask(self, text, mode="", *args, **kwargs): + return self.getmask2(text, mode, *args, **kwargs)[0] + + def font_variant(self, font=None, size=None, index=None, encoding=None, layout_engine=None): + if font is None and not isinstance(self.path, (str, bytes)) and self.path is not None: + v = FreeTypeFont.__new__(FreeTypeFont) + v.path, v.index, v.encoding, v.layout_engine = self.path, self.index, self.encoding, Layout.BASIC + v.size = self.size if size is None else size + if v.size <= 0: + raise ValueError("font size must be greater than 0, not " + str(v.size)) + v.font = _cn1image.ft_variant(self.font, float(v.size)) + return v + return FreeTypeFont(self.path if font is None else font, + self.size if size is None else size, + self.index if index is None else index, + self.encoding if encoding is None else encoding, + layout_engine) + + def load_default(size=None): - return load_default_imagefont() + """Aileron Regular at `size` pixels (10 by default), as a Pillow with + FreeType returns it.""" + return FreeTypeFont(io.BytesIO(_cn1image.ft_default_data()), 10 if size is None else size) def truetype(font=None, size=10, index=0, encoding="", layout_engine=None): - raise OSError("cannot open resource: TrueType fonts are not supported") + """Loads a TrueType font from a path, a file object or bytes.""" + if font is None: + raise OSError("cannot open resource") + try: + return FreeTypeFont(font, size, index, encoding, layout_engine) + except FileNotFoundError: + raise OSError("cannot open resource") def load(filename): diff --git a/maven/python-transpiler/src/test/resources/conformance/pil_truetype.out b/maven/python-transpiler/src/test/resources/conformance/pil_truetype.out new file mode 100644 index 00000000000..93ace887142 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/pil_truetype.out @@ -0,0 +1,35 @@ +FreeTypeFont ('Aileron', 'Regular') 20 +(20, 5) 10 (10, 3) +'H' 15.0 True +'AVA' 38.0 True +'Hello World' 107.0 True +'' 0.0 True +'gy j' 31.0 True +la True +lt True +ls True +lb True +ld True +ma True +mm True +ms True +rs True +rd True +ValueError bad anchor specified: xx +KeyError direction +10 (10, 3) [True, True, True] True +13 (13, 3) [True, True, True] True +37 (36, 9) [True, True, True] True +('Aileron', 'Regular') (20, 5) 48.0 True +40 (39, 10) True +OSError bad font +OSError missing file +True True +t1 True True +True +t2 True True +True +t3 True True +ValueError anchor not supported for multiline text +True True True +red pixels True diff --git a/maven/python-transpiler/src/test/resources/conformance/pil_truetype.py b/maven/python-transpiler/src/test/resources/conformance/pil_truetype.py new file mode 100644 index 00000000000..dbbe3303161 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/pil_truetype.py @@ -0,0 +1,126 @@ +# Pillow: scalable TrueType text -- ImageFont.load_default(size) (Aileron +# Regular), truetype() from a file object, FreeTypeFont metrics, and +# ImageDraw.text / multiline_text / textbbox / textlength with anchors. +# +# Tolerance. FreeType hints Aileron with its auto-hinter, which moves glyph +# edges and advances to whole pixels; Codename One's TrueTypeFont does not +# hint. So: +# - exact: getmetrics(), getname(), errors, and getlength() at size 20, +# where the hinted and unhinted advances agree; +# - within 1 px per glyph: getlength() at other sizes; +# - within 1 px per glyph: every box edge, for every anchor; +# - pixels: the ink's coverage sum (sum of mask values / 255) within 10% of +# what CPython+FreeType draws, and its centroid within 1 px. +# REF values below were measured with CPython 3.13 + Pillow 13.0 (basic +# layout); the program prints only whether each is within its tolerance. +import io + +from PIL import Image, ImageDraw, ImageFont + + +def near(a, b, tol): + return all(abs(x - y) <= tol for x, y in zip(a, b)) if isinstance(a, tuple) else abs(a - b) <= tol + + +f20 = ImageFont.load_default(20) +print(type(f20).__name__, f20.getname(), f20.size) +print(f20.getmetrics(), ImageFont.load_default().size, ImageFont.load_default().getmetrics()) +REF20 = {"H": (0, 6, 15, 20), "AVA": (0, 6, 38, 20), "Hello World": (0, 5, 107, 20), "": (0, 0, 0, 0), + "gy j": (0, 5, 31, 24)} +for s in ("H", "AVA", "Hello World", "", "gy j"): + print(repr(s), f20.getlength(s), near(f20.getbbox(s), REF20[s], 1)) +REFA = {"la": (0, 5, 19, 20), "lt": (0, 0, 19, 15), "ls": (0, -15, 19, 0), "lb": (0, -15, 19, 0), + "ld": (0, -20, 19, -5), "ma": (-10, 5, 9, 20), "mm": (-10, -7, 9, 8), "ms": (-10, -15, 9, 0), + "rs": (-19, -15, 0, 0), "rd": (-19, -20, 0, -5)} +for a in ("la", "lt", "ls", "lb", "ld", "ma", "mm", "ms", "rs", "rd"): + print(a, near(f20.getbbox("Hi", anchor=a), REFA[a], 1)) +try: + f20.getbbox("x", anchor="xx") +except ValueError as e: + print("ValueError", e) +try: + f20.getlength("x", direction="rtl") +except KeyError: + print("KeyError direction") + +REF_LEN = {10: (8.0, 18.0, 55.0), 13: (9.0, 24.0, 70.0), 37: (27.0, 71.0, 197.0)} +REF_BOX = {10: (0, 2, 25, 10), 13: (0, 4, 31, 13), 37: (0, 9, 87, 37)} +for size in (10, 13, 37): + f = ImageFont.load_default(size) + lens = (f.getlength("H"), f.getlength("AVA"), f.getlength("Hello World")) + print(size, f.getmetrics(), [near(x, r, n) for x, r, n in zip(lens, REF_LEN[size], (1, 3, 11))], + near(f.getbbox("Hello"), REF_BOX[size], 5)) + +# truetype() from a file object and from bytes; font_variant +tt = ImageFont.truetype(io.BytesIO(ImageFont.load_default(10).path.getvalue()), 20, + layout_engine=ImageFont.Layout.BASIC) +print(tt.getname(), tt.getmetrics(), tt.getlength("Hello"), near(tt.getbbox("Hello"), (0, 5, 48, 20), 1)) +v = tt.font_variant(size=40) +print(v.size, v.getmetrics(), near(v.getlength("Hello"), 94.0, 5)) +try: + ImageFont.truetype(io.BytesIO(b"not a font at all"), 12) +except OSError: + print("OSError bad font") +try: + ImageFont.truetype("/no/such/font.ttf", 12) +except OSError: + print("OSError missing file") + + +def ink(im): + """Coverage sum and centroid of a grayscale image's ink.""" + w, h = im.size + raw = im.tobytes() + total = sx = sy = 0 + for y in range(h): + for x in range(w): + v = raw[y * w + x] + total += v + sx += v * x + sy += v * y + return total / 255.0, (sx / total, sy / total) if total else (0, 0) + + +REF_INK = {"t1": (505.0, (63.4, 25.0)), "t2": (174.3, (48.6, 30.7)), "t3": (423.0, (53.2, 49.8))} + + +def check(name, im): + cov, (cx, cy) = ink(im) + rc, (rx, ry) = REF_INK[name] + print(name, abs(cov - rc) <= 0.10 * rc, near((cx, cy), (rx, ry), 1.0)) + + +im = Image.new("L", (130, 50), 0) +d = ImageDraw.Draw(im) +d.text((6, 12), "Hello, Pillow!", fill=255, font=ImageFont.load_default(20)) +print(near(d.textbbox((6, 12), "Hello, Pillow!", font=ImageFont.load_default(20)), (6, 17, 121, 34), 2), + near(d.textlength("Hello, Pillow!", font=ImageFont.load_default(20)), 115.0, 3)) +check("t1", im) + +im = Image.new("L", (100, 60), 0) +d = ImageDraw.Draw(im) +d.text((50, 30), "anchor", fill=255, font=ImageFont.load_default(16), anchor="mm") +print(near(d.textbbox((50, 30), "anchor", font=ImageFont.load_default(16), anchor="mm"), (24, 24, 76, 36), 2)) +check("t2", im) + +im = Image.new("L", (110, 80), 0) +d = ImageDraw.Draw(im) +d.multiline_text((55, 40), "two\nlines here", fill=255, font=ImageFont.load_default(18), + anchor="ms", align="center") +print(near(d.multiline_textbbox((55, 40), "two\nlines here", font=ImageFont.load_default(18), + anchor="ms", align="center"), (16.0, 28, 93.0, 62), 2)) +check("t3", im) +try: + d.multiline_text((0, 0), "a\nb", anchor="lt") +except ValueError as e: + print("ValueError", e) + +# Default font through ImageDraw, font_size, and fill colour on RGB +rgb = Image.new("RGB", (60, 20), "white") +d = ImageDraw.Draw(rgb) +print(near(d.textlength("abc"), 16.0, 3), near(d.textlength("abc", font_size=20), 34.0, 3), + near(d.textbbox((0, 0), "abc"), (0, 2, 16, 10), 2)) +d.text((2, 2), "abc", fill=(200, 0, 0), font_size=14) +raw = rgb.tobytes() +reds = sum(1 for i in range(0, len(raw), 3) if raw[i] > 150 and raw[i + 1] < 100) +print("red pixels", reds > 20) diff --git a/scripts/python-backend/e2e_client.py b/scripts/python-backend/e2e_client.py index a9a84540ac5..621eaca38c6 100755 --- a/scripts/python-backend/e2e_client.py +++ b/scripts/python-backend/e2e_client.py @@ -288,6 +288,17 @@ def flask(): check("GET / answers", st in (200, 404), st) body = streaming("/stream?who=bob", b"bob 0", b"bob 2") check("flask /stream body with request context", body == b"bob 0\nbob 1\nbob 2\n", body) + st, h, b = http("GET", "/badge.png?who=bob") + check("flask /badge.png is a 240x64 PNG", st == 200 and b[:8] == b"\x89PNG\r\n\x1a\n" + and struct.unpack(">II", b[16:24]) == (240, 64), (st, b[:24])) + st, h, b = http("GET", "/badge.jpg?who=bob") + check("flask /badge.jpg is a JPEG", st == 200 and b[:2] == b"\xff\xd8" and b[-2:] == b"\xff\xd9", (st, b[:4])) + st, h, b = http("GET", "/badge.json?who=bob") + info = json.loads(b) if st == 200 else {} + box = info.get("bbox") or [0, 0, 0, 0] + check("flask /badge.json text inked in the middle", st == 200 and info.get("name") == ["Aileron", "Regular"] + and 60 < box[0] < 120 < box[2] < 180 and 14 < box[1] < 32 < box[3] < 50 + and abs(info.get("length", 0) - (box[2] - box[0])) < 6, info) def main(): diff --git a/scripts/python-backend/samples/flask-todo/src/main.py b/scripts/python-backend/samples/flask-todo/src/main.py index ac2b8c42d91..74c53ff82b1 100644 --- a/scripts/python-backend/samples/flask-todo/src/main.py +++ b/scripts/python-backend/samples/flask-todo/src/main.py @@ -67,6 +67,35 @@ def gen(): return Response(stream_with_context(gen()), mimetype="text/plain") +@app.route("/badge.") +def badge(fmt): + """A PNG or JPEG drawn on the server: shapes, then text in the embedded + TrueType font (com.codename1.image underneath).""" + import io + from PIL import Image, ImageDraw, ImageFont + who = request.args.get("who", "todo") + font = ImageFont.load_default(24) + im = Image.new("RGB", (240, 64), (24, 32, 48)) + d = ImageDraw.Draw(im) + d.rectangle((4, 4, 235, 59), outline=(250, 200, 40), width=3) + d.text((120, 32), "Hi, " + who, fill=(255, 255, 255), font=font, anchor="mm") + ink = Image.new("L", (240, 64), 0) + ImageDraw.Draw(ink).text((120, 32), "Hi, " + who, fill=255, font=font, anchor="mm") + if fmt == "json": + return jsonify(bbox=list(ink.getbbox() or ()), length=font.getlength("Hi, " + who), + metrics=list(font.getmetrics()), name=list(font.getname())) + out = io.BytesIO() + if fmt == "png": + im.save(out, "PNG") + mime = "image/png" + elif fmt == "jpg": + im.save(out, "JPEG", quality=85) + mime = "image/jpeg" + else: + abort(404) + return Response(out.getvalue(), mimetype=mime) + + if __name__ == "__main__": import sys app.run(host="127.0.0.1", port=int(sys.argv[1]) if len(sys.argv) > 1 else 8080) From 6a0ba99af19019d7e0c638c188b6317e7bed4afd Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 14:27:51 +0200 Subject: [PATCH 081/112] Guide: Pillow reads and writes JPEG on servers, TrueType text Co-Authored-By: Claude Opus 5.5 --- docs/developer-guide/Python-Interop.asciidoc | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/developer-guide/Python-Interop.asciidoc b/docs/developer-guide/Python-Interop.asciidoc index 9e94754d0d1..4b5280e90f0 100644 --- a/docs/developer-guide/Python-Interop.asciidoc +++ b/docs/developer-guide/Python-Interop.asciidoc @@ -204,11 +204,14 @@ the real package: and rolling windows aren't supported. | `Pillow` | `PIL.Image` in the `RGB`, `RGBA`, `L`, `LA`, `P` and `1` modes: conversion, cropping, resizing, rotation, pasting, channels and point operations, - computed as Pillow computes them. PNG is read and written on every platform; - JPEG, GIF, BMP and WebP go through Codename One's image support and need the - application's display, so a server reads and writes PNG only. `ImageDraw` - draws points, lines, rectangles, ellipses, polygons and text in Pillow's - bitmap font, which `ImageFont.load_default()` returns. Every resizing + computed as Pillow computes them. PNG and JPEG are read and written on + every platform, servers included, by Codename One's `com.codename1.image` + package; GIF, BMP and WebP go through the platform's image support and need + the application's display. `ImageDraw` draws points, lines, rectangles, + ellipses, polygons and text. `ImageFont.truetype` loads TrueType fonts and + `ImageFont.load_default()` returns the scalable Aileron font, as in Pillow. + Text is laid out as Pillow's basic layout does, without hinting, so at small + sizes a glyph can sit a pixel away from where Pillow puts it. Every resizing filter, `LANCZOS` and `BICUBIC` included, gives Pillow's pixels. | `flask`, `fastapi`, `uvicorn` | See <>. | `typing_extensions`, `six`, `tabulate`, `idna`, `packaging` | From 195ddbeaf4ad7022bf981f72fa44b297894fca4d Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 15:27:25 +0200 Subject: [PATCH 082/112] Image API moves to the backend (com.codename1.backend.image); Pillow picks a server or client engine The pure-Java raster API is no longer client API: RasterImage, RasterGraphics, PngCodec, JpegCodec, Resampler, TrueTypeFont and the rest move from core to vm/backend as com.codename1.backend.image, with their tests, the Aileron license and NOTICE entry. The guide section moves to Backend-Views. PIL's _cn1image now asks a PyImageEngine for codecs, resampling and text. A backend build gets a generated BackendImage over the backend package (Pillow's exact pixels); a client build gets PyImageNative over Image, ImageIO, Graphics and the native font. python-runtime no longer references any image library. The pil_ conformance programs build as a server does. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/parparvm-tests.yml | 2 - NOTICE | 7 +- .../developerguide/backend/SocialCard.java | 12 +- docs/developer-guide/Backend-Views.asciidoc | 16 + docs/developer-guide/Python-Interop.asciidoc | 23 +- docs/developer-guide/graphics.asciidoc | 15 - .../backend}/image/RasterApiTest.java | 2 +- .../backend/image/SharedRasterTest.java | 14 +- .../backend}/image/TrueTypeFontTest.java | 2 +- .../backend}/third-party/aileron/LICENSE.txt | 0 .../com/codename1/python/runtime/PyImage.java | 318 ++++++------------ .../codename1/python/runtime/PyImageData.java | 46 +++ .../python/runtime/PyImageEngine.java | 100 ++++++ .../python/runtime/PyImageNative.java | 229 +++++++++++-- maven/python-transpiler/pom.xml | 7 + .../transpiler/api/PythonTranspiler.java | 37 +- .../com/codename1/python/site/PIL/Image.py | 30 +- .../codename1/python/site/PIL/ImageFont.py | 23 +- .../com/codename1/python/site/PIL/__init__.py | 10 +- .../transpiler/api/BackendImage.java.txt} | 219 ++++++++++-- .../python/transpiler/ConformanceTest.java | 18 +- scripts/python-backend/build.sh | 4 +- .../samples/flask-todo/src/main.py | 2 +- .../backend}/image/DefaultFontData.java | 5 +- .../codename1/backend}/image/JpegCodec.java | 3 +- .../codename1/backend}/image/PngCodec.java | 3 +- .../codename1/backend}/image/RasterFont.java | 3 +- .../backend}/image/RasterGradient.java | 3 +- .../backend}/image/RasterGraphics.java | 3 +- .../codename1/backend}/image/RasterImage.java | 3 +- .../codename1/backend}/image/RasterPaint.java | 3 +- .../codename1/backend}/image/RasterPath.java | 3 +- .../codename1/backend}/image/Resampler.java | 3 +- .../backend}/image/TrueTypeFont.java | 3 +- .../backend}/image/package-info.java | 13 +- 35 files changed, 811 insertions(+), 373 deletions(-) rename maven/{core-unittests/src/test/java/com/codename1 => backend/src/test/java/com/codename1/backend}/image/RasterApiTest.java (99%) rename maven/{core-unittests/src/test/java/com/codename1 => backend/src/test/java/com/codename1/backend}/image/TrueTypeFontTest.java (99%) rename {CodenameOne => maven/backend}/third-party/aileron/LICENSE.txt (100%) create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/PyImageData.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/PyImageEngine.java rename maven/{python-runtime/src/main/java/com/codename1/python/runtime/FreeTypeMetrics.java => python-transpiler/src/main/resources/com/codename1/python/transpiler/api/BackendImage.java.txt} (53%) rename {CodenameOne/src/com/codename1 => vm/backend/src/com/codename1/backend}/image/DefaultFontData.java (99%) rename {CodenameOne/src/com/codename1 => vm/backend/src/com/codename1/backend}/image/JpegCodec.java (99%) rename {CodenameOne/src/com/codename1 => vm/backend/src/com/codename1/backend}/image/PngCodec.java (99%) rename {CodenameOne/src/com/codename1 => vm/backend/src/com/codename1/backend}/image/RasterFont.java (97%) rename {CodenameOne/src/com/codename1 => vm/backend/src/com/codename1/backend}/image/RasterGradient.java (98%) rename {CodenameOne/src/com/codename1 => vm/backend/src/com/codename1/backend}/image/RasterGraphics.java (99%) rename {CodenameOne/src/com/codename1 => vm/backend/src/com/codename1/backend}/image/RasterImage.java (99%) rename {CodenameOne/src/com/codename1 => vm/backend/src/com/codename1/backend}/image/RasterPaint.java (96%) rename {CodenameOne/src/com/codename1 => vm/backend/src/com/codename1/backend}/image/RasterPath.java (99%) rename {CodenameOne/src/com/codename1 => vm/backend/src/com/codename1/backend}/image/Resampler.java (99%) rename {CodenameOne/src/com/codename1 => vm/backend/src/com/codename1/backend}/image/TrueTypeFont.java (99%) rename {CodenameOne/src/com/codename1 => vm/backend/src/com/codename1/backend}/image/package-info.java (81%) diff --git a/.github/workflows/parparvm-tests.yml b/.github/workflows/parparvm-tests.yml index bdcd4dbe7be..f77f52b04b3 100644 --- a/.github/workflows/parparvm-tests.yml +++ b/.github/workflows/parparvm-tests.yml @@ -51,7 +51,6 @@ on: # marker; a class marked outside these directories needs its path here. - 'CodenameOne/src/com/codename1/impl/SharedWithBackend.java' - 'CodenameOne/src/com/codename1/impl/orm/**' - - 'CodenameOne/src/com/codename1/image/**' - 'CodenameOne/src/com/codename1/orm/session/**' - 'CodenameOne/src/com/codename1/orm/repository/**' - 'CodenameOne/src/com/codename1/annotations/db/**' @@ -125,7 +124,6 @@ on: # marker; a class marked outside these directories needs its path here. - 'CodenameOne/src/com/codename1/impl/SharedWithBackend.java' - 'CodenameOne/src/com/codename1/impl/orm/**' - - 'CodenameOne/src/com/codename1/image/**' - 'CodenameOne/src/com/codename1/orm/session/**' - 'CodenameOne/src/com/codename1/orm/repository/**' - 'CodenameOne/src/com/codename1/annotations/db/**' diff --git a/NOTICE b/NOTICE index 39753e86dfb..abb2b548def 100644 --- a/NOTICE +++ b/NOTICE @@ -260,9 +260,10 @@ Pillow's notice: Aileron (font) ================================================================================ -CodenameOne/src/com/codename1/image/DefaultFontData.java embeds Aileron +vm/backend/src/com/codename1/backend/image/DefaultFontData.java embeds Aileron Regular by Sora Sagano (https://dotcolon.net/fonts/aileron), the subset Pillow embeds for ImageFont.load_default(). It is the default font of -com.codename1.image.TrueTypeFont and of the Python runtime's PIL.ImageFont. +com.codename1.backend.image.TrueTypeFont, the backend's image API, and +of PIL.ImageFont in a Python server build. Aileron is dedicated to the public domain under CC0 1.0 Universal; the legal -code is kept in CodenameOne/third-party/aileron/LICENSE.txt. +code is kept in maven/backend/third-party/aileron/LICENSE.txt. diff --git a/docs/demos/backend/src/main/java/com/codenameone/developerguide/backend/SocialCard.java b/docs/demos/backend/src/main/java/com/codenameone/developerguide/backend/SocialCard.java index 101fbbf76e7..2256b9839f5 100644 --- a/docs/demos/backend/src/main/java/com/codenameone/developerguide/backend/SocialCard.java +++ b/docs/demos/backend/src/main/java/com/codenameone/developerguide/backend/SocialCard.java @@ -22,12 +22,12 @@ */ package com.codenameone.developerguide.backend; -import com.codename1.image.JpegCodec; -import com.codename1.image.PngCodec; -import com.codename1.image.RasterGradient; -import com.codename1.image.RasterGraphics; -import com.codename1.image.RasterImage; -import com.codename1.image.Resampler; +import com.codename1.backend.image.JpegCodec; +import com.codename1.backend.image.PngCodec; +import com.codename1.backend.image.RasterGradient; +import com.codename1.backend.image.RasterGraphics; +import com.codename1.backend.image.RasterImage; +import com.codename1.backend.image.Resampler; import java.io.IOException; diff --git a/docs/developer-guide/Backend-Views.asciidoc b/docs/developer-guide/Backend-Views.asciidoc index f1b72b8b2e5..ed50e17d7d3 100644 --- a/docs/developer-guide/Backend-Views.asciidoc +++ b/docs/developer-guide/Backend-Views.asciidoc @@ -184,3 +184,19 @@ expressions or dialect features. Keep existing HTML and htmx attributes within t supported subset. Controller advice, method-level `@ModelAttribute`, flash attributes, `forward:` views, custom converters, dynamic fragment selectors, and internationalized message bundles aren't part of this first version. + +=== Images on the server + +A backend often has to produce an image: a sharing card for a link, a thumbnail of an upload, a badge, or a chart exported as PNG. A server has no display, so `com.codename1.backend.image` draws and encodes images in pure Java, with no platform code. It runs on the JVM and in a backend translated by ParparVM alike. A client application uses `com.codename1.ui.Image` and `Graphics` instead. + +* `RasterImage` holds non-premultiplied ARGB pixels, and crops, rotates, flips and resizes them. Its resize filters (`NEAREST`, `BILINEAR`, `BICUBIC`, `LANCZOS`, `BOX` and `HAMMING`) produce the same bytes as Pillow. +* `RasterGraphics` draws antialiased lines, rectangles, rounded rectangles, ellipses, arcs, polygons and Bezier paths, with stroke width, caps and joins, an affine transform, a clip rectangle, alpha, linear and radial gradients and scaled images. +* `TrueTypeFont` reads a TrueType font for `RasterGraphics.drawString`. `TrueTypeFont.createDefault(size)` needs no font file: it embeds Aileron, the font Pillow uses for `ImageFont.load_default()`. +* `PngCodec` reads every PNG variant and writes 8-bit PNG; `JpegCodec` reads baseline and progressive JPEG and writes baseline JPEG. + +[source,java] +---- +include::../demos/backend/src/main/java/com/codenameone/developerguide/backend/SocialCard.java[tag=raster-image-card,indent=0] +---- + +A handler returns the encoded bytes with the matching content type, such as `image/png`. A Python backend gets the same classes through Pillow: `PIL.Image`, `ImageDraw` and `ImageFont` run on them in a server build, pixel for pixel. diff --git a/docs/developer-guide/Python-Interop.asciidoc b/docs/developer-guide/Python-Interop.asciidoc index 4b5280e90f0..b0dabeece29 100644 --- a/docs/developer-guide/Python-Interop.asciidoc +++ b/docs/developer-guide/Python-Interop.asciidoc @@ -203,16 +203,19 @@ the real package: long and wide frames are shortened. Date indexes, `MultiIndex`, pivot tables, categoricals and rolling windows aren't supported. | `Pillow` | `PIL.Image` in the `RGB`, `RGBA`, `L`, `LA`, `P` and `1` modes: conversion, - cropping, resizing, rotation, pasting, channels and point operations, - computed as Pillow computes them. PNG and JPEG are read and written on - every platform, servers included, by Codename One's `com.codename1.image` - package; GIF, BMP and WebP go through the platform's image support and need - the application's display. `ImageDraw` draws points, lines, rectangles, - ellipses, polygons and text. `ImageFont.truetype` loads TrueType fonts and - `ImageFont.load_default()` returns the scalable Aileron font, as in Pillow. - Text is laid out as Pillow's basic layout does, without hinting, so at small - sizes a glyph can sit a pixel away from where Pillow puts it. Every resizing - filter, `LANCZOS` and `BICUBIC` included, gives Pillow's pixels. + cropping, rotation, pasting, channels and point operations, computed as + Pillow computes them. `ImageDraw` draws points, lines, rectangles, ellipses, + polygons and text. On a server, decoding and encoding PNG and JPEG, + resizing and text use the backend's `com.codename1.backend.image` package + and give Pillow's exact pixels: every resizing filter, `LANCZOS` and + `BICUBIC` included, matches Pillow, `ImageFont.truetype` loads TrueType + fonts and `ImageFont.load_default()` returns the scalable Aileron font. + Text follows Pillow's basic layout without hinting, so at small sizes a + glyph can sit a pixel away from where Pillow puts it. On a device, + decoding, encoding, resizing and text go through Codename One's image + support (`Image`, `ImageIO`, `Graphics` and the native font), which also + reads GIF, BMP and WebP. Those results are drawn by the platform, so they + look right but aren't Pillow's exact pixels. | `flask`, `fastapi`, `uvicorn` | See <>. | `typing_extensions`, `six`, `tabulate`, `idna`, `packaging` | |=== diff --git a/docs/developer-guide/graphics.asciidoc b/docs/developer-guide/graphics.asciidoc index 8bcc100745f..7e6f9127fe8 100644 --- a/docs/developer-guide/graphics.asciidoc +++ b/docs/developer-guide/graphics.asciidoc @@ -1081,18 +1081,3 @@ The following classes are available for different kinds of visualisations: | `XYMultipleSeriesDataset` / `XYMultipleSeriesRenderer` | Extends `LineChart` with date-aware labelling on the X axis for time series data. |=== - -=== Images without a display - -`com.codename1.image` draws and encodes images in pure Java, with no display and no platform code. The same calls work on a device, in the simulator and in a Codename One backend, which makes it the tool for server-generated sharing cards, thumbnails of uploads and chart exports. - -* `RasterImage` holds non-premultiplied ARGB pixels, and crops, rotates, flips and resizes them. Its resize filters (`NEAREST`, `BILINEAR`, `BICUBIC`, `LANCZOS`, `BOX` and `HAMMING`) produce the same bytes as Pillow. -* `RasterGraphics` draws antialiased lines, rectangles, rounded rectangles, ellipses, arcs, polygons and Bezier paths, with stroke width, caps and joins, an affine transform, a clip rectangle, alpha, linear and radial gradients and scaled images. -* `PngCodec` reads every PNG variant and writes 8-bit PNG; `JpegCodec` reads baseline and progressive JPEG and writes baseline JPEG. - -[source,java] ----- -include::../demos/backend/src/main/java/com/codenameone/developerguide/backend/SocialCard.java[tag=raster-image-card,indent=0] ----- - -To show the result in the UI, pass the pixels to `Image.createImage(raster.getPixels(), width, height)`. Text isn't drawn yet: `RasterGraphics.drawString` renders through a `RasterFont`, and Codename One doesn't ship one. diff --git a/maven/core-unittests/src/test/java/com/codename1/image/RasterApiTest.java b/maven/backend/src/test/java/com/codename1/backend/image/RasterApiTest.java similarity index 99% rename from maven/core-unittests/src/test/java/com/codename1/image/RasterApiTest.java rename to maven/backend/src/test/java/com/codename1/backend/image/RasterApiTest.java index ed705568c7f..a270cbc1fbc 100644 --- a/maven/core-unittests/src/test/java/com/codename1/image/RasterApiTest.java +++ b/maven/backend/src/test/java/com/codename1/backend/image/RasterApiTest.java @@ -20,7 +20,7 @@ * Please contact Codename One through http://www.codenameone.com/ if you * need additional information or have any questions. */ -package com.codename1.image; +package com.codename1.backend.image; import org.junit.jupiter.api.Test; diff --git a/maven/backend/src/test/java/com/codename1/backend/image/SharedRasterTest.java b/maven/backend/src/test/java/com/codename1/backend/image/SharedRasterTest.java index 0abcc7c905c..eca1a667114 100644 --- a/maven/backend/src/test/java/com/codename1/backend/image/SharedRasterTest.java +++ b/maven/backend/src/test/java/com/codename1/backend/image/SharedRasterTest.java @@ -22,12 +22,12 @@ */ package com.codename1.backend.image; -import com.codename1.image.JpegCodec; -import com.codename1.image.PngCodec; -import com.codename1.image.RasterGradient; -import com.codename1.image.RasterGraphics; -import com.codename1.image.RasterImage; -import com.codename1.image.Resampler; +import com.codename1.backend.image.JpegCodec; +import com.codename1.backend.image.PngCodec; +import com.codename1.backend.image.RasterGradient; +import com.codename1.backend.image.RasterGraphics; +import com.codename1.backend.image.RasterImage; +import com.codename1.backend.image.Resampler; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; @@ -37,7 +37,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -/// The core's `com.codename1.image` classes are shared with the backend: a +/// The core's `com.codename1.backend.image` classes are shared with the backend: a /// server draws, resizes and encodes images with the same code a device runs. class SharedRasterTest { diff --git a/maven/core-unittests/src/test/java/com/codename1/image/TrueTypeFontTest.java b/maven/backend/src/test/java/com/codename1/backend/image/TrueTypeFontTest.java similarity index 99% rename from maven/core-unittests/src/test/java/com/codename1/image/TrueTypeFontTest.java rename to maven/backend/src/test/java/com/codename1/backend/image/TrueTypeFontTest.java index abcd624db58..c46b24b59fa 100644 --- a/maven/core-unittests/src/test/java/com/codename1/image/TrueTypeFontTest.java +++ b/maven/backend/src/test/java/com/codename1/backend/image/TrueTypeFontTest.java @@ -20,7 +20,7 @@ * Please contact Codename One through http://www.codenameone.com/ if you * need additional information or have any questions. */ -package com.codename1.image; +package com.codename1.backend.image; import org.junit.jupiter.api.Test; diff --git a/CodenameOne/third-party/aileron/LICENSE.txt b/maven/backend/third-party/aileron/LICENSE.txt similarity index 100% rename from CodenameOne/third-party/aileron/LICENSE.txt rename to maven/backend/third-party/aileron/LICENSE.txt diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyImage.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyImage.java index 47d5509cbbc..a0f5bfbdf5e 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyImage.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyImage.java @@ -23,76 +23,45 @@ */ package com.codename1.python.runtime; -import com.codename1.image.JpegCodec; -import com.codename1.image.PngCodec; -import com.codename1.image.RasterImage; -import com.codename1.image.Resampler; -import com.codename1.image.TrueTypeFont; -import java.io.IOException; /// The `_cn1image` module: pixel storage and the hot loops behind the -/// bundled `PIL` package. An image is a [Img] handle holding interleaved +/// bundled `PIL` package. An image is a [PyImageData] handle holding interleaved /// 8-bit bands (1 for "L", 3 for "RGB", 4 for "RGBA"), exactly the layout of /// Pillow's `tobytes()`. Every arithmetic here follows Pillow's own fixed -/// point code so results are byte identical with it. -final class PyImage { +/// point code so results are byte identical with it. Codecs, resampling and +/// text go through the [PyImageEngine] the generated registry installs with +/// [#setEngine(PyImageEngine)]. +public final class PyImage { private PyImage() { } - /// One image: `w` by `h` pixels of `bands` interleaved bytes. - /// The platform image codec behind `native_decode`/`native_encode`: - /// Codename One's `Image` and `ImageIO`, installed by the generated - /// registry of a client build (`PyImageNative.install()`). It is a seam - /// rather than a direct call because a backend build has no UI package: - /// the backend translation takes every runtime class, and a reference - /// to `com.codename1.ui` from this one failed its C compile. There it - /// stays null and PIL uses its pure codecs. - interface Codec { - boolean available(); + private static PyImageEngine engine; - Img decode(byte[] data); - - byte[] encode(Img rgba, String format, float quality); - } - - private static Codec codec; - - static void setCodec(Codec c) { - codec = c; - } - - private static Codec requireCodec() { - Codec c = codec; - if (c == null) { - throw Py.error(Py.OS_ERROR, "this platform has no native image codec"); - } - return c; + /// Installs the engine behind codecs, resampling and text; the generated + /// `PythonModules` calls it with the server or the client one. + public static void setEngine(PyImageEngine e) { + engine = e; } - static final class Img { - final int w; - final int h; - final int bands; - final byte[] d; - - Img(int w, int h, int bands) { - this.w = w; - this.h = h; - this.bands = bands; - this.d = new byte[w * h * bands]; + private static PyImageEngine requireEngine() { + PyImageEngine e = engine; + if (e == null || !e.available()) { + throw Py.error(Py.OS_ERROR, "this build has no image engine: on a device PIL needs a running " + + "Codename One display"); } + return e; } - private static Img img(Object o) { - if (o instanceof PyJavaObject && ((PyJavaObject) o).value instanceof Img) { - return (Img) ((PyJavaObject) o).value; + private static PyImageData img(Object o) { + if (o instanceof PyJavaObject && ((PyJavaObject) o).value instanceof PyImageData) { + return (PyImageData) ((PyJavaObject) o).value; } throw Py.typeError("not an image"); } - private static Object wrap(Img i) { + private static Object wrap(PyImageData i) { return new PyJavaObject(i); } @@ -111,12 +80,12 @@ private static byte[] bytes(Object o) { static PyModule module() { PyModule m = new PyModule("_cn1image", new String[0]); m.setAttr("new", PyBuiltin.positional("new", a -> { - Img r = new Img(i(a[0]), i(a[1]), i(a[2])); + PyImageData r = new PyImageData(i(a[0]), i(a[1]), i(a[2])); fill(r, 0, 0, r.w, r.h, bytes(a[3]), null); return wrap(r); })); m.setAttr("frombytes", PyBuiltin.positional("frombytes", a -> { - Img r = new Img(i(a[0]), i(a[1]), i(a[2])); + PyImageData r = new PyImageData(i(a[0]), i(a[1]), i(a[2])); byte[] b = bytes(a[3]); if (b.length < r.d.length) { throw Py.error(Py.VALUE_ERROR, "not enough image data"); @@ -125,17 +94,17 @@ static PyModule module() { return wrap(r); })); m.setAttr("size", PyBuiltin.positional("size", a -> { - Img s = img(a[0]); + PyImageData s = img(a[0]); return new PyTuple(new Object[] {PyInt.of(s.w), PyInt.of(s.h)}); })); m.setAttr("tobytes", PyBuiltin.positional("tobytes", a -> { - Img s = img(a[0]); + PyImageData s = img(a[0]); byte[] out = new byte[s.d.length]; System.arraycopy(s.d, 0, out, 0, out.length); return new PyBytes(out); })); m.setAttr("getpixel", PyBuiltin.positional("getpixel", a -> { - Img s = img(a[0]); + PyImageData s = img(a[0]); int x = i(a[1]); int y = i(a[2]); if (x < 0 || y < 0 || x >= s.w || y >= s.h) { @@ -149,7 +118,7 @@ static PyModule module() { return new PyTuple(t); })); m.setAttr("putpixel", PyBuiltin.positional("putpixel", a -> { - Img s = img(a[0]); + PyImageData s = img(a[0]); int x = i(a[1]); int y = i(a[2]); if (x < 0 || y < 0 || x >= s.w || y >= s.h) { @@ -160,7 +129,7 @@ static PyModule module() { })); m.setAttr("crop", PyBuiltin.positional("crop", a -> wrap(crop(img(a[0]), i(a[1]), i(a[2]), i(a[3]), i(a[4]))))); m.setAttr("fill", PyBuiltin.positional("fill", a -> { - Img s = img(a[0]); + PyImageData s = img(a[0]); fill(s, i(a[1]), i(a[2]), i(a[3]), i(a[4]), bytes(a[5]), a[6] == Py.NONE ? null : img(a[6])); //NOPMD CompareObjectsWithEquals - identity is meant return Py.NONE; })); @@ -183,13 +152,8 @@ static PyModule module() { for (int j = 0; j < 4; j++) { box[j] = Py.asDouble(c[j]); } - Img src = img(a[0]); - boolean premultiply = a.length > 5 && Py.isTrue(a[5]); - if (premultiply) { - src = premultiply(src); - } - Img r = resample(src, i(a[1]), i(a[2]), box, a.length > 4 ? i(a[4]) : FILTER_BILINEAR); - return wrap(premultiply ? unpremultiply(r) : r); + return wrap(requireEngine().resample(img(a[0]), i(a[1]), i(a[2]), box, + a.length > 4 ? i(a[4]) : FILTER_BILINEAR, a.length > 5 && Py.isTrue(a[5]))); })); m.setAttr("palette", PyBuiltin.positional("palette", a -> wrap(palette(img(a[0]), bytes(a[1]))))); m.setAttr("scatter", PyBuiltin.positional("scatter", a -> { @@ -198,9 +162,9 @@ static PyModule module() { })); m.setAttr("convert", PyBuiltin.positional("convert", a -> wrap(convert(img(a[0]), i(a[1]))))); m.setAttr("band", PyBuiltin.positional("band", a -> { - Img s = img(a[0]); + PyImageData s = img(a[0]); int b = i(a[1]); - Img r = new Img(s.w, s.h, 1); + PyImageData r = new PyImageData(s.w, s.h, 1); for (int p = 0, n = s.w * s.h; p < n; p++) { r.d[p] = s.d[p * s.bands + b]; } @@ -208,10 +172,10 @@ static PyModule module() { })); m.setAttr("merge", PyBuiltin.positional("merge", a -> { Object[] parts = Py.toArray(a[0]); - Img first = img(parts[0]); - Img r = new Img(first.w, first.h, parts.length); + PyImageData first = img(parts[0]); + PyImageData r = new PyImageData(first.w, first.h, parts.length); for (int b = 0; b < parts.length; b++) { - Img s = img(parts[b]); + PyImageData s = img(parts[b]); if (s.w != r.w || s.h != r.h || s.bands != 1) { throw Py.error(Py.VALUE_ERROR, "images do not match"); } @@ -223,9 +187,9 @@ static PyModule module() { })); m.setAttr("getbbox", PyBuiltin.positional("getbbox", a -> getbbox(img(a[0]), Py.isTrue(a[1])))); m.setAttr("point", PyBuiltin.positional("point", a -> { - Img s = img(a[0]); + PyImageData s = img(a[0]); byte[] lut = bytes(a[1]); - Img r = new Img(s.w, s.h, s.bands); + PyImageData r = new PyImageData(s.w, s.h, s.bands); for (int p = 0; p < s.d.length; p += s.bands) { for (int b = 0; b < s.bands; b++) { r.d[p + b] = lut[(b << 8) + (s.d[p + b] & 0xff)]; @@ -234,21 +198,21 @@ static PyModule module() { return wrap(r); })); m.setAttr("boxblur", PyBuiltin.positional("boxblur", a -> { - Img s = img(a[0]); + PyImageData s = img(a[0]); int r = i(a[1]); - Img t = blurRows(s, r); - Img u = blurRows(transpose(t, 5), r); + PyImageData t = blurRows(s, r); + PyImageData u = blurRows(transpose(t, 5), r); return wrap(transpose(u, 5)); })); m.setAttr("png_unfilter", PyBuiltin.positional("png_unfilter", a -> - new PyBytes(pngUnfilter(bytes(a[0]), i(a[1]), i(a[2]), i(a[3]))))); + new PyBytes(requireEngine().pngUnfilter(bytes(a[0]), i(a[1]), i(a[2]), i(a[3]))))); m.setAttr("png_expand", PyBuiltin.positional("png_expand", a -> - wrap(pngExpand(bytes(a[0]), i(a[1]), i(a[2]), i(a[3]), i(a[4]))))); - m.setAttr("jpeg_decode", PyBuiltin.positional("jpeg_decode", a -> wrap(jpegDecode(bytes(a[0]))))); + wrap(requireEngine().pngExpand(bytes(a[0]), i(a[1]), i(a[2]), i(a[3]), i(a[4]))))); + m.setAttr("jpeg_decode", PyBuiltin.positional("jpeg_decode", a -> wrap(requireEngine().jpegDecode(bytes(a[0]))))); m.setAttr("jpeg_encode", PyBuiltin.positional("jpeg_encode", a -> - new PyBytes(jpegEncode(img(a[0]), (float) Py.asDouble(a[1]))))); + new PyBytes(requireEngine().jpegEncode(img(a[0]), (float) Py.asDouble(a[1]))))); m.setAttr("png_rows", PyBuiltin.positional("png_rows", a -> { - Img s = img(a[0]); + PyImageData s = img(a[0]); int row = s.w * s.bands; byte[] out = new byte[(row + 1) * s.h]; for (int y = 0; y < s.h; y++) { @@ -256,72 +220,74 @@ static PyModule module() { } return new PyBytes(out); })); - m.setAttr("ft_load", PyBuiltin.positional("ft_load", a -> { - try { - return new PyJavaObject(new FreeTypeMetrics( - TrueTypeFont.load(bytes(a[0]), i(a[1]), Py.asDouble(a[2])))); - } catch (IOException e) { - throw osError(e); - } + m.setAttr("ft_load", PyBuiltin.positional("ft_load", a -> + new PyJavaObject(requireEngine().loadFont(bytes(a[0]), i(a[1]), Py.asDouble(a[2]))))); + m.setAttr("ft_default_data", PyBuiltin.positional("ft_default_data", a -> { + byte[] d = requireEngine().defaultFontData(); + return d.length == 0 ? Py.NONE : new PyBytes(d); })); - m.setAttr("ft_default_data", PyBuiltin.positional("ft_default_data", a -> - new PyBytes(TrueTypeFont.getDefaultFontData()))); + m.setAttr("ft_default", PyBuiltin.positional("ft_default", a -> + new PyJavaObject(requireEngine().defaultFont(Py.asDouble(a[0]))))); m.setAttr("ft_variant", PyBuiltin.positional("ft_variant", a -> - new PyJavaObject(new FreeTypeMetrics(ft(a[0]).font.deriveFont(Py.asDouble(a[1])))))); + new PyJavaObject(requireEngine().deriveFont(ft(a[0]), Py.asDouble(a[1]))))); m.setAttr("ft_name", PyBuiltin.positional("ft_name", a -> { - TrueTypeFont f = ft(a[0]).font; - return new PyTuple(new Object[] {new PyStr(f.getFamilyName()), new PyStr(f.getStyleName())}); + String[] n = requireEngine().fontName(ft(a[0])); + return new PyTuple(new Object[] {new PyStr(n[0]), new PyStr(n[1])}); })); m.setAttr("ft_metrics", PyBuiltin.positional("ft_metrics", a -> { - FreeTypeMetrics f = ft(a[0]); - return new PyTuple(new Object[] {PyInt.of(FreeTypeMetrics.pixel(f.ascender)), - PyInt.of(-FreeTypeMetrics.pixel(f.descender))}); + int[] mt = requireEngine().fontMetrics(ft(a[0])); + return new PyTuple(new Object[] {PyInt.of(mt[0]), PyInt.of(mt[1])}); })); m.setAttr("ft_length", PyBuiltin.positional("ft_length", a -> - PyInt.of(ft(a[0]).length(Py.toStr(a[1]))))); + PyInt.of(requireEngine().textLength(ft(a[0]), Py.toStr(a[1]))))); m.setAttr("ft_bbox", PyBuiltin.positional("ft_bbox", a -> { - FreeTypeMetrics f = ft(a[0]); - int[] b = ftBox(f, f.layout(Py.toStr(a[1])), a[2]); + int[] b = requireEngine().textBox(ft(a[0]), Py.toStr(a[1]), anchor(a[2])); return new PyTuple(new Object[] {PyInt.of(b[0]), PyInt.of(b[1]), PyInt.of(b[2]), PyInt.of(b[3])}); })); m.setAttr("ft_mask", PyBuiltin.positional("ft_mask", a -> { - FreeTypeMetrics f = ft(a[0]); - FreeTypeMetrics.Layout l = f.layout(Py.toStr(a[1])); - int[] b = ftBox(f, l, a[2]); - Img r = new Img(Math.max(0, b[0]), Math.max(0, b[1]), 1); - byte[] cov = f.mask(l, b, Py.isTrue(a[3])); - System.arraycopy(cov, 0, r.d, 0, Math.min(cov.length, r.d.length)); - return new PyTuple(new Object[] {wrap(r), PyInt.of(b[2]), PyInt.of(b[3])}); + PyImageEngine e = requireEngine(); + Object f = ft(a[0]); + String text = Py.toStr(a[1]); + String anchor = anchor(a[2]); + int[] b = e.textBox(f, text, anchor); + return new PyTuple(new Object[] {wrap(e.textMask(f, text, anchor, Py.isTrue(a[3]))), PyInt.of(b[2]), + PyInt.of(b[3])}); + })); + m.setAttr("exact", PyBuiltin.positional("exact", a -> { + PyImageEngine e = engine; + return e != null && e.exact() ? Py.TRUE : Py.FALSE; })); m.setAttr("native_available", PyBuiltin.positional("native_available", a -> { - Codec c = codec; - return c != null && c.available() ? Py.TRUE : Py.FALSE; + PyImageEngine e = engine; + return e != null && e.available() ? Py.TRUE : Py.FALSE; })); m.setAttr("native_decode", PyBuiltin.positional("native_decode", a -> - wrap(requireCodec().decode(bytes(a[0]))))); + wrap(requireEngine().decode(bytes(a[0]))))); m.setAttr("native_encode", PyBuiltin.positional("native_encode", a -> - new PyBytes(requireCodec().encode(convert(img(a[0]), 4), Py.toStr(a[1]), + new PyBytes(requireEngine().encode(convert(img(a[0]), 4), Py.toStr(a[1]), (float) Py.asDouble(a[2]))))); return m; } - private static FreeTypeMetrics ft(Object o) { - if (o instanceof PyJavaObject && ((PyJavaObject) o).value instanceof FreeTypeMetrics) { - return (FreeTypeMetrics) ((PyJavaObject) o).value; + private static Object ft(Object o) { + if (o instanceof PyJavaObject && ((PyJavaObject) o).value != null) { + return ((PyJavaObject) o).value; } throw Py.typeError("not a TrueType font"); } - private static int[] ftBox(FreeTypeMetrics f, FreeTypeMetrics.Layout l, Object anchor) { - String a = Py.NONE.equals(anchor) ? null : Py.toStr(anchor); - if (!FreeTypeMetrics.validAnchor(a)) { + /// Pillow's anchor check: null (the default, `la`) or a horizontal + /// `l`/`m`/`r` followed by a vertical `a`/`t`/`m`/`s`/`b`/`d`. + private static String anchor(Object o) { + String a = Py.NONE.equals(o) ? null : Py.toStr(o); + if (a != null && (a.length() != 2 || "lmr".indexOf(a.charAt(0)) < 0 || "atmsbd".indexOf(a.charAt(1)) < 0)) { throw Py.error(Py.VALUE_ERROR, "bad anchor specified: " + a); } - return f.box(l, a); + return a; } - private static Img crop(Img s, int x0, int y0, int x1, int y1) { - Img r = new Img(Math.max(0, x1 - x0), Math.max(0, y1 - y0), s.bands); + private static PyImageData crop(PyImageData s, int x0, int y0, int x1, int y1) { + PyImageData r = new PyImageData(Math.max(0, x1 - x0), Math.max(0, y1 - y0), s.bands); for (int y = 0; y < r.h; y++) { int sy = y + y0; if (sy < 0 || sy >= s.h) { @@ -343,11 +309,11 @@ private static int blend(int out, int in, int mask) { return ((t >> 8) + t) >> 8; } - private static int maskAt(Img mask, int x, int y) { + private static int maskAt(PyImageData mask, int x, int y) { return mask.d[(y * mask.w + x) * mask.bands + mask.bands - 1] & 0xff; } - private static void fill(Img s, int x0, int y0, int x1, int y1, byte[] color, Img mask) { + private static void fill(PyImageData s, int x0, int y0, int x1, int y1, byte[] color, PyImageData mask) { int ax = Math.max(0, x0); int ay = Math.max(0, y0); int bx = Math.min(s.w, x1); @@ -367,7 +333,7 @@ private static void fill(Img s, int x0, int y0, int x1, int y1, byte[] color, Im } } - private static void paste(Img dst, Img src, int x0, int y0, Img mask) { + private static void paste(PyImageData dst, PyImageData src, int x0, int y0, PyImageData mask) { for (int y = 0; y < src.h; y++) { int dy = y + y0; if (dy < 0 || dy >= dst.h) { @@ -394,9 +360,9 @@ private static void paste(Img dst, Img src, int x0, int y0, Img mask) { /// Pillow's `Transpose` values: 0 flip left-right, 1 flip top-bottom, /// 2 rotate 90, 3 rotate 180, 4 rotate 270, 5 transpose, 6 transverse. - private static Img transpose(Img s, int op) { + private static PyImageData transpose(PyImageData s, int op) { boolean swap = op == 2 || op == 4 || op == 5 || op == 6; - Img r = swap ? new Img(s.h, s.w, s.bands) : new Img(s.w, s.h, s.bands); + PyImageData r = swap ? new PyImageData(s.h, s.w, s.bands) : new PyImageData(s.w, s.h, s.bands); for (int y = 0; y < r.h; y++) { for (int x = 0; x < r.w; x++) { int sx; @@ -424,8 +390,8 @@ private static int coord(double v) { /// Nearest neighbour affine transform, accumulating the coordinates the /// way Pillow's `affine_transform` does so the rounding is the same. - private static Img affine(Img s, int w, int h, double[] a, byte[] fill) { - Img r = new Img(w, h, s.bands); + private static PyImageData affine(PyImageData s, int w, int h, double[] a, byte[] fill) { + PyImageData r = new PyImageData(w, h, s.bands); double xo = a[2] + a[1] * 0.5 + a[0] * 0.5; double yo = a[5] + a[4] * 0.5 + a[3] * 0.5; for (int y = 0; y < h; y++) { @@ -455,82 +421,9 @@ private static Img affine(Img s, int w, int h, double[] a, byte[] fill) { static final int FILTER_BOX = 4; static final int FILTER_HAMMING = 5; - /// Pillow's two-pass resampling, shared with every Java program through - /// `com.codename1.image.Resampler` (same fixed point, same bytes). - private static Img resample(Img s, int w, int h, double[] box, int filter) { - Img r = new Img(w, h, s.bands); - byte[] d = Resampler.resample(s.d, s.w, s.h, s.bands, w, h, box, filter); - System.arraycopy(d, 0, r.d, 0, d.length); - return r; - } - - /// RGBA to RGBa (or LA to La): Pillow's `MULDIV255` premultiply. - private static Img premultiply(Img s) { - Img r = new Img(s.w, s.h, s.bands); - byte[] d = Resampler.premultiply(s.d, s.bands); - System.arraycopy(d, 0, r.d, 0, d.length); - return r; - } - - /// RGBa to RGBA (or La to LA), as Pillow's `rgbA2rgba`; in place. - private static Img unpremultiply(Img s) { - Resampler.unpremultiply(s.d, s.bands); - return s; - } - - private static Img fromBands(int w, int h, int bands, byte[] d) { - Img r = new Img(w, h, bands); - System.arraycopy(d, 0, r.d, 0, r.d.length); - return r; - } - - private static RuntimeException osError(IOException e) { - return Py.error(Py.OS_ERROR, String.valueOf(e.getMessage())); - } - - /// Decodes a JPEG file with the shared pure-Java codec, to "L" (one band) - /// for a gray file and "RGB" (three) otherwise, as Pillow opens them. - private static Img jpegDecode(byte[] data) { - RasterImage ri; - try { - ri = JpegCodec.decode(data); - } catch (IOException e) { - throw osError(e); // NOPMD PreserveStackTrace - surfaced as a Python OSError - } - int bands = JpegCodec.componentCount(data) == 1 ? 1 : 3; - int[] px = ri.getPixels(); - Img r = new Img(ri.getWidth(), ri.getHeight(), bands); - for (int p = 0; p < px.length; p++) { - int c = px[p]; - if (bands == 1) { - r.d[p] = (byte) c; - } else { - r.d[p * 3] = (byte) (c >> 16); - r.d[p * 3 + 1] = (byte) (c >> 8); - r.d[p * 3 + 2] = (byte) c; - } - } - return r; - } - - /// Encodes an "L" or "RGB" image as baseline JPEG. - private static byte[] jpegEncode(Img s, float quality) { - if (s.bands != 1 && s.bands != 3) { - throw Py.error(Py.OS_ERROR, "cannot write this mode as JPEG"); - } - int[] px = new int[s.w * s.h]; - for (int p = 0; p < px.length; p++) { - int r = s.d[p * s.bands] & 0xff; - int g = s.bands == 1 ? r : s.d[p * 3 + 1] & 0xff; - int b = s.bands == 1 ? r : s.d[p * 3 + 2] & 0xff; - px[p] = 0xff000000 | r << 16 | g << 8 | b; - } - return JpegCodec.encode(new RasterImage(s.w, s.h, px), quality, s.bands == 1); - } - /// Maps a 1-band index image through a 256 entry RGBA table. - private static Img palette(Img s, byte[] lut) { - Img r = new Img(s.w, s.h, 4); + private static PyImageData palette(PyImageData s, byte[] lut) { + PyImageData r = new PyImageData(s.w, s.h, 4); for (int p = 0, n = s.w * s.h; p < n; p++) { int idx = s.d[p * s.bands] & 0xff; System.arraycopy(lut, idx * 4, r.d, p * 4, 4); @@ -539,7 +432,7 @@ private static Img palette(Img s, byte[] lut) { } /// Writes `src` into `dst` at `x0 + x * dx, y0 + y * dy`: one Adam7 pass. - private static void scatter(Img dst, Img src, int x0, int y0, int dx, int dy) { + private static void scatter(PyImageData dst, PyImageData src, int x0, int y0, int dx, int dy) { int n = dst.bands; for (int y = 0; y < src.h; y++) { for (int x = 0; x < src.w; x++) { @@ -548,8 +441,8 @@ private static void scatter(Img dst, Img src, int x0, int y0, int dx, int dy) { } } - private static Img convert(Img s, int bands) { - Img r = new Img(s.w, s.h, bands); + private static PyImageData convert(PyImageData s, int bands) { + PyImageData r = new PyImageData(s.w, s.h, bands); for (int p = 0, n = s.w * s.h; p < n; p++) { int q = p * s.bands; int cr = s.d[q] & 0xff; @@ -576,7 +469,7 @@ private static Img convert(Img s, int bands) { return r; } - private static Object getbbox(Img s, boolean alphaOnly) { + private static Object getbbox(PyImageData s, boolean alphaOnly) { int x0 = s.w; int y0 = s.h; int x1 = -1; @@ -607,8 +500,8 @@ private static Object getbbox(Img s, boolean alphaOnly) { } /// One horizontal pass of Pillow's box blur for an integer radius. - private static Img blurRows(Img s, int r) { - Img o = new Img(s.w, s.h, s.bands); + private static PyImageData blurRows(PyImageData s, int r) { + PyImageData o = new PyImageData(s.w, s.h, s.bands); int ww = (1 << 24) / (r * 2 + 1); int fw = ((1 << 24) - (r * 2 + 1) * ww) / 2; int last = s.w - 1; @@ -628,17 +521,4 @@ private static Img blurRows(Img s, int r) { } return o; } - - private static byte[] pngUnfilter(byte[] data, int row, int h, int bpp) { - try { - return PngCodec.unfilter(data, 0, row, h, bpp); - } catch (IOException e) { - throw osError(e); // NOPMD PreserveStackTrace - surfaced as a Python OSError - } - } - - private static Img pngExpand(byte[] raw, int w, int h, int ctype, int depth) { - int bands = ctype == 0 || ctype == 3 ? 1 : ctype == 2 ? 3 : ctype == 4 ? 2 : 4; - return fromBands(w, h, bands, PngCodec.expand(raw, w, h, ctype, depth)); - } } diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyImageData.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyImageData.java new file mode 100644 index 00000000000..479e57d0dea --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyImageData.java @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +/// One image of PIL's `_cn1image` module: `w` by `h` pixels of `bands` +/// interleaved bytes (1 for "L", 2 for "LA", 3 for "RGB", 4 for "RGBA"). +/// It is what a [PyImageEngine] receives and returns. +public final class PyImageData { + /// Width in pixels. + public final int w; + /// Height in pixels. + public final int h; + /// Bytes per pixel. + public final int bands; + /// The pixels, row by row, `w * h * bands` bytes. + public final byte[] d; + + /// A zero-filled image. + public PyImageData(int w, int h, int bands) { + this.w = w; + this.h = h; + this.bands = bands; + this.d = new byte[w * h * bands]; + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyImageEngine.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyImageEngine.java new file mode 100644 index 00000000000..e60c23a5a96 --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyImageEngine.java @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +/// What PIL needs beyond its own pixel loops: codecs, resampling and text. +/// +/// The runtime keeps Pillow's simple pixel operations (modes, crop, paste, +/// point, the `ImageDraw` shapes) and asks this seam for the rest. There +/// are two implementations, and the generated `PythonModules` installs one: +/// +/// - On a **server** (a backend build) the transpiler generates +/// `BackendImage` over `com.codename1.backend.image`: Pillow's resampling +/// byte for byte, PNG and JPEG, and TrueType text laid out as FreeType +/// lays it out, so results match Pillow's pixels ([#exact()] is true). +/// - On a **device** and in the simulator, [PyImageNative] goes through +/// Codename One's `Image`, `ImageIO`, `Graphics` and `Font`. The platform +/// decodes, encodes, scales and draws text, so results are close to +/// Pillow's but not identical. +/// +/// The runtime itself references neither, so it compiles into both builds. +public interface PyImageEngine { + /// Whether results are Pillow's exact pixels (the server engine). + boolean exact(); + + /// Whether the engine can work now (the client one needs a display). + boolean available(); + + /// Decodes any image file the platform reads, to RGBA. + PyImageData decode(byte[] data); + + /// Encodes an RGBA image as `format` ("png", "jpeg", "gif"...). + byte[] encode(PyImageData rgba, String format, float quality); + + /// Resizes with Pillow's filter number `filter` (1 LANCZOS, 2 BILINEAR, + /// 3 BICUBIC, 4 BOX, 5 HAMMING) from the source rectangle `box`; + /// `premultiplied` resamples LA/RGBA with premultiplied alpha, as Pillow. + PyImageData resample(PyImageData s, int w, int h, double[] box, int filter, boolean premultiplied); + + /// Reverses PNG row filtering (exact engine only). + byte[] pngUnfilter(byte[] data, int row, int h, int bpp); + + /// Expands unfiltered PNG rows to 8-bit bands (exact engine only). + PyImageData pngExpand(byte[] raw, int w, int h, int ctype, int depth); + + /// Decodes a JPEG file to "L" (one band) or "RGB" (three). + PyImageData jpegDecode(byte[] data); + + /// Encodes an "L" or "RGB" image as JPEG. + byte[] jpegEncode(PyImageData s, float quality); + + /// Loads face `index` of a TrueType file at `size` pixels. + Object loadFont(byte[] data, int index, double size); + + /// The bytes of the font file `ImageFont.load_default()` reads, or an empty + /// array where there is none (the client), and [#defaultFont(double)] is used. + byte[] defaultFontData(); + + /// The font `ImageFont.load_default()` answers, at `size` pixels. + Object defaultFont(double size); + + /// `font` at another size. + Object deriveFont(Object font, double size); + + /// Family and style name. + String[] fontName(Object font); + + /// Ascent and descent in pixels, both positive. + int[] fontMetrics(Object font); + + /// The advance of `text` in 64ths of a pixel. + long textLength(Object font, String text); + + /// Width, height and the offset of the box from the anchor point, as + /// `{w, h, x, y}`; `anchor` is null or a valid two-letter anchor. + int[] textBox(Object font, String text, String anchor); + + /// The coverage of `text` as a one-band image of the [#textBox] size. + PyImageData textMask(Object font, String text, String anchor, boolean mono); +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyImageNative.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyImageNative.java index d8abab67efa..6c611e02682 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyImageNative.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyImageNative.java @@ -23,34 +23,49 @@ */ package com.codename1.python.runtime; +import com.codename1.ui.Display; +import com.codename1.ui.Font; +import com.codename1.ui.Graphics; +import com.codename1.ui.Image; +import com.codename1.ui.util.ImageIO; import java.io.ByteArrayOutputStream; import java.io.IOException; -/// PIL's native image codec over Codename One's `Image` and `ImageIO`. Kept -/// out of `PyImage` so that a backend build, which has no UI package, can -/// leave this class out; the generated registry of a client build installs -/// it. -public final class PyImageNative implements PyImage.Codec { +/// PIL's client [PyImageEngine]: Codename One's `Image`, `ImageIO`, +/// `Graphics` and `Font`, installed by the generated registry of a client +/// build. A backend build has no UI package and leaves this class out; it +/// gets the generated `BackendImage` instead. +/// +/// Everything here is drawn by the platform, so results are close to +/// Pillow's but are not its exact pixels: resizing is the platform's +/// scaling whatever filter was asked for, PNG files are decoded by the +/// platform, and text is the platform's native font. A device cannot load +/// a TrueType file from bytes, so `ImageFont.truetype` answers the native +/// font at the requested size as well. +public final class PyImageNative implements PyImageEngine { + private static final String NATIVE_FONT = "native:MainRegular"; + private PyImageNative() { } - /// Makes PIL decode and encode through the platform; idempotent. + /// Makes PIL decode, encode, resize and draw text through the platform; + /// idempotent. public static void install() { - PyImage.setCodec(new PyImageNative()); + PyImage.setEngine(new PyImageNative()); } @Override - public boolean available() { - return com.codename1.ui.Display.isInitialized(); + public boolean exact() { + return false; } @Override - public PyImage.Img decode(byte[] b) { - com.codename1.ui.Image ci = com.codename1.ui.Image.createImage(b, 0, b.length); - int w = ci.getWidth(); - int h = ci.getHeight(); - int[] argb = ci.getRGB(); - PyImage.Img r = new PyImage.Img(w, h, 4); + public boolean available() { + return Display.isInitialized(); + } + + private static PyImageData fromArgb(int[] argb, int w, int h) { + PyImageData r = new PyImageData(w, h, 4); for (int p = 0; p < w * h; p++) { int c = argb[p]; r.d[p * 4] = (byte) (c >> 16); @@ -61,15 +76,56 @@ public PyImage.Img decode(byte[] b) { return r; } - @Override - public byte[] encode(PyImage.Img s, String format, float quality) { + /// Any image as platform ARGB; `L`, `LA`, `RGB` and `RGBA` are accepted. + private static int[] toArgb(PyImageData s) { int[] argb = new int[s.w * s.h]; + int n = s.bands; for (int p = 0; p < argb.length; p++) { - argb[p] = (s.d[p * 4 + 3] & 0xff) << 24 | (s.d[p * 4] & 0xff) << 16 - | (s.d[p * 4 + 1] & 0xff) << 8 | (s.d[p * 4 + 2] & 0xff); + int o = p * n; + int r = s.d[o] & 0xff; + int g = n >= 3 ? s.d[o + 1] & 0xff : r; + int b = n >= 3 ? s.d[o + 2] & 0xff : r; + int a = n == 4 ? s.d[o + 3] & 0xff : n == 2 ? s.d[o + 1] & 0xff : 0xff; + argb[p] = a << 24 | r << 16 | g << 8 | b; + } + return argb; + } + + private static PyImageData bands(PyImageData rgba, int bands) { + if (bands == 4) { + return rgba; } - com.codename1.ui.Image ci = com.codename1.ui.Image.createImage(argb, s.w, s.h); - com.codename1.ui.util.ImageIO io = com.codename1.ui.util.ImageIO.getImageIO(); + PyImageData r = new PyImageData(rgba.w, rgba.h, bands); + for (int p = 0; p < rgba.w * rgba.h; p++) { + int red = rgba.d[p * 4] & 0xff; + int green = rgba.d[p * 4 + 1] & 0xff; + int blue = rgba.d[p * 4 + 2] & 0xff; + if (bands == 3) { + r.d[p * 3] = (byte) red; + r.d[p * 3 + 1] = (byte) green; + r.d[p * 3 + 2] = (byte) blue; + } else { + // ITU-R 601-2 luma, as Pillow's "L" conversion + int l = (red * 19595 + green * 38470 + blue * 7471 + 0x8000) >> 16; + r.d[p * bands] = (byte) l; + if (bands == 2) { + r.d[p * 2 + 1] = rgba.d[p * 4 + 3]; + } + } + } + return r; + } + + @Override + public PyImageData decode(byte[] b) { + Image ci = Image.createImage(b, 0, b.length); + return fromArgb(ci.getRGB(), ci.getWidth(), ci.getHeight()); + } + + @Override + public byte[] encode(PyImageData s, String format, float quality) { + Image ci = Image.createImage(toArgb(s), s.w, s.h); + ImageIO io = ImageIO.getImageIO(); if (io == null) { throw Py.error(Py.OS_ERROR, "this platform cannot encode " + format); } @@ -81,4 +137,135 @@ public byte[] encode(PyImage.Img s, String format, float quality) { } return bo.toByteArray(); } + + @Override + public PyImageData resample(PyImageData s, int w, int h, double[] box, int filter, boolean premultiplied) { + int x0 = (int) Math.floor(box[0]); + int y0 = (int) Math.floor(box[1]); + int x1 = (int) Math.ceil(box[2]); + int y1 = (int) Math.ceil(box[3]); + Image ci = Image.createImage(toArgb(s), s.w, s.h); + if (x0 > 0 || y0 > 0 || x1 < s.w || y1 < s.h) { + ci = ci.subImage(x0, y0, Math.max(1, x1 - x0), Math.max(1, y1 - y0), true); + } + Image scaled = ci.scaled(w, h); + return bands(fromArgb(scaled.getRGB(), w, h), s.bands); + } + + @Override + public byte[] pngUnfilter(byte[] data, int row, int h, int bpp) { + throw Py.error(Py.OS_ERROR, "PNG files are decoded by the platform on this target"); + } + + @Override + public PyImageData pngExpand(byte[] raw, int w, int h, int ctype, int depth) { + throw Py.error(Py.OS_ERROR, "PNG files are decoded by the platform on this target"); + } + + @Override + public PyImageData jpegDecode(byte[] data) { + PyImageData rgba = decode(data); + return bands(rgba, 3); + } + + @Override + public byte[] jpegEncode(PyImageData s, float quality) { + return encode(s, ImageIO.FORMAT_JPEG, quality); + } + + @Override + public Object loadFont(byte[] data, int index, double size) { + return defaultFont(size); + } + + @Override + public byte[] defaultFontData() { + return new byte[0]; + } + + @Override + public Object defaultFont(double size) { + Font f = Font.createTrueTypeFont(NATIVE_FONT, NATIVE_FONT); + return f.derive((float) size, Font.STYLE_PLAIN); + } + + private static Font font(Object f) { + if (f instanceof Font) { + return (Font) f; + } + throw Py.typeError("not a font"); + } + + @Override + public Object deriveFont(Object f, double size) { + return font(f).derive((float) size, Font.STYLE_PLAIN); + } + + @Override + public String[] fontName(Object f) { + font(f); + return new String[] {"Native", "Regular"}; + } + + @Override + public int[] fontMetrics(Object f) { + Font ft = font(f); + return new int[] {ft.getAscent(), ft.getDescent()}; + } + + @Override + public long textLength(Object f, String text) { + return font(f).stringWidth(text) * 64L; + } + + @Override + public int[] textBox(Object f, String text, String anchor) { + Font ft = font(f); + int w = ft.stringWidth(text); + int asc = ft.getAscent(); + int h = asc + ft.getDescent(); + String a = anchor == null ? "la" : anchor; + int x = a.charAt(0) == 'm' ? -w / 2 : a.charAt(0) == 'r' ? -w : 0; + int y; + switch (a.charAt(1)) { + case 'm': + y = -h / 2; + break; + case 's': + y = -asc; + break; + case 'b': + case 'd': + y = -h; + break; + default: + y = 0; + break; + } + return new int[] {w, h, x, y}; + } + + @Override + public PyImageData textMask(Object f, String text, String anchor, boolean mono) { + Font ft = font(f); + int[] b = textBox(ft, text, anchor); + int w = Math.max(1, b[0]); + int h = Math.max(1, b[1]); + Image canvas = Image.createImage(w, h, 0xff000000); + Graphics g = canvas.getGraphics(); + g.setAntiAliased(!mono); + g.setAntiAliasedText(!mono); + g.setFont(ft); + g.setColor(0xffffff); + g.drawString(text, 0, 0); + int[] argb = canvas.getRGB(); + PyImageData r = new PyImageData(Math.max(0, b[0]), Math.max(0, b[1]), 1); + for (int y = 0; y < r.h; y++) { + for (int x = 0; x < r.w; x++) { + int v = argb[y * w + x] >> 8 & 0xff; + r.d[y * r.w + x] = (byte) (mono ? (v >= 128 ? 255 : 0) : v); + } + } + return r; + } } diff --git a/maven/python-transpiler/pom.xml b/maven/python-transpiler/pom.xml index d8da8a3ce9a..0a588182826 100644 --- a/maven/python-transpiler/pom.xml +++ b/maven/python-transpiler/pom.xml @@ -164,6 +164,13 @@ ${project.version} test + + + com.codenameone + codenameone-backend + ${project.version} + test + diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java index 2892fe6d3cb..6cf597b444e 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java @@ -193,6 +193,7 @@ public static Map transpile(Map sources, String boolean server = false; boolean db = false; boolean image = false; + boolean serverImage = false; for (String text : sources.values()) { server |= text.contains("_cn1server"); db |= text.contains("_cn1db"); @@ -210,8 +211,11 @@ public static Map transpile(Map sources, String boolean backend = cp.find("com/codename1/backend/HttpServer") != null; server &= backend; db = backend && (db || server); - // PIL's platform codec is Codename One's UI Image, which a - // backend build does not have (and leaves out of its classes). + // PIL's engine: on a server the generated BackendImage over + // com.codename1.backend.image (Pillow's exact pixels); on a + // client PyImageNative over the UI Image, which a backend + // build does not have (and leaves out of its classes). + serverImage = image && backend; image &= !backend; } catch (IOException e) { throw new IllegalStateException("cannot read the class path: " + e.getMessage(), e); @@ -254,8 +258,11 @@ public static Map transpile(Map sources, String if (db) { out.put(PACKAGE.replace('.', '/') + "/BackendDb.java", backendDb()); } + if (serverImage) { + out.put(PACKAGE.replace('.', '/') + "/BackendImage.java", backendImage()); + } out.put(PACKAGE.replace('.', '/') + "/PythonModules.java", registry(classes, mainModule, interop != null, server, db, - image, nativeModules)); + image, serverImage, nativeModules)); return out; } @@ -586,6 +593,25 @@ private static String backendDb() { /// slow streamed response -- sleeping between chunks, or draining to a /// slow client -- leaves the other workers free. Background tasks run /// once the response has been handed back or its last chunk written. + /// PIL's server engine (`BackendImage` and its FreeType layout), kept as + /// a resource: it names `com.codename1.backend.image`, so it is compiled + /// only into a backend build and the runtime never depends on the backend. + private static String backendImage() { + try (InputStream in = PythonTranspiler.class.getResourceAsStream("BackendImage.java.txt")) { + if (in == null) { + throw new IllegalStateException("BackendImage.java.txt is missing from the transpiler"); + } + ByteArrayOutputStream out = new ByteArrayOutputStream(); + byte[] buf = new byte[8192]; + for (int n = in.read(buf); n > 0; n = in.read(buf)) { + out.write(buf, 0, n); + } + return new String(out.toByteArray(), StandardCharsets.UTF_8).replace("@PACKAGE@", PACKAGE); + } catch (IOException e) { + throw new IllegalStateException("cannot read BackendImage.java.txt: " + e.getMessage(), e); + } + } + private static String backendServer(int virtualThreadStack) { return "// Generated by the Codename One Python transpiler.\n" + "package " + PACKAGE + ";\n\n" @@ -969,7 +995,7 @@ static String withLineMap(String java, String pyFile) { } private static String registry(Map classes, String mainModule, boolean interop, boolean server, boolean db, - boolean image, Map nativeModules) { + boolean image, boolean serverImage, Map nativeModules) { StringBuilder sb = new StringBuilder(); sb.append("// Generated by the Codename One Python transpiler.\n"); sb.append("package ").append(PACKAGE).append(";\n\n"); @@ -993,6 +1019,9 @@ private static String registry(Map classes, String mainModule, b if (image) { sb.append(" com.codename1.python.runtime.PyImageNative.install();\n"); } + if (serverImage) { + sb.append(" BackendImage.install();\n"); + } if (classes.containsKey("_cn1_http")) { sb.append(" com.codename1.python.runtime.Cn1Http.installDefault();\n"); } diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/Image.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/Image.py index 6a36a0308dc..62e01b970c0 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/Image.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/Image.py @@ -256,6 +256,8 @@ def crop(self, box=None): return self._new(_cn1image.crop(self._im, x0, y0, x1, y1), (x1 - x0, y1 - y0)) def resize(self, size, resample=None, box=None, reducing_gap=None): + # Server builds resample with Pillow's own convolution, byte for byte; + # on a device every filter but NEAREST is the platform's scaling. w, h = int(size[0]), int(size[1]) if box is None: box = (0, 0) + self._size @@ -494,6 +496,12 @@ def open(fp, mode="r", formats=None): data = f.read() else: data = fp.read() + if data[:8] == _PNG_SIG and not _cn1image.exact(): + # A device decodes through the platform, then takes the file's mode + mode = {0: "L", 2: "RGB", 3: "RGBA", 4: "LA", 6: "RGBA"}.get(data[25], "RGBA") + im = _native_decode(data).convert(mode) + im.format = "PNG" + return im if data[:8] == _PNG_SIG: im = _png_decode(data) im.format = "PNG" @@ -513,10 +521,9 @@ def open(fp, mode="r", formats=None): kind = "WEBP" if kind is None: raise UnidentifiedImageError("cannot identify image file %r" % (fp if isinstance(fp, str) else fp,)) - if not _cn1image.native_available(): - raise OSError(kind + " needs a running Codename One display to decode; only PNG and JPEG are built in") - handle = _cn1image.native_decode(data) - im = _wrap("RGBA", _cn1image.size(handle), handle) + if _cn1image.exact(): + raise OSError(kind + " is not decoded on a server; only PNG and JPEG are") + im = _native_decode(data) im.format = kind if kind == "BMP": im = im.convert("RGB") @@ -524,15 +531,24 @@ def open(fp, mode="r", formats=None): return im +def _native_decode(data): + if not _cn1image.native_available(): + raise OSError("decoding an image on a device needs a running Codename One display") + handle = _cn1image.native_decode(data) + return _wrap("RGBA", _cn1image.size(handle), handle) + + def _native_encode(im, format, quality): + if _cn1image.exact(): + raise OSError(format + " is not encoded on a server; only PNG and JPEG are") if not _cn1image.native_available(): - raise OSError(format + " needs a running Codename One display to encode; only PNG and JPEG are built in") + raise OSError(format + " needs a running Codename One display to encode") return _cn1image.native_encode(im._im, format.lower(), quality / 100.0) def _jpeg_encode(im, quality): - # The shared pure-Java codec (com.codename1.image.JpegCodec): baseline, - # libjpeg's quality scaling, so it works on servers as well as devices. + # On a server the backend's JpegCodec (baseline, libjpeg's quality + # scaling); on a device the platform's encoder. if im.mode == "1": im = im.convert("L") if im.mode not in ("L", "RGB"): diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/ImageFont.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/ImageFont.py index e49c66715d0..8abc9aa1168 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/ImageFont.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/ImageFont.py @@ -36,11 +36,13 @@ # """PIL fonts: Pillow's built-in bitmap font and scalable TrueType fonts. -`truetype()` reads a TrueType font (a `.ttf`, an `.otf` with TrueType -outlines, or a face of a `.ttc`) with Codename One's pure-Java font engine, -`com.codename1.image.TrueTypeFont`. `load_default()` returns Aileron Regular, -the scalable font a Pillow with FreeType returns, embedded in that engine; -`load_default_imagefont()` returns the bitmap font. +On a server `truetype()` reads a TrueType font (a `.ttf`, an `.otf` with +TrueType outlines, or a face of a `.ttc`) with the backend's pure-Java font +engine, `com.codename1.backend.image.TrueTypeFont`, and `load_default()` +returns Aileron Regular, the scalable font a Pillow with FreeType returns. +On a device both answer the platform's native font at the requested size, +drawn by Codename One, so text is close to Pillow's but not pixel exact. +`load_default_imagefont()` returns the bitmap font everywhere. Layout is Pillow's basic layout (`Layout.BASIC`) whatever `layout_engine` asks for: one glyph per character, advances rounded to whole pixels, kerning @@ -135,6 +137,9 @@ def _text(text): return text +_DEFAULT = object() + + class FreeTypeFont: """A scalable TrueType font at one size.""" @@ -146,6 +151,9 @@ def __init__(self, font=None, size=10, index=0, encoding="", layout_engine=None) self.index = index self.encoding = encoding self.layout_engine = Layout.BASIC + if font is _DEFAULT: + self.font = _cn1image.ft_default(float(size)) + return if isinstance(font, (str, bytes)) or hasattr(font, "__fspath__"): with open(font, "rb") as f: data = f.read() @@ -209,7 +217,10 @@ def font_variant(self, font=None, size=None, index=None, encoding=None, layout_e def load_default(size=None): """Aileron Regular at `size` pixels (10 by default), as a Pillow with FreeType returns it.""" - return FreeTypeFont(io.BytesIO(_cn1image.ft_default_data()), 10 if size is None else size) + data = _cn1image.ft_default_data() + size = 10 if size is None else size + # a device has no font file: the native font stands in for Aileron + return FreeTypeFont(_DEFAULT if data is None else io.BytesIO(data), size) def truetype(font=None, size=10, index=0, encoding="", layout_engine=None): diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/__init__.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/__init__.py index 69ef6aa8344..c2b376d89da 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/__init__.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/__init__.py @@ -4,10 +4,12 @@ """A Pillow subset: images in "L", "RGB" and "RGBA" with PNG and JPEG built in. Pixels live in the runtime's `_cn1image` module, which also does every loop -over them. Resampling and the PNG and JPEG codecs are Codename One's pure-Java -`com.codename1.image` package, so they work on servers as well as devices; -GIF, BMP and WEBP go through the platform's image codecs and need a running -display. +over them. Codecs, resampling and text go through an engine the build picks: +on a server the backend's pure-Java `com.codename1.backend.image`, which +gives Pillow's exact pixels for resampling, PNG, JPEG and TrueType text; on +a device Codename One's `Image`, `ImageIO`, `Graphics` and `Font`, which +decode, encode, scale and draw text the platform's way, and also read GIF, +BMP and WEBP. """ __version__ = "11.0.0" diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/FreeTypeMetrics.java b/maven/python-transpiler/src/main/resources/com/codename1/python/transpiler/api/BackendImage.java.txt similarity index 53% rename from maven/python-runtime/src/main/java/com/codename1/python/runtime/FreeTypeMetrics.java rename to maven/python-transpiler/src/main/resources/com/codename1/python/transpiler/api/BackendImage.java.txt index cc69c880b97..4bcf18ad612 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/FreeTypeMetrics.java +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/transpiler/api/BackendImage.java.txt @@ -1,33 +1,192 @@ -/* - * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Codename One through http://www.codenameone.com/ if you - * need additional information or have any questions. - */ -package com.codename1.python.runtime; - -import com.codename1.image.RasterGraphics; -import com.codename1.image.RasterImage; -import com.codename1.image.RasterPath; -import com.codename1.image.TrueTypeFont; - -/// Pillow's basic text layout over a [TrueTypeFont], in FreeType's 26.6 +// Generated by the Codename One Python transpiler. +package @PACKAGE@; + +import com.codename1.backend.image.JpegCodec; +import com.codename1.backend.image.PngCodec; +import com.codename1.backend.image.RasterGraphics; +import com.codename1.backend.image.RasterImage; +import com.codename1.backend.image.RasterPath; +import com.codename1.backend.image.Resampler; +import com.codename1.backend.image.TrueTypeFont; +import com.codename1.python.runtime.Py; +import com.codename1.python.runtime.PyImage; +import com.codename1.python.runtime.PyImageData; +import com.codename1.python.runtime.PyImageEngine; +import java.io.IOException; + +/// PIL's server engine over the backend's `com.codename1.backend.image`: +/// Pillow's resampling byte for byte, PNG and JPEG, and TrueType text laid +/// out as FreeType lays it out. Generated only into a backend build, so a +/// client build never references the backend's classes. +final class BackendImage implements PyImageEngine { + private BackendImage() { + } + + static void install() { + PyImage.setEngine(new BackendImage()); + } + + private static RuntimeException osError(IOException e) { + return Py.error(Py.OS_ERROR, String.valueOf(e.getMessage())); + } + + @Override + public boolean exact() { + return true; + } + + @Override + public boolean available() { + return true; + } + + @Override + public PyImageData decode(byte[] data) { + throw Py.error(Py.OS_ERROR, "only PNG and JPEG are decoded on a server"); + } + + @Override + public byte[] encode(PyImageData rgba, String format, float quality) { + throw Py.error(Py.OS_ERROR, "only PNG and JPEG are encoded on a server, not " + format); + } + + private static PyImageData of(int w, int h, int bands, byte[] d) { + PyImageData r = new PyImageData(w, h, bands); + System.arraycopy(d, 0, r.d, 0, r.d.length); + return r; + } + + @Override + public PyImageData resample(PyImageData s, int w, int h, double[] box, int filter, boolean premultiplied) { + byte[] src = premultiplied ? Resampler.premultiply(s.d, s.bands) : s.d; + byte[] d = Resampler.resample(src, s.w, s.h, s.bands, w, h, box, filter); + if (premultiplied) { + d = Resampler.unpremultiply(d, s.bands); + } + return of(w, h, s.bands, d); + } + + @Override + public byte[] pngUnfilter(byte[] data, int row, int h, int bpp) { + try { + return PngCodec.unfilter(data, 0, row, h, bpp); + } catch (IOException e) { + throw osError(e); + } + } + + @Override + public PyImageData pngExpand(byte[] raw, int w, int h, int ctype, int depth) { + int bands = ctype == 0 || ctype == 3 ? 1 : ctype == 2 ? 3 : ctype == 4 ? 2 : 4; + return of(w, h, bands, PngCodec.expand(raw, w, h, ctype, depth)); + } + + @Override + public PyImageData jpegDecode(byte[] data) { + RasterImage ri; + try { + ri = JpegCodec.decode(data); + } catch (IOException e) { + throw osError(e); + } + int bands = JpegCodec.componentCount(data) == 1 ? 1 : 3; + int[] px = ri.getPixels(); + PyImageData r = new PyImageData(ri.getWidth(), ri.getHeight(), bands); + for (int p = 0; p < px.length; p++) { + int c = px[p]; + if (bands == 1) { + r.d[p] = (byte) c; + } else { + r.d[p * 3] = (byte) (c >> 16); + r.d[p * 3 + 1] = (byte) (c >> 8); + r.d[p * 3 + 2] = (byte) c; + } + } + return r; + } + + @Override + public byte[] jpegEncode(PyImageData s, float quality) { + if (s.bands != 1 && s.bands != 3) { + throw Py.error(Py.OS_ERROR, "cannot write this mode as JPEG"); + } + int[] px = new int[s.w * s.h]; + for (int p = 0; p < px.length; p++) { + int r = s.d[p * s.bands] & 0xff; + int g = s.bands == 1 ? r : s.d[p * 3 + 1] & 0xff; + int b = s.bands == 1 ? r : s.d[p * 3 + 2] & 0xff; + px[p] = 0xff000000 | r << 16 | g << 8 | b; + } + return JpegCodec.encode(new RasterImage(s.w, s.h, px), quality, s.bands == 1); + } + + private static FreeTypeMetrics ft(Object f) { + if (f instanceof FreeTypeMetrics) { + return (FreeTypeMetrics) f; + } + throw Py.typeError("not a TrueType font"); + } + + @Override + public Object loadFont(byte[] data, int index, double size) { + try { + return new FreeTypeMetrics(TrueTypeFont.load(data, index, size)); + } catch (IOException e) { + throw osError(e); + } + } + + @Override + public byte[] defaultFontData() { + return TrueTypeFont.getDefaultFontData(); + } + + @Override + public Object defaultFont(double size) { + return new FreeTypeMetrics(TrueTypeFont.createDefault(size)); + } + + @Override + public Object deriveFont(Object font, double size) { + return new FreeTypeMetrics(ft(font).font.deriveFont(size)); + } + + @Override + public String[] fontName(Object font) { + TrueTypeFont f = ft(font).font; + return new String[] {f.getFamilyName(), f.getStyleName()}; + } + + @Override + public int[] fontMetrics(Object font) { + FreeTypeMetrics f = ft(font); + return new int[] {FreeTypeMetrics.pixel(f.ascender), -FreeTypeMetrics.pixel(f.descender)}; + } + + @Override + public long textLength(Object font, String text) { + return ft(font).length(text); + } + + @Override + public int[] textBox(Object font, String text, String anchor) { + FreeTypeMetrics f = ft(font); + return f.box(f.layout(text), anchor); + } + + @Override + public PyImageData textMask(Object font, String text, String anchor, boolean mono) { + FreeTypeMetrics f = ft(font); + FreeTypeMetrics.Layout l = f.layout(text); + int[] b = f.box(l, anchor); + PyImageData r = new PyImageData(Math.max(0, b[0]), Math.max(0, b[1]), 1); + byte[] cov = f.mask(l, b, mono); + System.arraycopy(cov, 0, r.d, 0, Math.min(cov.length, r.d.length)); + return r; + } +} + +/// Pillow's basic text layout over a `TrueTypeFont`, in FreeType's 26.6 /// fixed point, so `getlength`, `getbbox`, `getmetrics` and the anchors give /// the integers Pillow gives. /// diff --git a/maven/python-transpiler/src/test/java/com/codename1/python/transpiler/ConformanceTest.java b/maven/python-transpiler/src/test/java/com/codename1/python/transpiler/ConformanceTest.java index 53443efc681..4be4211f122 100644 --- a/maven/python-transpiler/src/test/java/com/codename1/python/transpiler/ConformanceTest.java +++ b/maven/python-transpiler/src/test/java/com/codename1/python/transpiler/ConformanceTest.java @@ -41,7 +41,6 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.TreeMap; @@ -142,8 +141,16 @@ private static void run(String stem, Map sources, boolean http, List args = new ArrayList(); String runtime = location(com.codename1.python.runtime.Py.class); String core = location(com.codename1.io.Util.class); - // Programs may import Java classes: the JDK's, and the core's. - List classPath = Collections.singletonList(new File(core)); + // Programs may import Java classes: the JDK's, and the core's. The + // pil_ programs are built as a server is (the backend on the class + // path), so PIL runs on com.codename1.backend.image and its output + // stays Pillow's exact bytes. + String backend = stem.startsWith("pil_") ? location(com.codename1.backend.image.RasterImage.class) : null; + List classPath = new ArrayList(); + classPath.add(new File(core)); + if (backend != null) { + classPath.add(new File(backend)); + } for (Map.Entry e : PythonTranspiler.transpile(sources, "main", classPath, natives).entrySet()) { Path f = gen.resolve(e.getKey()); Files.createDirectories(f.getParent()); @@ -155,7 +162,7 @@ private static void run(String stem, Map sources, boolean http, args.add(1, "--release"); args.add(2, "17"); args.add(3, "-cp"); - args.add(4, runtime + File.pathSeparator + core); + args.add(4, runtime + File.pathSeparator + core + (backend == null ? "" : File.pathSeparator + backend)); args.add(5, "-d"); args.add(6, cls.toString()); JavaCompiler javac = ToolProvider.getSystemJavaCompiler(); @@ -166,7 +173,8 @@ private static void run(String stem, Map sources, boolean http, URL[] urls = {cls.toUri().toURL(), new File(runtime).toURI().toURL(), new File(core).toURI().toURL(), new File(location(JvmFileSystem.class)).toURI().toURL(), new File(location(org.sqlite.JDBC.class)).toURI().toURL(), - new File(location(org.slf4j.LoggerFactory.class)).toURI().toURL()}; + new File(location(org.slf4j.LoggerFactory.class)).toURI().toURL(), + new File(backend == null ? core : backend).toURI().toURL()}; Path fsRoot = work.resolve("fs"); Files.createDirectories(fsRoot.resolve("home")); ByteArrayOutputStream out = new ByteArrayOutputStream(); diff --git a/scripts/python-backend/build.sh b/scripts/python-backend/build.sh index e46f005d6cc..599b04bf399 100755 --- a/scripts/python-backend/build.sh +++ b/scripts/python-backend/build.sh @@ -71,8 +71,8 @@ public class PyBackendMain { } JAVA # The runtime, the one core package it uses, and the host file system shim. -# PyImageNative is PIL's GIF/BMP/WEBP codec over the client UI, which a server -# lacks; PNG and JPEG are com.codename1.image, which the backend jar carries. +# PyImageNative is PIL's client engine over the UI, which a server lacks; the +# transpiler generates BackendImage over com.codename1.backend.image instead. (cd "$CLASSES" && unzip -qo "$RUNTIME" 'com/*' -x 'com/codename1/python/runtime/PyImageNative*' \ && unzip -qo "$CORE" 'com/codename1/util/*') "$J17/bin/javac" -nowarn --release 17 -d "$CLASSES" $(find "$REPO/scripts/python-bench/hostfs" -name '*.java') diff --git a/scripts/python-backend/samples/flask-todo/src/main.py b/scripts/python-backend/samples/flask-todo/src/main.py index 74c53ff82b1..be900f5b805 100644 --- a/scripts/python-backend/samples/flask-todo/src/main.py +++ b/scripts/python-backend/samples/flask-todo/src/main.py @@ -70,7 +70,7 @@ def gen(): @app.route("/badge.") def badge(fmt): """A PNG or JPEG drawn on the server: shapes, then text in the embedded - TrueType font (com.codename1.image underneath).""" + TrueType font (com.codename1.backend.image underneath).""" import io from PIL import Image, ImageDraw, ImageFont who = request.args.get("who", "todo") diff --git a/CodenameOne/src/com/codename1/image/DefaultFontData.java b/vm/backend/src/com/codename1/backend/image/DefaultFontData.java similarity index 99% rename from CodenameOne/src/com/codename1/image/DefaultFontData.java rename to vm/backend/src/com/codename1/backend/image/DefaultFontData.java index 876d07afadd..7175cb6e3f1 100644 --- a/CodenameOne/src/com/codename1/image/DefaultFontData.java +++ b/vm/backend/src/com/codename1/backend/image/DefaultFontData.java @@ -20,7 +20,7 @@ * Please contact Codename One through http://www.codenameone.com/ if you * need additional information or have any questions. */ -package com.codename1.image; +package com.codename1.backend.image; import java.io.IOException; import java.util.zip.Inflater; @@ -30,11 +30,10 @@ /// The font is the subset Pillow embeds for `ImageFont.load_default()` /// (12676 bytes): Aileron by Sora Sagano (dotcolon.net), released into the /// public domain under CC0 1.0 ("No Rights Reserved", as its `name` table -/// says). The license is kept in `CodenameOne/third-party/aileron` and the +/// says). The license is kept in `maven/backend/third-party/aileron` and the /// root `NOTICE`. The bytes are zlib-compressed and base64-encoded here /// because classes are what the backend shares with the core, not /// resources. It is data, not code: replace it whole, never edit it. -@com.codename1.impl.SharedWithBackend final class DefaultFontData { private static final int SIZE = 12676; diff --git a/CodenameOne/src/com/codename1/image/JpegCodec.java b/vm/backend/src/com/codename1/backend/image/JpegCodec.java similarity index 99% rename from CodenameOne/src/com/codename1/image/JpegCodec.java rename to vm/backend/src/com/codename1/backend/image/JpegCodec.java index 2c6d79f7514..3232620d4f8 100644 --- a/CodenameOne/src/com/codename1/image/JpegCodec.java +++ b/vm/backend/src/com/codename1/backend/image/JpegCodec.java @@ -20,7 +20,7 @@ * Please contact Codename One through http://www.codenameone.com/ if you * need additional information or have any questions. */ -package com.codename1.image; +package com.codename1.backend.image; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -39,7 +39,6 @@ /// scaled to a quality the way libjpeg scales them, so quality 0.75 here /// gives the same tables as libjpeg's (and Pillow's) quality 75. Alpha is /// dropped. -@com.codename1.impl.SharedWithBackend public final class JpegCodec { static final int[] ZIGZAG = { 0, 1, 8, 16, 9, 2, 3, 10, 17, 24, 32, 25, 18, 11, 4, 5, 12, 19, 26, 33, 40, 48, 41, 34, 27, 20, 13, 6, 7, diff --git a/CodenameOne/src/com/codename1/image/PngCodec.java b/vm/backend/src/com/codename1/backend/image/PngCodec.java similarity index 99% rename from CodenameOne/src/com/codename1/image/PngCodec.java rename to vm/backend/src/com/codename1/backend/image/PngCodec.java index 185f5ddfffc..5547c1967ac 100644 --- a/CodenameOne/src/com/codename1/image/PngCodec.java +++ b/vm/backend/src/com/codename1/backend/image/PngCodec.java @@ -20,7 +20,7 @@ * Please contact Codename One through http://www.codenameone.com/ if you * need additional information or have any questions. */ -package com.codename1.image; +package com.codename1.backend.image; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -40,7 +40,6 @@ /// [#expand(byte[], int, int, int, int)] keep the file's own sample layout /// instead of converting to ARGB; the Python runtime's `PIL` uses them to /// keep a palette image a palette image. -@com.codename1.impl.SharedWithBackend public final class PngCodec { private static final byte[] SIGNATURE = {(byte) 0x89, 'P', 'N', 'G', 13, 10, 26, 10}; private static final int[][] ADAM7 = { diff --git a/CodenameOne/src/com/codename1/image/RasterFont.java b/vm/backend/src/com/codename1/backend/image/RasterFont.java similarity index 97% rename from CodenameOne/src/com/codename1/image/RasterFont.java rename to vm/backend/src/com/codename1/backend/image/RasterFont.java index ec698d61aa4..54e8b0cde38 100644 --- a/CodenameOne/src/com/codename1/image/RasterFont.java +++ b/vm/backend/src/com/codename1/backend/image/RasterFont.java @@ -20,7 +20,7 @@ * Please contact Codename One through http://www.codenameone.com/ if you * need additional information or have any questions. */ -package com.codename1.image; +package com.codename1.backend.image; /// The seam through which [RasterGraphics#drawString(String, double, double)] /// draws text: a font turns a string into an outline, and the graphics fills @@ -29,7 +29,6 @@ /// [TrueTypeFont] is the implementation that ships: TrueType fonts read from /// bytes, with an embedded default. Another font engine plugs in here /// without any change to the graphics. -@com.codename1.impl.SharedWithBackend public interface RasterFont { /// Returns the outline of `text` with its baseline starting at `(x, y)`. /// diff --git a/CodenameOne/src/com/codename1/image/RasterGradient.java b/vm/backend/src/com/codename1/backend/image/RasterGradient.java similarity index 98% rename from CodenameOne/src/com/codename1/image/RasterGradient.java rename to vm/backend/src/com/codename1/backend/image/RasterGradient.java index e53a47d46d2..c06a026c4b1 100644 --- a/CodenameOne/src/com/codename1/image/RasterGradient.java +++ b/vm/backend/src/com/codename1/backend/image/RasterGradient.java @@ -20,14 +20,13 @@ * Please contact Codename One through http://www.codenameone.com/ if you * need additional information or have any questions. */ -package com.codename1.image; +package com.codename1.backend.image; /// A linear or radial color gradient for [RasterGraphics#setPaint(RasterPaint)]. /// /// Colors are interpolated per channel, not premultiplied, between stops at /// increasing fractions from 0 to 1. Before the first stop and after the last /// one the end colors extend ("pad"). -@com.codename1.impl.SharedWithBackend public final class RasterGradient implements RasterPaint { private final boolean radial; private final double x1; diff --git a/CodenameOne/src/com/codename1/image/RasterGraphics.java b/vm/backend/src/com/codename1/backend/image/RasterGraphics.java similarity index 99% rename from CodenameOne/src/com/codename1/image/RasterGraphics.java rename to vm/backend/src/com/codename1/backend/image/RasterGraphics.java index 2f818f9af35..da8ebac21ee 100644 --- a/CodenameOne/src/com/codename1/image/RasterGraphics.java +++ b/vm/backend/src/com/codename1/backend/image/RasterGraphics.java @@ -20,7 +20,7 @@ * Please contact Codename One through http://www.codenameone.com/ if you * need additional information or have any questions. */ -package com.codename1.image; +package com.codename1.backend.image; /// Draws antialiased shapes, gradients and images into a [RasterImage]. /// @@ -43,7 +43,6 @@ /// - **Text** goes through the [RasterFont] seam set with [#setFont(RasterFont)]. /// /// A graphics object is not thread safe; create one per thread. -@com.codename1.impl.SharedWithBackend public final class RasterGraphics { /// Stroke ends stop exactly at the end points. public static final int CAP_BUTT = 0; diff --git a/CodenameOne/src/com/codename1/image/RasterImage.java b/vm/backend/src/com/codename1/backend/image/RasterImage.java similarity index 99% rename from CodenameOne/src/com/codename1/image/RasterImage.java rename to vm/backend/src/com/codename1/backend/image/RasterImage.java index 107c9d33b7c..95f3123496d 100644 --- a/CodenameOne/src/com/codename1/image/RasterImage.java +++ b/vm/backend/src/com/codename1/backend/image/RasterImage.java @@ -20,7 +20,7 @@ * Please contact Codename One through http://www.codenameone.com/ if you * need additional information or have any questions. */ -package com.codename1.image; +package com.codename1.backend.image; /// A mutable image held in memory as 32-bit ARGB pixels. /// @@ -33,7 +33,6 @@ /// server alike. Draw into it with [RasterGraphics], read and write files with /// [PngCodec] and [JpegCodec], and hand the pixels to the UI with /// `Image.createImage(raster.getPixels(), w, h)` when you need one. -@com.codename1.impl.SharedWithBackend public final class RasterImage { private final int width; private final int height; diff --git a/CodenameOne/src/com/codename1/image/RasterPaint.java b/vm/backend/src/com/codename1/backend/image/RasterPaint.java similarity index 96% rename from CodenameOne/src/com/codename1/image/RasterPaint.java rename to vm/backend/src/com/codename1/backend/image/RasterPaint.java index 9aca433cf77..596f9e9a827 100644 --- a/CodenameOne/src/com/codename1/image/RasterPaint.java +++ b/vm/backend/src/com/codename1/backend/image/RasterPaint.java @@ -20,11 +20,10 @@ * Please contact Codename One through http://www.codenameone.com/ if you * need additional information or have any questions. */ -package com.codename1.image; +package com.codename1.backend.image; /// What [RasterGraphics] fills with when it is not a single color: a /// gradient ([RasterGradient]) or your own pattern. -@com.codename1.impl.SharedWithBackend public interface RasterPaint { /// Returns the non-premultiplied ARGB color at a point. /// diff --git a/CodenameOne/src/com/codename1/image/RasterPath.java b/vm/backend/src/com/codename1/backend/image/RasterPath.java similarity index 99% rename from CodenameOne/src/com/codename1/image/RasterPath.java rename to vm/backend/src/com/codename1/backend/image/RasterPath.java index d9d60df8e89..936841fb1ea 100644 --- a/CodenameOne/src/com/codename1/image/RasterPath.java +++ b/vm/backend/src/com/codename1/backend/image/RasterPath.java @@ -20,7 +20,7 @@ * Please contact Codename One through http://www.codenameone.com/ if you * need additional information or have any questions. */ -package com.codename1.image; +package com.codename1.backend.image; /// A shape made of straight lines and quadratic and cubic Bezier curves, for /// [RasterGraphics#fill(RasterPath)] and [RasterGraphics#draw(RasterPath)]. @@ -32,7 +32,6 @@ /// The static factories build the common shapes. Angles are in degrees, /// counterclockwise from three o'clock, as on screen with `y` pointing down /// (so a positive extent turns toward the top). -@com.codename1.impl.SharedWithBackend public final class RasterPath { static final byte MOVE = 0; static final byte LINE = 1; diff --git a/CodenameOne/src/com/codename1/image/Resampler.java b/vm/backend/src/com/codename1/backend/image/Resampler.java similarity index 99% rename from CodenameOne/src/com/codename1/image/Resampler.java rename to vm/backend/src/com/codename1/backend/image/Resampler.java index e6f400c2f0a..ad2f2c76277 100644 --- a/CodenameOne/src/com/codename1/image/Resampler.java +++ b/vm/backend/src/com/codename1/backend/image/Resampler.java @@ -20,7 +20,7 @@ * Please contact Codename One through http://www.codenameone.com/ if you * need additional information or have any questions. */ -package com.codename1.image; +package com.codename1.backend.image; /// Resampling of interleaved 8-bit band data, the two-pass fixed-point /// convolution of Pillow's `Resample.c`. @@ -33,7 +33,6 @@ /// bleed their color into their neighbours. /// /// [RasterImage#scale(int, int, int)] wraps this for ARGB images. -@com.codename1.impl.SharedWithBackend public final class Resampler { /// Nearest neighbour: each output pixel takes the input pixel under its center. public static final int NEAREST = 0; diff --git a/CodenameOne/src/com/codename1/image/TrueTypeFont.java b/vm/backend/src/com/codename1/backend/image/TrueTypeFont.java similarity index 99% rename from CodenameOne/src/com/codename1/image/TrueTypeFont.java rename to vm/backend/src/com/codename1/backend/image/TrueTypeFont.java index e3fc515afc4..aca13c995a8 100644 --- a/CodenameOne/src/com/codename1/image/TrueTypeFont.java +++ b/vm/backend/src/com/codename1/backend/image/TrueTypeFont.java @@ -20,7 +20,7 @@ * Please contact Codename One through http://www.codenameone.com/ if you * need additional information or have any questions. */ -package com.codename1.image; +package com.codename1.backend.image; import java.io.IOException; import java.util.HashMap; @@ -53,7 +53,6 @@ /// An instance keeps a bounded cache of glyph outlines and is not safe to use /// from two threads at once; [#deriveFont(double)] is cheap and gives each /// thread its own. -@com.codename1.impl.SharedWithBackend public final class TrueTypeFont implements RasterFont { private static final int CACHE_LIMIT = 512; private static final int MAX_COMPONENT_DEPTH = 8; diff --git a/CodenameOne/src/com/codename1/image/package-info.java b/vm/backend/src/com/codename1/backend/image/package-info.java similarity index 81% rename from CodenameOne/src/com/codename1/image/package-info.java rename to vm/backend/src/com/codename1/backend/image/package-info.java index 1c16cc1921c..8e2a881a6bd 100644 --- a/CodenameOne/src/com/codename1/image/package-info.java +++ b/vm/backend/src/com/codename1/backend/image/package-info.java @@ -21,9 +21,10 @@ * need additional information or have any questions. */ -/// Pure-Java images: pixels, drawing, resampling and PNG/JPEG codecs that -/// need no display, so the same code runs on a device, in the simulator and -/// on a Codename One server. +/// Server-side images: pixels, drawing, resampling and PNG/JPEG codecs in +/// pure Java, for a Codename One backend that has no display -- social cards, +/// thumbnails, badges and charts rendered on the server. A client application +/// uses `com.codename1.ui.Image` and `com.codename1.ui.Graphics` instead. /// /// - [RasterImage] holds non-premultiplied ARGB pixels and rotates, flips, /// crops and resizes them. @@ -49,6 +50,6 @@ /// byte[] png = PngCodec.encode(card); /// ``` /// -/// Every class is shared with the backend runtime, and depends on nothing -/// but the subset of the JDK every Codename One target provides. -package com.codename1.image; +/// Every class depends on nothing but the subset of the JDK the backend +/// runtime provides, so it runs on the JVM and translated by ParparVM alike. +package com.codename1.backend.image; From 861036112355493b14600b62407f535e483b0df7 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 15:35:53 +0200 Subject: [PATCH 083/112] Pillow client engine: headless conformance run of PyImageNative; e2e --native derives JDK_8_HOME pilclient_ builds a Pillow program as a client app (no backend on the class path), so the transpiler installs PyImageNative, and runs it over compat-testing's HeadlessImplementation with a JDK-backed ImageIO: PNG and JPEG decode, convert, resize (with box), thumbnail, PNG/JPEG save round trips, and load_default()/truetype() text metrics, bbox, length and ink. Assertions are tolerant, so CPython's output is the expected one. Co-Authored-By: Claude Opus 5.5 --- maven/python-transpiler/pom.xml | 7 ++ .../python/transpiler/ConformanceTest.java | 21 +++- .../test/resources/conformance/pilclient_.out | 34 ++++++ .../resources/conformance/pilclient_/main.py | 101 ++++++++++++++++++ .../conformance/pilclient_/natives.properties | 2 + .../pilclient_/pilclienttest/JdkImageIO.java | 59 ++++++++++ scripts/python-backend/e2e-test.sh | 17 +++ 7 files changed, 240 insertions(+), 1 deletion(-) create mode 100644 maven/python-transpiler/src/test/resources/conformance/pilclient_.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/pilclient_/main.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/pilclient_/natives.properties create mode 100644 maven/python-transpiler/src/test/resources/conformance/pilclient_/pilclienttest/JdkImageIO.java diff --git a/maven/python-transpiler/pom.xml b/maven/python-transpiler/pom.xml index 0a588182826..81c07f9fd82 100644 --- a/maven/python-transpiler/pom.xml +++ b/maven/python-transpiler/pom.xml @@ -164,6 +164,13 @@ ${project.version} test + + + com.codenameone + codenameone-compat-testing + ${project.version} + test + com.codenameone diff --git a/maven/python-transpiler/src/test/java/com/codename1/python/transpiler/ConformanceTest.java b/maven/python-transpiler/src/test/java/com/codename1/python/transpiler/ConformanceTest.java index 4be4211f122..0dec891ffff 100644 --- a/maven/python-transpiler/src/test/java/com/codename1/python/transpiler/ConformanceTest.java +++ b/maven/python-transpiler/src/test/java/com/codename1/python/transpiler/ConformanceTest.java @@ -146,6 +146,9 @@ private static void run(String stem, Map sources, boolean http, // path), so PIL runs on com.codename1.backend.image and its output // stays Pillow's exact bytes. String backend = stem.startsWith("pil_") ? location(com.codename1.backend.image.RasterImage.class) : null; + // A pilclient_ program is built as a client app is (no backend), so + // PIL runs on PyImageNative over a headless Display. + boolean client = stem.startsWith("pilclient_"); List classPath = new ArrayList(); classPath.add(new File(core)); if (backend != null) { @@ -174,7 +177,9 @@ private static void run(String stem, Map sources, boolean http, new File(location(JvmFileSystem.class)).toURI().toURL(), new File(location(org.sqlite.JDBC.class)).toURI().toURL(), new File(location(org.slf4j.LoggerFactory.class)).toURI().toURL(), - new File(backend == null ? core : backend).toURI().toURL()}; + new File(backend == null ? core : backend).toURI().toURL(), + new File(client ? location(com.codename1.compat.testing.HeadlessImplementation.class) : core) + .toURI().toURL()}; Path fsRoot = work.resolve("fs"); Files.createDirectories(fsRoot.resolve("home")); ByteArrayOutputStream out = new ByteArrayOutputStream(); @@ -193,6 +198,20 @@ private static void run(String stem, Map sources, boolean http, Class sqlType = loader.loadClass("com.codename1.python.runtime.PySql"); sqlType.getMethod("install", sqlType).invoke(null, loader.loadClass(JvmSql.class.getName()) .getConstructor(String.class).newInstance(fsRoot.resolve("home").toString())); + if (client) { + // A Display over compat-testing's headless implementation, + // which decodes with the JDK's image readers, keeps + // pixels, scales and rasterizes text as boxes -- no window. + // The JDK's image readers and writers must not start a + // toolkit (a Dock icon on macOS) for it. + System.setProperty("java.awt.headless", "true"); + Class hi =loader.loadClass("com.codename1.compat.testing.HeadlessImplementation"); + hi.getField("pixelImages").setBoolean(null, true); + hi.getField("rasterImages").setBoolean(null, true); + hi.getField("imageIO").set(null, loader.loadClass("pilclienttest.JdkImageIO") + .getConstructor().newInstance()); + hi.getMethod("install").invoke(null); + } if (http) { Class httpType = loader.loadClass("com.codename1.python.runtime.PyHttp"); httpType.getMethod("install", httpType) diff --git a/maven/python-transpiler/src/test/resources/conformance/pilclient_.out b/maven/python-transpiler/src/test/resources/conformance/pilclient_.out new file mode 100644 index 00000000000..24fbf7c2cae --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/pilclient_.out @@ -0,0 +1,34 @@ +png PNG RGB (24, 16) +png red True +png green True +jpg JPEG RGB (24, 16) +jpg red True +convert L (24, 16) ('L',) +convert LA (24, 16) ('L', 'A') +convert RGBA (24, 16) ('R', 'G', 'B', 'A') +convert RGB (24, 16) ('R', 'G', 'B') +luma True +resize RGB (12, 8) True +resize RGBA (48, 32) True +resize L (6, 4) +resize box (8, 8) True +thumbnail (6, 4) +save PNG True PNG RGB (24, 16) +round trip PNG True +save JPEG True JPEG RGB (24, 16) +round trip JPEG True +save RGBA RGBA +default length True True +default bbox True True +default metrics True True +default textbbox True True True +default textlength True +default ink True +default mask L True True +truetype length True True +truetype bbox True True +truetype metrics True True +truetype textbbox True True True +truetype textlength True +truetype ink True +truetype mask L True True diff --git a/maven/python-transpiler/src/test/resources/conformance/pilclient_/main.py b/maven/python-transpiler/src/test/resources/conformance/pilclient_/main.py new file mode 100644 index 00000000000..c8c1cba0369 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/pilclient_/main.py @@ -0,0 +1,101 @@ +# Pillow on the client engine (PyImageNative): Codename One's Image, +# ImageIO, Graphics and native font, here over the headless implementation +# of compat-testing. The platform decodes, scales and draws, so pixels are +# not Pillow's: the program prints only sizes, modes, round trips, colors +# within a tolerance and whether text left ink -- facts CPython answers the +# same way, so its output is the expected one. +import base64 +import io + +from PIL import Image, ImageDraw, ImageFont + +PNG = base64.b64decode( + "iVBORw0KGgoAAAANSUhEUgAAABgAAAAQCAIAAACDRijCAAAAMUlEQVR4nGM8IafBQA3ARBVTGAajQSzIHJ99IlgVbXF6Qz8XMY0aNATD" + "iHE0rzEQAgBB5AU9DwB3PwAAAABJRU5ErkJggg==") +JPG = base64.b64decode( + "/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAMCAgMCAgMDAwMEAwMEBQgFBQQEBQoHBwYIDAoMDAsKCwsNDhIQDQ4RDgsLEBYQERMUFRUVDA8XGBYU" + "GBIUFRT/2wBDAQMEBAUEBQkFBQkUDQsNFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBT/wAARCAAQABgD" + "ASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKB" + "kaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZ" + "mqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQF" + "BgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5" + "OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX" + "2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwDyzwT4J/4TH7b/AKb9j+zbP+WW/du3f7Qx939aPG3gn/hDvsX+m/bPtO//AJZbNu3b" + "/tHP3v0o8E+Nv+EO+2/6F9s+07P+Wuzbt3f7Jz979KPG3jb/AITH7F/oX2P7Nv8A+Wu/du2/7Ix939a/MP8AaPrH9z5dvv3P27/jP/8AX/8A6kn/" + "AHA/58f+Dv439chy9FFFd5+5H//Z") + + +def near(a, b, tol): + return abs(a - b) <= tol + + +def rgb_near(px, ref, tol): + return all(near(a, b, tol) for a, b in zip(px[:3], ref)) + + +RED = (200, 30, 40) +GREEN = (20, 220, 60) + +png = Image.open(io.BytesIO(PNG)) +print("png", png.format, png.mode, png.size) +print("png red", rgb_near(png.getpixel((1, 1)), RED, 8)) +print("png green", rgb_near(png.getpixel((7, 7)), GREEN, 8)) + +jpg = Image.open(io.BytesIO(JPG)) +print("jpg", jpg.format, jpg.mode, jpg.size) +print("jpg red", rgb_near(jpg.getpixel((1, 1)), RED, 24)) + +for mode in ("L", "LA", "RGBA", "RGB"): + c = png.convert(mode) + print("convert", c.mode, c.size, c.getbands()) +print("luma", near(png.convert("L").getpixel((1, 1)), 82, 4)) + +small = png.resize((12, 8)) +print("resize", small.mode, small.size, rgb_near(small.getpixel((0, 0)), RED, 24)) +big = jpg.convert("RGBA").resize((48, 32)) +print("resize", big.mode, big.size, rgb_near(big.getpixel((2, 2)), RED, 32)) +gray = png.convert("L").resize((6, 4)) +print("resize", gray.mode, gray.size) +crop = png.resize((8, 8), box=(4, 4, 12, 12)) +print("resize box", crop.size, rgb_near(crop.getpixel((4, 4)), GREEN, 24)) +thumb = png.copy() +thumb.thumbnail((6, 6)) +print("thumbnail", thumb.size) + +for fmt in ("PNG", "JPEG"): + out = io.BytesIO() + png.save(out, fmt) + data = out.getvalue() + back = Image.open(io.BytesIO(data)) + print("save", fmt, len(data) > 0, back.format, back.mode, back.size) + print("round trip", fmt, rgb_near(back.convert("RGB").getpixel((1, 1)), RED, 24)) +out = io.BytesIO() +png.convert("RGBA").save(out, "PNG") +print("save RGBA", Image.open(io.BytesIO(out.getvalue())).mode) + + +def text_checks(name, f): + w = f.getlength("Hello") + print(name, "length", w > 0, f.getlength("Hello Hello") > w) + x0, y0, x1, y1 = f.getbbox("Hello") + print(name, "bbox", x1 > x0, y1 > y0) + asc, desc = f.getmetrics() + print(name, "metrics", asc > 0, desc >= 0) + canvas = Image.new("RGB", (200, 60), "white") + d = ImageDraw.Draw(canvas) + bx0, by0, bx1, by1 = d.textbbox((4, 4), "Hello", font=f) + print(name, "textbbox", bx1 > bx0, by1 > by0, bx0 >= 4) + print(name, "textlength", d.textlength("Hello", font=f) == w) + d.text((4, 4), "Hello", fill="black", font=f) + dark = sum(1 for y in range(60) for x in range(200) if canvas.getpixel((x, y))[0] < 128) + print(name, "ink", dark > 0) + mask = f.getmask("Hi") + print(name, "mask", mask.mode, mask.size[0] > 0, mask.size[1] > 0) + + +text_checks("default", ImageFont.load_default(18)) +# CPython hands back the TrueType bytes it loaded the default from; this +# runtime keeps none, and the client engine draws its native font whatever +# bytes truetype() is given -- so either way this is a real truetype() call. +data = getattr(ImageFont.load_default(), "font_bytes", b"") +text_checks("truetype", ImageFont.truetype(io.BytesIO(data), 18)) diff --git a/maven/python-transpiler/src/test/resources/conformance/pilclient_/natives.properties b/maven/python-transpiler/src/test/resources/conformance/pilclient_/natives.properties new file mode 100644 index 00000000000..30c6823d48b --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/pilclient_/natives.properties @@ -0,0 +1,2 @@ +# No module is replaced. The file is here so ConformanceTest compiles +# pilclienttest/JdkImageIO.java with the program. diff --git a/maven/python-transpiler/src/test/resources/conformance/pilclient_/pilclienttest/JdkImageIO.java b/maven/python-transpiler/src/test/resources/conformance/pilclient_/pilclienttest/JdkImageIO.java new file mode 100644 index 00000000000..9e1af0b7f9e --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/pilclient_/pilclienttest/JdkImageIO.java @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package pilclienttest; + +import com.codename1.ui.Image; +import com.codename1.ui.util.ImageIO; +import java.awt.image.BufferedImage; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +/// The image encoder of the headless client run: a port's `ImageIO`, written +/// over the JDK's (headless) image writers. Installed by ConformanceTest into +/// `HeadlessImplementation.imageIO`. +public class JdkImageIO extends ImageIO { + @Override + public void save(InputStream image, OutputStream response, String format, int width, int height, + float quality) throws IOException { + saveImage(Image.createImage(image).scaled(width, height), response, format, quality); + } + + @Override + protected void saveImage(Image img, OutputStream response, String format, float quality) throws IOException { + int w = img.getWidth(); + int h = img.getHeight(); + boolean jpeg = FORMAT_JPEG.equals(format); + BufferedImage b = new BufferedImage(w, h, jpeg ? BufferedImage.TYPE_INT_RGB : BufferedImage.TYPE_INT_ARGB); + b.setRGB(0, 0, w, h, img.getRGB(), 0, w); + if (!javax.imageio.ImageIO.write(b, jpeg ? "jpeg" : "png", response)) { + throw new IOException("no writer for " + format); + } + } + + @Override + public boolean isFormatSupported(String format) { + return FORMAT_JPEG.equals(format) || FORMAT_PNG.equals(format); + } +} diff --git a/scripts/python-backend/e2e-test.sh b/scripts/python-backend/e2e-test.sh index 9338be00af9..ee78d5322ec 100755 --- a/scripts/python-backend/e2e-test.sh +++ b/scripts/python-backend/e2e-test.sh @@ -13,9 +13,26 @@ # BackendServer installs and which exists only under ParparVM. # # Needs JAVA17_HOME, python3, and the inputs build.sh lists in its header. +# --native also needs a JDK 8 for the backend build and the translator: +# JDK_8_HOME, else JAVA_HOME when that is a JDK 8, else (macOS) the one +# /usr/libexec/java_home finds. A native run takes over ten minutes. set -e BUILD_MODE=--jvm if [ "$1" = "--native" ]; then BUILD_MODE=; shift; fi +is_jdk8() { + [ -x "$1/bin/javac" ] && "$1/bin/javac" -version 2>&1 | grep -q ' 1\.8\.' +} +if [ -z "$BUILD_MODE" ] && [ -z "$JDK_8_HOME" ]; then + if [ -n "$JAVA_HOME" ] && is_jdk8 "$JAVA_HOME"; then + JDK_8_HOME="$JAVA_HOME" + elif [ -x /usr/libexec/java_home ] && J8="$(/usr/libexec/java_home -v 1.8 2>/dev/null)" && is_jdk8 "$J8"; then + JDK_8_HOME="$J8" + else + echo "--native needs a JDK 8: set JDK_8_HOME" >&2 + exit 2 + fi + export JDK_8_HOME +fi HERE="$(cd "$(dirname "$0")" && pwd)" M2="${1:-$(cd "$HERE/../.." && pwd)/.m2-repo}" WORK="$(mktemp -d "${TMPDIR:-/tmp}/cn1-py-e2e.XXXXXX")" From 0da25db7941f1f115a7fc9ec1ec4ff67fcce0ef7 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 16:25:03 +0200 Subject: [PATCH 084/112] Font.createTrueTypeFontFromData: TrueType/OpenType fonts from a stream or bytes New public API Font.createTrueTypeFontFromData(String, InputStream|byte[]) and Font.isTrueTypeStreamSupported(), backed by the implementation hook loadTrueTypeFontFromData(String, byte[]). Deliberately not an overload of createTrueTypeFont: createTrueTypeFont(name, null) is a documented call that a second two-argument overload would make ambiguous. Ports: JavaSE (java.awt.Font.createFont), Android (content-hash cache file, Typeface.createFromFile), iOS/macOS (new IOSNative.registerFontData native: CGFontCreateWithDataProvider + CTFontManagerRegisterGraphicsFont, PostScript name), JavaScript (data: URL through the existing worker/host FontFace path), Windows and Linux (their existing in-memory loaders). UWP untouched. Pillow client engine: ImageFont.truetype loads through the new API, falling back to the native font; load_default stays native (Aileron is backend-only). Tests: core-unittests FontTest, JavaSE headless load/derive/measure of Roboto, pilclient_ conformance extended. Docs: CSS fonts section and Python Pillow row. Co-Authored-By: Claude Opus 5.5 --- .../impl/CodenameOneImplementation.java | 26 +++++ CodenameOne/src/com/codename1/ui/Font.java | 97 +++++++++++++++++++ .../impl/android/AndroidImplementation.java | 52 ++++++++++ .../com/codename1/impl/javase/JavaSEPort.java | 21 ++++ .../impl/html5/HTML5Implementation.java | 19 ++++ .../impl/linux/LinuxImplementation.java | 12 +++ .../impl/windows/WindowsImplementation.java | 12 +++ Ports/iOSPort/nativeSources/IOSNative.m | 39 ++++++++ .../codename1/impl/ios/IOSImplementation.java | 20 ++++ .../src/com/codename1/impl/ios/IOSNative.java | 1 + .../snippets/FontFromDataSnippets.java | 50 ++++++++++ docs/developer-guide/Python-Interop.asciidoc | 7 +- docs/developer-guide/css.asciidoc | 13 +++ .../testing/HeadlessImplementation.java | 16 +++ .../TestCodenameOneImplementation.java | 26 +++++ .../test/java/com/codename1/ui/FontTest.java | 33 +++++++ .../javase/JavaSEPortFontFromDataTest.java | 91 +++++++++++++++++ .../python/runtime/PyImageNative.java | 29 +++++- .../test/resources/conformance/pilclient_.out | 14 +++ .../resources/conformance/pilclient_/main.py | 12 ++- 20 files changed, 583 insertions(+), 7 deletions(-) create mode 100644 docs/demos/common/src/main/java/com/codenameone/developerguide/snippets/FontFromDataSnippets.java create mode 100644 maven/javase/src/test/java/com/codename1/impl/javase/JavaSEPortFontFromDataTest.java diff --git a/CodenameOne/src/com/codename1/impl/CodenameOneImplementation.java b/CodenameOne/src/com/codename1/impl/CodenameOneImplementation.java index 025e5cf6ebd..dd076ce7622 100644 --- a/CodenameOne/src/com/codename1/impl/CodenameOneImplementation.java +++ b/CodenameOne/src/com/codename1/impl/CodenameOneImplementation.java @@ -4740,6 +4740,32 @@ public Object loadTrueTypeFont(String fontName, String fileName) { return null; } + /// Creates a native TrueType or OpenType font from the content of a font file. + /// Ports that return true from `#isTrueTypeStreamSupported()` override this; + /// the result must be usable with `#deriveTrueTypeFont(Object, float, int)`. + /// + /// #### Parameters + /// + /// - `fontName`: the name the application gave the font + /// + /// - `data`: the bytes of a .ttf or .otf file + /// + /// #### Returns + /// + /// the native font, or null if it can't be created + public Object loadTrueTypeFontFromData(String fontName, byte[] data) { + return null; + } + + /// Returns true if `#loadTrueTypeFontFromData(String, byte[])` is implemented by this port. + /// + /// #### Returns + /// + /// true if fonts can be created from data + public boolean isTrueTypeStreamSupported() { + return false; + } + /// Indicates whether the implementation supports loading a font "natively" to handle one of the common /// native prefixes /// diff --git a/CodenameOne/src/com/codename1/ui/Font.java b/CodenameOne/src/com/codename1/ui/Font.java index d2251579f8c..5a29a7f8580 100644 --- a/CodenameOne/src/com/codename1/ui/Font.java +++ b/CodenameOne/src/com/codename1/ui/Font.java @@ -24,6 +24,10 @@ package com.codename1.ui; +import com.codename1.io.Util; + +import java.io.IOException; +import java.io.InputStream; import java.util.HashMap; import java.util.Hashtable; @@ -398,6 +402,99 @@ public static Font createTrueTypeFont(String fontName, String fileName) { return f; } + /// Returns true if the underlying platform can create a TrueType or OpenType font + /// from data in memory, see `#createTrueTypeFontFromData(String, InputStream)`. + /// + /// #### Returns + /// + /// true if fonts can be created from a stream or a byte array on this platform + public static boolean isTrueTypeStreamSupported() { + return Display.impl.isTrueTypeStreamSupported(); + } + + /// Creates a TrueType or OpenType font from data, for a font that was downloaded, + /// stored in `Storage` or generated at runtime rather than packaged in the src + /// root of the project. + /// + /// The stream is read fully and is closed by this method. The returned font + /// behaves like one created by `#createTrueTypeFont(String, String)`: derive + /// it to the size and weight you need. + /// + /// ```java + /// InputStream in = Storage.getInstance().createInputStream("Handlee-Regular.ttf"); + /// Font handlee = Font.createTrueTypeFontFromData("Handlee", in); + /// if (handlee != null) { + /// label.getAllStyles().setFont(handlee.derive(Display.getInstance().convertToPixels(4), Font.STYLE_PLAIN)); + /// } + /// ``` + /// + /// The font name identifies the font in the font cache: a second call with the + /// same name returns the font the first call created without reading the data. + /// Use a distinct name for each distinct font. + /// + /// This is not an overload of `createTrueTypeFont` on purpose: applications + /// call `createTrueTypeFont(name, null)` for an installed font, and a second + /// two-argument overload would make that call ambiguous. + /// + /// #### Parameters + /// + /// - `fontName`: a name that identifies this font + /// + /// - `data`: the content of a .ttf or .otf file + /// + /// #### Returns + /// + /// the font, or null when the platform can't create fonts from data + /// (see `#isTrueTypeStreamSupported()`) + /// + /// #### Throws + /// + /// - `IOException`: if reading the stream fails + public static Font createTrueTypeFontFromData(String fontName, InputStream data) throws IOException { + try { + return createTrueTypeFontFromData(fontName, Util.readInputStream(data)); + } finally { + Util.cleanup(data); + } + } + + /// Creates a TrueType or OpenType font from the bytes of a font file, see + /// `#createTrueTypeFontFromData(String, InputStream)`. + /// + /// #### Parameters + /// + /// - `fontName`: a name that identifies this font + /// + /// - `data`: the content of a .ttf or .otf file + /// + /// #### Returns + /// + /// the font, or null when the platform can't create fonts from data + public static Font createTrueTypeFontFromData(String fontName, byte[] data) { + if (fontName == null || data == null) { + throw new IllegalArgumentException("Font name and data are required"); + } + if (!Display.impl.isTrueTypeStreamSupported()) { + return null; + } + // A prefix keeps a font made from data apart from a file-based font that + // happens to use the same name, in both this cache and the derive cache. + String id = "data:" + fontName; + Font f = derivedFontCache.get(id); + if (f != null) { + return f; + } + Object font = Display.impl.loadTrueTypeFontFromData(fontName, data); + if (font == null) { + return null; + } + f = new Font(font); + f.ttf = true; + f.fontUniqueId = id; + derivedFontCache.put(id, f); + return f; + } + /// Creates a new font instance based on the platform specific string name of the /// font. This method isn't supported on some platforms. /// diff --git a/Ports/Android/src/com/codename1/impl/android/AndroidImplementation.java b/Ports/Android/src/com/codename1/impl/android/AndroidImplementation.java index 57761a6fa84..7c74e3606b4 100644 --- a/Ports/Android/src/com/codename1/impl/android/AndroidImplementation.java +++ b/Ports/Android/src/com/codename1/impl/android/AndroidImplementation.java @@ -2924,6 +2924,58 @@ public Object loadTrueTypeFont(String fontName, String fileName) { com.codename1.ui.Font.STYLE_PLAIN, com.codename1.ui.Font.SIZE_MEDIUM, newPaint, fileName, 0, 0); } + @Override + public boolean isTrueTypeStreamSupported() { + return true; + } + + @Override + public Object loadTrueTypeFontFromData(String fontName, byte[] data) { + // Typeface.createFromFile is the only loader every supported API level has; + // the memory based Typeface.Builder needs API 26. The file is named by a + // hash of the content, so the same font is written once however many + // times it is loaded, and two fonts can never overwrite each other. + try { + byte[] digest = MessageDigest.getInstance("SHA-256").digest(data); + StringBuilder name = new StringBuilder("font_"); + for (int i = 0; i < 16; i++) { + int b = digest[i] & 0xff; + name.append(Character.forDigit(b >> 4, 16)).append(Character.forDigit(b & 0xf, 16)); + } + name.append(".ttf"); + File dir = new File(getContext().getCacheDir(), "cn1_fonts"); + if (!dir.isDirectory() && !dir.mkdirs()) { + return null; + } + File file = new File(dir, name.toString()); + if (file.length() != data.length) { + File tmp = new File(dir, name + ".tmp"); + FileOutputStream out = new FileOutputStream(tmp); + try { + out.write(data); + } finally { + out.close(); + } + if (!tmp.renameTo(file)) { + return null; + } + } + Typeface t = Typeface.createFromFile(file); + if (t == null) { + return null; + } + CodenameOneTextPaint newPaint = new CodenameOneTextPaint(t); + newPaint.setAntiAlias(true); + newPaint.setSubpixelText(true); + return new NativeFont(com.codename1.ui.Font.FACE_SYSTEM, + com.codename1.ui.Font.STYLE_PLAIN, com.codename1.ui.Font.SIZE_MEDIUM, newPaint, + "data:" + fontName, 0, 0); + } catch (Exception err) { + com.codename1.io.Log.e(err); + return null; + } + } + public static class NativeFont { int face; int style; diff --git a/Ports/JavaSE/src/com/codename1/impl/javase/JavaSEPort.java b/Ports/JavaSE/src/com/codename1/impl/javase/JavaSEPort.java index ae14988382d..26336b32bfa 100644 --- a/Ports/JavaSE/src/com/codename1/impl/javase/JavaSEPort.java +++ b/Ports/JavaSE/src/com/codename1/impl/javase/JavaSEPort.java @@ -15654,6 +15654,27 @@ public Object loadTrueTypeFont(String fontName, String fileName) { throw new RuntimeException("The file wasn't found: " + fontName); } + @Override + public boolean isTrueTypeStreamSupported() { + return true; + } + + @Override + public Object loadTrueTypeFontFromData(String fontName, byte[] data) { + try { + // TRUETYPE_FONT reads the whole SFNT container, so CFF-flavoured + // OpenType loads here as well. deriveTrueTypeFont works on the + // returned java.awt.Font exactly as for a font loaded from a file. + return java.awt.Font.createFont(java.awt.Font.TRUETYPE_FONT, new java.io.ByteArrayInputStream(data)); + } catch (java.awt.FontFormatException err) { + System.err.println("Not a TrueType or OpenType font: " + fontName + " (" + err.getMessage() + ")"); + return null; + } catch (IOException err) { + System.err.println("Could not read font " + fontName + ": " + err.getMessage()); + return null; + } + } + private static double fontScale=1.0; public static double getFontScale() { diff --git a/Ports/JavaScriptPort/src/main/java/com/codename1/impl/html5/HTML5Implementation.java b/Ports/JavaScriptPort/src/main/java/com/codename1/impl/html5/HTML5Implementation.java index 98037c79ba3..55e4ed25b30 100644 --- a/Ports/JavaScriptPort/src/main/java/com/codename1/impl/html5/HTML5Implementation.java +++ b/Ports/JavaScriptPort/src/main/java/com/codename1/impl/html5/HTML5Implementation.java @@ -11387,6 +11387,25 @@ public Object loadTrueTypeFont(String fontName, String fileName) { } + @Override + public boolean isTrueTypeStreamSupported() { + return true; + } + + @Override + public Object loadTrueTypeFontFromData(String fontName, byte[] data) { + // The bytes travel as a data: URL, which both the worker's FontFace + // (measuring) and the host's document.fonts (painting) load as they load + // a bundled file. Encoded here in Java: the arrayBufferToBase64 helper + // goes through Window.current, which the worker cannot reach. + if (!loadedFonts.contains(fontName)) { + loadTrueTypeFont_(fontName, + "data:font/ttf;base64," + com.codename1.util.Base64.encodeNoNewline(data), "truetype"); + loadedFonts.add(fontName); + } + return createFallbackTrueTypeFont(fontName, null); + } + /** * The format hint handed to the FontFace constructor. Browsers treat * "truetype" and "opentype" as the same SFNT container, but naming the diff --git a/Ports/LinuxPort/src/com/codename1/impl/linux/LinuxImplementation.java b/Ports/LinuxPort/src/com/codename1/impl/linux/LinuxImplementation.java index 344463ff8b7..e0e860a4a32 100644 --- a/Ports/LinuxPort/src/com/codename1/impl/linux/LinuxImplementation.java +++ b/Ports/LinuxPort/src/com/codename1/impl/linux/LinuxImplementation.java @@ -2281,6 +2281,18 @@ public boolean isNativeFontSchemeSupported() { return true; } + @Override + public boolean isTrueTypeStreamSupported() { + return true; + } + + @Override + public Object loadTrueTypeFontFromData(String fontName, byte[] data) { + // The same in-memory loader bundled fonts go through. + long font = LinuxNative.loadTrueTypeFontFromMemory(fontName, data); + return font == 0 ? null : Long.valueOf(font); + } + @Override public Object loadTrueTypeFont(String fontName, String fileName) { long font = 0; diff --git a/Ports/WindowsPort/src/com/codename1/impl/windows/WindowsImplementation.java b/Ports/WindowsPort/src/com/codename1/impl/windows/WindowsImplementation.java index 372247d2b76..23caf5d8c83 100644 --- a/Ports/WindowsPort/src/com/codename1/impl/windows/WindowsImplementation.java +++ b/Ports/WindowsPort/src/com/codename1/impl/windows/WindowsImplementation.java @@ -2079,6 +2079,18 @@ public boolean isNativeFontSchemeSupported() { return true; } + @Override + public boolean isTrueTypeStreamSupported() { + return true; + } + + @Override + public Object loadTrueTypeFontFromData(String fontName, byte[] data) { + // The same in-memory loader bundled fonts go through. + long font = WindowsNative.loadTrueTypeFontFromMemory(fontName, data); + return font == 0 ? null : Long.valueOf(font); + } + @Override public Object loadTrueTypeFont(String fontName, String fileName) { long font = 0; diff --git a/Ports/iOSPort/nativeSources/IOSNative.m b/Ports/iOSPort/nativeSources/IOSNative.m index 7a14c573de4..e717c8e2833 100644 --- a/Ports/iOSPort/nativeSources/IOSNative.m +++ b/Ports/iOSPort/nativeSources/IOSNative.m @@ -14635,6 +14635,45 @@ void com_codename1_impl_ios_IOSNative_registerBundledFont___java_lang_String(CN1 POOL_END(); } +/// Registers a font held in memory (Font.createTrueTypeFontFromData) with +/// Core Text and answers its PostScript name, which createTruetypeFont then looks +/// up exactly as it does for a bundled font. Answers null when the bytes are not a +/// font. A font registered twice reports an error that is discarded: the name is +/// what the caller needs and it is valid either way. +JAVA_OBJECT com_codename1_impl_ios_IOSNative_registerFontData___byte_1ARRAY_R_java_lang_String(CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject, JAVA_OBJECT data) { + if (data == JAVA_NULL) { + return JAVA_NULL; + } + JAVA_OBJECT result = JAVA_NULL; + POOL_BEGIN(); + void *bytes = CN1_ARRAY_DATA((JAVA_ARRAY)data); + int length = ((JAVA_ARRAY)data)->length; + CFDataRef cfData = CFDataCreate(NULL, (const UInt8 *)bytes, length); + CGDataProviderRef provider = cfData == NULL ? NULL : CGDataProviderCreateWithCFData(cfData); + CGFontRef cgFont = provider == NULL ? NULL : CGFontCreateWithDataProvider(provider); + if (cgFont != NULL) { + CFErrorRef error = NULL; + CTFontManagerRegisterGraphicsFont(cgFont, &error); + if (error != NULL) { + CFRelease(error); + } + CFStringRef psName = CGFontCopyPostScriptName(cgFont); + if (psName != NULL) { + result = fromNSString(CN1_THREAD_STATE_PASS_ARG (BRIDGE_CAST NSString *)psName); + CFRelease(psName); + } + CGFontRelease(cgFont); + } + if (provider != NULL) { + CGDataProviderRelease(provider); + } + if (cfData != NULL) { + CFRelease(cfData); + } + POOL_END(); + return result; +} + #if TARGET_OS_OSX static NSFont *cn1MacSystemFontForAlias(NSString *name, CGFloat size) { BOOL italic = [name hasPrefix:@"native:Italic"]; diff --git a/Ports/iOSPort/src/com/codename1/impl/ios/IOSImplementation.java b/Ports/iOSPort/src/com/codename1/impl/ios/IOSImplementation.java index e54b513fbb9..c4c8645929a 100644 --- a/Ports/iOSPort/src/com/codename1/impl/ios/IOSImplementation.java +++ b/Ports/iOSPort/src/com/codename1/impl/ios/IOSImplementation.java @@ -11957,6 +11957,26 @@ public Object loadTrueTypeFont(String fontName, String fileName) { return fnt; } + @Override + public boolean isTrueTypeStreamSupported() { + return true; + } + + @Override + public Object loadTrueTypeFontFromData(String fontName, byte[] data) { + String postScriptName = nativeInstance.registerFontData(data); + if (postScriptName == null) { + return null; + } + NativeFont fnt = new NativeFont(); + fnt.face = com.codename1.ui.Font.FACE_SYSTEM; + fnt.size = com.codename1.ui.Font.SIZE_MEDIUM; + fnt.style = com.codename1.ui.Font.STYLE_PLAIN; + fnt.name = postScriptName; + fnt.peer = nativeInstance.createTruetypeFont(postScriptName); + return fnt; + } + @Override public Object deriveTrueTypeFont(Object font, float size, int weight) { NativeFont original = (NativeFont)font; diff --git a/Ports/iOSPort/src/com/codename1/impl/ios/IOSNative.java b/Ports/iOSPort/src/com/codename1/impl/ios/IOSNative.java index 03294d8897d..c074cde12df 100644 --- a/Ports/iOSPort/src/com/codename1/impl/ios/IOSNative.java +++ b/Ports/iOSPort/src/com/codename1/impl/ios/IOSNative.java @@ -1037,6 +1037,7 @@ native void gl3dDrawArrays(long contextPeer, long pipelinePeer, long vboPeer, in /// lookup -- 33 files and 5.7MB in an application shipping a font family, /// measured at 25ms on the start-up path. native void registerBundledFont(String fileName); + native String registerFontData(byte[] data); native long deriveTruetypeFont(long uiFont, boolean bold, boolean italic, float size); native void log(String text); diff --git a/docs/demos/common/src/main/java/com/codenameone/developerguide/snippets/FontFromDataSnippets.java b/docs/demos/common/src/main/java/com/codenameone/developerguide/snippets/FontFromDataSnippets.java new file mode 100644 index 00000000000..184b1b5feb3 --- /dev/null +++ b/docs/demos/common/src/main/java/com/codenameone/developerguide/snippets/FontFromDataSnippets.java @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codenameone.developerguide.snippets; + +import com.codename1.io.Storage; +import com.codename1.ui.CN; +import com.codename1.ui.Font; +import com.codename1.ui.Label; + +import java.io.IOException; +import java.io.InputStream; + +/// Snippets for creating a font from data at runtime, included by the +/// developer guide's CSS fonts section. +public class FontFromDataSnippets { + Label fontFromStorage(Label label) throws IOException { + // tag::font-from-data[] + Font base = null; + if (Font.isTrueTypeStreamSupported()) { + InputStream in = Storage.getInstance().createInputStream("Handlee-Regular.ttf"); + base = Font.createTrueTypeFontFromData("Handlee", in); + } + if (base == null) { + base = Font.createTrueTypeFont("native:MainRegular"); + } + label.getAllStyles().setFont(base.derive(CN.convertToPixels(4), Font.STYLE_PLAIN)); + // end::font-from-data[] + return label; + } +} diff --git a/docs/developer-guide/Python-Interop.asciidoc b/docs/developer-guide/Python-Interop.asciidoc index b0dabeece29..02d636e6897 100644 --- a/docs/developer-guide/Python-Interop.asciidoc +++ b/docs/developer-guide/Python-Interop.asciidoc @@ -214,8 +214,11 @@ the real package: glyph can sit a pixel away from where Pillow puts it. On a device, decoding, encoding, resizing and text go through Codename One's image support (`Image`, `ImageIO`, `Graphics` and the native font), which also - reads GIF, BMP and WebP. Those results are drawn by the platform, so they - look right but aren't Pillow's exact pixels. + reads GIF, BMP and WebP. `ImageFont.truetype` loads the font through + `Font.createTrueTypeFontFromData` where the platform supports it and falls + back to the native font elsewhere, and `ImageFont.load_default()` is the + native font. Those results are drawn by the platform, so they look right + but aren't Pillow's exact pixels. | `flask`, `fastapi`, `uvicorn` | See <>. | `typing_extensions`, `six`, `tabulate`, `idna`, `packaging` | |=== diff --git a/docs/developer-guide/css.asciidoc b/docs/developer-guide/css.asciidoc index 9a0ffd5cfb3..a6655c0623b 100644 --- a/docs/developer-guide/css.asciidoc +++ b/docs/developer-guide/css.asciidoc @@ -719,6 +719,19 @@ include::../demos/common/src/main/css/guide-snippets-theme.css[tag=css-css-031,i NOTE: If a third-party font repository moves or changes licensing, vendor the font into your project and reference it locally so builds stay reproducible. +===== Fonts loaded at runtime + +A font that isn't known at build time, such as one the app downloads or a user picks, can't go through `@font-face`. `Font.createTrueTypeFontFromData` creates a font from the bytes of a `.ttf` or `.otf` file, given either as an `InputStream` or as a `byte[]`. The result works like a bundled font: derive it to the size and weight you need and set it on a style. + +[source,java] +---- +include::../demos/common/src/main/java/com/codenameone/developerguide/snippets/FontFromDataSnippets.java[tag=font-from-data,indent=0] +---- + +The name you pass identifies the font in the font cache, so a second call with the same name returns the first font without reading the data again. Give each distinct font its own name. + +`Font.isTrueTypeStreamSupported()` reports whether the platform can do this. The simulator, Android, iOS, macOS, the JavaScript port and the native Windows and Linux ports all can. Elsewhere `createTrueTypeFontFromData` returns `null`, so keep a fallback font as the example does. + ==== `font-size` It's best practice to size your fonts using millimetres (`rem`) (or another "real-world" measurement unit such as inches (`in`), centimetres (`cm`), millimetres (`mm`). This will allow the font to be sized appropriate for all display densities. If you specify size in pixels (`px`), it will treat it the same as if you sized it in points (`pt`), where 1pt == 1/72 inches (one seventy-second of an inch). diff --git a/maven/compat-testing/src/main/java/com/codename1/compat/testing/HeadlessImplementation.java b/maven/compat-testing/src/main/java/com/codename1/compat/testing/HeadlessImplementation.java index 622de8090f9..d71009ba5ce 100644 --- a/maven/compat-testing/src/main/java/com/codename1/compat/testing/HeadlessImplementation.java +++ b/maven/compat-testing/src/main/java/com/codename1/compat/testing/HeadlessImplementation.java @@ -110,6 +110,22 @@ public Object loadTrueTypeFont(String fontName, String fileName) { return FONT; } + @Override + public boolean isTrueTypeStreamSupported() { + return true; + } + + /// Accepts anything that starts like an SFNT file (TrueType, OpenType or + /// Apple 'true') and answers the box font every other load answers. + @Override + public Object loadTrueTypeFontFromData(String fontName, byte[] data) { + if (data == null || data.length < 4) { + return null; + } + int tag = (data[0] & 0xff) << 24 | (data[1] & 0xff) << 16 | (data[2] & 0xff) << 8 | data[3] & 0xff; + return tag == 0x00010000 || tag == 0x4f54544f || tag == 0x74727565 ? FONT : null; + } + @Override public Object deriveTrueTypeFont(Object font, float size, int weight) { return FONT; diff --git a/maven/core-unittests/src/test/java/com/codename1/testing/TestCodenameOneImplementation.java b/maven/core-unittests/src/test/java/com/codename1/testing/TestCodenameOneImplementation.java index 12d0bf16acd..048bb1e7b34 100644 --- a/maven/core-unittests/src/test/java/com/codename1/testing/TestCodenameOneImplementation.java +++ b/maven/core-unittests/src/test/java/com/codename1/testing/TestCodenameOneImplementation.java @@ -3309,6 +3309,32 @@ public void setTrueTypeSupported(boolean trueTypeSupported) { this.trueTypeSupported = trueTypeSupported; } + private boolean trueTypeStreamSupported = true; + private int trueTypeDataLoads; + + public void setTrueTypeStreamSupported(boolean supported) { + this.trueTypeStreamSupported = supported; + } + + /// How many times loadTrueTypeFontFromData(String, byte[]) reached this implementation. + public int getTrueTypeDataLoads() { + return trueTypeDataLoads; + } + + @Override + public boolean isTrueTypeStreamSupported() { + return trueTypeStreamSupported; + } + + @Override + public Object loadTrueTypeFontFromData(String fontName, byte[] data) { + trueTypeDataLoads++; + if (data.length == 0) { + return null; + } + return new TestFont(defaultFont.charWidth, defaultFont.height); + } + @Override public boolean isNativeFontSchemeSupported() { return nativeFontSchemeSupported; diff --git a/maven/core-unittests/src/test/java/com/codename1/ui/FontTest.java b/maven/core-unittests/src/test/java/com/codename1/ui/FontTest.java index 703603bac65..201ce7127cc 100644 --- a/maven/core-unittests/src/test/java/com/codename1/ui/FontTest.java +++ b/maven/core-unittests/src/test/java/com/codename1/ui/FontTest.java @@ -28,6 +28,8 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import java.io.ByteArrayInputStream; +import java.io.IOException; import java.lang.reflect.Field; import java.util.HashMap; @@ -40,6 +42,7 @@ void clearCaches() throws Exception { setFontReturnedHeight(0f); implementation.setTrueTypeSupported(true); implementation.setNativeFontSchemeSupported(true); + implementation.setTrueTypeStreamSupported(true); } @EdtTest @@ -121,6 +124,36 @@ void testIsTrueTypeFileSupportedDelegatesToImplementation() { assertTrue(Font.isTrueTypeFileSupported()); } + @EdtTest + void testCreateTrueTypeFontFromStreamCachesByNameAndDerives() throws IOException { + int loads = implementation.getTrueTypeDataLoads(); + Font first = Font.createTrueTypeFontFromData("DataFont", new ByteArrayInputStream(new byte[]{1, 2, 3})); + assertNotNull(first); + assertTrue(first.isTTFNativeFont()); + assertTrue(Font.isTrueTypeStreamSupported()); + Font second = Font.createTrueTypeFontFromData("DataFont", new byte[]{1, 2, 3}); + assertSame(first, second); + assertEquals(loads + 1, implementation.getTrueTypeDataLoads()); + + Font derived = first.derive(24f, Font.STYLE_BOLD); + assertSame(derived, first.derive(24f, Font.STYLE_BOLD)); + assertEquals(24f, derived.getPixelSize(), 0.001f); + + // A file-based font with the same name is a different font. + Font fromFile = Font.createTrueTypeFont("DataFont", "DataFont.ttf"); + assertNotSame(first, fromFile); + assertNotSame(derived, fromFile.derive(24f, Font.STYLE_BOLD)); + } + + @EdtTest + void testCreateTrueTypeFontFromDataReturnsNullWhenUnsupportedOrInvalid() { + assertNull(Font.createTrueTypeFontFromData("Empty", new byte[0])); + implementation.setTrueTypeStreamSupported(false); + assertFalse(Font.isTrueTypeStreamSupported()); + assertNull(Font.createTrueTypeFontFromData("Unsupported", new byte[]{1})); + assertThrows(IllegalArgumentException.class, () -> Font.createTrueTypeFontFromData("NoData", (byte[]) null)); + } + private HashMap getDerivedFontCache() throws Exception { Field field = Font.class.getDeclaredField("derivedFontCache"); field.setAccessible(true); diff --git a/maven/javase/src/test/java/com/codename1/impl/javase/JavaSEPortFontFromDataTest.java b/maven/javase/src/test/java/com/codename1/impl/javase/JavaSEPortFontFromDataTest.java new file mode 100644 index 00000000000..90d69f3b492 --- /dev/null +++ b/maven/javase/src/test/java/com/codename1/impl/javase/JavaSEPortFontFromDataTest.java @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.impl.javase; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Font.createTrueTypeFont(String, InputStream) on the simulator: a real TTF + * loaded from bytes, derived and measured headless. + */ +public class JavaSEPortFontFromDataTest { + private JavaSEPort port; + private JavaSEPort originalInstance; + + @BeforeEach + void setUp() { + originalInstance = JavaSEPort.instance; + port = new JavaSEPort(); + } + + @AfterEach + void tearDown() { + JavaSEPort.instance = originalInstance; + } + + @Test + void loadsDerivesAndMeasuresAFontFromBytes() throws IOException { + assertTrue(port.isTrueTypeStreamSupported()); + Object font = port.loadTrueTypeFontFromData("Roboto", readResource("/com/codename1/impl/javase/Roboto-Regular.ttf")); + assertNotNull(font); + assertEquals("Roboto", ((java.awt.Font) font).getFamily()); + + Object small = port.deriveTrueTypeFont(font, 12f, com.codename1.ui.Font.STYLE_PLAIN); + Object large = port.deriveTrueTypeFont(font, 48f, com.codename1.ui.Font.STYLE_PLAIN); + int smallWidth = port.stringWidth(small, "Hello World"); + int largeWidth = port.stringWidth(large, "Hello World"); + assertTrue(smallWidth > 0, "measured " + smallWidth); + assertTrue(largeWidth > smallWidth * 3, smallWidth + " vs " + largeWidth); + } + + @Test + void bytesThatAreNotAFontLoadNothing() { + assertNull(port.loadTrueTypeFontFromData("Junk", new byte[]{1, 2, 3, 4, 5, 6, 7, 8})); + } + + private static byte[] readResource(String path) throws IOException { + InputStream in = JavaSEPortFontFromDataTest.class.getResourceAsStream(path); + assertNotNull(in, path); + try { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + byte[] buf = new byte[8192]; + for (int n = in.read(buf); n > 0; n = in.read(buf)) { + out.write(buf, 0, n); + } + return out.toByteArray(); + } finally { + in.close(); + } + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyImageNative.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyImageNative.java index 6c611e02682..79bf631b8c5 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyImageNative.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyImageNative.java @@ -39,9 +39,13 @@ /// Everything here is drawn by the platform, so results are close to /// Pillow's but are not its exact pixels: resizing is the platform's /// scaling whatever filter was asked for, PNG files are decoded by the -/// platform, and text is the platform's native font. A device cannot load -/// a TrueType file from bytes, so `ImageFont.truetype` answers the native -/// font at the requested size as well. +/// platform, and text is the platform's native font unless a font is +/// loaded. `ImageFont.truetype` hands the file's bytes to +/// `Font.createTrueTypeFontFromData(String, byte[])` where the platform supports +/// fonts from data, and answers the native font at the requested size where +/// it does not or the bytes are not a font. `ImageFont.load_default` is the +/// native font: the Aileron data the server engine embeds lives in the +/// backend runtime, which a client build does not ship. public final class PyImageNative implements PyImageEngine { private static final String NATIVE_FONT = "native:MainRegular"; @@ -175,9 +179,28 @@ public byte[] jpegEncode(PyImageData s, float quality) { @Override public Object loadFont(byte[] data, int index, double size) { + if (data != null && data.length > 0 && Font.isTrueTypeStreamSupported()) { + // The name keys Font's cache, so it has to tell fonts apart by + // content: the same file loaded twice is one platform font. + Font f = Font.createTrueTypeFontFromData("pil:" + contentKey(data), data); + if (f != null) { + return f.derive((float) size, Font.STYLE_PLAIN); + } + } return defaultFont(size); } + private static String contentKey(byte[] data) { + // 64 bit FNV-1a plus the length; collisions between two real fonts + // in one process are not a practical concern. + long h = 0xcbf29ce484222325L; + for (byte b : data) { + h ^= b & 0xff; + h *= 0x100000001b3L; + } + return Long.toString(h, 16) + "_" + data.length; + } + @Override public byte[] defaultFontData() { return new byte[0]; diff --git a/maven/python-transpiler/src/test/resources/conformance/pilclient_.out b/maven/python-transpiler/src/test/resources/conformance/pilclient_.out index 24fbf7c2cae..19e2893b5a8 100644 --- a/maven/python-transpiler/src/test/resources/conformance/pilclient_.out +++ b/maven/python-transpiler/src/test/resources/conformance/pilclient_.out @@ -32,3 +32,17 @@ truetype textbbox True True True truetype textlength True truetype ink True truetype mask L True True +sfnt length True True +sfnt bbox True True +sfnt metrics True True +sfnt textbbox True True True +sfnt textlength True +sfnt ink True +sfnt mask L True True +sfnt again length True True +sfnt again bbox True True +sfnt again metrics True True +sfnt again textbbox True True True +sfnt again textlength True +sfnt again ink True +sfnt again mask L True True diff --git a/maven/python-transpiler/src/test/resources/conformance/pilclient_/main.py b/maven/python-transpiler/src/test/resources/conformance/pilclient_/main.py index c8c1cba0369..226c45c5e1a 100644 --- a/maven/python-transpiler/src/test/resources/conformance/pilclient_/main.py +++ b/maven/python-transpiler/src/test/resources/conformance/pilclient_/main.py @@ -95,7 +95,15 @@ def text_checks(name, f): text_checks("default", ImageFont.load_default(18)) # CPython hands back the TrueType bytes it loaded the default from; this -# runtime keeps none, and the client engine draws its native font whatever -# bytes truetype() is given -- so either way this is a real truetype() call. +# runtime keeps none on a client, and the client engine falls back to its +# native font for bytes that are not a font -- so either way this is a real +# truetype() call. data = getattr(ImageFont.load_default(), "font_bytes", b"") text_checks("truetype", ImageFont.truetype(io.BytesIO(data), 18)) +# An SFNT header goes through Font.createTrueTypeFontFromData(String, byte[]); the +# headless implementation answers its box font for it. CPython rejects a +# header with no tables, so the TrueType bytes it loaded the default from +# stand in for it there. +sfnt = data if data else b"\x00\x01\x00\x00" + b"\x00" * 60 +text_checks("sfnt", ImageFont.truetype(io.BytesIO(sfnt), 18)) +text_checks("sfnt again", ImageFont.truetype(io.BytesIO(sfnt), 24)) From 20a7cab8c4ea6952b153c079d41370661c8dec05 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 16:33:35 +0200 Subject: [PATCH 085/112] Linux/Windows createTrueTypeFontFromData: null for bytes without an SFNT header The in-memory loaders answer a system family when they cannot register the bytes; the API contract is null, so the ports check the SFNT tag (0x00010000, 'true', 'OTTO', 'ttcf') first. Corrects the stale Linux header comment that said in-memory loading was not wired. Co-Authored-By: Claude Opus 5.5 --- .../LinuxPort/nativeSources/cn1_linux_text.c | 6 ++++-- .../impl/linux/LinuxImplementation.java | 19 ++++++++++++++++++- .../impl/windows/WindowsImplementation.java | 19 ++++++++++++++++++- 3 files changed, 40 insertions(+), 4 deletions(-) diff --git a/Ports/LinuxPort/nativeSources/cn1_linux_text.c b/Ports/LinuxPort/nativeSources/cn1_linux_text.c index e27c06acd52..d72884eff5f 100644 --- a/Ports/LinuxPort/nativeSources/cn1_linux_text.c +++ b/Ports/LinuxPort/nativeSources/cn1_linux_text.c @@ -30,8 +30,10 @@ * * NOTE: implemented against Pango/PangoCairo but not yet compiled/run on a * Linux/GTK host. In-memory TrueType registration (loadTrueTypeFontFromMemory) - * currently falls back to a system family -- wiring the bytes through - * FcConfigAppFontAddMemFace/FreeType is a later refinement. + * spools the bytes to a temp file, adds it to FontConfig and names the family + * from the face via FreeType. Bytes that are not a font still answer a system + * family here, so LinuxImplementation.loadTrueTypeFontFromData rejects anything + * without an SFNT header before calling it. */ #define _GNU_SOURCE diff --git a/Ports/LinuxPort/src/com/codename1/impl/linux/LinuxImplementation.java b/Ports/LinuxPort/src/com/codename1/impl/linux/LinuxImplementation.java index e0e860a4a32..16a6828190a 100644 --- a/Ports/LinuxPort/src/com/codename1/impl/linux/LinuxImplementation.java +++ b/Ports/LinuxPort/src/com/codename1/impl/linux/LinuxImplementation.java @@ -2286,9 +2286,26 @@ public boolean isTrueTypeStreamSupported() { return true; } + /// True when data starts with a TrueType (0x00010000 or 'true'), OpenType + /// CFF ('OTTO') or collection ('ttcf') tag. + private static boolean isSfntHeader(byte[] data) { + if (data == null || data.length < 12) { + return false; + } + int tag = ((data[0] & 0xff) << 24) | ((data[1] & 0xff) << 16) + | ((data[2] & 0xff) << 8) | (data[3] & 0xff); + return tag == 0x00010000 || tag == 0x74727565 || tag == 0x4f54544f + || tag == 0x74746366; + } + @Override public Object loadTrueTypeFontFromData(String fontName, byte[] data) { - // The same in-memory loader bundled fonts go through. + // The in-memory loader answers a system family for bytes it cannot + // register, while the contract for data that is not a font is null, + // so anything without an SFNT header is rejected before it. + if (!isSfntHeader(data)) { + return null; + } long font = LinuxNative.loadTrueTypeFontFromMemory(fontName, data); return font == 0 ? null : Long.valueOf(font); } diff --git a/Ports/WindowsPort/src/com/codename1/impl/windows/WindowsImplementation.java b/Ports/WindowsPort/src/com/codename1/impl/windows/WindowsImplementation.java index 23caf5d8c83..6f98bf0fbf8 100644 --- a/Ports/WindowsPort/src/com/codename1/impl/windows/WindowsImplementation.java +++ b/Ports/WindowsPort/src/com/codename1/impl/windows/WindowsImplementation.java @@ -2084,9 +2084,26 @@ public boolean isTrueTypeStreamSupported() { return true; } + /// True when data starts with a TrueType (0x00010000 or 'true'), OpenType + /// CFF ('OTTO') or collection ('ttcf') tag. + private static boolean isSfntHeader(byte[] data) { + if (data == null || data.length < 12) { + return false; + } + int tag = ((data[0] & 0xff) << 24) | ((data[1] & 0xff) << 16) + | ((data[2] & 0xff) << 8) | (data[3] & 0xff); + return tag == 0x00010000 || tag == 0x74727565 || tag == 0x4f54544f + || tag == 0x74746366; + } + @Override public Object loadTrueTypeFontFromData(String fontName, byte[] data) { - // The same in-memory loader bundled fonts go through. + // The in-memory loader answers a system family for bytes it cannot + // register, while the contract for data that is not a font is null, + // so anything without an SFNT header is rejected before it. + if (!isSfntHeader(data)) { + return null; + } long font = WindowsNative.loadTrueTypeFontFromMemory(fontName, data); return font == 0 ? null : Long.valueOf(font); } From 320b57b5a68ffef372cb053cd4f5742eb2a3f59d Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 18:38:28 +0200 Subject: [PATCH 086/112] JavaScript createTrueTypeFontFromData: null for bytes without an SFNT header FontFace rejects non-font bytes asynchronously, after the port had already returned a font; check the SFNT tag up front as Linux/Windows do. Adds TrueTypeFontFromDataTest to the hellocodenameone device suite. Co-Authored-By: Claude Opus 5.5 --- .../impl/html5/HTML5Implementation.java | 15 +++ .../tests/Cn1ssDeviceRunner.java | 1 + .../tests/TrueTypeFontFromDataTest.java | 104 ++++++++++++++++++ 3 files changed, 120 insertions(+) create mode 100644 scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/TrueTypeFontFromDataTest.java diff --git a/Ports/JavaScriptPort/src/main/java/com/codename1/impl/html5/HTML5Implementation.java b/Ports/JavaScriptPort/src/main/java/com/codename1/impl/html5/HTML5Implementation.java index 55e4ed25b30..feb8fa59501 100644 --- a/Ports/JavaScriptPort/src/main/java/com/codename1/impl/html5/HTML5Implementation.java +++ b/Ports/JavaScriptPort/src/main/java/com/codename1/impl/html5/HTML5Implementation.java @@ -11398,6 +11398,12 @@ public Object loadTrueTypeFontFromData(String fontName, byte[] data) { // (measuring) and the host's document.fonts (painting) load as they load // a bundled file. Encoded here in Java: the arrayBufferToBase64 helper // goes through Window.current, which the worker cannot reach. + // FontFace rejects bytes that are not a font asynchronously, after this + // has already answered; the contract is null, so check the SFNT tag + // (0x00010000, 'true', 'OTTO', 'ttcf') up front as the other ports do. + if (!isSfntHeader(data)) { + return null; + } if (!loadedFonts.contains(fontName)) { loadTrueTypeFont_(fontName, "data:font/ttf;base64," + com.codename1.util.Base64.encodeNoNewline(data), "truetype"); @@ -11406,6 +11412,15 @@ public Object loadTrueTypeFontFromData(String fontName, byte[] data) { return createFallbackTrueTypeFont(fontName, null); } + private static boolean isSfntHeader(byte[] data) { + if (data == null || data.length < 12) { + return false; + } + int tag = ((data[0] & 0xff) << 24) | ((data[1] & 0xff) << 16) + | ((data[2] & 0xff) << 8) | (data[3] & 0xff); + return tag == 0x00010000 || tag == 0x74727565 || tag == 0x4f54544f || tag == 0x74746366; + } + /** * The format hint handed to the FontFace constructor. Browsers treat * "truetype" and "opentype" as the same SFNT container, but naming the diff --git a/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/Cn1ssDeviceRunner.java b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/Cn1ssDeviceRunner.java index aba2a1209c4..ce6cf5f6f99 100644 --- a/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/Cn1ssDeviceRunner.java +++ b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/Cn1ssDeviceRunner.java @@ -619,6 +619,7 @@ private static int testTimeoutMs(BaseTest testClass) { // SurfacesRasterizerScreenshotTest above. new SurfacesSerializerRoundTripTest(), new SurfacesTimelineLogicTest(), + new TrueTypeFontFromDataTest(), new SurfacesActionDispatchTest(), new SurfacesPublishTest(), // Document provider on the device VM: the serializer that is the on-disk contract diff --git a/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/TrueTypeFontFromDataTest.java b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/TrueTypeFontFromDataTest.java new file mode 100644 index 00000000000..aa4f30db4a8 --- /dev/null +++ b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/TrueTypeFontFromDataTest.java @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + */ +package com.codenameone.examples.hellocodenameone.tests; + +import com.codename1.io.Util; +import com.codename1.ui.Display; +import com.codename1.ui.Font; +import com.codename1.ui.Graphics; +import com.codename1.ui.Image; +import com.codename1.ui.util.UITimer; + +import java.io.InputStream; + +/// Font.createTrueTypeFontFromData on the device: a real .ttf from bytes measures and paints +/// with its own glyphs, and bytes that are not a font answer null. A browser loads the +/// FontFace asynchronously, so the measuring half polls until the face is in. Assertion-only. +public class TrueTypeFontFromDataTest extends BaseTest { + // A Material icon codepoint: the default font has no glyph for it. + private static final String SAMPLE = ""; + private Font font; + private Font plain; + private int attempts; + + @Override + public boolean shouldTakeScreenshot() { + return false; + } + + @Override + public boolean runTest() { + try { + if (!Font.isTrueTypeStreamSupported()) { + done(); + return true; + } + if (Font.createTrueTypeFontFromData("cn1ssNotAFont", new byte[64]) != null) { + fail("bytes without an SFNT header must answer null"); + return false; + } + InputStream in = Display.getInstance().getResourceAsStream(getClass(), "/material-design-font.ttf"); + if (in == null) { + fail("material-design-font.ttf is not in the bundle"); + return false; + } + byte[] data = Util.readInputStream(in); + Util.cleanup(in); + Font base = Font.createTrueTypeFontFromData("cn1ssMaterialFromData", data); + if (base == null) { + fail("createTrueTypeFontFromData answered null for a real .ttf"); + return false; + } + font = base.derive(48, Font.STYLE_PLAIN); + plain = Font.createSystemFont(Font.FACE_SYSTEM, Font.STYLE_PLAIN, Font.SIZE_LARGE); + poll(); + } catch (Throwable t) { + fail("unexpected " + t); + } + return true; + } + + private void poll() { + int w = font.stringWidth(SAMPLE); + int inked = inkedPixels(); + System.out.println("CN1SS:TTFDATA width=" + w + " plainWidth=" + plain.stringWidth(SAMPLE) + + " inked=" + inked + " attempt=" + attempts); + if (w > 0 && w != plain.stringWidth(SAMPLE) && inked > 0) { + done(); + return; + } + if (++attempts > 50) { + fail("font from data never took effect: width=" + w + " inked=" + inked); + return; + } + UITimer.timer(100, false, Display.getInstance().getCurrent(), new Runnable() { + public void run() { + poll(); + } + }); + } + + private int inkedPixels() { + Image img = Image.createImage(200, 80, 0xffffffff); + Graphics g = img.getGraphics(); + g.setColor(0x000000); + g.setFont(font); + g.drawString(SAMPLE, 4, 4); + int[] rgb = img.getRGB(); + int n = 0; + for (int i = 0; i < rgb.length; i++) { + if ((rgb[i] & 0xffffff) != 0xffffff) { + n++; + } + } + return n; + } +} From dbca9a287c1b8ceeec05b6f3d8dafb6a637d5142 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 19:11:13 +0200 Subject: [PATCH 087/112] Column types cached per row; data fonts keyed by content; font loaders report corrupt bytes AbstractDBCursor takes a column's storage class before any getter reads it and answers getColumnType from that per-row cache, since sqlite3_column_type is undefined after a conversion. Font.createTrueTypeFontFromData keys its cache on name, length and a hash of the bytes (the JavaScript port's browser family too). iOS registerFontData answers null when registration fails, accepting an already-registered name only for the same font. Linux and Windows answer 0 (null) for bytes FreeType/DirectWrite cannot load instead of a system family. Co-Authored-By: Claude Opus 5.5 --- .../com/codename1/impl/AbstractDBCursor.java | 45 ++++- CodenameOne/src/com/codename1/ui/Font.java | 21 +- .../impl/html5/HTML5Implementation.java | 20 +- .../LinuxPort/nativeSources/cn1_linux_text.c | 70 ++++--- .../impl/linux/LinuxImplementation.java | 6 +- .../nativeSources/cn1_windows_text.c | 15 +- .../impl/windows/WindowsImplementation.java | 6 +- Ports/iOSPort/nativeSources/IOSNative.m | 56 +++++- .../impl/AbstractDBCursorColumnTypeTest.java | 181 ++++++++++++++++++ .../test/java/com/codename1/ui/FontTest.java | 14 ++ .../javase/JavaSEPortFontFromDataTest.java | 10 + 11 files changed, 389 insertions(+), 55 deletions(-) create mode 100644 maven/core-unittests/src/test/java/com/codename1/impl/AbstractDBCursorColumnTypeTest.java diff --git a/CodenameOne/src/com/codename1/impl/AbstractDBCursor.java b/CodenameOne/src/com/codename1/impl/AbstractDBCursor.java index 57bec15951a..5cc318ab677 100644 --- a/CodenameOne/src/com/codename1/impl/AbstractDBCursor.java +++ b/CodenameOne/src/com/codename1/impl/AbstractDBCursor.java @@ -81,6 +81,18 @@ public abstract class AbstractDBCursor implements Cursor, CursorExt, Row, RowExt /// per-read path. private int knownColumnCount = -1; + /// Counts the rows stepped onto, so a cached column type can tell which row it belongs to. + private int rowStamp; + + /// The storage class of each column of the current row, as `columnType` reported it before + /// any getter read the value; valid where the matching `typeStamps` entry is `rowStamp`. + /// SQLite's `sqlite3_column_type` is undefined once a getter has converted the value (reading + /// an INTEGER as text), so the type is taken before the first read and answered from here. + private int[] cachedTypes; + + /// The `rowStamp` each entry of `cachedTypes` was taken on. + private int[] typeStamps; + /// Repositions the underlying statement before its first row. /// /// #### Throws @@ -234,7 +246,7 @@ public boolean next() throws IOException { // position beyond the row count. Only an explicit rewind clears this. return false; } - if (stepForward()) { + if (advance()) { position++; onRow = true; return true; @@ -308,7 +320,7 @@ private boolean seek(int row, boolean newPass) throws IOException { rewindInternal(); } while (position < row) { - if (!stepForward()) { + if (!advance()) { markExhausted(); return false; } @@ -351,7 +363,7 @@ public int getCount() throws IOException { rewindInternal(); int total = 0; - while (stepForward()) { + while (advance()) { total++; } knownCount = total; @@ -444,6 +456,28 @@ protected boolean legacyWasNullBeforeAnyRead() { return false; } + /// Steps the engine forward and starts a new row for the column-type cache. + private boolean advance() throws IOException { + rowStamp++; + return stepForward(); + } + + /// The storage class of a column on the current row, taken from the engine the first time the + /// row is asked and answered from the cache afterwards, so a getter that converted the value + /// in between cannot change the answer. + private int cachedColumnType(int index) throws IOException { + if (cachedTypes == null || cachedTypes.length <= index) { + int size = Math.max(index + 1, knownColumnCount); + cachedTypes = new int[size]; + typeStamps = new int[size]; + } + if (typeStamps[index] != rowStamp) { + cachedTypes[index] = columnType(index); + typeStamps[index] = rowStamp; + } + return cachedTypes[index]; + } + /// Returns the storage class of a value as one of the `Database.COLUMN_*` constants, or /// `Database.COLUMN_UNKNOWN` (the default) where the engine cannot say. Only called on an /// open cursor positioned on a row, with a valid index. @@ -458,7 +492,7 @@ public int getColumnType(int index) throws IOException { throw new IOException("The cursor is not on a row"); } checkColumn(index); - return columnType(index); + return cachedColumnType(index); } /// Guards a value read and updates the null latch. @@ -469,6 +503,9 @@ private void beginRead(int index) throws IOException { } checkColumn(index); anythingRead = true; + // Taken before the value is read: the read may convert it, after which the engine cannot + // say what it was. + cachedColumnType(index); lastReadWasNull = isNullAt(index); } diff --git a/CodenameOne/src/com/codename1/ui/Font.java b/CodenameOne/src/com/codename1/ui/Font.java index 5a29a7f8580..40dd772ea1d 100644 --- a/CodenameOne/src/com/codename1/ui/Font.java +++ b/CodenameOne/src/com/codename1/ui/Font.java @@ -470,6 +470,14 @@ public static Font createTrueTypeFontFromData(String fontName, InputStream data) /// #### Returns /// /// the font, or null when the platform can't create fonts from data + /// + /// Fonts are cached by name and content together: calling this again with the same name and + /// the same bytes answers the font made the first time, while the same name with different + /// bytes makes a new font. + /// + /// On the JavaScript port the browser loads the font asynchronously, so the bytes are only + /// checked for a font header here; a font that has a valid header but is corrupt is not + /// reported as null and renders in the browser's fallback font. public static Font createTrueTypeFontFromData(String fontName, byte[] data) { if (fontName == null || data == null) { throw new IllegalArgumentException("Font name and data are required"); @@ -479,7 +487,9 @@ public static Font createTrueTypeFontFromData(String fontName, byte[] data) { } // A prefix keeps a font made from data apart from a file-based font that // happens to use the same name, in both this cache and the derive cache. - String id = "data:" + fontName; + // The length and a hash of the bytes are part of the key, so the same name given + // different bytes is a different font rather than the one cached first. + String id = "data:" + fontName + ":" + data.length + ":" + Integer.toHexString(hashBytes(data)); Font f = derivedFontCache.get(id); if (f != null) { return f; @@ -495,6 +505,15 @@ public static Font createTrueTypeFontFromData(String fontName, byte[] data) { return f; } + /// A cheap hash of a font's bytes for the data-font cache key. + private static int hashBytes(byte[] data) { + int h = 1; + for (byte b : data) { + h = 31 * h + b; + } + return h; + } + /// Creates a new font instance based on the platform specific string name of the /// font. This method isn't supported on some platforms. /// diff --git a/Ports/JavaScriptPort/src/main/java/com/codename1/impl/html5/HTML5Implementation.java b/Ports/JavaScriptPort/src/main/java/com/codename1/impl/html5/HTML5Implementation.java index feb8fa59501..7d8d4208164 100644 --- a/Ports/JavaScriptPort/src/main/java/com/codename1/impl/html5/HTML5Implementation.java +++ b/Ports/JavaScriptPort/src/main/java/com/codename1/impl/html5/HTML5Implementation.java @@ -11401,15 +11401,27 @@ public Object loadTrueTypeFontFromData(String fontName, byte[] data) { // FontFace rejects bytes that are not a font asynchronously, after this // has already answered; the contract is null, so check the SFNT tag // (0x00010000, 'true', 'OTTO', 'ttcf') up front as the other ports do. + // + // Because the browser loads the face asynchronously, a font whose header + // is valid but whose tables are corrupt is not detected here: it answers a + // font, and the browser draws its text in the fallback family. if (!isSfntHeader(data)) { return null; } - if (!loadedFonts.contains(fontName)) { - loadTrueTypeFont_(fontName, + // The browser family is keyed by content as well as name, so the same name + // given different bytes registers a second face instead of drawing with + // the first one loaded under that name. + int h = 1; + for (byte b : data) { + h = 31 * h + b; + } + String family = fontName + "_" + data.length + "_" + Integer.toHexString(h); + if (!loadedFonts.contains(family)) { + loadTrueTypeFont_(family, "data:font/ttf;base64," + com.codename1.util.Base64.encodeNoNewline(data), "truetype"); - loadedFonts.add(fontName); + loadedFonts.add(family); } - return createFallbackTrueTypeFont(fontName, null); + return createFallbackTrueTypeFont(family, null); } private static boolean isSfntHeader(byte[] data) { diff --git a/Ports/LinuxPort/nativeSources/cn1_linux_text.c b/Ports/LinuxPort/nativeSources/cn1_linux_text.c index d72884eff5f..6df441e27f0 100644 --- a/Ports/LinuxPort/nativeSources/cn1_linux_text.c +++ b/Ports/LinuxPort/nativeSources/cn1_linux_text.c @@ -483,37 +483,51 @@ JAVA_LONG com_codename1_impl_linux_LinuxNative_loadTrueTypeFontFromMemory___java } bytes = (unsigned char*) CN1_ARRAY_DATA(data); - fd = mkstemp(tmpl); - if (fd >= 0) { - if (write(fd, bytes, (size_t) len) == len) { - close(fd); - FcConfigAppFontAddFile(FcConfigGetCurrent(), (const FcChar8*) tmpl); - /* Reset Pango's default font map (and our cached context) so the next - * layout sees the freshly registered family. */ - pango_cairo_font_map_set_default(0); - cn1Pango = 0; - /* A family that fell back to a system face until now may resolve to - * the new one, so every cached answer is suspect. */ - if (cn1MetricsCache != NULL) { - g_hash_table_remove_all(cn1MetricsCache); - } - cn1WidthCacheClear(); - } else { - close(fd); - } + /* Parse the face first: bytes FreeType cannot open are not a font, and the + * caller is told so (0) instead of being handed a system family that Pango + * would substitute for a name nothing registered. The font's own family name + * is the reliable handle for Pango. */ + if (FT_Init_FreeType(&lib) != 0) { + return 0; } - - /* The font's own family name is the reliable handle for Pango. */ - if (FT_Init_FreeType(&lib) == 0) { - if (FT_New_Memory_Face(lib, bytes, len, 0, &face) == 0) { - if (face->family_name) { - strncpy(family, face->family_name, sizeof(family) - 1); - family[sizeof(family) - 1] = 0; - } - FT_Done_Face(face); - } + if (FT_New_Memory_Face(lib, bytes, len, 0, &face) != 0) { FT_Done_FreeType(lib); + return 0; + } + if (face->family_name) { + strncpy(family, face->family_name, sizeof(family) - 1); + family[sizeof(family) - 1] = 0; } + FT_Done_Face(face); + FT_Done_FreeType(lib); + + /* The spooled file is deliberately not unlinked: FontConfig records the path + * and Pango/cairo open the face from it lazily, at every later layout. */ + fd = mkstemp(tmpl); + if (fd < 0) { + return 0; + } + if (write(fd, bytes, (size_t) len) != len) { + close(fd); + unlink(tmpl); + return 0; + } + close(fd); + if (!FcConfigAppFontAddFile(FcConfigGetCurrent(), (const FcChar8*) tmpl)) { + unlink(tmpl); + return 0; + } + /* Reset Pango's default font map (and our cached context) so the next + * layout sees the freshly registered family. */ + pango_cairo_font_map_set_default(0); + cn1Pango = 0; + /* A family that fell back to a system face until now may resolve to + * the new one, so every cached answer is suspect. */ + if (cn1MetricsCache != NULL) { + g_hash_table_remove_all(cn1MetricsCache); + } + cn1WidthCacheClear(); + if (family[0] == 0) { /* Fall back to the requested name (minus the native: scheme). */ strncpy(family, strncmp(name, "native:", 7) == 0 ? name + 7 : name, sizeof(family) - 1); diff --git a/Ports/LinuxPort/src/com/codename1/impl/linux/LinuxImplementation.java b/Ports/LinuxPort/src/com/codename1/impl/linux/LinuxImplementation.java index 16a6828190a..ac610c5bdfa 100644 --- a/Ports/LinuxPort/src/com/codename1/impl/linux/LinuxImplementation.java +++ b/Ports/LinuxPort/src/com/codename1/impl/linux/LinuxImplementation.java @@ -2300,9 +2300,9 @@ private static boolean isSfntHeader(byte[] data) { @Override public Object loadTrueTypeFontFromData(String fontName, byte[] data) { - // The in-memory loader answers a system family for bytes it cannot - // register, while the contract for data that is not a font is null, - // so anything without an SFNT header is rejected before it. + // The contract for data that is not a font is null. Bytes without an + // SFNT header are rejected here; bytes with one that the native loader + // still cannot parse or register make it answer 0, which is null too. if (!isSfntHeader(data)) { return null; } diff --git a/Ports/WindowsPort/nativeSources/cn1_windows_text.c b/Ports/WindowsPort/nativeSources/cn1_windows_text.c index 128eaeeb2ee..d70d2ccc825 100644 --- a/Ports/WindowsPort/nativeSources/cn1_windows_text.c +++ b/Ports/WindowsPort/nativeSources/cn1_windows_text.c @@ -312,7 +312,9 @@ JAVA_LONG com_codename1_impl_windows_WindowsNative_loadTrueTypeFont___java_lang_ * PE-resource data, read out of the exe) via the DirectWrite in-memory loader -- * the single-exe path, no file on disk. Falls back to a system family resolved * from the font name (so "native:" scheme names still work) when there are no - * bytes or registration fails. + * bytes. Bytes that DirectWrite cannot register answer 0: a corrupt font is a + * failure the caller reports (Font.createTrueTypeFontFromData answers null, a + * bundled font falls back to the file loader), never a silent system family. */ JAVA_LONG com_codename1_impl_windows_WindowsNative_loadTrueTypeFontFromMemory___java_lang_String_byte_1ARRAY_R_long( CODENAME_ONE_THREAD_STATE, JAVA_OBJECT __cn1Arg1, JAVA_OBJECT __cn1Arg2) { @@ -325,9 +327,14 @@ JAVA_LONG com_codename1_impl_windows_WindowsNative_loadTrueTypeFontFromMemory___ void* data = CN1_ARRAY_DATA(__cn1Arg2); wchar_t registered[128]; registered[0] = L'\0'; - if (len > 0 && data != NULL - && cn1dwRegisterFontMemory(data, (unsigned int) len, registered, 128) && registered[0] != L'\0') { - font = cn1WinMakeFont(registered, 15.0f * dpi, 0, 0); + if (len > 0 && data != NULL) { + if (cn1dwRegisterFontMemory(data, (unsigned int) len, registered, 128) && registered[0] != L'\0') { + font = cn1WinMakeFont(registered, 15.0f * dpi, 0, 0); + } + if (font == NULL) { + free(name); + return 0; + } } } diff --git a/Ports/WindowsPort/src/com/codename1/impl/windows/WindowsImplementation.java b/Ports/WindowsPort/src/com/codename1/impl/windows/WindowsImplementation.java index 6f98bf0fbf8..73bc966139e 100644 --- a/Ports/WindowsPort/src/com/codename1/impl/windows/WindowsImplementation.java +++ b/Ports/WindowsPort/src/com/codename1/impl/windows/WindowsImplementation.java @@ -2098,9 +2098,9 @@ private static boolean isSfntHeader(byte[] data) { @Override public Object loadTrueTypeFontFromData(String fontName, byte[] data) { - // The in-memory loader answers a system family for bytes it cannot - // register, while the contract for data that is not a font is null, - // so anything without an SFNT header is rejected before it. + // The contract for data that is not a font is null. Bytes without an + // SFNT header are rejected here; bytes with one that the native loader + // still cannot parse or register make it answer 0, which is null too. if (!isSfntHeader(data)) { return null; } diff --git a/Ports/iOSPort/nativeSources/IOSNative.m b/Ports/iOSPort/nativeSources/IOSNative.m index e717c8e2833..7ebd8d63859 100644 --- a/Ports/iOSPort/nativeSources/IOSNative.m +++ b/Ports/iOSPort/nativeSources/IOSNative.m @@ -14635,11 +14635,45 @@ void com_codename1_impl_ios_IOSNative_registerBundledFont___java_lang_String(CN1 POOL_END(); } +/// True when Core Text already has a font registered under `psName` and it is the same +/// font as `candidate`, judged by the `head` table (which carries the whole-file checksum +/// adjustment and the modification date). Core Text answers a name it does not know with a +/// fallback font, so the registered font's own PostScript name is checked as well. +static BOOL cn1RegisteredFontMatches(CFStringRef psName, CGFontRef candidate) { + BOOL same = NO; + CTFontRef existing = CTFontCreateWithName(psName, 12, NULL); + if (existing == NULL) { + return NO; + } + CFStringRef existingName = CTFontCopyPostScriptName(existing); + if (existingName != NULL && CFStringCompare(existingName, psName, 0) == kCFCompareEqualTo) { + CGFontRef existingGraphics = CTFontCopyGraphicsFont(existing, NULL); + if (existingGraphics != NULL) { + CFDataRef a = CGFontCopyTableForTag(existingGraphics, 'head'); + CFDataRef b = CGFontCopyTableForTag(candidate, 'head'); + same = a != NULL && b != NULL && CFEqual(a, b); + if (a != NULL) { + CFRelease(a); + } + if (b != NULL) { + CFRelease(b); + } + CGFontRelease(existingGraphics); + } + } + if (existingName != NULL) { + CFRelease(existingName); + } + CFRelease(existing); + return same; +} + /// Registers a font held in memory (Font.createTrueTypeFontFromData) with /// Core Text and answers its PostScript name, which createTruetypeFont then looks /// up exactly as it does for a bundled font. Answers null when the bytes are not a -/// font. A font registered twice reports an error that is discarded: the name is -/// what the caller needs and it is valid either way. +/// font or registration fails. A font already registered under the same PostScript +/// name is accepted only when it is the same font; a different font holding that +/// name would otherwise be what the name lookup returns. JAVA_OBJECT com_codename1_impl_ios_IOSNative_registerFontData___byte_1ARRAY_R_java_lang_String(CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject, JAVA_OBJECT data) { if (data == JAVA_NULL) { return JAVA_NULL; @@ -14652,14 +14686,20 @@ JAVA_OBJECT com_codename1_impl_ios_IOSNative_registerFontData___byte_1ARRAY_R_ja CGDataProviderRef provider = cfData == NULL ? NULL : CGDataProviderCreateWithCFData(cfData); CGFontRef cgFont = provider == NULL ? NULL : CGFontCreateWithDataProvider(provider); if (cgFont != NULL) { - CFErrorRef error = NULL; - CTFontManagerRegisterGraphicsFont(cgFont, &error); - if (error != NULL) { - CFRelease(error); - } CFStringRef psName = CGFontCopyPostScriptName(cgFont); if (psName != NULL) { - result = fromNSString(CN1_THREAD_STATE_PASS_ARG (BRIDGE_CAST NSString *)psName); + CFErrorRef error = NULL; + BOOL usable = CTFontManagerRegisterGraphicsFont(cgFont, &error) ? YES : NO; + if (!usable && error != NULL + && CFErrorGetCode(error) == kCTFontManagerErrorAlreadyRegistered) { + usable = cn1RegisteredFontMatches(psName, cgFont); + } + if (error != NULL) { + CFRelease(error); + } + if (usable) { + result = fromNSString(CN1_THREAD_STATE_PASS_ARG (BRIDGE_CAST NSString *)psName); + } CFRelease(psName); } CGFontRelease(cgFont); diff --git a/maven/core-unittests/src/test/java/com/codename1/impl/AbstractDBCursorColumnTypeTest.java b/maven/core-unittests/src/test/java/com/codename1/impl/AbstractDBCursorColumnTypeTest.java new file mode 100644 index 00000000000..bfb3467934d --- /dev/null +++ b/maven/core-unittests/src/test/java/com/codename1/impl/AbstractDBCursorColumnTypeTest.java @@ -0,0 +1,181 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.impl; + +import com.codename1.db.Database; +import com.codename1.db.Row; +import org.junit.jupiter.api.Test; + +import java.io.IOException; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/// The column type a cursor reports is the storage class the value had before +/// any getter read it. SQLite's sqlite3_column_type is undefined once a getter +/// converted the value, so the type is cached per row. +class AbstractDBCursorColumnTypeTest { + + /// Two rows of one INTEGER column. Reading the value as text "converts" it, + /// after which the engine answers TEXT, as SQLite may. + private static final class ConvertingCursor extends AbstractDBCursor { + int row = -1; + boolean converted; + int typeQueries; + + protected void rewind() { + row = -1; + converted = false; + } + + protected boolean stepForward() { + row++; + converted = false; + return row < 2; + } + + protected void closeImpl() { + } + + protected int columnCount() { + return 1; + } + + protected String columnLabel(int columnIndex) { + return "c"; + } + + protected boolean isNullAt(int index) { + return false; + } + + protected int columnType(int index) { + typeQueries++; + return converted ? Database.COLUMN_TEXT : Database.COLUMN_INTEGER; + } + + protected String readString(int index) { + converted = true; + return String.valueOf(row); + } + + protected byte[] readBlob(int index) { + return new byte[0]; + } + + protected double readDouble(int index) { + return row; + } + + protected long readLong(int index) { + return row; + } + } + + /// An engine that cannot say: the base class default. + private static final class UntypedCursor extends AbstractDBCursor { + private int row = -1; + + protected void rewind() { + row = -1; + } + + protected boolean stepForward() { + row++; + return row < 1; + } + + protected void closeImpl() { + } + + protected int columnCount() { + return 1; + } + + protected String columnLabel(int columnIndex) { + return "c"; + } + + protected boolean isNullAt(int index) { + return false; + } + + protected String readString(int index) { + return "x"; + } + + protected byte[] readBlob(int index) { + return new byte[0]; + } + + protected double readDouble(int index) { + return 0; + } + + protected long readLong(int index) { + return 0; + } + } + + @Test + void getterBeforeGetColumnTypeStillReportsTheStoredClass() throws IOException { + ConvertingCursor cursor = new ConvertingCursor(); + cursor.next(); + Row row = cursor.getRow(); + assertEquals("0", row.getString(0)); + assertEquals(Database.COLUMN_INTEGER, Database.getColumnType(row, 0)); + assertEquals(Database.COLUMN_INTEGER, cursor.getColumnType(0)); + } + + @Test + void cacheIsPerRow() throws IOException { + ConvertingCursor cursor = new ConvertingCursor(); + cursor.next(); + assertEquals(Database.COLUMN_INTEGER, cursor.getColumnType(0)); + int queries = cursor.typeQueries; + cursor.getString(0); + assertEquals(Database.COLUMN_INTEGER, cursor.getColumnType(0)); + assertEquals(queries, cursor.typeQueries); + cursor.next(); + assertEquals(Database.COLUMN_INTEGER, cursor.getColumnType(0)); + assertEquals(queries + 1, cursor.typeQueries); + cursor.getString(0); + assertEquals(Database.COLUMN_INTEGER, cursor.getColumnType(0)); + } + + @Test + void cacheSurvivesRewind() throws IOException { + ConvertingCursor cursor = new ConvertingCursor(); + cursor.next(); + cursor.getString(0); + cursor.first(); + assertEquals(Database.COLUMN_INTEGER, cursor.getColumnType(0)); + } + + @Test + void engineThatCannotSayAnswersUnknown() throws IOException { + UntypedCursor cursor = new UntypedCursor(); + cursor.next(); + cursor.getString(0); + assertEquals(Database.COLUMN_UNKNOWN, Database.getColumnType(cursor.getRow(), 0)); + } +} diff --git a/maven/core-unittests/src/test/java/com/codename1/ui/FontTest.java b/maven/core-unittests/src/test/java/com/codename1/ui/FontTest.java index 201ce7127cc..c298cfa1cab 100644 --- a/maven/core-unittests/src/test/java/com/codename1/ui/FontTest.java +++ b/maven/core-unittests/src/test/java/com/codename1/ui/FontTest.java @@ -145,6 +145,20 @@ void testCreateTrueTypeFontFromStreamCachesByNameAndDerives() throws IOException assertNotSame(derived, fromFile.derive(24f, Font.STYLE_BOLD)); } + @EdtTest + void testSameNameWithDifferentBytesIsADifferentFont() { + int loads = implementation.getTrueTypeDataLoads(); + Font first = Font.createTrueTypeFontFromData("SharedName", new byte[]{1, 2, 3}); + Font other = Font.createTrueTypeFontFromData("SharedName", new byte[]{1, 2, 4}); + Font longer = Font.createTrueTypeFontFromData("SharedName", new byte[]{1, 2, 3, 0}); + assertNotNull(first); + assertNotNull(other); + assertNotSame(first, other); + assertNotSame(first, longer); + assertSame(first, Font.createTrueTypeFontFromData("SharedName", new byte[]{1, 2, 3})); + assertEquals(loads + 3, implementation.getTrueTypeDataLoads()); + } + @EdtTest void testCreateTrueTypeFontFromDataReturnsNullWhenUnsupportedOrInvalid() { assertNull(Font.createTrueTypeFontFromData("Empty", new byte[0])); diff --git a/maven/javase/src/test/java/com/codename1/impl/javase/JavaSEPortFontFromDataTest.java b/maven/javase/src/test/java/com/codename1/impl/javase/JavaSEPortFontFromDataTest.java index 90d69f3b492..fe149b247ce 100644 --- a/maven/javase/src/test/java/com/codename1/impl/javase/JavaSEPortFontFromDataTest.java +++ b/maven/javase/src/test/java/com/codename1/impl/javase/JavaSEPortFontFromDataTest.java @@ -74,6 +74,16 @@ void bytesThatAreNotAFontLoadNothing() { assertNull(port.loadTrueTypeFontFromData("Junk", new byte[]{1, 2, 3, 4, 5, 6, 7, 8})); } + @Test + void aValidHeaderOverCorruptTablesLoadsNothing() throws IOException { + byte[] header = new byte[]{0, 1, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 7, 7, 7, 7}; + assertNull(port.loadTrueTypeFontFromData("Corrupt", header)); + byte[] roboto = readResource("/com/codename1/impl/javase/Roboto-Regular.ttf"); + byte[] truncated = new byte[64]; + System.arraycopy(roboto, 0, truncated, 0, truncated.length); + assertNull(port.loadTrueTypeFontFromData("Truncated", truncated)); + } + private static byte[] readResource(String path) throws IOException { InputStream in = JavaSEPortFontFromDataTest.class.getResourceAsStream(path); assertNotNull(in, path); From 9a6c102fe0c9984d3082d427c152b1ffe57bd6d8 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 19:11:13 +0200 Subject: [PATCH 088/112] Backend: sslmode in TLS errors, timetz kind, case-insensitive header merge, late stack raise refused Co-Authored-By: Claude Opus 5.5 --- .../backend/DatabaseUrlSslModeTest.java | 64 +++++++++++++ .../HttpServerHeadersAndStackTest.java | 89 +++++++++++++++++++ .../backend/sql/SqlLimitsAccess.java | 33 +++++++ .../backend/sql/SqlLimitsAndKindsTest.java | 79 ++++++++++++++++ .../src/com/codename1/backend/Database.java | 14 ++- .../src/com/codename1/backend/HttpServer.java | 49 ++++++++-- .../src/com/codename1/backend/sql/MySql.java | 4 +- .../com/codename1/backend/sql/Postgres.java | 6 +- 8 files changed, 326 insertions(+), 12 deletions(-) create mode 100644 maven/backend/src/test/java/com/codename1/backend/DatabaseUrlSslModeTest.java create mode 100644 maven/backend/src/test/java/com/codename1/backend/HttpServerHeadersAndStackTest.java create mode 100644 maven/backend/src/test/java/com/codename1/backend/sql/SqlLimitsAccess.java create mode 100644 maven/backend/src/test/java/com/codename1/backend/sql/SqlLimitsAndKindsTest.java diff --git a/maven/backend/src/test/java/com/codename1/backend/DatabaseUrlSslModeTest.java b/maven/backend/src/test/java/com/codename1/backend/DatabaseUrlSslModeTest.java new file mode 100644 index 00000000000..b40e0b588eb --- /dev/null +++ b/maven/backend/src/test/java/com/codename1/backend/DatabaseUrlSslModeTest.java @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend; + +import com.codename1.backend.sql.SqlLimitsAccess; +import org.junit.jupiter.api.Test; + +import java.io.IOException; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/// Every accepted sslmode spelling and what it folds to. verify-full folding into +/// require is only safe while require checks the host name, so that is asserted +/// here beside the mapping. +class DatabaseUrlSslModeTest { + @Test + void canonicalSpellings() throws IOException { + assertEquals("disable", Database.Url.canonicalSslMode("disable")); + assertEquals("disable", Database.Url.canonicalSslMode("DISABLED")); + assertEquals("disable", Database.Url.canonicalSslMode("false")); + assertEquals("prefer", Database.Url.canonicalSslMode("prefer")); + assertEquals("prefer", Database.Url.canonicalSslMode("PREFERRED")); + assertEquals("require", Database.Url.canonicalSslMode("require")); + assertEquals("require", Database.Url.canonicalSslMode("REQUIRED")); + assertEquals("require", Database.Url.canonicalSslMode("true")); + assertEquals("verify-ca", Database.Url.canonicalSslMode("verify-ca")); + assertEquals("verify-ca", Database.Url.canonicalSslMode("VERIFY_CA")); + } + + @Test + void verifyFullFoldsIntoARequireThatChecksTheHost() throws IOException { + assertEquals("require", Database.Url.canonicalSslMode("verify-full")); + assertEquals("require", Database.Url.canonicalSslMode("VERIFY_IDENTITY")); + assertTrue(SqlLimitsAccess.verifiesHost("require")); + } + + @Test + void unknownModeIsRefused() { + assertThrows(IOException.class, () -> Database.Url.canonicalSslMode("allow")); + assertThrows(IOException.class, () -> Database.Url.canonicalSslMode(null)); + } +} diff --git a/maven/backend/src/test/java/com/codename1/backend/HttpServerHeadersAndStackTest.java b/maven/backend/src/test/java/com/codename1/backend/HttpServerHeadersAndStackTest.java new file mode 100644 index 00000000000..97b28e6cdef --- /dev/null +++ b/maven/backend/src/test/java/com/codename1/backend/HttpServerHeadersAndStackTest.java @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend; + +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/// Header merging for upgrade responses, and the virtual-thread stack raise that +/// must happen before any server starts. +class HttpServerHeadersAndStackTest { + @Test + void namesDifferingOnlyByCaseMergeIntoTheFirstSpelling() { + Map first = new LinkedHashMap(); + first.put("Set-Cookie", "a=1"); + Map second = new LinkedHashMap(); + second.put("set-cookie", "b=2"); + second.put("X-Other", "x"); + Map merged = HttpServer.mergeHeaders(first, second); + assertEquals(2, merged.size()); + assertEquals(Arrays.asList("a=1", "b=2"), merged.get("Set-Cookie")); + assertNull(merged.get("set-cookie")); + assertEquals("x", merged.get("X-Other")); + } + + @Test + void listAndScalarValuesMerge() { + Map first = new LinkedHashMap(); + first.put("Vary", Arrays.asList("Origin", "Accept")); + Map second = new LinkedHashMap(); + second.put("VARY", "Cookie"); + Map merged = HttpServer.mergeHeaders(first, second); + assertEquals(Arrays.asList("Origin", "Accept", "Cookie"), (List) merged.get("Vary")); + } + + @Test + void emptySideAnswersTheOther() { + Map only = new LinkedHashMap(); + only.put("A", "1"); + assertSame(only, HttpServer.mergeHeaders(null, only)); + assertSame(only, HttpServer.mergeHeaders(only, new LinkedHashMap())); + assertNull(HttpServer.mergeHeaders(null, null)); + } + + @Test + void raisingTheStackAfterAServerStartedIsRefused() throws Exception { + HttpServer server = HttpServer.start("127.0.0.1", 0, 16, 1, new HttpServer.Handler() { + public HttpServer.Response handle(HttpServer.Request request) { + return HttpServer.Response.text(200, "ok"); + } + }); + try { + assertThrows(IllegalStateException.class, + () -> HttpServer.requireVirtualThreadStack(Integer.MAX_VALUE)); + // Asking for no more than is already there stays a no-op. + HttpServer.requireVirtualThreadStack(1); + } finally { + server.stop(1000); + } + } +} diff --git a/maven/backend/src/test/java/com/codename1/backend/sql/SqlLimitsAccess.java b/maven/backend/src/test/java/com/codename1/backend/sql/SqlLimitsAccess.java new file mode 100644 index 00000000000..0e9ba7676ce --- /dev/null +++ b/maven/backend/src/test/java/com/codename1/backend/sql/SqlLimitsAccess.java @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.sql; + +/// Test-only bridge to the package-private `SqlLimits` from the parent package. +public final class SqlLimitsAccess { + private SqlLimitsAccess() { + } + + public static boolean verifiesHost(String sslMode) { + return SqlLimits.verifiesHost(sslMode); + } +} diff --git a/maven/backend/src/test/java/com/codename1/backend/sql/SqlLimitsAndKindsTest.java b/maven/backend/src/test/java/com/codename1/backend/sql/SqlLimitsAndKindsTest.java new file mode 100644 index 00000000000..cfac471fadd --- /dev/null +++ b/maven/backend/src/test/java/com/codename1/backend/sql/SqlLimitsAndKindsTest.java @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.sql; + +import org.junit.jupiter.api.Test; + +import java.io.IOException; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/// The sslmode predicates both drivers share, and the column kinds each driver +/// reports for the values that arrive as text. +class SqlLimitsAndKindsTest { + @Test + void demandsTlsOnlyForRequireAndVerifyCa() { + assertTrue(SqlLimits.demandsTls("require")); + assertTrue(SqlLimits.demandsTls("verify-ca")); + assertFalse(SqlLimits.demandsTls("prefer")); + assertFalse(SqlLimits.demandsTls("disable")); + } + + @Test + void verifiesHostForEveryModeButVerifyCa() { + assertTrue(SqlLimits.verifiesHost("require")); + assertTrue(SqlLimits.verifiesHost("prefer")); + assertFalse(SqlLimits.verifiesHost("verify-ca")); + } + + @Test + void verifyCaNeedsItsOwnRoot() throws IOException { + assertThrows(IOException.class, () -> SqlLimits.requireRootForVerifyCa("verify-ca", null)); + SqlLimits.requireRootForVerifyCa("verify-ca", "/ca.pem"); + SqlLimits.requireRootForVerifyCa("require", null); + } + + @Test + void postgresKindsNameTheZonedTypesConsistently() { + assertEquals("decimal", Postgres.kind(1700)); + assertEquals("date", Postgres.kind(1082)); + assertEquals("datetime", Postgres.kind(1114)); + assertEquals("datetimetz", Postgres.kind(1184)); + assertEquals("time", Postgres.kind(1083)); + assertEquals("timetz", Postgres.kind(1266)); + assertNull(Postgres.kind(25)); + } + + @Test + void mySqlKinds() { + assertEquals("decimal", MySql.kind(0xf6)); + assertEquals("date", MySql.kind(0x0a)); + assertEquals("datetime", MySql.kind(0x07)); + assertEquals("time", MySql.kind(0x0b)); + assertNull(MySql.kind(0x03)); + } +} diff --git a/vm/backend/src/com/codename1/backend/Database.java b/vm/backend/src/com/codename1/backend/Database.java index e3e4d402a55..83e2b1357d9 100644 --- a/vm/backend/src/com/codename1/backend/Database.java +++ b/vm/backend/src/com/codename1/backend/Database.java @@ -329,8 +329,9 @@ public List query(String sql, Object[] params) throws IOException { /// statement projects them, for the columns whose values come back as text /// that still has to be parsed: `"decimal"` (an exact DECIMAL or NUMERIC, /// kept as its digits), `"date"`, `"datetime"` (no zone), `"datetimetz"` - /// (PostgreSQL's timestamp with time zone, which carries its offset) and - /// `"time"`. Every other column is null, because its value is already the + /// (PostgreSQL's timestamp with time zone, which carries its offset), + /// `"time"` and `"timetz"` (PostgreSQL's time with time zone, which carries + /// its offset). Every other column is null, because its value is already the /// Long, Double, String or byte\[\] it is. /// /// Answers null for rows this class did not produce, and for SQLite, which @@ -1225,7 +1226,7 @@ public String toString() { /// pulled from java.net: URI is not on the server-safe surface, and the /// password in the userinfo has to be percent-decoded, which a hand-rolled /// split usually forgets. - private static final class Url { + static final class Url { String host = "localhost"; int port; String path = ""; @@ -1406,6 +1407,13 @@ private static String strip(String host) { /// the drivers implement: `disable`, `prefer`, `require` (verify-full) and /// `verify-ca`. Anything else is refused rather than read as prefer, which /// would accept plaintext. + /// + /// Folding verify-full (and MySQL's VERIFY_IDENTITY) into `require` is + /// safe only because `require` here is stricter than libpq's: it verifies + /// the certificate chain AND the host name (`SqlLimits.verifiesHost` + /// answers true for it). If `require` ever stops checking the host, this + /// mapping silently weakens verify-full, so `DatabaseUrlSslModeTest` + /// holds both facts together. static String canonicalSslMode(String value) throws IOException { String v = value == null ? "" : value.trim(); if ("disable".equals(v) || "DISABLED".equals(v) || "false".equals(v)) { diff --git a/vm/backend/src/com/codename1/backend/HttpServer.java b/vm/backend/src/com/codename1/backend/HttpServer.java index 443160ad84c..15496877c24 100644 --- a/vm/backend/src/com/codename1/backend/HttpServer.java +++ b/vm/backend/src/com/codename1/backend/HttpServer.java @@ -2162,12 +2162,30 @@ static boolean isSpoolable(String contentType, String contentEncoding) { /// #### Parameters /// /// - `bytes`: the C stack a connection's handler needs + /// + /// #### Throws + /// + /// - `IllegalStateException`: when a server has already started and `bytes` + /// would raise the size. Hosts and connections that exist keep the stack + /// they were created with, so a late raise would protect only some of the + /// handlers and overflow the others the same way. A call that asks for no + /// more than the current size is a no-op at any time. public static void requireVirtualThreadStack(int bytes) { if (bytes > vtStackBytes) { + if (ANY_SERVER_STARTED.get()) { + throw new IllegalStateException("requireVirtualThreadStack(" + bytes + + ") was called after a server started; call it before " + + "HttpServer.start so every connection gets the stack"); + } vtStackBytes = bytes; } } + /// Set by the first `start`, so a stack raise that can no longer reach every + /// connection is refused rather than silently partial. + private static final java.util.concurrent.atomic.AtomicBoolean ANY_SERVER_STARTED = + new java.util.concurrent.atomic.AtomicBoolean(false); + /// How often a virtual-thread host sweeps its deadlines, however busy it is. /// The same 250ms the idle poll waits, so a quiet host behaves exactly as /// before and a busy one stops being exempt. @@ -2503,6 +2521,7 @@ public static HttpServer start(String host, int port, int backlog, int workerCou static HttpServer start(String host, int port, int backlog, int workerCount, Handler handler, Tls tls, WebSocketRoutes webSockets, Tracer tracer) throws IOException { + ANY_SERVER_STARTED.set(true); // BEFORE THE BIND, because the two arms failed this differently and both // badly. Java SE's Executors.newFixedThreadPool throws for a non-positive // count -- but only after the listener and the reactor are open, so the @@ -5421,8 +5440,11 @@ && isHeaderSafe(text) && !isServerOwnedHeader(name) } /// Both header maps in one, `first`'s entries before `second`'s; a name in - /// both keeps every value. Null when neither has any. - private static Map mergeHeaders(Map first, Map second) { + /// both keeps every value. Header names are case-insensitive, so a name in + /// `second` that differs from one in `first` only by case is merged into + /// `first`'s spelling rather than written as a second header. Null when + /// neither has any. + static Map mergeHeaders(Map first, Map second) { if (first == null || first.isEmpty()) { return second; } @@ -5433,19 +5455,36 @@ private static Map mergeHeaders(Map first, Map second) { java.util.Iterator entries = second.entrySet().iterator(); while (entries.hasNext()) { Map.Entry entry = (Map.Entry) entries.next(); - Object had = merged.get(entry.getKey()); + Object key = sameHeaderKey(merged, entry.getKey()); + Object had = merged.get(key); if (had == null) { - merged.put(entry.getKey(), entry.getValue()); + merged.put(key, entry.getValue()); continue; } List values = new ArrayList(); addValues(values, had); addValues(values, entry.getValue()); - merged.put(entry.getKey(), values); + merged.put(key, values); } return merged; } + /// The key of `map` that names the same header as `name`, compared without + /// regard to case; `name` itself when there is none. + private static Object sameHeaderKey(Map map, Object name) { + if (map.containsKey(name) || !(name instanceof String)) { + return name; + } + java.util.Iterator keys = map.keySet().iterator(); + while (keys.hasNext()) { + Object key = keys.next(); + if (key instanceof String && ((String) key).equalsIgnoreCase((String) name)) { + return key; + } + } + return name; + } + private static void addValues(List into, Object value) { if (value instanceof List) { into.addAll((List) value); diff --git a/vm/backend/src/com/codename1/backend/sql/MySql.java b/vm/backend/src/com/codename1/backend/sql/MySql.java index 9b3537ea71c..cefc8b4c137 100644 --- a/vm/backend/src/com/codename1/backend/sql/MySql.java +++ b/vm/backend/src/com/codename1/backend/sql/MySql.java @@ -197,7 +197,7 @@ private void handshake(String host, String database, String user, String passwor if (useTls && (serverCapabilities & CLIENT_SSL) == 0) { if (SqlLimits.demandsTls(sslMode)) { throw new IOException("The MySQL server at " + host - + " does not offer TLS and sslmode=require"); + + " does not offer TLS and sslmode=" + sslMode); } useTls = false; } @@ -665,7 +665,7 @@ private long executePrepared(int statementId, Object[] params, Column[] columns, /// The portable column kind (`Database.columnTypes(List)`) of a binary /// protocol type: only the kinds whose values arrive as text a caller must /// parse. - private static String kind(int type) { + static String kind(int type) { switch (type) { case 0x00: // DECIMAL case 0xf6: // NEWDECIMAL diff --git a/vm/backend/src/com/codename1/backend/sql/Postgres.java b/vm/backend/src/com/codename1/backend/sql/Postgres.java index 98a003a3768..9e0a646affa 100644 --- a/vm/backend/src/com/codename1/backend/sql/Postgres.java +++ b/vm/backend/src/com/codename1/backend/sql/Postgres.java @@ -120,7 +120,7 @@ public static Postgres connect(String host, int port, String database, String us boolean required = SqlLimits.demandsTls(sslMode); if (!offered && required) { throw new IOException("The server at " + host - + " refused TLS and sslmode=require"); + + " refused TLS and sslmode=" + sslMode); } if (offered) { try { @@ -738,7 +738,7 @@ private static byte[] encodeParameter(Object value) { /// The portable column kind (`Database.columnTypes(List)`) of a type OID: /// only the kinds whose values arrive as text a caller must parse. - private static String kind(int typeOid) { + static String kind(int typeOid) { switch (typeOid) { case 1700: return "decimal"; @@ -750,6 +750,8 @@ private static String kind(int typeOid) { return "datetimetz"; case 1083: return "time"; + case 1266: + return "timetz"; default: return null; } From aabd6dd5d1e8179b7ace60610d954d1f39efafe4 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 19:14:31 +0200 Subject: [PATCH 089/112] Python runtime: lock-free locks and websocket handshake, fair GIL yield, red-team fixes PyLock and the generated websocket Socket no longer use monitors, which block a virtual thread's host; they spin on atomics through Gil.pauseWaiter. The handshake has a terminal timed-out state, so a late accept or message does nothing. Gil.yieldNow hands the lock to a waiter instead of retaking it. Instance __dict__ keeps non-str keys and syncs in place; dict iteration sees stores made through the object. hasCustomGetattribute is cached per type. PyServer logs any failing background task and closes a stream that raised; non-bytes chunks fail as in Starlette. FastAPI keeps a response's own background. _cn1dbapi skips comments, dollar quotes and escapes when binding and detecting queries, and parses time offsets and infinity. hmac rejects non-ASCII str. copyreg is bundled for stdlib modules that need it. Co-Authored-By: Claude Opus 5.5 --- .../com/codename1/python/runtime/Gil.java | 69 ++++++-- .../com/codename1/python/runtime/PyDict.java | 5 + .../codename1/python/runtime/PyInstance.java | 80 ++++++++- .../com/codename1/python/runtime/PyLock.java | 58 +++---- .../codename1/python/runtime/PyServer.java | 35 +++- .../com/codename1/python/runtime/PyType.java | 21 ++- .../transpiler/api/PythonTranspiler.java | 163 +++++++++++------- .../codename1/python/site/fastapi/routing.py | 4 +- .../python/site/sqlalchemy/_cn1dbapi.py | 150 +++++++++++++--- .../python/site/starlette/responses.py | 6 +- .../com/codename1/python/stdlib/hmac.py | 7 +- .../conformance/fastapi_background_.out | 6 + .../conformance/fastapi_background_.py | 95 ++++++++++ .../conformance/object_protocol_.out | 15 ++ .../resources/conformance/object_protocol_.py | 110 ++++++++++++ .../resources/conformance/threading_lock_.out | 12 ++ .../resources/conformance/threading_lock_.py | 113 ++++++++++++ 17 files changed, 803 insertions(+), 146 deletions(-) create mode 100644 maven/python-transpiler/src/test/resources/conformance/fastapi_background_.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/fastapi_background_.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/object_protocol_.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/object_protocol_.py create mode 100644 maven/python-transpiler/src/test/resources/conformance/threading_lock_.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/threading_lock_.py diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Gil.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Gil.java index 763ccf4e2c7..afab22bc90e 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Gil.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Gil.java @@ -23,6 +23,7 @@ */ package com.codename1.python.runtime; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; /// The global interpreter lock: Python code runs on one thread at a time. @@ -54,6 +55,15 @@ public final class Gil { /// through a compare-and-set on [#OWNER], which orders it. private static int depth; + /// How many threads are inside [#acquire(long)] waiting for the lock. + /// [#yieldNow()] reads it to hand the lock over rather than take it + /// straight back. + private static final AtomicInteger WAITERS = new AtomicInteger(); + + /// How many pauses [#yieldNow()] waits at most for a waiter to take the + /// lock before it competes for it again. + private static final int HANDOFF_PAUSES = 16; + private static Scheduler scheduler; /// What a server with virtual threads tells the lock about them. @@ -104,21 +114,36 @@ public static void enter() { } private static void acquire(long me) { - int spins = 0; - while (!OWNER.compareAndSet(0, me)) { - Scheduler s = scheduler; - if (s != null && s.pause()) { - continue; + if (OWNER.compareAndSet(0, me)) { + return; + } + WAITERS.incrementAndGet(); + try { + int attempt = 0; + while (!OWNER.compareAndSet(0, me)) { + pauseWaiter(attempt++); } - if (spins < SPINS) { - spins++; - Thread.yield(); - } else { - try { - Thread.sleep(1); - } catch (InterruptedException e) { - // keep waiting: the lock is the only way back into Python - } + } finally { + WAITERS.decrementAndGet(); + } + } + + /// One step aside for a thread waiting on something another thread + /// will change: the virtual thread yields its host, a platform thread + /// yields and, after [#SPINS] attempts, sleeps a millisecond. Never a + /// monitor wait, which would block a virtual thread's host. + public static void pauseWaiter(int attempt) { + Scheduler s = scheduler; + if (s != null && s.pause()) { + return; + } + if (attempt < SPINS) { + Thread.yield(); + } else { + try { + Thread.sleep(1); + } catch (InterruptedException e) { + // keep waiting: the caller re-checks its condition } } } @@ -157,11 +182,21 @@ public static void restore(int saved) { /// Lets another thread waiting for the lock run, as `time.sleep(0)` /// does. + /// + /// Fair to a waiter: a bare yield followed by a compare-and-set lets + /// the yielding thread win the lock straight back almost every time, + /// starving the others. So when a thread is waiting, the lock is left + /// free until somebody else has taken it (bounded by + /// [#HANDOFF_PAUSES], in case the waiter gave up), and when nobody is + /// waiting the lock is not released at all. public static void yieldNow() { + if (WAITERS.get() == 0) { + pauseWaiter(0); + return; + } int saved = release(); - Scheduler s = scheduler; - if (s == null || !s.pause()) { - Thread.yield(); + for (int i = 0; i < HANDOFF_PAUSES && OWNER.get() == 0 && WAITERS.get() > 0; i++) { + pauseWaiter(i); } restore(saved); } diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyDict.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyDict.java index 2a0aa0f8e71..4abe50175af 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyDict.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyDict.java @@ -450,6 +450,11 @@ Object iterate(final int kind) { @Override public Object next() { + // A view (an instance's __dict__) picks up what changed in the + // object behind it, so a store made through the object while + // iterating is caught as one made through the dict is. A plain + // dict's sync does nothing. + sync(); if (size != startSize) { throw Py.runtimeError("dictionary changed size during iteration"); } diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyInstance.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyInstance.java index 391cf59656c..63686c92f24 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyInstance.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyInstance.java @@ -52,6 +52,11 @@ public class PyInstance extends PyObject { Shape shape; Object[] slots; + /// Keys of `__dict__` that are not strings (`o.__dict__[1] = x`), which + /// CPython keeps in the dict but no attribute can name. Null for nearly + /// every instance. + PyDict nonStrKeys; + /// The shape, the empty one for null. final Shape shape() { return shape == null ? Shape.EMPTY : shape; @@ -663,20 +668,76 @@ void sync() { } syncing = true; try { - super.clear(); String[] names = self.attributeNames(); + PyDict other = self.nonStrKeys; + if (sameKeys(names, other)) { + // The same keys in the same order: the values are + // brought up to date in place, so an iteration in + // progress over this dict is not disturbed. + int n = 0; + for (int i = 0; i < used && n < names.length; i++) { + if (keys[i] != null) { + values[i] = self.own(names[n++]); + } + } + return; + } + super.clear(); for (String e : names) { super.setItem(e, self.own(e)); } + if (other != null) { + for (int i = 0; i < other.used; i++) { + if (other.keys[i] != null) { + super.setItem(other.keys[i], other.values[i]); + } + } + } } finally { syncing = false; } } + /// Whether the keys already are `names` and then `other`'s, in that + /// order. + private boolean sameKeys(String[] names, PyDict other) { + int extra = other == null ? 0 : other.size; + if (size != names.length + extra) { + return false; + } + int n = 0; + for (int i = 0; i < used; i++) { + if (keys[i] == null) { + continue; + } + if (n < names.length) { + if (!names[n].equals(keys[i])) { + return false; + } + } else if (!other.contains(keys[i])) { + return false; + } + n++; + } + return true; + } + + private static boolean isStr(Object key) { + return key instanceof String || Py.isinstance(key, Py.STR_TYPE); + } + @Override public void setItem(Object key, Object value) { + sync(); super.setItem(key, value); - self.store(Py.toStr(key), value); + if (isStr(key)) { + self.store(Py.toStr(key), value); + } else { + if (self.nonStrKeys == null) { + self.nonStrKeys = new PyDict(); + } + self.nonStrKeys.setItem(key, value); + } } @Override @@ -692,7 +753,16 @@ public void delItem(Object key) { if (super.remove(key) == null) { throw Py.keyError(key); } - self.delAttrDirect(Py.toStr(key)); + forget(key); + } + + /// Removes `key` from what the dict stands for. + private void forget(Object key) { + if (isStr(key)) { + self.delAttrDirect(Py.toStr(key)); + } else if (self.nonStrKeys != null) { + self.nonStrKeys.remove(key); + } } // pop() and popitem() remove through here, clear() through clear(): without these @@ -702,7 +772,7 @@ public Object remove(Object key) { sync(); Object v = super.remove(key); if (v != null) { - self.delAttrDirect(Py.toStr(key)); + forget(key); } return v; } @@ -719,7 +789,7 @@ public void clear() { } super.clear(); for (int i = 0; i < n; i++) { - self.delAttrDirect(Py.toStr(snapshot[i])); + forget(snapshot[i]); } } } diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyLock.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyLock.java index 8e42cb4d2c5..613a0e1bc75 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyLock.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyLock.java @@ -23,62 +23,50 @@ */ package com.codename1.python.runtime; +import java.util.concurrent.atomic.AtomicBoolean; + /// `_thread.LockType`: a lock that is not owned by the thread that took it, /// so any thread may release it. Waiting for it releases the [Gil]. +/// +/// A compare-and-set, never a monitor: `Object.wait` or a contended +/// `synchronized` on a backend virtual thread blocks the HOST thread, and +/// with tasks queued on every host the whole server wedges. A waiter steps +/// aside through [Gil#pauseWaiter(int)] between attempts instead. final class PyLock { - private boolean locked; + private final AtomicBoolean locked = new AtomicBoolean(); /// Takes the lock: at once, without waiting when `blocking` is false, /// or waiting at most `timeoutMillis` when that is not negative. boolean acquire(boolean blocking, long timeoutMillis) { - synchronized (this) { - if (!locked) { - locked = true; - return true; - } - if (!blocking) { - return false; - } + if (locked.compareAndSet(false, true)) { + return true; } + if (!blocking) { + return false; + } + long deadline = timeoutMillis < 0 ? 0 : System.currentTimeMillis() + timeoutMillis; int saved = Gil.release(); try { - synchronized (this) { - long deadline = timeoutMillis < 0 ? 0 : System.currentTimeMillis() + timeoutMillis; - while (locked) { - long wait = 0; - if (timeoutMillis >= 0) { - wait = deadline - System.currentTimeMillis(); - if (wait <= 0) { - return false; - } - } - try { - wait(wait); - } catch (InterruptedException e) { - // a spurious wake-up: the loop checks again - } + int attempt = 0; + while (!locked.compareAndSet(false, true)) { + if (timeoutMillis >= 0 && System.currentTimeMillis() >= deadline) { + return false; } - locked = true; - return true; + Gil.pauseWaiter(attempt++); } + return true; } finally { Gil.restore(saved); } } void release() { - synchronized (this) { - if (!locked) { - throw Py.error(Py.RUNTIME_ERROR, "release unlocked lock"); - } - locked = false; - notifyAll(); + if (!locked.compareAndSet(true, false)) { + throw Py.error(Py.RUNTIME_ERROR, "release unlocked lock"); } } boolean locked() { - synchronized (this) { - return locked; - } + return locked.get(); } } diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyServer.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyServer.java index c0da8315e32..9064d9f8fab 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyServer.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyServer.java @@ -228,13 +228,24 @@ public static void runAfter(Response response) { Gil.enter(); try { Py.obj(after).call(new Object[0], Py.NO_KW); - } catch (PyException e) { + } catch (RuntimeException e) { + // A background task that fails runs after the response is out: + // nobody is left to answer, so it is logged, as Starlette's + // server logs it. A Java exception escaping a task must not + // kill the server thread that ran it either. e.printStackTrace(); } finally { Gil.exit(); } } + /// One step aside for generated server code waiting on the Python + /// side (a websocket handshake): never a monitor wait, which would block + /// a virtual thread's host. See [Gil#pauseWaiter(int)]. + public static void pauseWaiter(int attempt) { + Gil.pauseWaiter(attempt); + } + /// Drops the work to run after a response that was never completed -- /// the client went away -- as Starlette skips its background tasks when /// a send fails. @@ -268,6 +279,7 @@ public static byte[] nextChunk(Response response) { return new byte[0]; } Gil.enter(); + boolean done = false; try { while (true) { Object item = Py.next(response.stream); @@ -277,15 +289,36 @@ public static byte[] nextChunk(Response response) { } byte[] chunk = PyBytes.like(item); if (chunk == null) { + if (!Py.isinstance(item, Py.STR_TYPE)) { + // Starlette encodes anything that is not bytes with + // chunk.encode(charset): an int fails right there. + throw Py.attributeError("'" + Py.typeOf(item).name + "' object has no attribute 'encode'"); + } chunk = Py.toStr(item).getBytes("UTF-8"); } if (chunk.length > 0) { + done = true; return chunk; } } } catch (java.io.UnsupportedEncodingException e) { throw Py.error(Py.OS_ERROR, "UTF-8 is unsupported"); // NOPMD PreserveStackTrace - surfaced as a Python OSError } finally { + if (!done && response.stream != null) { + // The generator raised: close it, so its finally blocks run + // and a later call answers the end rather than resuming a + // generator that already failed. The exception propagates. + Object it = response.stream; + response.stream = null; + try { + Object close = Py.obj(it).getAttrOrNull("close"); + if (close != null) { + Py.obj(close).call(new Object[0], Py.NO_KW); + } + } catch (RuntimeException e) { + e.printStackTrace(); + } + } Gil.exit(); } } diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyType.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyType.java index 235b9784391..735c1d327f2 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyType.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyType.java @@ -55,6 +55,11 @@ public interface Factory { /// nothing in practice and needs no list of subclasses. static int version; + /// [#hasCustomGetattribute()]'s answer, valid while [#version] equals + /// [#customGetattributeVersion]. + private boolean customGetattribute; + private int customGetattributeVersion = -1; + private static final Object MISSING = new PyObject() { @Override public PyType type() { @@ -637,9 +642,21 @@ public Object lookup(String attr) { /// Whether the class overrides `__getattribute__`, which takes every /// attribute read out of the inline caches' hands. + /// + /// Asked on every plain attribute read, so the answer is kept per class + /// against [#version]: a read costs two field compares until some class + /// changes. The version stays one global counter rather than one per + /// class with propagation to subclasses: writes to a class after it is + /// built are rare (and mostly at import), a type does not track its + /// subclasses, and a per-class version would need that list kept for + /// every class just to make the rare write cheaper. boolean hasCustomGetattribute() { - Object g = lookup("__getattribute__"); - return g != null && !(g instanceof PyBuiltin); + if (customGetattributeVersion != version) { + Object g = lookup("__getattribute__"); + customGetattribute = g != null && !(g instanceof PyBuiltin); + customGetattributeVersion = version; + } + return customGetattribute; } /// `issubclass(self, other)`. diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java index 6cf597b444e..5b10ee87c10 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java @@ -215,6 +215,16 @@ public static Map transpile(Map sources, String // com.codename1.backend.image (Pillow's exact pixels); on a // client PyImageNative over the UI Image, which a backend // build does not have (and leaves out of its classes). + // Chosen by the class path rather than by a build target on + // purpose: the transpiler runs once, in the common module's + // generate-sources, before any target is known, and the + // classes it writes serve every target built from them. What + // separates a backend from a client application is the + // artifact it compiles against -- codenameone-backend is on + // a backend's class path and on no client's -- so the probe + // answers exactly the question "can these classes reach + // com.codename1.backend.image", which is what the engine + // needs; a target name could disagree with the classes. serverImage = image && backend; image &= !backend; } catch (IOException e) { @@ -332,6 +342,12 @@ private static Set addStdlib(Map sources, Map addStdlib(Map sources, Map= end) {\n" + + " state.compareAndSet(DECIDING, TIMED_OUT);\n" + + " break;\n" + " }\n" - + " accepted = state == 1;\n" + + " PyServer.pauseWaiter(attempt++);\n" + " }\n" - + " if (!accepted) {\n" + + " if (state.get() != ACCEPTED) {\n" + " PyServer.webSocketClosed(c, 1006, \"\");\n" - + " synchronized (this) {\n" - + " return refusal != null ? refusal\n" - + " : HttpServer.refuseUpgrade(500, \"Internal Server Error\");\n" - + " }\n" + + " WebSocket r = refusal;\n" + + " return r != null ? r : HttpServer.refuseUpgrade(500, \"Internal Server Error\");\n" + " }\n" + " return this;\n" + " }\n" + "\n" - + " public synchronized String[] getSubprotocols() {\n" + + " public String[] getSubprotocols() {\n" + " return subprotocol == null ? null : new String[] {subprotocol};\n" + " }\n" + "\n" - + " public synchronized java.util.Map getHandshakeHeaders() {\n" + + " public java.util.Map getHandshakeHeaders() {\n" + " if (acceptHeaders == null || acceptHeaders.length == 0) {\n" + " return null;\n" + " }\n" @@ -756,12 +777,21 @@ private static String backendServer(int virtualThreadStack) { + " }\n" + "\n" + " public void onOpen(WebSocketSession s) {\n" - + " synchronized (this) {\n" - + " session = s;\n" - + " for (int i = 0; i < early.size(); i++) {\n" - + " deliver(early.get(i));\n" + + " session = s;\n" + + " // Drain until the queue is empty, then mark it OPEN in the same\n" + + " // compare-and-set: a message sent meanwhile is drained here, so\n" + + " // the order the endpoint sent them in is kept.\n" + + " while (true) {\n" + + " Object[] cur = (Object[]) queue.get();\n" + + " if (cur.length == 0) {\n" + + " if (queue.compareAndSet(cur, OPEN)) {\n" + + " return;\n" + + " }\n" + + " } else if (queue.compareAndSet(cur, NONE)) {\n" + + " for (int i = 0; i < cur.length; i++) {\n" + + " deliver(cur[i]);\n" + + " }\n" + " }\n" - + " early.clear();\n" + " }\n" + " }\n" + "\n" @@ -779,54 +809,67 @@ private static String backendServer(int virtualThreadStack) { + " PyServer.webSocketClosed(connection, code, reason);\n" + " }\n" + "\n" - + " public synchronized void accept(String subprotocol, String[] headers) {\n" + + " public void accept(String subprotocol, String[] headers) {\n" + + " if (state.get() != DECIDING) {\n" + + " return;\n" + + " }\n" + " this.subprotocol = subprotocol;\n" + " this.acceptHeaders = headers;\n" - + " state = 1;\n" - + " notifyAll();\n" + + " state.compareAndSet(DECIDING, ACCEPTED);\n" + " }\n" + "\n" - + " public synchronized void deny(int status, String[] headers, byte[] body) {\n" - + " if (state == 0) {\n" - + " String type = null;\n" - + " for (int i = 0; i + 1 < headers.length; i += 2) {\n" - + " if (\"content-type\".equalsIgnoreCase(headers[i])) {\n" - + " type = headers[i + 1];\n" - + " }\n" + + " public void deny(int status, String[] headers, byte[] body) {\n" + + " if (state.get() != DECIDING) {\n" + + " return;\n" + + " }\n" + + " String type = null;\n" + + " for (int i = 0; i + 1 < headers.length; i += 2) {\n" + + " if (\"content-type\".equalsIgnoreCase(headers[i])) {\n" + + " type = headers[i + 1];\n" + " }\n" - + " Map extra = pairs(headers);\n" - + " extra.remove(\"content-type\");\n" - + " extra.remove(\"content-length\");\n" - + " refusal = HttpServer.refuseUpgrade(new HttpServer.Response(status,\n" - + " type == null ? \"text/plain; charset=utf-8\" : type, body, extra));\n" - + " state = 2;\n" - + " notifyAll();\n" + " }\n" + + " Map extra = pairs(headers);\n" + + " extra.remove(\"content-type\");\n" + + " extra.remove(\"content-length\");\n" + + " refusal = HttpServer.refuseUpgrade(new HttpServer.Response(status,\n" + + " type == null ? \"text/plain; charset=utf-8\" : type, body, extra));\n" + + " state.compareAndSet(DECIDING, REFUSED);\n" + " }\n" + "\n" - + " public synchronized void text(String value) {\n" + + " public void text(String value) {\n" + " outgoing(value);\n" + " }\n" + "\n" - + " public synchronized void binary(byte[] value) {\n" + + " public void binary(byte[] value) {\n" + " outgoing(value);\n" + " }\n" + "\n" - + " public synchronized void close(int code, String reason) {\n" - + " if (state == 0) {\n" + + " public void close(int code, String reason) {\n" + + " if (state.compareAndSet(DECIDING, REFUSED)) {\n" + " // Closed before accepting: the upgrade is refused.\n" - + " state = 2;\n" - + " notifyAll();\n" + " return;\n" + " }\n" + " outgoing(new Object[] {Integer.valueOf(code), reason});\n" + " }\n" + "\n" + " private void outgoing(Object message) {\n" - + " if (session == null) {\n" - + " early.add(message);\n" - + " } else {\n" - + " deliver(message);\n" + + " if (state.get() != ACCEPTED) {\n" + + " // Refused or timed out: nobody will open a session, so\n" + + " // keeping the message would only grow the queue.\n" + + " return;\n" + + " }\n" + + " while (true) {\n" + + " Object[] cur = (Object[]) queue.get();\n" + + " if (cur == OPEN) {\n" + + " deliver(message);\n" + + " return;\n" + + " }\n" + + " Object[] next = new Object[cur.length + 1];\n" + + " System.arraycopy(cur, 0, next, 0, cur.length);\n" + + " next[cur.length] = message;\n" + + " if (queue.compareAndSet(cur, next)) {\n" + + " return;\n" + + " }\n" + " }\n" + " }\n" + "\n" diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/routing.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/routing.py index 92f39e519b3..16053fd63b1 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/routing.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/fastapi/routing.py @@ -551,7 +551,9 @@ async def run_route(route, req, default_class): if k == "content-length": continue resp.headers.append(k, v) - if ctx.background.tasks: + if ctx.background.tasks and resp.background is None: + # As FastAPI: a response that carries its own background task + # keeps it, and the injected BackgroundTasks are dropped. resp.background = ctx.background _finish_cleanups(ctx, None) except Exception as exc: diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/_cn1dbapi.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/_cn1dbapi.py index d716d1abecb..5e2f32a473c 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/_cn1dbapi.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/_cn1dbapi.py @@ -43,24 +43,65 @@ def _error(e): return OperationalError(msg) -def _named(sql, params): +def _skip(sql, i, backslash): + """The end of the quoted text, comment or dollar-quoted body starting at ``sql[i]``, or + -1 when ``sql[i]`` starts none. ``backslash`` is whether a backslash escapes the next + character inside quotes (MySQL and MariaDB always, PostgreSQL in an ``E''`` string).""" + n = len(sql) + ch = sql[i] + if ch in "'\"`": + esc = backslash or (ch == "'" and i > 0 and sql[i - 1] in "eE" + and (i < 2 or not (sql[i - 2].isalnum() or sql[i - 2] == "_"))) + j = i + 1 + while j < n: + c = sql[j] + if esc and c == "\\": + j += 2 + continue + if c == ch: + if j + 1 < n and sql[j + 1] == ch: + j += 2 + continue + return j + 1 + j += 1 + return n + if ch == "-" and sql.startswith("--", i): + j = sql.find("\n", i) + return n if j < 0 else j + 1 + if ch == "/" and sql.startswith("/*", i): + j = sql.find("*/", i + 2) + return n if j < 0 else j + 2 + if ch == "$" and (i == 0 or not (sql[i - 1].isalnum() or sql[i - 1] == "_")): + j = i + 1 + while j < n and (sql[j].isalnum() or sql[j] == "_"): + j += 1 + if j < n and sql[j] == "$" and not sql[i + 1:j][:1].isdigit(): + tag = sql[i:j + 1] + k = sql.find(tag, j + 1) + return n if k < 0 else k + len(tag) + return -1 + + +def _named(sql, params, backslash=False): """``:name`` binds (``text()``, the named style sqlite3 takes) as ``?`` and a tuple: the - backend's drivers bind by position. Quoted text, identifiers and PostgreSQL's ``::`` - casts are left alone.""" + backend's drivers bind by position. Quoted text and identifiers, comments, PostgreSQL's + dollar-quoted bodies and its ``::`` casts are left alone.""" out = [] values = [] i = 0 n = len(sql) while i < n: ch = sql[i] - if ch in "'\"`": - j = sql.find(ch, i + 1) - j = n - 1 if j < 0 else j - out.append(sql[i:j + 1]) - i = j + 1 + j = _skip(sql, i, backslash) + if j >= 0: + out.append(sql[i:j]) + i = j continue - if ch == ":" and i + 1 < n and (sql[i + 1].isalpha() or sql[i + 1] == "_") \ - and (i == 0 or sql[i - 1] != ":"): + if ch == ":" and i + 1 < n and sql[i + 1] == ":": + out.append("::") + i += 2 + continue + if ch == ":" and i + 1 < n and (sql[i + 1].isalpha() or sql[i + 1] == "_"): j = i + 1 while j < n and (sql[j].isalnum() or sql[j] == "_"): j += 1 @@ -76,11 +117,75 @@ def _named(sql, params): return "".join(out), tuple(values) -def _datetime(text): - # PostgreSQL writes a timestamptz offset as +HH when it has no minutes. - if len(text) > 3 and text[-3] in "+-" and text[-2:].isdigit(): +def _code(sql, backslash=False): + """``sql`` upper-cased with its quoted text, comments and dollar-quoted bodies replaced by + a space, so a keyword test sees only keywords.""" + out = [] + i = 0 + n = len(sql) + while i < n: + j = _skip(sql, i, backslash) + if j >= 0: + out.append(" ") + i = j + continue + out.append(sql[i]) + i += 1 + return "".join(out).upper() + + +_QUERY_HEADS = ("SELECT", "WITH", "VALUES", "SHOW", "EXPLAIN", "PRAGMA", "TABLE", "DESCRIBE", "DESC") + + +def _words(code): + w = [] + cur = [] + for c in code: + if c.isalnum() or c == "_": + cur.append(c) + elif cur: + w.append("".join(cur)) + cur = [] + if cur: + w.append("".join(cur)) + return w + + +def _is_query(sql, backslash=False): + """Whether ``sql`` answers rows. The backend's run() is told up front (its drivers read + a result set only when asked), so this decides by the statement: its first keyword after + comments and opening parentheses, or a RETURNING clause anywhere outside quotes.""" + words = _words(_code(sql, backslash)) + if not words: + return False + return words[0] in _QUERY_HEADS or "RETURNING" in words + + +def _offset(text): + # PostgreSQL writes an offset as +HH when it has no minutes; fromisoformat wants +HH:MM. + if len(text) > 3 and text[-3] in "+-" and text[-2:].isdigit() and ":" in text[:-3]: text += ":00" - return _dt.datetime.fromisoformat(text) + return text + + +def _datetime(text): + if text == "infinity": + return _dt.datetime.max + if text == "-infinity": + return _dt.datetime.min + return _dt.datetime.fromisoformat(_offset(text)) + + +def _date(text): + if text == "infinity": + return _dt.date.max + if text == "-infinity": + return _dt.date.min + return _dt.date.fromisoformat(text) + + +def _time(text): + return _dt.time.fromisoformat(_offset(text)) # A server database sends DECIMAL/NUMERIC and the date and time types as text; @@ -88,10 +193,11 @@ def _datetime(text): # psycopg2 and mysqlclient return. _PARSERS = { "decimal": _decimal.Decimal, - "date": _dt.date.fromisoformat, + "date": _date, "datetime": _datetime, "datetimetz": _datetime, - "time": _dt.time.fromisoformat, + "time": _time, + "timetz": _time, } @@ -117,14 +223,16 @@ def __init__(self, conn): self._at = 0 def execute(self, sql, params=()): - if isinstance(params, dict): - sql, params = _named(sql, params) c = self.connection + backslash = c.server_dialect in ("mysql", "mariadb") + if isinstance(params, dict): + sql, params = _named(sql, params, backslash) if c._handle is None: raise OperationalError("Cannot operate on a closed database.") - head = sql.lstrip()[:16].upper() - query = head.startswith("SELECT") or head.startswith("WITH") or " RETURNING " in sql.upper() - if not c._in_tx and not head.startswith("CREATE") and not head.startswith("DROP"): + words = _words(_code(sql, backslash)) + head = words[0] if words else "" + query = head in _QUERY_HEADS or "RETURNING" in words + if not c._in_tx and head != "CREATE" and head != "DROP": _cn1db.begin(c._handle) c._in_tx = True try: diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/responses.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/responses.py index f155a2adf41..31c25ec1c74 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/responses.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/starlette/responses.py @@ -140,10 +140,12 @@ def _sync_chunks(content, charset): if hasattr(content, "__aiter__"): content = _drive_async(content.__aiter__()) for c in content: - if isinstance(c, (bytes, bytearray)): + if isinstance(c, (bytes, bytearray, memoryview)): yield bytes(c) else: - yield str(c).encode(charset) + # As Starlette: anything that is not bytes is encoded, so a str + # works and an int fails with AttributeError here. + yield c.encode(charset) def _drive_async(it): diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/hmac.py b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/hmac.py index be79870d434..72a0fff2854 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/hmac.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/stdlib/hmac.py @@ -73,8 +73,11 @@ def digest(key, msg, digest): def compare_digest(a, b): if isinstance(a, str) and isinstance(b, str): - a = a.encode("ascii") - b = b.encode("ascii") + try: + a = a.encode("ascii") + b = b.encode("ascii") + except UnicodeEncodeError: + raise TypeError("comparing strings with non-ASCII characters is not supported") elif isinstance(a, str) or isinstance(b, str): raise TypeError("unsupported operand types(s) or combination of types") a = bytes(a) diff --git a/maven/python-transpiler/src/test/resources/conformance/fastapi_background_.out b/maven/python-transpiler/src/test/resources/conformance/fastapi_background_.out new file mode 100644 index 00000000000..b01bcddb273 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/fastapi_background_.out @@ -0,0 +1,6 @@ +200 {'ok': True} ['handler', 'injected'] +200 {'own': True} ['own'] +ValueError task failed +200 b'abc' +ValueError stream failed ['stream finally'] +AttributeError 'int' object has no attribute 'encode' diff --git a/maven/python-transpiler/src/test/resources/conformance/fastapi_background_.py b/maven/python-transpiler/src/test/resources/conformance/fastapi_background_.py new file mode 100644 index 00000000000..93bccc69f71 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/fastapi_background_.py @@ -0,0 +1,95 @@ +from fastapi import BackgroundTasks, FastAPI +from fastapi.responses import JSONResponse, StreamingResponse +from fastapi.testclient import TestClient +from starlette.background import BackgroundTask + +app = FastAPI() +log = [] + + +def note(msg): + log.append(msg) + + +def boom(): + raise ValueError("task failed") + + +@app.get("/injected") +def injected(tasks: BackgroundTasks): + tasks.add_task(note, "injected") + log.append("handler") + return {"ok": True} + + +@app.get("/own") +def own(tasks: BackgroundTasks): + tasks.add_task(note, "injected-dropped") + return JSONResponse({"own": True}, background=BackgroundTask(note, "own")) + + +@app.get("/raising") +def raising(tasks: BackgroundTasks): + tasks.add_task(boom) + return {"ok": 1} + + +def chunks(): + yield b"a" + yield "b" + yield b"" + yield b"c" + + +def failing(): + try: + yield b"start" + raise ValueError("stream failed") + finally: + log.append("stream finally") + + +def bad_type(): + yield b"x" + yield 5 + + +@app.get("/stream") +def stream(): + return StreamingResponse(chunks()) + + +@app.get("/failing") +def failing_route(): + return StreamingResponse(failing()) + + +@app.get("/badtype") +def bad_route(): + return StreamingResponse(bad_type()) + + +client = TestClient(app) +r = client.get("/injected") +print(r.status_code, r.json(), log) +del log[:] +r = client.get("/own") +print(r.status_code, r.json(), log) +del log[:] +try: + client.get("/raising") + print("no error") +except ValueError as e: + print("ValueError", e) +r = client.get("/stream") +print(r.status_code, r.content) +try: + client.get("/failing") + print("no error") +except ValueError as e: + print("ValueError", e, log) +try: + client.get("/badtype") + print("no error") +except AttributeError as e: + print("AttributeError", e) diff --git a/maven/python-transpiler/src/test/resources/conformance/object_protocol_.out b/maven/python-transpiler/src/test/resources/conformance/object_protocol_.out new file mode 100644 index 00000000000..e0217461eff --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/object_protocol_.out @@ -0,0 +1,15 @@ +1 fallback:hidden_x fallback:missing +fallback:hidden_y True +reads 2 +1 2 +10 +['1', 'name'] n +False one +False ['name'] +RuntimeError dictionary changed size during iteration +['a', 'b'] 5 +__newobj__ True +3 4 +True False +TypeError comparing strings with non-ASCII characters is not supported +TypeError mixed diff --git a/maven/python-transpiler/src/test/resources/conformance/object_protocol_.py b/maven/python-transpiler/src/test/resources/conformance/object_protocol_.py new file mode 100644 index 00000000000..af82d252996 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/object_protocol_.py @@ -0,0 +1,110 @@ +import copy +import hmac + + +# __getattribute__ raising AttributeError falls back to __getattr__. +class Fallback: + def __init__(self): + self.real = 1 + + def __getattribute__(self, name): + if name.startswith("hidden"): + raise AttributeError(name) + return object.__getattribute__(self, name) + + def __getattr__(self, name): + return "fallback:" + name + + +f = Fallback() +print(f.real, f.hidden_x, f.missing) +print(getattr(f, "hidden_y"), hasattr(f, "anything")) + + +class Counting: + def __init__(self): + object.__setattr__(self, "reads", 0) + + def __getattribute__(self, name): + if name != "reads": + object.__setattr__(self, "reads", object.__getattribute__(self, "reads") + 1) + return object.__getattribute__(self, name) + + +c = Counting() +c.a = 5 +_ = c.a +_ = c.a +print("reads", c.reads) + + +# Writing an unrelated class does not change what an instance reads. +class Plain: + x = 1 + + +p = Plain() +p.y = 2 + + +class Other: + pass + + +Other.z = 3 +print(p.x, p.y) +Plain.x = 10 +print(p.x) + +# __dict__ with keys that are not strings. +o = Plain() +o.__dict__[1] = "one" +o.__dict__["name"] = "n" +print(sorted(map(str, o.__dict__.keys())), o.name) +print(hasattr(o, "1"), o.__dict__.get(1)) +d = o.__dict__ +del d[1] +print(1 in o.__dict__, list(o.__dict__)) + +# Mutating the instance while iterating its __dict__ raises, as for a dict. +o.a = 1 +o.b = 2 +try: + for k in o.__dict__: + o.__dict__["new_" + str(k)] = 0 + print("no error") +except RuntimeError as e: + print("RuntimeError", e) +o2 = Plain() +o2.a = 1 +o2.b = 2 +seen = [] +for k in o2.__dict__: + seen.append(k) + o2.a = 5 +print(seen, o2.a) + + +# copyreg reached through a computed getattr. +class Point: + def __init__(self, x, y): + self.x = x + self.y = y + + +name = "__reduce" + "_ex__" +red = getattr(Point(1, 2), name)(2) +print(red[0].__name__, red[1][0] is Point) +q = copy.copy(Point(3, 4)) +print(q.x, q.y) + +# hmac.compare_digest on strings. +print(hmac.compare_digest("abc", "abc"), hmac.compare_digest("abc", "abd")) +try: + hmac.compare_digest("café", "café") +except TypeError as e: + print("TypeError", e) +try: + hmac.compare_digest("abc", b"abc") +except TypeError: + print("TypeError mixed") diff --git a/maven/python-transpiler/src/test/resources/conformance/threading_lock_.out b/maven/python-transpiler/src/test/resources/conformance/threading_lock_.out new file mode 100644 index 00000000000..4958cb32554 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/threading_lock_.out @@ -0,0 +1,12 @@ +counter 800 +timed out False True +non-blocking False +acquired after release True +RuntimeError release unlocked lock +rlock order ['owner', 'other'] +rlock release unowned: RuntimeError +spin [(0, 10000), (1, 19999), (2, 30000), (3, 40000), (4, 49996), (5, 59997)] +condition True [1] +event timeout False +event set True +semaphore timeout False diff --git a/maven/python-transpiler/src/test/resources/conformance/threading_lock_.py b/maven/python-transpiler/src/test/resources/conformance/threading_lock_.py new file mode 100644 index 00000000000..9ba0b4b3fe5 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/threading_lock_.py @@ -0,0 +1,113 @@ +import threading +import time + +# Contention on a plain Lock between real threads. +lock = threading.Lock() +counter = [0] + + +def worker(n): + for _ in range(n): + with lock: + v = counter[0] + time.sleep(0) + counter[0] = v + 1 + + +ts = [threading.Thread(target=worker, args=(200,)) for _ in range(4)] +for t in ts: + t.start() +for t in ts: + t.join() +print("counter", counter[0]) + +# A timeout on a lock another thread holds. +held = threading.Lock() +held.acquire() +t0 = time.monotonic() +got = held.acquire(timeout=0.05) +waited = time.monotonic() - t0 +print("timed out", got, waited >= 0.04) +print("non-blocking", held.acquire(False)) + +# Released by another thread while this one waits. +def later(): + time.sleep(0.02) + held.release() + + +threading.Thread(target=later).start() +print("acquired after release", held.acquire(timeout=5)) +held.release() +try: + held.release() +except RuntimeError as e: + print("RuntimeError", e) + +# RLock: reentrant for its owner, and other threads wait for the last release. +r = threading.RLock() +order = [] + + +def other(): + with r: + order.append("other") + + +with r: + with r: + t = threading.Thread(target=other) + t.start() + time.sleep(0.02) + order.append("owner") +t.join() +print("rlock order", order) +try: + threading.RLock().release() +except RuntimeError: + print("rlock release unowned: RuntimeError") + +# GIL stress: CPU-bound threads all finish, each sees its own results. +results = {} + + +def spin(k): + s = 0 + for i in range(20000): + s += i % (k + 2) + if i % 1000 == 0: + time.sleep(0) + results[k] = s + + +ts = [threading.Thread(target=spin, args=(k,)) for k in range(6)] +for t in ts: + t.start() +for t in ts: + t.join() +print("spin", sorted(results.items())) + +# Condition and Event wake a waiting thread. +cond = threading.Condition() +items = [] + + +def produce(): + time.sleep(0.01) + with cond: + items.append(1) + cond.notify() + + +threading.Thread(target=produce).start() +with cond: + ok = cond.wait_for(lambda: items, timeout=5) +print("condition", bool(ok), items) +ev = threading.Event() +print("event timeout", ev.wait(0.01)) +threading.Thread(target=ev.set).start() +print("event set", ev.wait(5)) +sem = threading.Semaphore(1) +sem.acquire() +print("semaphore timeout", sem.acquire(timeout=0.01)) +sem.release() From bd6d027f59f32b7d0773c23c9239e02de13791c1 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 19:22:57 +0200 Subject: [PATCH 090/112] Column-type capture before reads only for the iOS cursor; Windows reports whether fonts load from memory AbstractDBCursor captures a column's type before its first read only when the cursor opts in (capturesColumnTypeBeforeRead, true for the iOS SQLite cursor); others are asked lazily. SECursor derives the type from the value isNullAt already fetched instead of fetching it again. Windows isTrueTypeStreamSupported asks the new isFontMemoryLoadingSupported native whether IDWriteFactory5 exists. Co-Authored-By: Claude Opus 5.5 --- .../com/codename1/impl/AbstractDBCursor.java | 21 +++++++++-- .../com/codename1/impl/javase/SECursor.java | 35 +++++++++++++++++-- .../nativeSources/cn1_windows_dwrite.cpp | 13 +++++++ .../nativeSources/cn1_windows_dwrite.h | 4 +++ .../nativeSources/cn1_windows_text.c | 10 ++++++ .../impl/windows/WindowsImplementation.java | 4 ++- .../codename1/impl/windows/WindowsNative.java | 6 ++++ .../com/codename1/impl/ios/DatabaseImpl.java | 6 ++++ .../impl/AbstractDBCursorColumnTypeTest.java | 18 ++++++++++ 9 files changed, 111 insertions(+), 6 deletions(-) diff --git a/CodenameOne/src/com/codename1/impl/AbstractDBCursor.java b/CodenameOne/src/com/codename1/impl/AbstractDBCursor.java index 5cc318ab677..0a625113aab 100644 --- a/CodenameOne/src/com/codename1/impl/AbstractDBCursor.java +++ b/CodenameOne/src/com/codename1/impl/AbstractDBCursor.java @@ -478,6 +478,19 @@ private int cachedColumnType(int index) throws IOException { return cachedTypes[index]; } + /// Whether the engine's answer to `columnType` can change once a getter has read the value, + /// so the type must be taken before every first read of a column on a row. True for the + /// SQLite C API, where `sqlite3_column_type` is undefined after a conversion. Off by + /// default because the capture is one more engine call per read; a cursor that answers + /// `columnType` reliably at any time is asked lazily, only when `getColumnType` is. + /// + /// #### Returns + /// + /// true to capture each column's type before its first read on a row + protected boolean capturesColumnTypeBeforeRead() { + return false; + } + /// Returns the storage class of a value as one of the `Database.COLUMN_*` constants, or /// `Database.COLUMN_UNKNOWN` (the default) where the engine cannot say. Only called on an /// open cursor positioned on a row, with a valid index. @@ -503,9 +516,11 @@ private void beginRead(int index) throws IOException { } checkColumn(index); anythingRead = true; - // Taken before the value is read: the read may convert it, after which the engine cannot - // say what it was. - cachedColumnType(index); + if (capturesColumnTypeBeforeRead()) { + // Taken before the value is read: the read may convert it, after which the engine + // cannot say what it was. + cachedColumnType(index); + } lastReadWasNull = isNullAt(index); } diff --git a/Ports/JavaSE/src/com/codename1/impl/javase/SECursor.java b/Ports/JavaSE/src/com/codename1/impl/javase/SECursor.java index 9cda35da6b8..57328b665ef 100644 --- a/Ports/JavaSE/src/com/codename1/impl/javase/SECursor.java +++ b/Ports/JavaSE/src/com/codename1/impl/javase/SECursor.java @@ -96,6 +96,7 @@ protected void rewind() throws IOException { @Override protected boolean stepForward() throws IOException { + forgetFetched(); try { return resultSet.next(); } catch (SQLException ex) { @@ -158,11 +159,41 @@ protected String columnLabel(int columnIndex) throws IOException { } } + /// The value `isNullAt` fetched for the current row, by column, so `columnType` derives the + /// storage class from it rather than fetching the value a second time (a second copy of a + /// blob, and a second read some drivers refuse). Cleared on every step. + private Object[] fetched; + + /// Which entries of `fetched` hold a value of the current row. + private boolean[] fetchedSet; + + private Object fetch(int index) throws SQLException { + if (fetched != null && index < fetched.length && fetchedSet[index]) { + return fetched[index]; + } + Object v = resultSet.getObject(index + 1); + if (fetched == null || index >= fetched.length) { + int size = Math.max(index + 1, resultSet.getMetaData().getColumnCount()); + fetched = new Object[size]; + fetchedSet = new boolean[size]; + } + fetched[index] = v; + fetchedSet[index] = true; + return v; + } + + private void forgetFetched() { + if (fetched != null) { + java.util.Arrays.fill(fetched, null); + java.util.Arrays.fill(fetchedSet, false); + } + } + @Override protected int columnType(int index) throws IOException { Object v; try { - v = resultSet.getObject(index + 1); + v = fetch(index); } catch (SQLException ex) { throw new IOException(ex.getMessage(), ex); } @@ -188,7 +219,7 @@ protected int columnType(int index) throws IOException { @Override protected boolean isNullAt(int index) throws IOException { try { - return resultSet.getObject(index + 1) == null; + return fetch(index) == null; } catch (SQLException ex) { throw new IOException(ex.getMessage(), ex); } diff --git a/Ports/WindowsPort/nativeSources/cn1_windows_dwrite.cpp b/Ports/WindowsPort/nativeSources/cn1_windows_dwrite.cpp index dc8173b1c9a..a7c068a8e12 100644 --- a/Ports/WindowsPort/nativeSources/cn1_windows_dwrite.cpp +++ b/Ports/WindowsPort/nativeSources/cn1_windows_dwrite.cpp @@ -183,6 +183,19 @@ extern "C" int cn1dwRegisterFontFile(const wchar_t* path, wchar_t* outFamily, in * caller's buffer need not outlive this call. Returns 1 and the family name on * success. */ +extern "C" int cn1dwFontMemorySupported(void) { + IDWriteFactory* f = dwFactory(); + if (f == nullptr) { + return 0; + } + IDWriteFactory5* f5 = nullptr; + if (FAILED(f->QueryInterface(__uuidof(IDWriteFactory5), reinterpret_cast(&f5))) || f5 == nullptr) { + return 0; + } + f5->Release(); + return 1; +} + extern "C" int cn1dwRegisterFontMemory(const void* data, unsigned int len, wchar_t* outFamily, int outLen) { IDWriteFactory* f = dwFactory(); if (f == nullptr || data == nullptr || len == 0) { diff --git a/Ports/WindowsPort/nativeSources/cn1_windows_dwrite.h b/Ports/WindowsPort/nativeSources/cn1_windows_dwrite.h index f5ba626176c..17da75c4b9b 100644 --- a/Ports/WindowsPort/nativeSources/cn1_windows_dwrite.h +++ b/Ports/WindowsPort/nativeSources/cn1_windows_dwrite.h @@ -58,6 +58,10 @@ int cn1dwRegisterFontFile(const wchar_t* path, wchar_t* outFamily, int outLen); * The loader copies the data, so the buffer need not outlive the call. */ int cn1dwRegisterFontMemory(const void* data, unsigned int len, wchar_t* outFamily, int outLen); +/* 1 when cn1dwRegisterFontMemory can work at all: the DirectWrite factory offers + * IDWriteFactory5 (Windows 10 1703 and later), which owns the in-memory loader. */ +int cn1dwFontMemorySupported(void); + /* Vertical metrics, in DIPs, for a text format handle. */ float cn1dwFontHeight(void* format); float cn1dwFontAscent(void* format); diff --git a/Ports/WindowsPort/nativeSources/cn1_windows_text.c b/Ports/WindowsPort/nativeSources/cn1_windows_text.c index d70d2ccc825..a11edee8c9e 100644 --- a/Ports/WindowsPort/nativeSources/cn1_windows_text.c +++ b/Ports/WindowsPort/nativeSources/cn1_windows_text.c @@ -345,6 +345,16 @@ JAVA_LONG com_codename1_impl_windows_WindowsNative_loadTrueTypeFontFromMemory___ return (JAVA_LONG) (intptr_t) font; } +/* + * Whether fonts can be loaded from bytes on this system at all, which is what + * isTrueTypeStreamSupported reports: without IDWriteFactory5 every load from + * memory fails. + */ +JAVA_BOOLEAN com_codename1_impl_windows_WindowsNative_isFontMemoryLoadingSupported___R_boolean( + CODENAME_ONE_THREAD_STATE) { + return cn1dwFontMemorySupported() ? JAVA_TRUE : JAVA_FALSE; +} + /* * deriveTrueTypeFont produces a new peer at the requested pixel size and weight, * reusing the base font's family. weight carries the CN1 STYLE_* bits. diff --git a/Ports/WindowsPort/src/com/codename1/impl/windows/WindowsImplementation.java b/Ports/WindowsPort/src/com/codename1/impl/windows/WindowsImplementation.java index 73bc966139e..274c3415823 100644 --- a/Ports/WindowsPort/src/com/codename1/impl/windows/WindowsImplementation.java +++ b/Ports/WindowsPort/src/com/codename1/impl/windows/WindowsImplementation.java @@ -2081,7 +2081,9 @@ public boolean isNativeFontSchemeSupported() { @Override public boolean isTrueTypeStreamSupported() { - return true; + // Asked of the system rather than assumed: without DirectWrite's in-memory + // loader every createTrueTypeFontFromData would answer null. + return WindowsNative.isFontMemoryLoadingSupported(); } /// True when data starts with a TrueType (0x00010000 or 'true'), OpenType diff --git a/Ports/WindowsPort/src/com/codename1/impl/windows/WindowsNative.java b/Ports/WindowsPort/src/com/codename1/impl/windows/WindowsNative.java index 95bbfe819b5..178f10c6816 100644 --- a/Ports/WindowsPort/src/com/codename1/impl/windows/WindowsNative.java +++ b/Ports/WindowsPort/src/com/codename1/impl/windows/WindowsNative.java @@ -484,6 +484,12 @@ public static native void accessibilityNode(long id, long parentId, String role, */ public static native long loadTrueTypeFontFromMemory(String fontName, byte[] data); + /** + * Whether DirectWrite can load a font from memory on this system (it needs + * IDWriteFactory5, Windows 10 1703 and later). + */ + public static native boolean isFontMemoryLoadingSupported(); + /** Derives a new font peer at the given pixel size and CN1 style/weight bits. */ public static native long deriveTrueTypeFont(long font, float size, int weight); diff --git a/Ports/iOSPort/src/com/codename1/impl/ios/DatabaseImpl.java b/Ports/iOSPort/src/com/codename1/impl/ios/DatabaseImpl.java index 10b7d9f7b02..a5962954aff 100644 --- a/Ports/iOSPort/src/com/codename1/impl/ios/DatabaseImpl.java +++ b/Ports/iOSPort/src/com/codename1/impl/ios/DatabaseImpl.java @@ -607,6 +607,12 @@ protected boolean isNullAt(int index) throws IOException { return IOSImplementation.nativeInstance.sqlCursorNullValueAtColumn(stmt, index); } + @Override + protected boolean capturesColumnTypeBeforeRead() { + // sqlite3_column_type is undefined once a getter has converted the value. + return true; + } + @Override protected int columnType(int index) throws IOException { return IOSImplementation.nativeInstance.sqlCursorColumnType(stmt, index); diff --git a/maven/core-unittests/src/test/java/com/codename1/impl/AbstractDBCursorColumnTypeTest.java b/maven/core-unittests/src/test/java/com/codename1/impl/AbstractDBCursorColumnTypeTest.java index bfb3467934d..b344f5202c5 100644 --- a/maven/core-unittests/src/test/java/com/codename1/impl/AbstractDBCursorColumnTypeTest.java +++ b/maven/core-unittests/src/test/java/com/codename1/impl/AbstractDBCursorColumnTypeTest.java @@ -41,6 +41,11 @@ private static final class ConvertingCursor extends AbstractDBCursor { int row = -1; boolean converted; int typeQueries; + boolean eager = true; + + protected boolean capturesColumnTypeBeforeRead() { + return eager; + } protected void rewind() { row = -1; @@ -171,6 +176,19 @@ void cacheSurvivesRewind() throws IOException { assertEquals(Database.COLUMN_INTEGER, cursor.getColumnType(0)); } + @Test + void cursorThatDoesNotOptInIsAskedOnlyByGetColumnType() throws IOException { + ConvertingCursor cursor = new ConvertingCursor(); + cursor.eager = false; + cursor.next(); + cursor.getLong(0); + cursor.getLong(0); + assertEquals(0, cursor.typeQueries); + assertEquals(Database.COLUMN_INTEGER, cursor.getColumnType(0)); + cursor.getColumnType(0); + assertEquals(1, cursor.typeQueries); + } + @Test void engineThatCannotSayAnswersUnknown() throws IOException { UntypedCursor cursor = new UntypedCursor(); From 5fb4edd229f9c5201e4561963080733eb686d54c Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 19:22:57 +0200 Subject: [PATCH 091/112] Backend: stack-raise latch counts live bound servers; TLS errors name sslmode as written Co-Authored-By: Claude Opus 5.5 --- .../codename1/backend/DatabaseUrlAccess.java | 35 +++++++++++++++ .../HttpServerHeadersAndStackTest.java | 36 +++++++++++++++ .../backend/sql/SqlLimitsAccess.java | 4 ++ .../backend/sql/SqlLimitsAndKindsTest.java | 45 +++++++++++++++++++ .../src/com/codename1/backend/Database.java | 5 ++- .../src/com/codename1/backend/HttpServer.java | 20 ++++++--- .../src/com/codename1/backend/sql/MySql.java | 10 +++-- .../com/codename1/backend/sql/Postgres.java | 8 +++- .../com/codename1/backend/sql/SqlLimits.java | 33 +++++++++++--- 9 files changed, 175 insertions(+), 21 deletions(-) create mode 100644 maven/backend/src/test/java/com/codename1/backend/DatabaseUrlAccess.java diff --git a/maven/backend/src/test/java/com/codename1/backend/DatabaseUrlAccess.java b/maven/backend/src/test/java/com/codename1/backend/DatabaseUrlAccess.java new file mode 100644 index 00000000000..38fdaf74e70 --- /dev/null +++ b/maven/backend/src/test/java/com/codename1/backend/DatabaseUrlAccess.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend; + +import java.io.IOException; + +/// Test-only bridge to `Database.Url` for tests in other packages. +public final class DatabaseUrlAccess { + private DatabaseUrlAccess() { + } + + public static String canonicalSslMode(String value) throws IOException { + return Database.Url.canonicalSslMode(value); + } +} diff --git a/maven/backend/src/test/java/com/codename1/backend/HttpServerHeadersAndStackTest.java b/maven/backend/src/test/java/com/codename1/backend/HttpServerHeadersAndStackTest.java index 97b28e6cdef..c34cfd79c07 100644 --- a/maven/backend/src/test/java/com/codename1/backend/HttpServerHeadersAndStackTest.java +++ b/maven/backend/src/test/java/com/codename1/backend/HttpServerHeadersAndStackTest.java @@ -33,6 +33,7 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; /// Header merging for upgrade responses, and the virtual-thread stack raise that /// must happen before any server starts. @@ -85,5 +86,40 @@ public HttpServer.Response handle(HttpServer.Request request) { } finally { server.stop(1000); } + // The last live server stopped, so a raise is accepted again; the size + // is put back with the smallest legal raise (it never lowers). + HttpServer.requireVirtualThreadStack(256 * 1024); + } + + @Test + void aStartThatFailsToBindDoesNotLatch() throws Exception { + HttpServer holder = HttpServer.start("127.0.0.1", 0, 16, 1, new HttpServer.Handler() { + public HttpServer.Response handle(HttpServer.Request request) { + return HttpServer.Response.text(200, "ok"); + } + }); + int taken = holder.getPort(); + holder.stop(1000); + HttpServer blocker = HttpServer.start("127.0.0.1", 0, 16, 1, new HttpServer.Handler() { + public HttpServer.Response handle(HttpServer.Request request) { + return HttpServer.Response.text(200, "ok"); + } + }); + int busy = blocker.getPort(); + blocker.stop(1000); + java.net.ServerSocket occupied = new java.net.ServerSocket(busy, 1, + java.net.InetAddress.getByName("127.0.0.1")); + try { + assertThrows(java.io.IOException.class, () -> HttpServer.start("127.0.0.1", busy, 16, 1, + new HttpServer.Handler() { + public HttpServer.Response handle(HttpServer.Request request) { + return HttpServer.Response.text(200, "ok"); + } + })); + HttpServer.requireVirtualThreadStack(320 * 1024); + } finally { + occupied.close(); + } + assertTrue(taken > 0); } } diff --git a/maven/backend/src/test/java/com/codename1/backend/sql/SqlLimitsAccess.java b/maven/backend/src/test/java/com/codename1/backend/sql/SqlLimitsAccess.java index 0e9ba7676ce..a55b6a71639 100644 --- a/maven/backend/src/test/java/com/codename1/backend/sql/SqlLimitsAccess.java +++ b/maven/backend/src/test/java/com/codename1/backend/sql/SqlLimitsAccess.java @@ -27,6 +27,10 @@ public final class SqlLimitsAccess { private SqlLimitsAccess() { } + public static String urlCanonical(String sslMode) throws java.io.IOException { + return com.codename1.backend.DatabaseUrlAccess.canonicalSslMode(sslMode); + } + public static boolean verifiesHost(String sslMode) { return SqlLimits.verifiesHost(sslMode); } diff --git a/maven/backend/src/test/java/com/codename1/backend/sql/SqlLimitsAndKindsTest.java b/maven/backend/src/test/java/com/codename1/backend/sql/SqlLimitsAndKindsTest.java index cfac471fadd..f8d5e79d403 100644 --- a/maven/backend/src/test/java/com/codename1/backend/sql/SqlLimitsAndKindsTest.java +++ b/maven/backend/src/test/java/com/codename1/backend/sql/SqlLimitsAndKindsTest.java @@ -57,6 +57,51 @@ void verifyCaNeedsItsOwnRoot() throws IOException { SqlLimits.requireRootForVerifyCa("require", null); } + @Test + void driverSpellingsMatchTheUrlParser() throws IOException { + String[] spellings = {"disable", "DISABLED", "false", "prefer", "PREFERRED", "require", + "REQUIRED", "true", "verify-full", "VERIFY_IDENTITY", "verify-ca", "VERIFY_CA"}; + for (String s : spellings) { + assertEquals(SqlLimitsAccess.urlCanonical(s), SqlLimits.canonicalSslMode(s), s); + } + assertNull(SqlLimits.canonicalSslMode(null)); + assertThrows(IOException.class, () -> SqlLimits.canonicalSslMode("allow")); + } + + @Test + void tlsRefusalNamesTheModeAsWritten() throws Exception { + // A peer that answers the SSLRequest with 'N', as a server without TLS does. + final java.net.ServerSocket peer = new java.net.ServerSocket(0, 1, + java.net.InetAddress.getByName("127.0.0.1")); + Thread answer = new Thread(() -> { + try (java.net.Socket s = peer.accept()) { + byte[] request = new byte[8]; + int read = 0; + while (read < 8) { + int n = s.getInputStream().read(request, read, 8 - read); + if (n < 0) { + return; + } + read += n; + } + s.getOutputStream().write('N'); + s.getOutputStream().flush(); + s.getInputStream().read(); + } catch (IOException ignored) { + // the client hung up, which is the expected end + } + }); + answer.start(); + try { + IOException err = assertThrows(IOException.class, () -> Postgres.connect("127.0.0.1", + peer.getLocalPort(), "db", "u", "pw", "verify-full", null, 5000, 5000)); + assertTrue(err.getMessage().contains("sslmode=verify-full"), err.getMessage()); + } finally { + peer.close(); + answer.join(5000); + } + } + @Test void postgresKindsNameTheZonedTypesConsistently() { assertEquals("decimal", Postgres.kind(1700)); diff --git a/vm/backend/src/com/codename1/backend/Database.java b/vm/backend/src/com/codename1/backend/Database.java index 83e2b1357d9..0776e53e9ca 100644 --- a/vm/backend/src/com/codename1/backend/Database.java +++ b/vm/backend/src/com/codename1/backend/Database.java @@ -1336,7 +1336,10 @@ private static void applyQuery(Url out, String query) throws IOException { String value = decode(pair.substring(equals + 1)); if ("sslmode".equals(key) || "ssl".equals(key) || "ssl-mode".equals(key) || "sslMode".equals(key)) { - out.sslMode = canonicalSslMode(value); + // Validated here, before any socket, and handed on as + // written so the drivers' errors name the user's spelling. + canonicalSslMode(value); + out.sslMode = value.trim(); } else if ("user".equals(key)) { out.user = value; } else if ("password".equals(key)) { diff --git a/vm/backend/src/com/codename1/backend/HttpServer.java b/vm/backend/src/com/codename1/backend/HttpServer.java index 15496877c24..f8debec4ba4 100644 --- a/vm/backend/src/com/codename1/backend/HttpServer.java +++ b/vm/backend/src/com/codename1/backend/HttpServer.java @@ -2172,7 +2172,7 @@ static boolean isSpoolable(String contentType, String contentEncoding) { /// more than the current size is a no-op at any time. public static void requireVirtualThreadStack(int bytes) { if (bytes > vtStackBytes) { - if (ANY_SERVER_STARTED.get()) { + if (LIVE_SERVERS.get() > 0) { throw new IllegalStateException("requireVirtualThreadStack(" + bytes + ") was called after a server started; call it before " + "HttpServer.start so every connection gets the stack"); @@ -2181,10 +2181,12 @@ public static void requireVirtualThreadStack(int bytes) { } } - /// Set by the first `start`, so a stack raise that can no longer reach every - /// connection is refused rather than silently partial. - private static final java.util.concurrent.atomic.AtomicBoolean ANY_SERVER_STARTED = - new java.util.concurrent.atomic.AtomicBoolean(false); + /// Servers that bound their listener and have not been stopped. While any + /// is live a stack raise could no longer reach every connection, so it is + /// refused rather than silently partial; a start that failed to bind never + /// counts, and once the last server stops a raise is accepted again. + private static final java.util.concurrent.atomic.AtomicInteger LIVE_SERVERS = + new java.util.concurrent.atomic.AtomicInteger(); /// How often a virtual-thread host sweeps its deadlines, however busy it is. /// The same 250ms the idle poll waits, so a quiet host behaves exactly as @@ -2521,7 +2523,6 @@ public static HttpServer start(String host, int port, int backlog, int workerCou static HttpServer start(String host, int port, int backlog, int workerCount, Handler handler, Tls tls, WebSocketRoutes webSockets, Tracer tracer) throws IOException { - ANY_SERVER_STARTED.set(true); // BEFORE THE BIND, because the two arms failed this differently and both // badly. Java SE's Executors.newFixedThreadPool throws for a non-positive // count -- but only after the listener and the reactor are open, so the @@ -2596,6 +2597,9 @@ static HttpServer start(String host, int port, int backlog, int workerCount, final HttpServer server = new HttpServer(listener, reactor, useVirtualThreads ? null : Executors.newFixedThreadPool(workerCount), workerCount, handler, tls); + // Counted only now, with the listener bound: a start that threw above + // leaves the stack size free to raise. + LIVE_SERVERS.incrementAndGet(); if (tracer != null) { server.serverTracer = tracer; } @@ -2834,7 +2838,9 @@ public void stop(int drainMillis) { // wedge itself against a drain that is waiting for that same handler: the // bound is the whole teardown the winner is allowed, and after it this // returns whether or not the winner finished. - if (!stopClaimed.compareAndSet(false, true)) { + if (stopClaimed.compareAndSet(false, true)) { + LIVE_SERVERS.decrementAndGet(); + } else { long limit = System.currentTimeMillis() + drainMillis + SESSION_RELEASE_GRACE_MILLIS + POLL_LOOP_JOIN_MILLIS; synchronized (stopped) { diff --git a/vm/backend/src/com/codename1/backend/sql/MySql.java b/vm/backend/src/com/codename1/backend/sql/MySql.java index cefc8b4c137..20447b64c2f 100644 --- a/vm/backend/src/com/codename1/backend/sql/MySql.java +++ b/vm/backend/src/com/codename1/backend/sql/MySql.java @@ -115,7 +115,9 @@ public static MySql connect(String host, int port, String database, String user, String password, String sslMode, String caFile, int timeoutMillis, int socketTimeoutMillis) throws IOException { // BEFORE THE SOCKET, for the reason Postgres gives one engine over. - SqlLimits.requireSslMode(sslMode); + // Kept as written for the errors, as Postgres does. + final String sslModeAsWritten = sslMode; + sslMode = SqlLimits.canonicalSslMode(sslMode); SqlLimits.requireRootForVerifyCa(sslMode, caFile); Tcp connection = Tcp.connect(host, port <= 0 ? 3306 : port, timeoutMillis); // AFTER THE CONNECT, and before anything is said on it: the deadline @@ -127,7 +129,7 @@ public static MySql connect(String host, int port, String database, String user, } try { MySql session = new MySql(new Wire(connection)); - session.handshake(host, database, user, password, sslMode, caFile); + session.handshake(host, database, user, password, sslMode, sslModeAsWritten, caFile); // UTC, whatever the server's default. A TIMESTAMP column is converted // to and from the session zone on every read and write, and a // @Temporal value is written as UTC text, so a session left in the @@ -142,7 +144,7 @@ public static MySql connect(String host, int port, String database, String user, } private void handshake(String host, String database, String user, String password, - String sslMode, String caFile) throws IOException { + String sslMode, String sslModeAsWritten, String caFile) throws IOException { Packet greeting = readPacket(); Reader reader = new Reader(greeting.body); int protocol = reader.u8(); @@ -197,7 +199,7 @@ private void handshake(String host, String database, String user, String passwor if (useTls && (serverCapabilities & CLIENT_SSL) == 0) { if (SqlLimits.demandsTls(sslMode)) { throw new IOException("The MySQL server at " + host - + " does not offer TLS and sslmode=" + sslMode); + + " does not offer TLS and sslmode=" + sslModeAsWritten); } useTls = false; } diff --git a/vm/backend/src/com/codename1/backend/sql/Postgres.java b/vm/backend/src/com/codename1/backend/sql/Postgres.java index 9e0a646affa..0fb1a37cfb9 100644 --- a/vm/backend/src/com/codename1/backend/sql/Postgres.java +++ b/vm/backend/src/com/codename1/backend/sql/Postgres.java @@ -101,7 +101,11 @@ public static Postgres connect(String host, int port, String database, String us int socketTimeoutMillis) throws IOException { // BEFORE THE SOCKET: a mode this code does not implement is a typo, and // reading it as prefer is how "require" spelled wrong accepts plaintext. - SqlLimits.requireSslMode(sslMode); + // The mode as written is kept for the errors: "sslmode=require" in a + // message about a URL that said verify-full would send the reader + // looking for a setting they never wrote. + final String sslModeAsWritten = sslMode; + sslMode = SqlLimits.canonicalSslMode(sslMode); SqlLimits.requireRootForVerifyCa(sslMode, caFile); Tcp connection = Tcp.connect(host, port <= 0 ? 5432 : port, timeoutMillis); // AFTER THE CONNECT, and before anything is said on it: the deadline @@ -120,7 +124,7 @@ public static Postgres connect(String host, int port, String database, String us boolean required = SqlLimits.demandsTls(sslMode); if (!offered && required) { throw new IOException("The server at " + host - + " refused TLS and sslmode=" + sslMode); + + " refused TLS and sslmode=" + sslModeAsWritten); } if (offered) { try { diff --git a/vm/backend/src/com/codename1/backend/sql/SqlLimits.java b/vm/backend/src/com/codename1/backend/sql/SqlLimits.java index f690043f382..71ae244f6a9 100644 --- a/vm/backend/src/com/codename1/backend/sql/SqlLimits.java +++ b/vm/backend/src/com/codename1/backend/sql/SqlLimits.java @@ -79,16 +79,35 @@ static long maxMessageBytes() { /// Null is the default and stays it: both entry points document the /// absent mode as "prefer", and a URL without the parameter is the ordinary /// case rather than a mistake. - static void requireSslMode(String sslMode) throws IOException { + /// + /// Answers the canonical mode the drivers branch on: every spelling + /// `Database` accepts in a URL (verify-full, MySQL's REQUIRED and so on) is + /// accepted here too, so a driver can be handed the mode exactly as the user + /// wrote it and name it that way in its errors. verify-full folds into + /// require, which is only safe because require verifies the chain and the + /// host (`verifiesHost`). The table must match `Database.Url.canonicalSslMode`; + /// `SqlLimitsAndKindsTest` holds the two together. + static String canonicalSslMode(String sslMode) throws IOException { if (sslMode == null) { - return; + return null; + } + String v = sslMode.trim(); + if ("disable".equals(v) || "DISABLED".equals(v) || "false".equals(v)) { + return "disable"; + } + if ("prefer".equals(v) || "PREFERRED".equals(v)) { + return "prefer"; + } + if ("require".equals(v) || "REQUIRED".equals(v) || "true".equals(v) + || "verify-full".equals(v) || "VERIFY_IDENTITY".equals(v)) { + return "require"; } - if (!"require".equals(sslMode) && !"prefer".equals(sslMode) - && !"disable".equals(sslMode) && !"verify-ca".equals(sslMode)) { - throw new IOException("sslmode=" + sslMode + " is not one this runtime " - + "implements: use require, verify-ca, prefer or disable. It is " - + "refused rather than read as prefer, which would accept plaintext"); + if ("verify-ca".equals(v) || "VERIFY_CA".equals(v)) { + return "verify-ca"; } + throw new IOException("sslmode=" + sslMode + " is not one this runtime " + + "implements: use require, verify-full, verify-ca, prefer or disable. It is " + + "refused rather than read as prefer, which would accept plaintext"); } /// Whether `sslMode` refuses a connection that cannot be encrypted. From fa4eb626b1305defaabdbe80a7c95fdc52a069f1 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 19:25:51 +0200 Subject: [PATCH 092/112] HttpServer: an abandoned start gives back its live-server count A start that failed after the listener bound (websocket routes, VT hosts) never decremented the count, so every later stack raise was refused for the life of the process. Co-Authored-By: Claude Opus 5.5 --- .../HttpServerHeadersAndStackTest.java | 25 +++++++++++++------ .../src/com/codename1/backend/HttpServer.java | 5 ++++ 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/maven/backend/src/test/java/com/codename1/backend/HttpServerHeadersAndStackTest.java b/maven/backend/src/test/java/com/codename1/backend/HttpServerHeadersAndStackTest.java index c34cfd79c07..51ce08fabe5 100644 --- a/maven/backend/src/test/java/com/codename1/backend/HttpServerHeadersAndStackTest.java +++ b/maven/backend/src/test/java/com/codename1/backend/HttpServerHeadersAndStackTest.java @@ -93,13 +93,8 @@ public HttpServer.Response handle(HttpServer.Request request) { @Test void aStartThatFailsToBindDoesNotLatch() throws Exception { - HttpServer holder = HttpServer.start("127.0.0.1", 0, 16, 1, new HttpServer.Handler() { - public HttpServer.Response handle(HttpServer.Request request) { - return HttpServer.Response.text(200, "ok"); - } - }); - int taken = holder.getPort(); - holder.stop(1000); + // The raise below is permanent for this JVM; the module runs each test + // class in its own fork (reuseForks=false), so no other class sees it. HttpServer blocker = HttpServer.start("127.0.0.1", 0, 16, 1, new HttpServer.Handler() { public HttpServer.Response handle(HttpServer.Request request) { return HttpServer.Response.text(200, "ok"); @@ -120,6 +115,20 @@ public HttpServer.Response handle(HttpServer.Request request) { } finally { occupied.close(); } - assertTrue(taken > 0); + } + + @Test + void aStartAbandonedAfterBindDoesNotLatch() { + assertThrows(java.io.IOException.class, () -> HttpServer.start("127.0.0.1", 0, 16, 1, + new HttpServer.Handler() { + public HttpServer.Response handle(HttpServer.Request request) { + return HttpServer.Response.text(200, "ok"); + } + }, null, new HttpServer.WebSocketRoutes() { + public void register(HttpServer.WebSocketRegistry registry) { + throw new IllegalStateException("no routes"); + } + })); + HttpServer.requireVirtualThreadStack(384 * 1024); } } diff --git a/vm/backend/src/com/codename1/backend/HttpServer.java b/vm/backend/src/com/codename1/backend/HttpServer.java index f8debec4ba4..89721b7c8bb 100644 --- a/vm/backend/src/com/codename1/backend/HttpServer.java +++ b/vm/backend/src/com/codename1/backend/HttpServer.java @@ -2711,6 +2711,11 @@ public void run() { /// goes out of its way to avoid -- a failed start is not a reason to take the /// process down with it. private static void abandonStart(ServerSocket listener, HttpServer server) { + // The caller never gets this server, so nothing will stop() it: give + // back the live count start() took, or every later stack raise is refused. + if (server.stopClaimed.compareAndSet(false, true)) { + LIVE_SERVERS.decrementAndGet(); + } server.running = false; if (server.pollLoopsEnded(POLL_LOOP_JOIN_MILLIS)) { server.closePollers(); From 975383505f993765c7d496db0b6b57296e462f4c Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 19:26:29 +0200 Subject: [PATCH 093/112] Python runtime: round-2 red-team fixes and driver/websocket tests _cn1dbapi classifies WITH by the statement after the CTE list, follows MySQL comment rules (-- needs whitespace, #), maps 24:00:00 as psycopg2 does. Gil.yieldNow waits up to 2ms for a waiter and waiters poll while a handoff is pending. Instance __dict__ sync re-reads values by index while the shape and set fields are unchanged. The websocket Socket raises on accept or send after a refusal or timeout. Cn1DbApiTest runs the driver's pure functions under CPython; WebSocketSocketTest compiles the generated Socket against a fake server. Co-Authored-By: Claude Opus 5.5 --- .../com/codename1/python/runtime/Gil.java | 24 +- .../codename1/python/runtime/PyInstance.java | 62 +++ .../codename1/python/runtime/PyServer.java | 6 + .../transpiler/api/PythonTranspiler.java | 429 +++++++++--------- .../python/site/sqlalchemy/_cn1dbapi.py | 59 ++- .../python/transpiler/Cn1DbApiTest.java | 64 +++ .../transpiler/api/WebSocketSocketTest.java | 143 ++++++ .../src/test/resources/cn1dbapi/_cn1db.py | 1 + .../test/resources/cn1dbapi/test_cn1dbapi.py | 65 +++ .../conformance/object_protocol_.out | 2 + .../resources/conformance/object_protocol_.py | 15 + .../resources/conformance/threading_lock_.out | 1 + .../resources/conformance/threading_lock_.py | 22 + 13 files changed, 672 insertions(+), 221 deletions(-) create mode 100644 maven/python-transpiler/src/test/java/com/codename1/python/transpiler/Cn1DbApiTest.java create mode 100644 maven/python-transpiler/src/test/java/com/codename1/python/transpiler/api/WebSocketSocketTest.java create mode 100644 maven/python-transpiler/src/test/resources/cn1dbapi/_cn1db.py create mode 100644 maven/python-transpiler/src/test/resources/cn1dbapi/test_cn1dbapi.py diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Gil.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Gil.java index afab22bc90e..cea820e1ff2 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Gil.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Gil.java @@ -62,7 +62,13 @@ public final class Gil { /// How many pauses [#yieldNow()] waits at most for a waiter to take the /// lock before it competes for it again. - private static final int HANDOFF_PAUSES = 16; + private static final long HANDOFF_NANOS = 2000000L; + + /// Set while [#yieldNow()] leaves the lock free for a waiter: waiters + /// then poll instead of sleeping, so a platform thread asleep in + /// `Thread.sleep(1)` is not outwaited by the yielder. + private static final java.util.concurrent.atomic.AtomicBoolean HANDOFF = + new java.util.concurrent.atomic.AtomicBoolean(); private static Scheduler scheduler; @@ -121,7 +127,7 @@ private static void acquire(long me) { try { int attempt = 0; while (!OWNER.compareAndSet(0, me)) { - pauseWaiter(attempt++); + pauseWaiter(HANDOFF.get() ? 0 : attempt++); } } finally { WAITERS.decrementAndGet(); @@ -186,18 +192,22 @@ public static void restore(int saved) { /// Fair to a waiter: a bare yield followed by a compare-and-set lets /// the yielding thread win the lock straight back almost every time, /// starving the others. So when a thread is waiting, the lock is left - /// free until somebody else has taken it (bounded by - /// [#HANDOFF_PAUSES], in case the waiter gave up), and when nobody is - /// waiting the lock is not released at all. + /// free until somebody else has taken it -- for at most + /// [#HANDOFF_NANOS], longer than a platform waiter's one-millisecond + /// sleep, and waiters stop sleeping while [#HANDOFF] is set -- and + /// when nobody is waiting the lock is not released at all. public static void yieldNow() { if (WAITERS.get() == 0) { pauseWaiter(0); return; } int saved = release(); - for (int i = 0; i < HANDOFF_PAUSES && OWNER.get() == 0 && WAITERS.get() > 0; i++) { - pauseWaiter(i); + HANDOFF.set(true); + long end = System.nanoTime() + HANDOFF_NANOS; + while (OWNER.get() == 0 && WAITERS.get() > 0 && System.nanoTime() - end < 0) { + pauseWaiter(0); } + HANDOFF.set(false); restore(saved); } } diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyInstance.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyInstance.java index 63686c92f24..f9f8c591d43 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyInstance.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyInstance.java @@ -657,6 +657,21 @@ private static final class InstanceDict extends PyDict { /// Set while refilling, so the refill's own stores do not sync again. private boolean syncing; + /// What the keys were last read from: the instance's shape and which + /// of its fields were set. While both are unchanged the keys are, and + /// sync only re-reads the values -- by index, with no name lookup. + /// Values cannot be stamped: inline caches and generated code store + /// into slots and fields directly. + private Object seenShape; + private long seenFields; + private boolean seen; + + // No "dictionary keys changed during iteration" check: CPython 3.14 + // does not raise it for an instance's __dict__ either (deleting one + // attribute and adding another while iterating runs on, as + // object_protocol_ in the conformance tests shows); only a change of + // size is reported, as there. + InstanceDict(PyInstance self) { this.self = self; } @@ -668,6 +683,24 @@ void sync() { } syncing = true; try { + String[] fields = self.fieldNames(); + long mask = 0; + boolean small = fields.length <= 64; + for (int i = 0; small && i < fields.length; i++) { + if (self.getField(i) != null) { + mask |= 1L << i; + } + } + PyDict other0 = self.nonStrKeys; + int expected = Long.bitCount(mask) + self.shape().size() + (other0 == null ? 0 : other0.size); + if (small && seen && seenShape == self.shape && seenFields == mask //NOPMD CompareObjectsWithEquals - identity is meant + && size == expected) { + refreshValues(fields.length, mask); + return; + } + seenShape = self.shape; + seenFields = mask; + seen = small; String[] names = self.attributeNames(); PyDict other = self.nonStrKeys; if (sameKeys(names, other)) { @@ -698,6 +731,31 @@ void sync() { } } + /// Re-reads the values in key order: the set fields, then the shape's + /// slots (the order [PyInstance#attributeNames()] lists them in). + private void refreshValues(int fieldCount, long mask) { + int f = 0; + int slot = 0; + Object[] sl = self.slots; + for (int i = 0; i < used; i++) { + if (keys[i] == null) { + continue; + } + while (f < fieldCount && (mask & (1L << f)) == 0) { + f++; + } + if (f < fieldCount) { + values[i] = self.getField(f++); + } else if (sl != null && slot < sl.length && self.shape != null && slot < self.shape.size()) { + values[i] = sl[slot++]; + } else { + // The non-str keys, after every attribute: they change + // only through this dict, which keeps them current. + break; + } + } + } + /// Whether the keys already are `names` and then `other`'s, in that /// order. private boolean sameKeys(String[] names, PyDict other) { @@ -733,6 +791,10 @@ public void setItem(Object key, Object value) { if (isStr(key)) { self.store(Py.toStr(key), value); } else { + // Kept apart from the attributes, so the dict lists them + // after every attribute rather than in insertion order as + // CPython does: a known difference, for keys no program + // reaches through an attribute anyway. if (self.nonStrKeys == null) { self.nonStrKeys = new PyDict(); } diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyServer.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyServer.java index 9064d9f8fab..980dc502826 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyServer.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyServer.java @@ -246,6 +246,12 @@ public static void pauseWaiter(int attempt) { Gil.pauseWaiter(attempt); } + /// The error a websocket endpoint gets for using a connection the server + /// no longer holds open (refused, or the handshake timed out). + public static RuntimeException webSocketGone(String why) { + return Py.error(Py.RUNTIME_ERROR, why); + } + /// Drops the work to run after a response that was never completed -- /// the client went away -- as Starlette skips its background tasks when /// a send fails. diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java index 5b10ee87c10..b6e34788269 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java @@ -358,6 +358,231 @@ private static Set addStdlib(Map sources, Map= end) {\n" + + " state.compareAndSet(DECIDING, TIMED_OUT);\n" + + " break;\n" + + " }\n" + + " PyServer.pauseWaiter(attempt++);\n" + + " }\n" + + " if (state.get() != ACCEPTED) {\n" + + " PyServer.webSocketClosed(c, 1006, \"\");\n" + + " WebSocket r = refusal;\n" + + " return r != null ? r : HttpServer.refuseUpgrade(500, \"Internal Server Error\");\n" + + " }\n" + + " return this;\n" + + " }\n" + + "\n" + + " public String[] getSubprotocols() {\n" + + " return subprotocol == null ? null : new String[] {subprotocol};\n" + + " }\n" + + "\n" + + " public java.util.Map getHandshakeHeaders() {\n" + + " if (acceptHeaders == null || acceptHeaders.length == 0) {\n" + + " return null;\n" + + " }\n" + + " return pairs(acceptHeaders);\n" + + " }\n" + + "\n" + + " static java.util.Map pairs(String[] h) {\n" + + " Map out = new LinkedHashMap();\n" + + " for (int i = 0; i + 1 < h.length; i += 2) {\n" + + " Object had = out.get(h[i]);\n" + + " if (had == null) {\n" + + " out.put(h[i], h[i + 1]);\n" + + " } else {\n" + + " java.util.List all = had instanceof java.util.List ? (java.util.List) had : new java.util.ArrayList();\n" + + " if (all != had) {\n" + + " all.add(had);\n" + + " }\n" + + " all.add(h[i + 1]);\n" + + " out.put(h[i], all);\n" + + " }\n" + + " }\n" + + " return out;\n" + + " }\n" + + "\n" + + " public void onOpen(WebSocketSession s) {\n" + + " session = s;\n" + + " // Drain until the queue is empty, then mark it OPEN in the same\n" + + " // compare-and-set: a message sent meanwhile is drained here, so\n" + + " // the order the endpoint sent them in is kept.\n" + + " while (true) {\n" + + " Object[] cur = (Object[]) queue.get();\n" + + " if (cur.length == 0) {\n" + + " if (queue.compareAndSet(cur, OPEN)) {\n" + + " return;\n" + + " }\n" + + " } else if (queue.compareAndSet(cur, NONE)) {\n" + + " for (int i = 0; i < cur.length; i++) {\n" + + " deliver(cur[i]);\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "\n" + + " public void onText(WebSocketSession s, String message) {\n" + + " PyServer.webSocketText(connection, message);\n" + + " }\n" + + "\n" + + " public void onBinary(WebSocketSession s, byte[] message, int offset, int length) {\n" + + " byte[] copy = new byte[length];\n" + + " System.arraycopy(message, offset, copy, 0, length);\n" + + " PyServer.webSocketBinary(connection, copy);\n" + + " }\n" + + "\n" + + " public void onClose(WebSocketSession s, int code, String reason) {\n" + + " PyServer.webSocketClosed(connection, code, reason);\n" + + " }\n" + + "\n" + + " public void accept(String subprotocol, String[] headers) {\n" + + " int st = state.get();\n" + + " if (st == TIMED_OUT || st == REFUSED) {\n" + + " // The server already answered the upgrade and delivered a 1006\n" + + " // disconnect to the endpoint's receive; an accept now must not\n" + + " // look like it worked.\n" + + " throw PyServer.webSocketGone(st == TIMED_OUT\n" + + " ? \"websocket handshake timed out before accept\" : \"websocket was refused\");\n" + + " }\n" + + " if (st != DECIDING) {\n" + + " return;\n" + + " }\n" + + " this.subprotocol = subprotocol;\n" + + " this.acceptHeaders = headers;\n" + + " state.compareAndSet(DECIDING, ACCEPTED);\n" + + " }\n" + + "\n" + + " public void deny(int status, String[] headers, byte[] body) {\n" + + " // Racing the timeout is benign: refusal is written before the\n" + + " // compare-and-set, and handshake reads it only after seeing\n" + + " // REFUSED; if the timeout won, refusal is never read and the\n" + + " // client gets the 500 refusal instead.\n" + + " if (state.get() != DECIDING) {\n" + + " return;\n" + + " }\n" + + " String type = null;\n" + + " for (int i = 0; i + 1 < headers.length; i += 2) {\n" + + " if (\"content-type\".equalsIgnoreCase(headers[i])) {\n" + + " type = headers[i + 1];\n" + + " }\n" + + " }\n" + + " Map extra = pairs(headers);\n" + + " extra.remove(\"content-type\");\n" + + " extra.remove(\"content-length\");\n" + + " refusal = HttpServer.refuseUpgrade(new HttpServer.Response(status,\n" + + " type == null ? \"text/plain; charset=utf-8\" : type, body, extra));\n" + + " state.compareAndSet(DECIDING, REFUSED);\n" + + " }\n" + + "\n" + + " public void text(String value) {\n" + + " outgoing(value);\n" + + " }\n" + + "\n" + + " public void binary(byte[] value) {\n" + + " outgoing(value);\n" + + " }\n" + + "\n" + + " public void close(int code, String reason) {\n" + + " if (state.compareAndSet(DECIDING, REFUSED)) {\n" + + " // Closed before accepting: the upgrade is refused.\n" + + " return;\n" + + " }\n" + + " outgoing(new Object[] {Integer.valueOf(code), reason});\n" + + " }\n" + + "\n" + + " private void outgoing(Object message) {\n" + + " int st = state.get();\n" + + " if (st != ACCEPTED) {\n" + + " // Before accept, or refused or timed out: nobody will open a\n" + + " // session, so the message cannot go anywhere. Raised, as\n" + + " // Starlette raises a send outside an accepted connection.\n" + + " throw PyServer.webSocketGone(st == DECIDING ? \"websocket send before accept\"\n" + + " : st == TIMED_OUT ? \"websocket handshake timed out\" : \"websocket was refused\");\n" + + " }\n" + + " while (true) {\n" + + " Object[] cur = (Object[]) queue.get();\n" + + " if (cur == OPEN) {\n" + + " deliver(message);\n" + + " return;\n" + + " }\n" + + " Object[] next = new Object[cur.length + 1];\n" + + " System.arraycopy(cur, 0, next, 0, cur.length);\n" + + " next[cur.length] = message;\n" + + " if (queue.compareAndSet(cur, next)) {\n" + + " return;\n" + + " }\n" + + " }\n" + + " }\n" + + "\n" + + " private void deliver(Object message) {\n" + + " try {\n" + + " if (message instanceof String) {\n" + + " session.sendText((String) message);\n" + + " } else if (message instanceof byte[]) {\n" + + " session.sendBinary((byte[]) message);\n" + + " } else if (message instanceof Object[]) {\n" + + " Object[] c = (Object[]) message;\n" + + " Object code = c[0];\n" + + " session.close(code instanceof Integer ? ((Integer) code).intValue() : 1000, String.valueOf(c[1]));\n" + + " }\n" + + " } catch (java.io.IOException e) {\n" + + " session.abort();\n" + + " }\n" + + " }\n" + + " }\n"; + } + /// Whether `text` may call the builtin `open` (a cheap textual test: /// bundling `io` for a false positive costs only its size). /// Whether `text` names a method of the reduction protocol that @@ -686,209 +911,7 @@ private static String backendServer(int virtualThreadStack) { + " }\n" + " return headers;\n" + " }\n\n" - + " /**\n" - + " * One websocket connection driven by the Python endpoint, which runs on\n" - + " * Python's websocket event loop thread. The handshake waits for the\n" - + " * endpoint to accept or refuse, so a refusal is answered before the\n" - + " * upgrade and an accepted subprotocol is the one negotiated. Messages\n" - + " * the endpoint sends before the server has opened the session are kept\n" - + " * and sent from onOpen, in order.\n" - + " */\n" - + " static final class Socket implements WebSocket, PyServer.WebSocketSink {\n" - + " private static final long HANDSHAKE_MILLIS = 30000L;\n" - + " private static final int DECIDING = 0;\n" - + " private static final int ACCEPTED = 1;\n" - + " private static final int REFUSED = 2;\n" - + " /** The handshake gave up; a late accept or message does nothing. */\n" - + " private static final int TIMED_OUT = 3;\n" - + " private static final Object[] NONE = new Object[0];\n" - + " /** Stands in the queue once the session is open and drained. */\n" - + " private static final Object[] OPEN = new Object[0];\n" - + " private final Object open;\n" - + " private final String target;\n" - + " private final String[] headers;\n" - + " // Atomics, never a monitor: this runs on backend virtual threads,\n" - + " // where Object.wait or a contended synchronized blocks the HOST\n" - + " // thread and wedges every task queued on it.\n" - + " private final java.util.concurrent.atomic.AtomicInteger state = new java.util.concurrent.atomic.AtomicInteger();\n" - + " /** Messages sent before onOpen, as an immutable array; OPEN after. */\n" - + " private final java.util.concurrent.atomic.AtomicReference queue = new java.util.concurrent.atomic.AtomicReference(NONE);\n" - + " /** Written before the queue becomes OPEN, which publishes it. */\n" - + " private WebSocketSession session;\n" - + " private Object connection;\n" - + " private String subprotocol;\n" - + " private String[] acceptHeaders;\n" - + " private WebSocket refusal;\n" - + "\n" - + " Socket(Object open, String target, String[] headers) {\n" - + " this.open = open;\n" - + " this.target = target;\n" - + " this.headers = headers;\n" - + " }\n" - + "\n" - + " /** Runs the endpoint up to its accept; answers this socket, or the refusal. */\n" - + " WebSocket handshake() {\n" - + " Object c = PyServer.openWebSocket(open, target, headers, this);\n" - + " connection = c;\n" - + " long end = System.currentTimeMillis() + HANDSHAKE_MILLIS;\n" - + " int attempt = 0;\n" - + " while (state.get() == DECIDING) {\n" - + " if (System.currentTimeMillis() >= end) {\n" - + " state.compareAndSet(DECIDING, TIMED_OUT);\n" - + " break;\n" - + " }\n" - + " PyServer.pauseWaiter(attempt++);\n" - + " }\n" - + " if (state.get() != ACCEPTED) {\n" - + " PyServer.webSocketClosed(c, 1006, \"\");\n" - + " WebSocket r = refusal;\n" - + " return r != null ? r : HttpServer.refuseUpgrade(500, \"Internal Server Error\");\n" - + " }\n" - + " return this;\n" - + " }\n" - + "\n" - + " public String[] getSubprotocols() {\n" - + " return subprotocol == null ? null : new String[] {subprotocol};\n" - + " }\n" - + "\n" - + " public java.util.Map getHandshakeHeaders() {\n" - + " if (acceptHeaders == null || acceptHeaders.length == 0) {\n" - + " return null;\n" - + " }\n" - + " return pairs(acceptHeaders);\n" - + " }\n" - + "\n" - + " static java.util.Map pairs(String[] h) {\n" - + " Map out = new LinkedHashMap();\n" - + " for (int i = 0; i + 1 < h.length; i += 2) {\n" - + " Object had = out.get(h[i]);\n" - + " if (had == null) {\n" - + " out.put(h[i], h[i + 1]);\n" - + " } else {\n" - + " java.util.List all = had instanceof java.util.List ? (java.util.List) had : new java.util.ArrayList();\n" - + " if (all != had) {\n" - + " all.add(had);\n" - + " }\n" - + " all.add(h[i + 1]);\n" - + " out.put(h[i], all);\n" - + " }\n" - + " }\n" - + " return out;\n" - + " }\n" - + "\n" - + " public void onOpen(WebSocketSession s) {\n" - + " session = s;\n" - + " // Drain until the queue is empty, then mark it OPEN in the same\n" - + " // compare-and-set: a message sent meanwhile is drained here, so\n" - + " // the order the endpoint sent them in is kept.\n" - + " while (true) {\n" - + " Object[] cur = (Object[]) queue.get();\n" - + " if (cur.length == 0) {\n" - + " if (queue.compareAndSet(cur, OPEN)) {\n" - + " return;\n" - + " }\n" - + " } else if (queue.compareAndSet(cur, NONE)) {\n" - + " for (int i = 0; i < cur.length; i++) {\n" - + " deliver(cur[i]);\n" - + " }\n" - + " }\n" - + " }\n" - + " }\n" - + "\n" - + " public void onText(WebSocketSession s, String message) {\n" - + " PyServer.webSocketText(connection, message);\n" - + " }\n" - + "\n" - + " public void onBinary(WebSocketSession s, byte[] message, int offset, int length) {\n" - + " byte[] copy = new byte[length];\n" - + " System.arraycopy(message, offset, copy, 0, length);\n" - + " PyServer.webSocketBinary(connection, copy);\n" - + " }\n" - + "\n" - + " public void onClose(WebSocketSession s, int code, String reason) {\n" - + " PyServer.webSocketClosed(connection, code, reason);\n" - + " }\n" - + "\n" - + " public void accept(String subprotocol, String[] headers) {\n" - + " if (state.get() != DECIDING) {\n" - + " return;\n" - + " }\n" - + " this.subprotocol = subprotocol;\n" - + " this.acceptHeaders = headers;\n" - + " state.compareAndSet(DECIDING, ACCEPTED);\n" - + " }\n" - + "\n" - + " public void deny(int status, String[] headers, byte[] body) {\n" - + " if (state.get() != DECIDING) {\n" - + " return;\n" - + " }\n" - + " String type = null;\n" - + " for (int i = 0; i + 1 < headers.length; i += 2) {\n" - + " if (\"content-type\".equalsIgnoreCase(headers[i])) {\n" - + " type = headers[i + 1];\n" - + " }\n" - + " }\n" - + " Map extra = pairs(headers);\n" - + " extra.remove(\"content-type\");\n" - + " extra.remove(\"content-length\");\n" - + " refusal = HttpServer.refuseUpgrade(new HttpServer.Response(status,\n" - + " type == null ? \"text/plain; charset=utf-8\" : type, body, extra));\n" - + " state.compareAndSet(DECIDING, REFUSED);\n" - + " }\n" - + "\n" - + " public void text(String value) {\n" - + " outgoing(value);\n" - + " }\n" - + "\n" - + " public void binary(byte[] value) {\n" - + " outgoing(value);\n" - + " }\n" - + "\n" - + " public void close(int code, String reason) {\n" - + " if (state.compareAndSet(DECIDING, REFUSED)) {\n" - + " // Closed before accepting: the upgrade is refused.\n" - + " return;\n" - + " }\n" - + " outgoing(new Object[] {Integer.valueOf(code), reason});\n" - + " }\n" - + "\n" - + " private void outgoing(Object message) {\n" - + " if (state.get() != ACCEPTED) {\n" - + " // Refused or timed out: nobody will open a session, so\n" - + " // keeping the message would only grow the queue.\n" - + " return;\n" - + " }\n" - + " while (true) {\n" - + " Object[] cur = (Object[]) queue.get();\n" - + " if (cur == OPEN) {\n" - + " deliver(message);\n" - + " return;\n" - + " }\n" - + " Object[] next = new Object[cur.length + 1];\n" - + " System.arraycopy(cur, 0, next, 0, cur.length);\n" - + " next[cur.length] = message;\n" - + " if (queue.compareAndSet(cur, next)) {\n" - + " return;\n" - + " }\n" - + " }\n" - + " }\n" - + "\n" - + " private void deliver(Object message) {\n" - + " try {\n" - + " if (message instanceof String) {\n" - + " session.sendText((String) message);\n" - + " } else if (message instanceof byte[]) {\n" - + " session.sendBinary((byte[]) message);\n" - + " } else if (message instanceof Object[]) {\n" - + " Object[] c = (Object[]) message;\n" - + " Object code = c[0];\n" - + " session.close(code instanceof Integer ? ((Integer) code).intValue() : 1000, String.valueOf(c[1]));\n" - + " }\n" - + " } catch (java.io.IOException e) {\n" - + " session.abort();\n" - + " }\n" - + " }\n" - + " }\n" + + webSocketSocketSource() + "\n" + " /**\n" + " * A streamed body: each chunk the application yields is written and\n" diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/_cn1dbapi.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/_cn1dbapi.py index 5e2f32a473c..f4d8ce13ffb 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/_cn1dbapi.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/_cn1dbapi.py @@ -46,7 +46,9 @@ def _error(e): def _skip(sql, i, backslash): """The end of the quoted text, comment or dollar-quoted body starting at ``sql[i]``, or -1 when ``sql[i]`` starts none. ``backslash`` is whether a backslash escapes the next - character inside quotes (MySQL and MariaDB always, PostgreSQL in an ``E''`` string).""" + character inside quotes (MySQL and MariaDB always, PostgreSQL in an ``E''`` string); it + also selects MySQL's comment rules, where ``--`` opens a comment only when followed by + whitespace (``1--1`` is arithmetic) and ``#`` runs to the end of the line.""" n = len(sql) ch = sql[i] if ch in "'\"`": @@ -65,7 +67,9 @@ def _skip(sql, i, backslash): return j + 1 j += 1 return n - if ch == "-" and sql.startswith("--", i): + if (ch == "-" and sql.startswith("--", i) + and (not backslash or i + 2 >= n or sql[i + 2] in " \t\r\n\f\v")) \ + or (ch == "#" and backslash): j = sql.find("\n", i) return n if j < 0 else j + 1 if ch == "/" and sql.startswith("/*", i): @@ -138,27 +142,54 @@ def _code(sql, backslash=False): def _words(code): + """The words of ``code``, each paired with its parenthesis depth.""" w = [] cur = [] + depth = 0 for c in code: if c.isalnum() or c == "_": cur.append(c) - elif cur: - w.append("".join(cur)) + continue + if cur: + w.append(("".join(cur), depth)) cur = [] + if c == "(": + depth += 1 + elif c == ")": + depth = max(depth - 1, 0) if cur: - w.append("".join(cur)) + w.append(("".join(cur), depth)) return w +_STATEMENTS = ("SELECT", "VALUES", "TABLE", "INSERT", "UPDATE", "DELETE", "MERGE") + + +def _query_words(words): + """Whether a statement of these words answers rows: by its first keyword, or for a + ``WITH`` by the first statement keyword after the CTE list (outside its parentheses), + or by a ``RETURNING`` clause anywhere.""" + if not words: + return False + if any(w == "RETURNING" for w, _ in words): + return True + base = words[0][1] + head = words[0][0] + if head == "WITH": + head = "" + for w, d in words[1:]: + if d == base and w in _STATEMENTS: + head = w + break + return head in ("SELECT", "VALUES", "TABLE") + return head in _QUERY_HEADS + + def _is_query(sql, backslash=False): """Whether ``sql`` answers rows. The backend's run() is told up front (its drivers read a result set only when asked), so this decides by the statement: its first keyword after comments and opening parentheses, or a RETURNING clause anywhere outside quotes.""" - words = _words(_code(sql, backslash)) - if not words: - return False - return words[0] in _QUERY_HEADS or "RETURNING" in words + return _query_words(_words(_code(sql, backslash))) def _offset(text): @@ -176,6 +207,8 @@ def _datetime(text): return _dt.datetime.fromisoformat(_offset(text)) +# A BC date or timestamp ("0044-03-15 BC") is outside what datetime can hold. psycopg2 +# raises ValueError for it too, so it is not mapped to anything: fromisoformat raises. def _date(text): if text == "infinity": return _dt.date.max @@ -185,6 +218,10 @@ def _date(text): def _time(text): + # PostgreSQL allows 24:00:00 (the end of a day); psycopg2 answers it as 00:00:00, + # since datetime.time cannot hold it. + if text.startswith("24:00:00"): + text = "00:00:00" + text[8:] return _dt.time.fromisoformat(_offset(text)) @@ -230,8 +267,8 @@ def execute(self, sql, params=()): if c._handle is None: raise OperationalError("Cannot operate on a closed database.") words = _words(_code(sql, backslash)) - head = words[0] if words else "" - query = head in _QUERY_HEADS or "RETURNING" in words + head = words[0][0] if words else "" + query = _query_words(words) if not c._in_tx and head != "CREATE" and head != "DROP": _cn1db.begin(c._handle) c._in_tx = True diff --git a/maven/python-transpiler/src/test/java/com/codename1/python/transpiler/Cn1DbApiTest.java b/maven/python-transpiler/src/test/java/com/codename1/python/transpiler/Cn1DbApiTest.java new file mode 100644 index 00000000000..e93932bb428 --- /dev/null +++ b/maven/python-transpiler/src/test/java/com/codename1/python/transpiler/Cn1DbApiTest.java @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.transpiler; + +import org.junit.jupiter.api.Test; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +/// Runs `test_cn1dbapi.py` under CPython: the SQL scanning and value parsing +/// of the bundled `_cn1dbapi` driver, which needs a server database to run +/// transpiled but whose pure functions import under CPython with a stub +/// `_cn1db`. Skipped where no `python3` is installed, except on CI. +class Cn1DbApiTest { + @Test + void pureFunctions() throws IOException, InterruptedException { + Path site = Paths.get("src/main/resources/com/codename1/python/site/sqlalchemy").toAbsolutePath(); + Path stubs = Paths.get("src/test/resources/cn1dbapi").toAbsolutePath(); + Path log = Files.createTempFile("cn1dbapi", ".log"); + Process p; + try { + p = new ProcessBuilder("python3", "-B", stubs.resolve("test_cn1dbapi.py").toString(), + site.toString(), stubs.toString()).redirectErrorStream(true) + .redirectOutput(log.toFile()).start(); + } catch (IOException e) { + assumeTrue(System.getenv("CI") != null, "python3 is not installed"); + throw e; + } + boolean done = p.waitFor(120, TimeUnit.SECONDS); + String out = new String(Files.readAllBytes(log), StandardCharsets.UTF_8); + Files.delete(log); + assertEquals("ok" + System.lineSeparator(), done ? out : "timed out" + File.separator + out); + assertEquals(0, p.exitValue()); + } +} diff --git a/maven/python-transpiler/src/test/java/com/codename1/python/transpiler/api/WebSocketSocketTest.java b/maven/python-transpiler/src/test/java/com/codename1/python/transpiler/api/WebSocketSocketTest.java new file mode 100644 index 00000000000..03ecb8a5d86 --- /dev/null +++ b/maven/python-transpiler/src/test/java/com/codename1/python/transpiler/api/WebSocketSocketTest.java @@ -0,0 +1,143 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.transpiler.api; + +import org.junit.jupiter.api.Test; + +import javax.tools.JavaCompiler; +import javax.tools.ToolProvider; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/// Compiles the generated server's websocket `Socket` against a fake server +/// and drives its handshake and message state machine. +class WebSocketSocketTest { + private static final String STUBS = String.join("\n", + "import java.util.*;", + "interface WebSocket {}", + "interface WebSocketSession {", + " void sendText(String s) throws java.io.IOException;", + " void sendBinary(byte[] b) throws java.io.IOException;", + " void close(int c, String r) throws java.io.IOException;", + " void abort();", + "}", + "class HttpServer {", + " static final WebSocket REFUSED = new WebSocket() {};", + " static WebSocket refuseUpgrade(int a, String b) { return REFUSED; }", + " static WebSocket refuseUpgrade(Response r) { return REFUSED; }", + " static class Response { Response(int s, String t, byte[] b, Map m) {} }", + "}", + "class PyServer {", + " interface WebSocketSink {}", + " static final StringBuffer LOG = new StringBuffer();", + " static volatile Object sink;", + " static volatile boolean closeAtOnce;", + " static Object openWebSocket(Object a, String b, String[] c, Object d) {", + " sink = d;", + " if (closeAtOnce) { ((Outer.Socket) d).close(1000, \"\"); }", + " return \"conn\";", + " }", + " static void webSocketClosed(Object c, int code, String r) { LOG.append(\"closed \" + code + \";\"); }", + " static void webSocketText(Object c, String m) {}", + " static void webSocketBinary(Object c, byte[] b) {}", + " static void pauseWaiter(int a) { Thread.yield(); }", + " static RuntimeException webSocketGone(String why) { return new IllegalStateException(why); }", + "}", + "class Outer {"); + + private static final String DRIVER = String.join("\n", + "public class Driver {", + " static String gone(Runnable r) {", + " try { r.run(); return \"no error\"; } catch (IllegalStateException e) { return e.getMessage(); }", + " }", + " public static String run() throws Exception {", + " StringBuilder out = new StringBuilder();", + // Early messages keep their order across onOpen, sent concurrently. + " final List sent = Collections.synchronizedList(new ArrayList());", + " final WebSocketSession session = new WebSocketSession() {", + " public void sendText(String s) { sent.add(s); }", + " public void sendBinary(byte[] b) {}", + " public void close(int c, String r) { sent.add(\"close\" + c); }", + " public void abort() {}", + " };", + " for (int round = 0; round < 50; round++) {", + " sent.clear();", + " final Outer.Socket s = new Outer.Socket(null, \"/\", new String[0]);", + " Thread t = new Thread(new Runnable() { public void run() { s.handshake(); } });", + " t.start();", + " while (PyServer.sink != s) { Thread.yield(); }", + " s.accept(null, new String[0]);", + " t.join();", + " s.text(\"0\");", + " Thread opener = new Thread(new Runnable() { public void run() { s.onOpen(session); } });", + " opener.start();", + " for (int i = 1; i < 20; i++) { s.text(String.valueOf(i)); }", + " s.close(1000, \"\");", + " opener.join();", + " StringBuilder want = new StringBuilder();", + " for (int i = 0; i < 20; i++) { want.append(i).append(','); }", + " want.append(\"close1000,\");", + " StringBuilder got = new StringBuilder();", + " for (String m : sent) { got.append(m).append(','); }", + " if (!want.toString().equals(got.toString())) { return \"out of order: \" + got; }", + " }", + " out.append(\"ordered;\");", + // A late accept after the handshake timed out. + " Outer.Socket.handshakeMillis = 30;", + " PyServer.LOG.setLength(0);", + " final Outer.Socket late = new Outer.Socket(null, \"/\", new String[0]);", + " out.append(late.handshake() == HttpServer.REFUSED).append(';').append(PyServer.LOG);", + " out.append(gone(new Runnable() { public void run() { late.accept(null, new String[0]); } })).append(';');", + " out.append(gone(new Runnable() { public void run() { late.text(\"x\"); } })).append(';');", + // Closed before accepting: refused at once, sends raise. + " PyServer.closeAtOnce = true;", + " Outer.Socket.handshakeMillis = 30000;", + " PyServer.LOG.setLength(0);", + " final Outer.Socket shut = new Outer.Socket(null, \"/\", new String[0]);", + " out.append(shut.handshake() == HttpServer.REFUSED).append(';').append(PyServer.LOG);", + " out.append(gone(new Runnable() { public void run() { shut.text(\"x\"); } }));", + " return out.toString();", + " }", + "}"); + + @Test + void stateMachine() throws Exception { + Path dir = Files.createTempDirectory("ws-socket"); + String src = STUBS + "\n" + PythonTranspiler.webSocketSocketSource() + "}\n" + DRIVER + "\n"; + Path file = dir.resolve("Driver.java"); + Files.write(file, src.getBytes(StandardCharsets.UTF_8)); + JavaCompiler javac = ToolProvider.getSystemJavaCompiler(); + assertEquals(0, javac.run(null, null, null, "-nowarn", "-d", dir.toString(), file.toString())); + try (URLClassLoader loader = new URLClassLoader(new URL[] {dir.toUri().toURL()}, null)) { + Object r = loader.loadClass("Driver").getMethod("run").invoke(null); + assertEquals("ordered;true;closed 1006;websocket handshake timed out before accept;" + + "websocket handshake timed out;true;closed 1006;websocket was refused", r); + } + } +} diff --git a/maven/python-transpiler/src/test/resources/cn1dbapi/_cn1db.py b/maven/python-transpiler/src/test/resources/cn1dbapi/_cn1db.py new file mode 100644 index 00000000000..48d2d07d2e2 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/cn1dbapi/_cn1db.py @@ -0,0 +1 @@ +"""Stands in for the backend bridge so _cn1dbapi imports under CPython.""" diff --git a/maven/python-transpiler/src/test/resources/cn1dbapi/test_cn1dbapi.py b/maven/python-transpiler/src/test/resources/cn1dbapi/test_cn1dbapi.py new file mode 100644 index 00000000000..25532a4622e --- /dev/null +++ b/maven/python-transpiler/src/test/resources/cn1dbapi/test_cn1dbapi.py @@ -0,0 +1,65 @@ +"""CPython checks of _cn1dbapi's pure functions; run by Cn1DbApiTest.""" +import datetime as dt +import sys + +sys.path.insert(0, sys.argv[1]) +sys.path.insert(0, sys.argv[2]) +import _cn1dbapi as d # noqa: E402 + +failures = [] + + +def check(label, got, want): + if got != want: + failures.append("%s: got %r, want %r" % (label, got, want)) + + +P = {"a": 1, "b": 2} +N = d._named +check("quotes", N("select :a, ':b', \"x:b\", `y:b`, 'it''s :a'", P), ("select ?, ':b', \"x:b\", `y:b`, 'it''s :a'", (1,))) +check("cast", N("select x::text, :b::int", P), ("select x::text, ?::int", (2,))) +check("positional", N("select $1, :a", P), ("select $1, ?", (1,))) +check("dollar", N("select $$ :a $$, $t$ :b $t$, :b", P), ("select $$ :a $$, $t$ :b $t$, ?", (2,))) +check("comments", N("select :a -- :b\n/* :b */ , :b", P), ("select ? -- :b\n/* :b */ , ?", (1, 2))) +check("pg backslash is literal", N("select 'C:\\', :b", P), ("select 'C:\\', ?", (2,))) +check("E-string", N("select E'x\\' :a', :b", P), ("select E'x\\' :a', ?", (2,))) +check("mysql backslash", N("select 'it\\'s :a', :b", P, True), ("select 'it\\'s :a', ?", (2,))) +check("mysql --1", N("select 1--1, :a", P, True), ("select 1--1, ?", (1,))) +check("mysql -- comment", N("select :a -- :b\n", P, True), ("select ? -- :b\n", (1,))) +check("mysql #", N("select :a # :b\n, :b", P, True), ("select ? # :b\n, ?", (1, 2))) +check("pg # is code", N("select :a # :b", P), ("select ? # ?", (1, 2))) +try: + N("select :zz", P) + failures.append("missing bind did not raise") +except d.ProgrammingError: + pass + +Q = d._is_query +check("select", Q(" /* c */ (SELECT 1)"), True) +check("returning newline", Q("-- x\nINSERT INTO t VALUES (1)\nRETURNING\nid"), True) +check("returning in quotes", Q("insert into t values ('returning')"), False) +check("values", Q("VALUES (1)"), True) +check("show", Q("show tables"), True) +check("update", Q("update t set a=1"), False) +check("with select", Q("WITH x AS (UPDATE t SET a=1 RETURNING a) SELECT * FROM x"), True) +check("with update", Q("WITH x AS (SELECT 1) UPDATE t SET a = (SELECT * FROM x)"), False) +check("with recursive select", Q("with recursive r(n) as (select 1 union select n+1 from r) select n from r"), True) +check("with insert", Q("WITH x AS (SELECT 1 AS v) INSERT INTO t SELECT v FROM x"), False) +check("with delete returning", Q("WITH x AS (SELECT 1) DELETE FROM t RETURNING id"), True) +check("mysql # hides keyword", Q("# select\nupdate t set a=1", True), False) + +tz2 = dt.timezone(dt.timedelta(hours=2)) +check("typed", d._typed([("12:00:00+02", "24:00:00", "infinity", "-infinity", "2020-01-02 03:04:05+05", "2.5")], + ["timetz", "time", "datetime", "date", "datetimetz", "decimal"]), + [(dt.time(12, 0, tzinfo=tz2), dt.time(0, 0), dt.datetime.max, dt.date.min, + dt.datetime(2020, 1, 2, 3, 4, 5, tzinfo=dt.timezone(dt.timedelta(hours=5))), d._decimal.Decimal("2.5"))]) +try: + d._typed([("0044-03-15 BC",)], ["date"]) + failures.append("BC date did not raise") +except ValueError: + pass + +if failures: + print("\n".join(failures)) + sys.exit(1) +print("ok") diff --git a/maven/python-transpiler/src/test/resources/conformance/object_protocol_.out b/maven/python-transpiler/src/test/resources/conformance/object_protocol_.out index e0217461eff..69caee50248 100644 --- a/maven/python-transpiler/src/test/resources/conformance/object_protocol_.out +++ b/maven/python-transpiler/src/test/resources/conformance/object_protocol_.out @@ -13,3 +13,5 @@ __newobj__ True True False TypeError comparing strings with non-ASCII characters is not supported TypeError mixed +no error +['b', 'c'] 3 diff --git a/maven/python-transpiler/src/test/resources/conformance/object_protocol_.py b/maven/python-transpiler/src/test/resources/conformance/object_protocol_.py index af82d252996..dc002cbd8ff 100644 --- a/maven/python-transpiler/src/test/resources/conformance/object_protocol_.py +++ b/maven/python-transpiler/src/test/resources/conformance/object_protocol_.py @@ -108,3 +108,18 @@ def __init__(self, x, y): hmac.compare_digest("abc", b"abc") except TypeError: print("TypeError mixed") + +# Deleting one key and adding another through the dict keeps the size: still an error. +o3 = Plain() +o3.a = 1 +o3.b = 2 +d3 = o3.__dict__ +try: + for k in d3: + if k == "a": + del d3["a"] + d3["c"] = 3 + print("no error") +except RuntimeError as e: + print("RuntimeError", e) +print(sorted(o3.__dict__), o3.c) diff --git a/maven/python-transpiler/src/test/resources/conformance/threading_lock_.out b/maven/python-transpiler/src/test/resources/conformance/threading_lock_.out index 4958cb32554..8feab43cf63 100644 --- a/maven/python-transpiler/src/test/resources/conformance/threading_lock_.out +++ b/maven/python-transpiler/src/test/resources/conformance/threading_lock_.out @@ -10,3 +10,4 @@ condition True [1] event timeout False event set True semaphore timeout False +both progressed True True diff --git a/maven/python-transpiler/src/test/resources/conformance/threading_lock_.py b/maven/python-transpiler/src/test/resources/conformance/threading_lock_.py index 9ba0b4b3fe5..dbfe8efd90e 100644 --- a/maven/python-transpiler/src/test/resources/conformance/threading_lock_.py +++ b/maven/python-transpiler/src/test/resources/conformance/threading_lock_.py @@ -111,3 +111,25 @@ def produce(): sem.acquire() print("semaphore timeout", sem.acquire(timeout=0.01)) sem.release() + +# A platform thread waiting for the GIL gets turns while another thread runs a +# CPU-bound loop that only yields with sleep(0). +progress = {"loop": 0, "other": 0} +stop = [False] + + +def other_thread(): + while not stop[0]: + progress["other"] += 1 + time.sleep(0) + + +ot = threading.Thread(target=other_thread) +ot.start() +for i in range(3000): + progress["loop"] += 1 + if i % 10 == 0: + time.sleep(0) +stop[0] = True +ot.join() +print("both progressed", progress["loop"] == 3000, progress["other"] > 0) From 214540b998fa76bf9ace2bb2abf2a451551c5748 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 19:38:47 +0200 Subject: [PATCH 094/112] Python runtime: round-3 red-team fixes Instance __dict__: dict-level writes invalidate the sync stamp, the fast path checks every key against the expected name, popitem syncs first, a store happens before the dict changes, and a pure-__slots__ instance has no __dict__. Gil handoff is a counter cleared in finally. The websocket handshake timeout is per socket. RETURNING inside a CTE no longer makes a statement a query. Cn1DbApiTest kills a hung python3. Co-Authored-By: Claude Opus 5.5 --- .../com/codename1/python/runtime/Gil.java | 23 ++++++----- .../codename1/python/runtime/PyInstance.java | 41 ++++++++++++++++--- .../codename1/python/runtime/TypeMethods.java | 3 ++ .../transpiler/api/PythonTranspiler.java | 4 +- .../python/site/sqlalchemy/_cn1dbapi.py | 8 ++-- .../python/transpiler/Cn1DbApiTest.java | 6 ++- .../transpiler/api/WebSocketSocketTest.java | 3 +- .../test/resources/cn1dbapi/test_cn1dbapi.py | 2 + .../conformance/object_protocol_.out | 4 ++ .../resources/conformance/object_protocol_.py | 26 ++++++++++++ 10 files changed, 97 insertions(+), 23 deletions(-) diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Gil.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Gil.java index cea820e1ff2..5c4beccbbe9 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Gil.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Gil.java @@ -64,11 +64,10 @@ public final class Gil { /// lock before it competes for it again. private static final long HANDOFF_NANOS = 2000000L; - /// Set while [#yieldNow()] leaves the lock free for a waiter: waiters + /// Non-zero while a [#yieldNow()] leaves the lock free for a waiter: waiters /// then poll instead of sleeping, so a platform thread asleep in /// `Thread.sleep(1)` is not outwaited by the yielder. - private static final java.util.concurrent.atomic.AtomicBoolean HANDOFF = - new java.util.concurrent.atomic.AtomicBoolean(); + private static final AtomicInteger HANDOFF = new AtomicInteger(); private static Scheduler scheduler; @@ -127,7 +126,7 @@ private static void acquire(long me) { try { int attempt = 0; while (!OWNER.compareAndSet(0, me)) { - pauseWaiter(HANDOFF.get() ? 0 : attempt++); + pauseWaiter(HANDOFF.get() > 0 ? 0 : attempt++); } } finally { WAITERS.decrementAndGet(); @@ -194,7 +193,7 @@ public static void restore(int saved) { /// starving the others. So when a thread is waiting, the lock is left /// free until somebody else has taken it -- for at most /// [#HANDOFF_NANOS], longer than a platform waiter's one-millisecond - /// sleep, and waiters stop sleeping while [#HANDOFF] is set -- and + /// sleep, and waiters stop sleeping while [#HANDOFF] is non-zero -- and /// when nobody is waiting the lock is not released at all. public static void yieldNow() { if (WAITERS.get() == 0) { @@ -202,12 +201,16 @@ public static void yieldNow() { return; } int saved = release(); - HANDOFF.set(true); - long end = System.nanoTime() + HANDOFF_NANOS; - while (OWNER.get() == 0 && WAITERS.get() > 0 && System.nanoTime() - end < 0) { - pauseWaiter(0); + // A count, not a flag: two threads may yield at once. + HANDOFF.incrementAndGet(); + try { + long end = System.nanoTime() + HANDOFF_NANOS; + while (OWNER.get() == 0 && WAITERS.get() > 0 && System.nanoTime() - end < 0) { + pauseWaiter(0); + } + } finally { + HANDOFF.decrementAndGet(); } - HANDOFF.set(false); restore(saved); } } diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyInstance.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyInstance.java index f9f8c591d43..8faf906fbe0 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyInstance.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyInstance.java @@ -52,6 +52,7 @@ public class PyInstance extends PyObject { Shape shape; Object[] slots; + /// Keys of `__dict__` that are not strings (`o.__dict__[1] = x`), which /// CPython keeps in the dict but no attribute can name. Null for nearly /// every instance. @@ -231,6 +232,16 @@ Object genericAttrOrNull(String name) { // A live view: its stores and deletes reach the instance, and it // refills itself from the instance before every read, so a write // made through the attribute shows in a view fetched earlier. + if (cls.slotNames() != null) { + // Every class declares __slots__ without __dict__: CPython + // has no dict to answer either. + throw Py.attributeError("'" + cls.name + "' object has no attribute '__dict__'"); + } + // A new view per read, so `o.__dict__ is o.__dict__` is False + // here (True in CPython). Keeping one view per instance was + // tried and broke zipfile's write-through-open in the compress_ + // conformance program; until that is understood each read + // gets its own view, which refills itself from the instance. return new InstanceDict(this); } if ("__class__".equals(name)) { @@ -691,11 +702,13 @@ void sync() { mask |= 1L << i; } } + // A net, not the mechanism: every write through this dict + // clears `seen`, but a path that changes the instance without + // moving its shape or set fields would otherwise go unseen. PyDict other0 = self.nonStrKeys; int expected = Long.bitCount(mask) + self.shape().size() + (other0 == null ? 0 : other0.size); if (small && seen && seenShape == self.shape && seenFields == mask //NOPMD CompareObjectsWithEquals - identity is meant - && size == expected) { - refreshValues(fields.length, mask); + && size == expected && refreshValues(fields, mask)) { return; } seenShape = self.shape; @@ -733,7 +746,11 @@ void sync() { /// Re-reads the values in key order: the set fields, then the shape's /// slots (the order [PyInstance#attributeNames()] lists them in). - private void refreshValues(int fieldCount, long mask) { + /// Answers false, leaving the caller to rebuild, when a key is not + /// the name expected at its place. + private boolean refreshValues(String[] fieldNames, long mask) { + int fieldCount = fieldNames.length; + String[] shapeNames = self.shape().names; int f = 0; int slot = 0; Object[] sl = self.slots; @@ -745,8 +762,14 @@ private void refreshValues(int fieldCount, long mask) { f++; } if (f < fieldCount) { + if (!fieldNames[f].equals(keys[i])) { + return false; + } values[i] = self.getField(f++); - } else if (sl != null && slot < sl.length && self.shape != null && slot < self.shape.size()) { + } else if (sl != null && slot < sl.length && slot < shapeNames.length) { + if (!shapeNames[slot].equals(keys[i])) { + return false; + } values[i] = sl[slot++]; } else { // The non-str keys, after every attribute: they change @@ -754,6 +777,7 @@ private void refreshValues(int fieldCount, long mask) { break; } } + return true; } /// Whether the keys already are `names` and then `other`'s, in that @@ -787,10 +811,13 @@ private static boolean isStr(Object key) { @Override public void setItem(Object key, Object value) { sync(); - super.setItem(key, value); + // The attribute first: a store that fails (a slot it may not + // hold) leaves the dict as it was. if (isStr(key)) { self.store(Py.toStr(key), value); + super.setItem(key, value); } else { + super.setItem(key, value); // Kept apart from the attributes, so the dict lists them // after every attribute rather than in insertion order as // CPython does: a known difference, for keys no program @@ -800,6 +827,9 @@ public void setItem(Object key, Object value) { } self.nonStrKeys.setItem(key, value); } + // The write may restore a shape the stamp already holds (del + // d['b'] then c.b = 9), so the next sync must not trust it. + seen = false; } @Override @@ -820,6 +850,7 @@ public void delItem(Object key) { /// Removes `key` from what the dict stands for. private void forget(Object key) { + seen = false; if (isStr(key)) { self.delAttrDirect(Py.toStr(key)); } else if (self.nonStrKeys != null) { diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/TypeMethods.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/TypeMethods.java index 6842573e374..602e6dad60b 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/TypeMethods.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/TypeMethods.java @@ -880,6 +880,9 @@ private static void installDict() { }); def(t, "popitem", a -> { PyDict self = d(a[0]); + // An instance's __dict__ refills itself here; reading the + // entries before that could answer a stale pair. + self.sync(); for (int i = self.used - 1; i >= 0; i--) { if (self.keys[i] != null) { Object k = self.keys[i]; diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java index b6e34788269..ce56deb3d9d 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/api/PythonTranspiler.java @@ -372,7 +372,9 @@ public static String webSocketSocketSource() { + " * and sent from onOpen, in order.\n" + " */\n" + " static final class Socket implements WebSocket, PyServer.WebSocketSink {\n" - + " static long handshakeMillis = 30000L;\n" + + " private static final long HANDSHAKE_MILLIS = 30000L;\n" + + " /** How long the handshake waits for the endpoint; a test shortens it. */\n" + + " long handshakeMillis = HANDSHAKE_MILLIS;\n" + " private static final int DECIDING = 0;\n" + " private static final int ACCEPTED = 1;\n" + " private static final int REFUSED = 2;\n" diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/_cn1dbapi.py b/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/_cn1dbapi.py index f4d8ce13ffb..afe80140936 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/_cn1dbapi.py +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/site/sqlalchemy/_cn1dbapi.py @@ -168,12 +168,14 @@ def _words(code): def _query_words(words): """Whether a statement of these words answers rows: by its first keyword, or for a ``WITH`` by the first statement keyword after the CTE list (outside its parentheses), - or by a ``RETURNING`` clause anywhere.""" + or by a ``RETURNING`` clause of the statement itself (not of a CTE).""" if not words: return False - if any(w == "RETURNING" for w, _ in words): - return True base = words[0][1] + # RETURNING counts only in the statement itself, not inside a CTE: WITH d AS + # (DELETE ... RETURNING id) DELETE FROM u USING d answers no rows. + if any(w == "RETURNING" and d == base for w, d in words): + return True head = words[0][0] if head == "WITH": head = "" diff --git a/maven/python-transpiler/src/test/java/com/codename1/python/transpiler/Cn1DbApiTest.java b/maven/python-transpiler/src/test/java/com/codename1/python/transpiler/Cn1DbApiTest.java index e93932bb428..3b5a4463c11 100644 --- a/maven/python-transpiler/src/test/java/com/codename1/python/transpiler/Cn1DbApiTest.java +++ b/maven/python-transpiler/src/test/java/com/codename1/python/transpiler/Cn1DbApiTest.java @@ -25,7 +25,6 @@ import org.junit.jupiter.api.Test; -import java.io.File; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -56,9 +55,12 @@ void pureFunctions() throws IOException, InterruptedException { throw e; } boolean done = p.waitFor(120, TimeUnit.SECONDS); + if (!done) { + p.destroyForcibly().waitFor(10, TimeUnit.SECONDS); + } String out = new String(Files.readAllBytes(log), StandardCharsets.UTF_8); Files.delete(log); - assertEquals("ok" + System.lineSeparator(), done ? out : "timed out" + File.separator + out); + assertEquals("ok" + System.lineSeparator(), done ? out : "timed out after 120s:" + System.lineSeparator() + out); assertEquals(0, p.exitValue()); } } diff --git a/maven/python-transpiler/src/test/java/com/codename1/python/transpiler/api/WebSocketSocketTest.java b/maven/python-transpiler/src/test/java/com/codename1/python/transpiler/api/WebSocketSocketTest.java index 03ecb8a5d86..ed68afba80c 100644 --- a/maven/python-transpiler/src/test/java/com/codename1/python/transpiler/api/WebSocketSocketTest.java +++ b/maven/python-transpiler/src/test/java/com/codename1/python/transpiler/api/WebSocketSocketTest.java @@ -109,15 +109,14 @@ class WebSocketSocketTest { " }", " out.append(\"ordered;\");", // A late accept after the handshake timed out. - " Outer.Socket.handshakeMillis = 30;", " PyServer.LOG.setLength(0);", " final Outer.Socket late = new Outer.Socket(null, \"/\", new String[0]);", + " late.handshakeMillis = 30;", " out.append(late.handshake() == HttpServer.REFUSED).append(';').append(PyServer.LOG);", " out.append(gone(new Runnable() { public void run() { late.accept(null, new String[0]); } })).append(';');", " out.append(gone(new Runnable() { public void run() { late.text(\"x\"); } })).append(';');", // Closed before accepting: refused at once, sends raise. " PyServer.closeAtOnce = true;", - " Outer.Socket.handshakeMillis = 30000;", " PyServer.LOG.setLength(0);", " final Outer.Socket shut = new Outer.Socket(null, \"/\", new String[0]);", " out.append(shut.handshake() == HttpServer.REFUSED).append(';').append(PyServer.LOG);", diff --git a/maven/python-transpiler/src/test/resources/cn1dbapi/test_cn1dbapi.py b/maven/python-transpiler/src/test/resources/cn1dbapi/test_cn1dbapi.py index 25532a4622e..46127f7de6a 100644 --- a/maven/python-transpiler/src/test/resources/cn1dbapi/test_cn1dbapi.py +++ b/maven/python-transpiler/src/test/resources/cn1dbapi/test_cn1dbapi.py @@ -46,6 +46,8 @@ def check(label, got, want): check("with recursive select", Q("with recursive r(n) as (select 1 union select n+1 from r) select n from r"), True) check("with insert", Q("WITH x AS (SELECT 1 AS v) INSERT INTO t SELECT v FROM x"), False) check("with delete returning", Q("WITH x AS (SELECT 1) DELETE FROM t RETURNING id"), True) +check("returning only in cte", Q("WITH d AS (DELETE FROM t RETURNING id) DELETE FROM u USING d WHERE u.id = d.id"), False) +check("with select from returning cte", Q("WITH d AS (DELETE FROM t RETURNING id) SELECT id FROM d"), True) check("mysql # hides keyword", Q("# select\nupdate t set a=1", True), False) tz2 = dt.timezone(dt.timedelta(hours=2)) diff --git a/maven/python-transpiler/src/test/resources/conformance/object_protocol_.out b/maven/python-transpiler/src/test/resources/conformance/object_protocol_.out index 69caee50248..894ba113df1 100644 --- a/maven/python-transpiler/src/test/resources/conformance/object_protocol_.out +++ b/maven/python-transpiler/src/test/resources/conformance/object_protocol_.out @@ -15,3 +15,7 @@ TypeError comparing strings with non-ASCII characters is not supported TypeError mixed no error ['b', 'c'] 3 +2 +{'a': 1, 'b': 9} +('z', 5) ['a', 'b'] False +AttributeError 'Slotted' object has no attribute '__dict__' diff --git a/maven/python-transpiler/src/test/resources/conformance/object_protocol_.py b/maven/python-transpiler/src/test/resources/conformance/object_protocol_.py index dc002cbd8ff..b006d054a4f 100644 --- a/maven/python-transpiler/src/test/resources/conformance/object_protocol_.py +++ b/maven/python-transpiler/src/test/resources/conformance/object_protocol_.py @@ -123,3 +123,29 @@ def __init__(self, x, y): except RuntimeError as e: print("RuntimeError", e) print(sorted(o3.__dict__), o3.c) + +# A dict-level delete, then an attribute write that restores the old shape. +c = Plain() +c.a = 1 +c.b = 2 +dd = c.__dict__ +print(len(dd)) +del dd["b"] +c.b = 9 +print(dd) +# popitem after an attribute was added behind the dict's back. +c.z = 5 +print(dd.popitem(), sorted(c.__dict__), hasattr(c, "z")) + + +class Slotted: + __slots__ = ("x",) + + +s = Slotted() +s.x = 1 +try: + s.__dict__ + print("has dict") +except AttributeError as e: + print("AttributeError", e) From c19617a39a8ee82575388720ebd9b9607808b78e Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 20:10:57 +0200 Subject: [PATCH 095/112] Inflate decodes a stream's last short code; one __dict__ view per instance JZlib's InfCodes waited for a full table width of bits before decoding a literal/length or distance code, so a complete raw deflate stream whose last code ended a few bits into its final byte answered Z_BUF_ERROR. The compress_ conformance program hit it through zipfile: the archived gzip file carries its mtime, so whether the entry decompressed depended on the wall clock. Py.dictCount's one-probe path is now taken only for a plain dict; an instance's __dict__ view goes through get and setitem so the store reaches the instance. With the flake explained, an instance keeps one __dict__ view, so o.__dict__ is o.__dict__ as in CPython. Co-Authored-By: Claude Opus 5.5 --- .../src/com/codename1/io/gzip/InfCodes.java | 24 ++++++++++++++++++ .../java/com/codename1/io/gzip/JZlibTest.java | 25 +++++++++++++++++++ .../java/com/codename1/python/runtime/Py.java | 8 ++++-- .../codename1/python/runtime/PyInstance.java | 21 ++++++++++------ .../conformance/object_protocol_.out | 3 +++ .../resources/conformance/object_protocol_.py | 23 +++++++++++++++++ .../resources/conformance/zlib_raw_end_.out | 7 ++++++ .../resources/conformance/zlib_raw_end_.py | 21 ++++++++++++++++ 8 files changed, 123 insertions(+), 9 deletions(-) create mode 100644 maven/python-transpiler/src/test/resources/conformance/zlib_raw_end_.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/zlib_raw_end_.py diff --git a/CodenameOne/src/com/codename1/io/gzip/InfCodes.java b/CodenameOne/src/com/codename1/io/gzip/InfCodes.java index e9fe8d2d204..730011d9bbe 100644 --- a/CodenameOne/src/com/codename1/io/gzip/InfCodes.java +++ b/CodenameOne/src/com/codename1/io/gzip/InfCodes.java @@ -161,6 +161,18 @@ int proc(int r) { while (k < (j)) { if (n != 0) { r = Z_OK; + } else if (tree[(tree_index + (b & inflate_mask[j])) * 3 + 1] <= k) { + // The input is exhausted but the bits held already + // complete a code: every entry of the table is + // replicated over the bits past its own length, so + // the zero bits above k select the right one. A + // stream's last code is usually shorter than the + // table's index width; waiting for bytes that never + // come made a complete raw stream answer + // Z_BUF_ERROR under Z_FINISH ("incomplete or + // truncated stream"), depending on how its final + // bits fell. Modern zlib decodes the same way. + break; } else { s.bitb = b; @@ -251,6 +263,18 @@ int proc(int r) { while (k < (j)) { if (n != 0) { r = Z_OK; + } else if (tree[(tree_index + (b & inflate_mask[j])) * 3 + 1] <= k) { + // The input is exhausted but the bits held already + // complete a code: every entry of the table is + // replicated over the bits past its own length, so + // the zero bits above k select the right one. A + // stream's last code is usually shorter than the + // table's index width; waiting for bytes that never + // come made a complete raw stream answer + // Z_BUF_ERROR under Z_FINISH ("incomplete or + // truncated stream"), depending on how its final + // bits fell. Modern zlib decodes the same way. + break; } else { s.bitb = b; diff --git a/maven/core-unittests/src/test/java/com/codename1/io/gzip/JZlibTest.java b/maven/core-unittests/src/test/java/com/codename1/io/gzip/JZlibTest.java index abf7f4c6d91..fc2cd99d849 100644 --- a/maven/core-unittests/src/test/java/com/codename1/io/gzip/JZlibTest.java +++ b/maven/core-unittests/src/test/java/com/codename1/io/gzip/JZlibTest.java @@ -24,4 +24,29 @@ public void testJZlibConstants() { Assertions.assertEquals(-5, JZlib.Z_BUF_ERROR); Assertions.assertEquals(-6, JZlib.Z_VERSION_ERROR); } + /// A complete raw stream whose last code is shorter than the literal + /// table's index width, and whose final byte holds fewer bits than that + /// width: inflate used to wait for input that never comes and answer + /// Z_BUF_ERROR, with Z_FINISH and without it. + @FormTest + public void testRawStreamEndingInShortCode() { + String hex = "93efe66060582a91c5f4fff4c933dea127ce7b3f3dc568e875eaf34e86eeeb5f4a8518181800"; + byte[] in = new byte[hex.length() / 2]; + for (int i = 0; i < in.length; i++) { + in[i] = (byte) Integer.parseInt(hex.substring(2 * i, 2 * i + 2), 16); + } + int[] flushes = {JZlib.Z_FINISH, JZlib.Z_NO_FLUSH}; + for (int flush : flushes) { + ZStream z = new ZStream(); + Assertions.assertEquals(JZlib.Z_OK, z.inflateInit(-15)); + z.setInput(in, 0, in.length, false); + byte[] out = new byte[1024]; + z.setOutput(out, 0, out.length); + Assertions.assertEquals(JZlib.Z_STREAM_END, z.inflate(flush)); + Assertions.assertEquals(0, z.availIn); + Assertions.assertEquals(35, out.length - z.availOut); + Assertions.assertEquals(0x1f, out[0] & 0xff); + Assertions.assertEquals(0x8b, out[1] & 0xff); + } + } } diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Py.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Py.java index 126665e6413..1a4eee35da3 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Py.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Py.java @@ -814,9 +814,13 @@ private static boolean rightFirst(Object a, Object b) { return ta != tb && tb.isSubtype(ta) && b instanceof PyInstance; //NOPMD CompareObjectsWithEquals - identity is meant } - /// `d[k] = d.get(k, dflt) + inc`, probing an exact dict once. + /// `d[k] = d.get(k, dflt) + inc`, probing an exact dict once. Only a + /// plain dict takes the one-probe path: it writes the value array + /// directly, which would bypass a view such as an instance's `__dict__` + /// (whose stores must reach the instance) -- those go through get and + /// setitem. public static void dictCount(Object d, Object k, Object dflt, Object inc) { - if (d instanceof PyDict && ((PyDict) d).type() == DICT_TYPE) { + if (d instanceof PyDict && d.getClass() == PyDict.class && ((PyDict) d).type() == DICT_TYPE) { ((PyDict) d).count(k, dflt, inc); return; } diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyInstance.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyInstance.java index 8faf906fbe0..5402c513b6d 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyInstance.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/PyInstance.java @@ -58,6 +58,10 @@ public class PyInstance extends PyObject { /// every instance. PyDict nonStrKeys; + /// The `__dict__` view, made on first use and kept, so that + /// `o.__dict__ is o.__dict__` as in CPython. Null until then. + private InstanceDict dictView; + /// The shape, the empty one for null. final Shape shape() { return shape == null ? Shape.EMPTY : shape; @@ -237,12 +241,12 @@ Object genericAttrOrNull(String name) { // has no dict to answer either. throw Py.attributeError("'" + cls.name + "' object has no attribute '__dict__'"); } - // A new view per read, so `o.__dict__ is o.__dict__` is False - // here (True in CPython). Keeping one view per instance was - // tried and broke zipfile's write-through-open in the compress_ - // conformance program; until that is understood each read - // gets its own view, which refills itself from the instance. - return new InstanceDict(this); + // One view per instance, refilled from the instance before + // every read, so a view fetched earlier sees later writes. + if (dictView == null) { + dictView = new InstanceDict(this); + } + return dictView; } if ("__class__".equals(name)) { return cls; @@ -747,7 +751,10 @@ void sync() { /// Re-reads the values in key order: the set fields, then the shape's /// slots (the order [PyInstance#attributeNames()] lists them in). /// Answers false, leaving the caller to rebuild, when a key is not - /// the name expected at its place. + /// the name expected at its place. The caller has already matched + /// the shape, the set fields and the size, which fix the keys; the + /// name check is a cheap safety net against a path that changes the + /// keys without moving any of those, never the expected case. private boolean refreshValues(String[] fieldNames, long mask) { int fieldCount = fieldNames.length; String[] shapeNames = self.shape().names; diff --git a/maven/python-transpiler/src/test/resources/conformance/object_protocol_.out b/maven/python-transpiler/src/test/resources/conformance/object_protocol_.out index 894ba113df1..216503a0fa4 100644 --- a/maven/python-transpiler/src/test/resources/conformance/object_protocol_.out +++ b/maven/python-transpiler/src/test/resources/conformance/object_protocol_.out @@ -19,3 +19,6 @@ no error {'a': 1, 'b': 9} ('z', 5) ['a', 'b'] False AttributeError 'Slotted' object has no attribute '__dict__' +True True +11 11 +5 [('a', 11), ('b', 5)] diff --git a/maven/python-transpiler/src/test/resources/conformance/object_protocol_.py b/maven/python-transpiler/src/test/resources/conformance/object_protocol_.py index b006d054a4f..e34e5c985c0 100644 --- a/maven/python-transpiler/src/test/resources/conformance/object_protocol_.py +++ b/maven/python-transpiler/src/test/resources/conformance/object_protocol_.py @@ -149,3 +149,26 @@ class Slotted: print("has dict") except AttributeError as e: print("AttributeError", e) + + +# The __dict__ view is one object per instance, and a counting store through +# it (d[k] = d.get(k, 0) + n, which plain dicts run with one probe) reaches +# the instance. +class Counted: + def __init__(self): + self.a = 1 + + +def count_through_dict(): + c = Counted() + print(c.__dict__ is c.__dict__, vars(c) is c.__dict__) + d = c.__dict__ + k = "a" + d[k] = d.get(k, 0) + 10 + print(c.a, d[k]) + k = "b" + d[k] = d.get(k, 0) + 5 + print(c.b, sorted(d.items())) + + +count_through_dict() diff --git a/maven/python-transpiler/src/test/resources/conformance/zlib_raw_end_.out b/maven/python-transpiler/src/test/resources/conformance/zlib_raw_end_.out new file mode 100644 index 00000000000..3c5cb56799f --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/zlib_raw_end_.out @@ -0,0 +1,7 @@ +raw round trips 300 300 +1 True True +2 True True +3 True True +7 True True +50 True True +300 True True diff --git a/maven/python-transpiler/src/test/resources/conformance/zlib_raw_end_.py b/maven/python-transpiler/src/test/resources/conformance/zlib_raw_end_.py new file mode 100644 index 00000000000..6b6afee6aaa --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/zlib_raw_end_.py @@ -0,0 +1,21 @@ +# A raw deflate stream's last code may end a few bits into its final byte, +# short of the width the decoder's table is indexed with. The inflater used +# to wait for input that never comes and report the complete stream as +# truncated -- for roughly half of these inputs, depending on the mtime the +# gzip header carries (which is how zipfile lost an entry intermittently). +import zlib, gzip +ok = streamed = 0 +for k in range(1780000000, 1780000000 + 2100, 7): + data = gzip.compress(b"line one\nline two\n", mtime=k) + c = zlib.compressobj(-1, zlib.DEFLATED, -15) + comp = c.compress(data) + c.flush() + if zlib.decompress(comp, -15) == data: + ok += 1 + d = zlib.decompressobj(-15) + if d.decompress(comp) == data and d.eof and d.unused_data == b"": + streamed += 1 +print("raw round trips", ok, streamed) +for size in (1, 2, 3, 7, 50, 300): + data = bytes((i * 7 + size) & 0xff for i in range(size)) + comp = zlib.compress(data, 9, -15) + print(size, zlib.decompress(comp, -15) == data, zlib.decompress(zlib.compress(data)) == data) From a098e0b07f75a718c1548fd34bad1c1c69416088 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 20:16:08 +0200 Subject: [PATCH 096/112] Python: len() on a dict view syncs before answering Typed.len read PyDict.size directly, so len(o.__dict__) after an attribute write answered the count of the last sync. Co-Authored-By: Claude Opus 5.5 --- .../java/com/codename1/python/runtime/Typed.java | 2 +- .../resources/conformance/object_protocol_.out | 2 ++ .../test/resources/conformance/object_protocol_.py | 14 ++++++++++++++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Typed.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Typed.java index af50b65b982..73fbcde892f 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Typed.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Typed.java @@ -482,7 +482,7 @@ public static void dset(Object m, long k, Object v) { public static long len(Object f, Object o) { if (f == Builtins.LEN) { //NOPMD CompareObjectsWithEquals - identity is meant if (o instanceof PyDict) { - return ((PyDict) o).size; + return ((PyDict) o).size(); } return Py.len(o); } diff --git a/maven/python-transpiler/src/test/resources/conformance/object_protocol_.out b/maven/python-transpiler/src/test/resources/conformance/object_protocol_.out index 216503a0fa4..7fd04cdfedd 100644 --- a/maven/python-transpiler/src/test/resources/conformance/object_protocol_.out +++ b/maven/python-transpiler/src/test/resources/conformance/object_protocol_.out @@ -22,3 +22,5 @@ AttributeError 'Slotted' object has no attribute '__dict__' True True 11 11 5 [('a', 11), ('b', 5)] +1 +2 2 diff --git a/maven/python-transpiler/src/test/resources/conformance/object_protocol_.py b/maven/python-transpiler/src/test/resources/conformance/object_protocol_.py index e34e5c985c0..2e572aba9f3 100644 --- a/maven/python-transpiler/src/test/resources/conformance/object_protocol_.py +++ b/maven/python-transpiler/src/test/resources/conformance/object_protocol_.py @@ -172,3 +172,17 @@ def count_through_dict(): count_through_dict() + + +def len_through_cached_dict(): + class C: + pass + o = C() + o.a = 1 + d = o.__dict__ + print(len(d)) + o.x = 1 + print(len(o.__dict__), len(d)) + + +len_through_cached_dict() From 982eccb329f48b12bf4c976f8638b192896a4820 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 28 Oct 2026 20:16:34 +0200 Subject: [PATCH 097/112] TrueTypeFontFromDataTest: icon glyphs as unicode escapes (ASCII-only sources) Co-Authored-By: Claude Opus 5.5 --- .../hellocodenameone/tests/TrueTypeFontFromDataTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/TrueTypeFontFromDataTest.java b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/TrueTypeFontFromDataTest.java index aa4f30db4a8..08a40ee4ce0 100644 --- a/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/TrueTypeFontFromDataTest.java +++ b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/TrueTypeFontFromDataTest.java @@ -24,7 +24,7 @@ /// FontFace asynchronously, so the measuring half polls until the face is in. Assertion-only. public class TrueTypeFontFromDataTest extends BaseTest { // A Material icon codepoint: the default font has no glyph for it. - private static final String SAMPLE = ""; + private static final String SAMPLE = "\ue87c\ue87c\ue87c"; private Font font; private Font plain; private int attempts; From 7f80a3acfaf429e95fbc578247e227d84e087d8d Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Thu, 29 Oct 2026 03:55:55 +0200 Subject: [PATCH 098/112] PR review: CI paths for the hostfs shims and the core, version-free bench jars, Pillow kerning note Co-Authored-By: Claude Opus 5.5 --- .github/workflows/parparvm-tests.yml | 4 ++++ .github/workflows/python-bench.yml | 11 ++++++++--- .../python/transpiler/api/BackendImage.java.txt | 2 ++ 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/.github/workflows/parparvm-tests.yml b/.github/workflows/parparvm-tests.yml index e775e434a65..977917c3957 100644 --- a/.github/workflows/parparvm-tests.yml +++ b/.github/workflows/parparvm-tests.yml @@ -42,6 +42,8 @@ on: - 'scripts/python-backend/e2e-test.sh' - 'scripts/python-backend/e2e_client.py' - 'scripts/python-backend/samples/**' + # sqlalchemy-realdb-test.sh compiles the host file system shims from here. + - 'scripts/python-bench/hostfs/**' - 'scripts/hellocodenameone/pom.xml' - 'scripts/hellocodenameone/backend/**' # The core classes the backend compiles as its own (the ORM and the entity @@ -119,6 +121,8 @@ on: - 'scripts/python-backend/e2e-test.sh' - 'scripts/python-backend/e2e_client.py' - 'scripts/python-backend/samples/**' + # sqlalchemy-realdb-test.sh compiles the host file system shims from here. + - 'scripts/python-bench/hostfs/**' - 'scripts/hellocodenameone/pom.xml' - 'scripts/hellocodenameone/backend/**' # The core classes the backend compiles as its own (the ORM and the entity diff --git a/.github/workflows/python-bench.yml b/.github/workflows/python-bench.yml index 27dfe48e43e..3e4aa3caa8e 100644 --- a/.github/workflows/python-bench.yml +++ b/.github/workflows/python-bench.yml @@ -25,6 +25,10 @@ on: - 'vm/ByteCodeTranslator/**' - 'vm/JavaAPI/**' - 'vm/selfhost/compile-dist.sh' + # Transpiled Python runs on the core's classes (collections, io/gzip). + - 'CodenameOne/src/**' + - 'maven/core/**' + - 'maven/pom.xml' workflow_dispatch: permissions: @@ -114,11 +118,12 @@ jobs: - name: Measure and gate run: | repo="$GITHUB_WORKSPACE/m2/com/codenameone" + v=$(cd "$repo/codenameone-python-runtime" && ls -d */ | head -1 | tr -d /) python3 scripts/python-bench/run_bench.py \ --java-home "$JAVA_HOME" \ - --runtime-jar "$repo/codenameone-python-runtime/8.0-SNAPSHOT/codenameone-python-runtime-8.0-SNAPSHOT.jar" \ - --core-jar "$repo/codenameone-core/8.0-SNAPSHOT/codenameone-core-8.0-SNAPSHOT.jar" \ - --transpiler-cp "$repo/codenameone-python-transpiler/8.0-SNAPSHOT/codenameone-python-transpiler-8.0-SNAPSHOT.jar" \ + --runtime-jar "$repo/codenameone-python-runtime/$v/codenameone-python-runtime-$v.jar" \ + --core-jar "$repo/codenameone-core/$v/codenameone-core-$v.jar" \ + --transpiler-cp "$repo/codenameone-python-transpiler/$v/codenameone-python-transpiler-$v.jar" \ --json python-bench.json --check | tee -a "$GITHUB_STEP_SUMMARY" - name: Upload the report diff --git a/maven/python-transpiler/src/main/resources/com/codename1/python/transpiler/api/BackendImage.java.txt b/maven/python-transpiler/src/main/resources/com/codename1/python/transpiler/api/BackendImage.java.txt index 4bcf18ad612..59a06212949 100644 --- a/maven/python-transpiler/src/main/resources/com/codename1/python/transpiler/api/BackendImage.java.txt +++ b/maven/python-transpiler/src/main/resources/com/codename1/python/transpiler/api/BackendImage.java.txt @@ -276,6 +276,8 @@ final class FreeTypeMetrics { int k = l.count; if (last != 0 && g != 0) { long delta = round(mulFix(font.kerning(last, g), scale)); + // Whole pixels added to a 26.6 advance on purpose: Pillow's + // own x_advance += PIXEL(delta.x), so text lays out the same. l.advances[k - 1] += pixel(delta); } l.glyphs[k] = g; From 811f2635a4aa667f239a828c5665f125c6dfe266 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Thu, 29 Oct 2026 04:10:22 +0200 Subject: [PATCH 099/112] Guide name check reads Python sources too (starlette's TestClient is real) Co-Authored-By: Claude Opus 5.5 --- scripts/developer-guide/check-guide-api-names.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/developer-guide/check-guide-api-names.py b/scripts/developer-guide/check-guide-api-names.py index ad4a71a570c..193e1261bab 100755 --- a/scripts/developer-guide/check-guide-api-names.py +++ b/scripts/developer-guide/check-guide-api-names.py @@ -73,8 +73,10 @@ for d in ('CodenameOne', 'Ports', 'vm', 'maven', os.path.join('docs', 'demos'), os.path.join('scripts', 'wayline'))] +# '.py' because the Python support ships real Python packages under maven/ +# (starlette's TestClient is one edit from RestClient and is a real class). IDENTIFIER_SUFFIXES = ('.java', '.m', '.h', '.mm', '.kt', '.js', '.c', '.cpp', - '.xml', '.gradle', '.swift', '.cs') + '.xml', '.gradle', '.swift', '.cs', '.py') # The last segment may name a nested class -- URLImage.ImageAdapter, # ActionEvent.Type, LayeredLayout.LayeredLayoutConstraint.Inset -- so the dot From 15525099a0e9b861691b8b5dee0684dad348be9b Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Thu, 29 Oct 2026 04:10:43 +0200 Subject: [PATCH 100/112] python-bench: packages read, so the job can pull the CI container Co-Authored-By: Claude Opus 5.5 --- .github/workflows/python-bench.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/python-bench.yml b/.github/workflows/python-bench.yml index 3e4aa3caa8e..ce61698a3f5 100644 --- a/.github/workflows/python-bench.yml +++ b/.github/workflows/python-bench.yml @@ -33,6 +33,9 @@ on: permissions: contents: read + # The pr-ci-container image lives on ghcr; without this the job dies in + # "Initialize containers" with a bare "denied". + packages: read env: # Pinned, like flutter-bench's FLUTTER_REF: the ratios are against this From 9bc29003b02449daf1721259e7bffaad1058ecc7 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Thu, 29 Oct 2026 04:16:16 +0200 Subject: [PATCH 101/112] Copyright headers: full header on three test files; drop the .py exclusion the gate rejects Co-Authored-By: Claude Opus 5.5 --- .../java/com/codename1/io/gzip/JZlibTest.java | 23 +++++++++++++++++++ .../conformance/cext_/FastlibCore.java | 23 +++++++++++++++++++ scripts/copyright-header-exclusions.txt | 1 - .../tests/TrueTypeFontFromDataTest.java | 13 +++++++++++ 4 files changed, 59 insertions(+), 1 deletion(-) diff --git a/maven/core-unittests/src/test/java/com/codename1/io/gzip/JZlibTest.java b/maven/core-unittests/src/test/java/com/codename1/io/gzip/JZlibTest.java index fc2cd99d849..b03b8554d30 100644 --- a/maven/core-unittests/src/test/java/com/codename1/io/gzip/JZlibTest.java +++ b/maven/core-unittests/src/test/java/com/codename1/io/gzip/JZlibTest.java @@ -1,3 +1,26 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ package com.codename1.io.gzip; import com.codename1.junit.UITestBase; diff --git a/maven/python-transpiler/src/test/resources/conformance/cext_/FastlibCore.java b/maven/python-transpiler/src/test/resources/conformance/cext_/FastlibCore.java index 0fc56e87f22..553af1a09c3 100644 --- a/maven/python-transpiler/src/test/resources/conformance/cext_/FastlibCore.java +++ b/maven/python-transpiler/src/test/resources/conformance/cext_/FastlibCore.java @@ -1,3 +1,26 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ package com.codename1.python.ext; import com.codename1.python.runtime.NativeModule; diff --git a/scripts/copyright-header-exclusions.txt b/scripts/copyright-header-exclusions.txt index 4568a1883fe..3b53078afc4 100644 --- a/scripts/copyright-header-exclusions.txt +++ b/scripts/copyright-header-exclusions.txt @@ -427,4 +427,3 @@ CodenameOne/src/com/codename1/util/TBigDecimal.java | Apache Harmony source reta CodenameOne/src/com/codename1/util/TBigInteger.java | Apache Harmony source retaining its original Apache-2.0 notice CodenameOne/src/com/codename1/util/TBitLevel.java | Apache Harmony source retaining its original Apache-2.0 notice CodenameOne/src/com/codename1/util/TDivision.java | Apache Harmony source retaining its original Apache-2.0 notice -maven/python-transpiler/src/main/resources/com/codename1/python/site/PIL/ImageFont.py | Pillow ImageFont rewrite with its embedded courB08 font, retaining the Secret Labs AB / Fredrik Lundh HPND notice diff --git a/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/TrueTypeFontFromDataTest.java b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/TrueTypeFontFromDataTest.java index 08a40ee4ce0..83976407578 100644 --- a/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/TrueTypeFontFromDataTest.java +++ b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/TrueTypeFontFromDataTest.java @@ -7,6 +7,19 @@ * published by the Free Software Foundation. Codename One designates this * particular file as subject to the "Classpath" exception as provided * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codenameone.examples.hellocodenameone.tests; From 3adbb23235bde8f87fc3c23b14edbfb216a7f304 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Thu, 29 Oct 2026 04:21:12 +0200 Subject: [PATCH 102/112] Port status: TrueTypeFontFromDataTest belongs to text and image drawing Co-Authored-By: Claude Opus 5.5 --- docs/website/data/port_status.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/website/data/port_status.json b/docs/website/data/port_status.json index 9d3e19373b3..67740389421 100644 --- a/docs/website/data/port_status.json +++ b/docs/website/data/port_status.json @@ -276,12 +276,13 @@ "id": "graphics-text-images", "category": "2D graphics", "name": "Text and image drawing", - "description": "Checks text decoration, glyph rendering, image rectangles, and tiled images.", + "description": "Checks text decoration, glyph rendering, TrueType fonts loaded from bytes, image rectangles, and tiled images.", "tests": [ "DrawImage", "DrawString", "DrawStringDecorated", - "TileImage" + "TileImage", + "TrueTypeFontFromDataTest" ] }, { From 60a9c54bdea308f322902d040b346fee9ae007a6 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Thu, 29 Oct 2026 05:45:50 +0200 Subject: [PATCH 103/112] python-bench: install the transpiler's test dependencies before the JDK 17 step Co-Authored-By: Claude Opus 5.5 --- .github/workflows/python-bench.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-bench.yml b/.github/workflows/python-bench.yml index ce61698a3f5..af004db85c4 100644 --- a/.github/workflows/python-bench.yml +++ b/.github/workflows/python-bench.yml @@ -56,13 +56,16 @@ jobs: - name: Link cn1-binaries run: ln -s /opt/cn1-binaries ../cn1-binaries + # compat-testing and backend are the transpiler's test dependencies + # (the pilclient_ and pil_ conformance programs); the JDK 17 step + # below runs without -am, so they have to be installed here. - name: Build the core and the transpiler on JDK 8 run: | cd maven JAVA_HOME="$JAVA_HOME_8" bash ../scripts/ci/retry.sh mvn -B -q install -DskipTests \ -Dmaven.javadoc.skip=true -Dmaven.source.skip=true \ -Dmaven.repo.local="$GITHUB_WORKSPACE/m2" \ - -pl core,python-transpiler -am + -pl core,python-transpiler,compat-testing,backend -am # Without -am: core was installed above, and re-resolving it here would # rebuild it on the wrong JDK. From 270df5d93b92df13a3b2d7faa69eb4a78cb66f62 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Thu, 29 Oct 2026 06:27:21 +0200 Subject: [PATCH 104/112] Guide: prose flags, self-contained font snippets, server snippets marked no-try Co-Authored-By: Claude Opus 5.5 --- CodenameOne/src/com/codename1/ui/Font.java | 10 ++++++++-- .../snippets/FontFromDataSnippets.java | 12 +++++++++--- .../Backend-Operations.asciidoc | 2 +- docs/developer-guide/Backend-Views.asciidoc | 2 +- docs/developer-guide/Python-Interop.asciidoc | 18 +++++++++--------- docs/developer-guide/languagetool-accept.txt | 11 +++++++++++ .../build/shared/BuildHintsGeneral.java | 4 ++-- 7 files changed, 41 insertions(+), 18 deletions(-) diff --git a/CodenameOne/src/com/codename1/ui/Font.java b/CodenameOne/src/com/codename1/ui/Font.java index 40dd772ea1d..892c407f9b2 100644 --- a/CodenameOne/src/com/codename1/ui/Font.java +++ b/CodenameOne/src/com/codename1/ui/Font.java @@ -421,8 +421,14 @@ public static boolean isTrueTypeStreamSupported() { /// it to the size and weight you need. /// /// ```java - /// InputStream in = Storage.getInstance().createInputStream("Handlee-Regular.ttf"); - /// Font handlee = Font.createTrueTypeFontFromData("Handlee", in); + /// Label label = new Label("Handlee"); + /// Font handlee = null; + /// try { + /// InputStream in = Storage.getInstance().createInputStream("Handlee-Regular.ttf"); + /// handlee = Font.createTrueTypeFontFromData("Handlee", in); + /// } catch (IOException err) { + /// Log.e(err); + /// } /// if (handlee != null) { /// label.getAllStyles().setFont(handlee.derive(Display.getInstance().convertToPixels(4), Font.STYLE_PLAIN)); /// } diff --git a/docs/demos/common/src/main/java/com/codenameone/developerguide/snippets/FontFromDataSnippets.java b/docs/demos/common/src/main/java/com/codenameone/developerguide/snippets/FontFromDataSnippets.java index 184b1b5feb3..ff5ae9d6f95 100644 --- a/docs/demos/common/src/main/java/com/codenameone/developerguide/snippets/FontFromDataSnippets.java +++ b/docs/demos/common/src/main/java/com/codenameone/developerguide/snippets/FontFromDataSnippets.java @@ -22,6 +22,7 @@ */ package com.codenameone.developerguide.snippets; +import com.codename1.io.Log; import com.codename1.io.Storage; import com.codename1.ui.CN; import com.codename1.ui.Font; @@ -33,12 +34,17 @@ /// Snippets for creating a font from data at runtime, included by the /// developer guide's CSS fonts section. public class FontFromDataSnippets { - Label fontFromStorage(Label label) throws IOException { + Label fontFromStorage() { // tag::font-from-data[] + Label label = new Label("Handlee"); Font base = null; if (Font.isTrueTypeStreamSupported()) { - InputStream in = Storage.getInstance().createInputStream("Handlee-Regular.ttf"); - base = Font.createTrueTypeFontFromData("Handlee", in); + try { + InputStream in = Storage.getInstance().createInputStream("Handlee-Regular.ttf"); + base = Font.createTrueTypeFontFromData("Handlee", in); + } catch (IOException err) { + Log.e(err); + } } if (base == null) { base = Font.createTrueTypeFont("native:MainRegular"); diff --git a/docs/developer-guide/Backend-Operations.asciidoc b/docs/developer-guide/Backend-Operations.asciidoc index 95044bfaa83..1d963667262 100644 --- a/docs/developer-guide/Backend-Operations.asciidoc +++ b/docs/developer-guide/Backend-Operations.asciidoc @@ -179,7 +179,7 @@ no TLS, and what it does need is exactly what a translated binary is good at. in bytes or with a `KB` or `MB` suffix, and when several classes ask, the largest wins: + -[source,java] +[source,java,role=cn1-no-try] ---- include::../demos/backend/src/main/java/com/codenameone/developerguide/backend/DeepReports.java[tag=backend-virtual-thread-stack,indent=0] ---- diff --git a/docs/developer-guide/Backend-Views.asciidoc b/docs/developer-guide/Backend-Views.asciidoc index ed50e17d7d3..a6533d06940 100644 --- a/docs/developer-guide/Backend-Views.asciidoc +++ b/docs/developer-guide/Backend-Views.asciidoc @@ -194,7 +194,7 @@ A backend often has to produce an image: a sharing card for a link, a thumbnail * `TrueTypeFont` reads a TrueType font for `RasterGraphics.drawString`. `TrueTypeFont.createDefault(size)` needs no font file: it embeds Aileron, the font Pillow uses for `ImageFont.load_default()`. * `PngCodec` reads every PNG variant and writes 8-bit PNG; `JpegCodec` reads baseline and progressive JPEG and writes baseline JPEG. -[source,java] +[source,java,role=cn1-no-try] ---- include::../demos/backend/src/main/java/com/codenameone/developerguide/backend/SocialCard.java[tag=raster-image-card,indent=0] ---- diff --git a/docs/developer-guide/Python-Interop.asciidoc b/docs/developer-guide/Python-Interop.asciidoc index 02d636e6897..524b9e8ed8d 100644 --- a/docs/developer-guide/Python-Interop.asciidoc +++ b/docs/developer-guide/Python-Interop.asciidoc @@ -20,7 +20,7 @@ Two things follow from compiling ahead of time rather than interpreting: * Packages with C extensions are supported. The package's Python is translated like your own code, and each compiled module is replaced by a Java class with the same functions; see <>. - The most used ones -- `numpy`, `pydantic`'s core, `yaml`'s scanner -- are + The most used ones -- `numpy`, ``pydantic``'s core, ``yaml``'s scanner -- are already provided, as Codename One implementations of the same public API. === Adding Python to a project @@ -35,8 +35,8 @@ include::../demos/common/src/main/snippets/developer-guide/python-interop.sh[tag The import copies the `.py` files, adds the Python runtime dependency and the `transpile-python` goal to the common module, records the entry module and -makes the application's main class run it. Build and run the project as usual -afterwards; the translation runs in `generate-sources`. Gradle projects get the +makes the application's main class run it. Then build and run the project as +usual; the translation runs in `generate-sources`. Gradle projects get the same step as the `transpilePython` task. The goal reads these properties: @@ -76,8 +76,8 @@ include::../demos/common/src/main/snippets/developer-guide/python-interop.sh[tag The mapping is applied to the compiled classes by the build's `remap-compat` step (`cn1RemapCompat` in Gradle), so the generated Java in -`target/generated-sources/python` still shows which Python line each -statement came from, as a `/* line N */` comment above it. +`target/generated-sources/python` still shows the Python line that +each statement came from, as a `/* line N */` comment above it. === Calling Codename One and Java from Python @@ -119,7 +119,7 @@ translated module. When Python is the application's entry point, its main class calls `PythonModules.registerAll()` at startup; a Java application that uses Python as a library calls it once before importing anything: -[source,java] +[source,java,role=cn1-no-try] ---- include::../demos/common/src/main/java/com/codenameone/developerguide/python/PythonInteropSnippets.java[tag=python-interop-java-001,indent=0] ---- @@ -264,7 +264,7 @@ uses from each one. For every module it generates a class in `common/src/main/java/com/codename1/python/ext` with a stub per function, and lists it in `python-packages/cn1-native-modules.properties`: -[source,java] +[source,java,role=cn1-no-try] ---- include::../demos/common/src/main/java/com/codenameone/developerguide/python/MarkupsafeSpeedups.java[tag=python-interop-java-002,indent=0] ---- @@ -383,7 +383,7 @@ of the two applies. === Compared with CPython, PyPy and GraalPy -The same programs, run on one Apple silicon Mac with +The same programs, run on one Apple Silicon Mac with `scripts/python-compare/compare.py` and `scripts/python-compare/common_compare.py`. Codename One rows are native executables. The interpreters are CPython 3.14, PyPy 3.11 (7.3.20) and GraalPy @@ -493,7 +493,7 @@ What the numbers say: * **Python reaches Java's memory.** On every workload the compiled Python uses about the memory the Java program uses, and sometimes less: - ints, floats and int lists are held unboxed, a list of ints that fit 32 bits + integers, floats and integer lists are held unboxed, a list of integers that fit 32 bits takes four bytes an item as a Java `int[]` does, and objects that don't escape a function are never allocated. * **And mostly its speed.** Six of the eleven run within 15% of Java, among diff --git a/docs/developer-guide/languagetool-accept.txt b/docs/developer-guide/languagetool-accept.txt index 303c3798a93..8399836134d 100644 --- a/docs/developer-guide/languagetool-accept.txt +++ b/docs/developer-guide/languagetool-accept.txt @@ -866,3 +866,14 @@ passlib Werkzeug enqueues backoff +# Python support: package and stdlib names, and plain technical words. +antialiased +libpq +pydantic +yaml +dataclass +autoescaping +Starlette +asyncio +subqueries +recurses diff --git a/maven/build-hint-catalog/src/main/java/com/codename1/build/shared/BuildHintsGeneral.java b/maven/build-hint-catalog/src/main/java/com/codename1/build/shared/BuildHintsGeneral.java index 8641c01f61b..9ce0f0bc136 100644 --- a/maven/build-hint-catalog/src/main/java/com/codename1/build/shared/BuildHintsGeneral.java +++ b/maven/build-hint-catalog/src/main/java/com/codename1/build/shared/BuildHintsGeneral.java @@ -313,8 +313,8 @@ static void register(List h) { .def("1MB") .platform("general") .doc("The C stack each request's virtual thread gets in a Python backend: bytes, " - + "or a number followed by `KB` or `MB`, such as `2MB`. Raise it when a " - + "handler recurses deeply. `CN1_HTTP_VT_STACK` can still raise it at " + + "or a number followed by `KB` or `MB`, such as `2MB`. Raise it for a " + + "handler with deep recursion. `CN1_HTTP_VT_STACK` can still raise it at " + "start-up; the larger of the two applies.")); h.add(new Hint("watchMain") From 37b436902b8dc0917d4d10da1cf1f009ed94b63a Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Thu, 29 Oct 2026 07:25:47 +0200 Subject: [PATCH 105/112] CodeQL: bound sdist tar entry sizes, possessive import regex; archetype-smoke gets headroom Co-Authored-By: Claude Opus 5.5 --- .github/workflows/archetype-smoke.yml | 4 ++- .../codename1/maven/PythonExtensionStubs.java | 7 ++-- .../maven/PythonPackageResolver.java | 17 ++++++--- .../maven/PythonPackageResolverTest.java | 36 +++++++++++++++++++ 4 files changed, 56 insertions(+), 8 deletions(-) diff --git a/.github/workflows/archetype-smoke.yml b/.github/workflows/archetype-smoke.yml index 424545f63ce..c5878ef7c38 100644 --- a/.github/workflows/archetype-smoke.yml +++ b/.github/workflows/archetype-smoke.yml @@ -73,7 +73,9 @@ concurrency: jobs: archetype-smoke: runs-on: ubuntu-24.04 - timeout-minutes: 30 + # master already ran 29.5 minutes against the old 30-minute limit, so any + # module added to the build tipped it over mid-step. + timeout-minutes: 45 steps: - uses: actions/checkout@v6 # A JDK that is NOT 8, for the `cn1:backend-package` check at the end of the diff --git a/maven/build-engine/src/main/java/com/codename1/maven/PythonExtensionStubs.java b/maven/build-engine/src/main/java/com/codename1/maven/PythonExtensionStubs.java index a8db3ca1a2e..5eb946c0a42 100644 --- a/maven/build-engine/src/main/java/com/codename1/maven/PythonExtensionStubs.java +++ b/maven/build-engine/src/main/java/com/codename1/maven/PythonExtensionStubs.java @@ -53,8 +53,11 @@ final class PythonExtensionStubs { static final String INDEX = "cn1-native-modules.properties"; static final String PACKAGE = "com.codename1.python.ext"; + // Possessive: the module part may be empty, so with greedy quantifiers the + // two blank runs around it could split a long run of blanks every way + // there is, which is quadratic on a hostile line. private static final Pattern FROM = Pattern.compile( - "^([ \\t]*)from[ \\t]+(\\.*)([\\w.]*)[ \\t]+import[ \\t]+\\(?([^#)\\n]*)", Pattern.MULTILINE); + "^([ \\t]*+)from[ \\t]++(\\.*+)([\\w.]*+)[ \\t]+import[ \\t]+\\(?([^#)\\n]*)", Pattern.MULTILINE); private static final Pattern IMPORT = Pattern.compile( "^([ \\t]*)import[ \\t]+([\\w.]+)(?:[ \\t]+as[ \\t]+(\\w+))?", Pattern.MULTILINE); @@ -203,7 +206,7 @@ private static String absolute(String pkg, String dots, String rest) { /// The start offset and indent of every import of `module` in `text`, /// adding the names taken from it to `names` when that is given. - private static java.util.List imports(String text, String pkg, String module, TreeSet names) { + static java.util.List imports(String text, String pkg, String module, TreeSet names) { java.util.List at = new java.util.ArrayList(); String leaf = module.substring(module.lastIndexOf('.') + 1); Matcher m = FROM.matcher(text); diff --git a/maven/build-engine/src/main/java/com/codename1/maven/PythonPackageResolver.java b/maven/build-engine/src/main/java/com/codename1/maven/PythonPackageResolver.java index be4f0cef406..4f25107578f 100644 --- a/maven/build-engine/src/main/java/com/codename1/maven/PythonPackageResolver.java +++ b/maven/build-engine/src/main/java/com/codename1/maven/PythonPackageResolver.java @@ -475,7 +475,7 @@ private static Map unzip(byte[] archive) throws IOException { } /// A ustar/pax reader: enough for source distributions. - private static Map untar(byte[] archive) throws IOException { + static Map untar(byte[] archive) throws IOException { Map out = new TreeMap(); byte[] tar; try (InputStream in = new GZIPInputStream(new ByteArrayInputStream(archive))) { @@ -492,8 +492,15 @@ private static Map untar(byte[] archive) throws IOException { long size = Long.parseLong(field(tar, pos + 124, 12).trim().isEmpty() ? "0" : field(tar, pos + 124, 12).trim(), 8); char type = (char) tar[pos + 156]; int data = pos + 512; + // The header's size is the archive's word, not ours: an entry that + // claims more than is left would truncate the casts below. + if (size < 0 || size > tar.length - data) { + throw new IOException("Corrupt sdist archive: an entry claims " + size + + " bytes with " + (tar.length - data) + " left"); + } + int len = (int) size; if (type == 'x' || type == 'L') { - String body = new String(tar, data, (int) size, StandardCharsets.UTF_8); + String body = new String(tar, data, len, StandardCharsets.UTF_8); if (type == 'L') { longName = body.trim(); } else { @@ -508,14 +515,14 @@ private static Map untar(byte[] archive) throws IOException { String full = longName != null ? longName : prefix.isEmpty() ? name : prefix + "/" + name; longName = null; if (safe(full)) { - byte[] b = new byte[(int) size]; - System.arraycopy(tar, data, b, 0, (int) size); + byte[] b = new byte[len]; + System.arraycopy(tar, data, b, 0, len); out.put(full, b); } } else { longName = null; } - pos = data + (int) ((size + 511) / 512 * 512); + pos = data + (len + 511) / 512 * 512; } return out; } diff --git a/maven/build-engine/src/test/java/com/codename1/maven/PythonPackageResolverTest.java b/maven/build-engine/src/test/java/com/codename1/maven/PythonPackageResolverTest.java index 3e5ed8ee1dd..c151fea8653 100644 --- a/maven/build-engine/src/test/java/com/codename1/maven/PythonPackageResolverTest.java +++ b/maven/build-engine/src/test/java/com/codename1/maven/PythonPackageResolverTest.java @@ -220,4 +220,40 @@ public void requirementSpecsKeepVersions() { "[project]\nname = \"x\"\ndependencies = [\"requests[socks]>=2\", 'attrs']\n", "# c\nnumpy==2.1 # pinned\n-r other.txt\n")); } + + /// A tar header's size is untrusted: one claiming more bytes than the + /// archive holds is refused, not truncated into a negative cast. + @Test + public void aTarEntryLargerThanTheArchiveIsRefused() throws Exception { + byte[] header = new byte[512]; + byte[] name = "pkg/big.py".getBytes(StandardCharsets.US_ASCII); + System.arraycopy(name, 0, header, 0, name.length); + byte[] size = "77777777777\0".getBytes(StandardCharsets.US_ASCII); + System.arraycopy(size, 0, header, 124, size.length); + header[156] = '0'; + byte[] tar = new byte[2048]; + System.arraycopy(header, 0, tar, 0, 512); + ByteArrayOutputStream gz = new ByteArrayOutputStream(); + try (java.util.zip.GZIPOutputStream out = new java.util.zip.GZIPOutputStream(gz)) { + out.write(tar); + } + try { + PythonPackageResolver.untar(gz.toByteArray()); + fail("an oversized entry was accepted"); + } catch (IOException expected) { + assertTrue(expected.getMessage(), expected.getMessage().contains("Corrupt sdist archive")); + } + } + + /// `from` followed by a long run of blanks and no module must not + /// backtrack quadratically through the import pattern. + @Test(timeout = 5000) + public void aLongBlankRunAfterFromIsLinear() { + StringBuilder line = new StringBuilder("from"); + for (int i = 0; i < 200000; i++) { + line.append('\t'); + } + line.append("x\nfrom ._core import fast_sum\n"); + assertEquals(1, PythonExtensionStubs.imports(line.toString(), "fastlib", "fastlib._core", null).size()); + } } From af8d1494c37991064213fa7cd82d7eca94e28366 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Thu, 29 Oct 2026 09:02:07 +0200 Subject: [PATCH 106/112] Backend SelfTest: verify-full, verify-ca and VERIFY_IDENTITY are implemented modes now Co-Authored-By: Claude Opus 5.5 --- vm/backend/demo/selftest/com/demo/SelfTest.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/vm/backend/demo/selftest/com/demo/SelfTest.java b/vm/backend/demo/selftest/com/demo/SelfTest.java index f699cf4abc9..119bf1732a7 100644 --- a/vm/backend/demo/selftest/com/demo/SelfTest.java +++ b/vm/backend/demo/selftest/com/demo/SelfTest.java @@ -5441,15 +5441,16 @@ public void run() { * written to prevent, reached by spelling it slightly wrong. */ private static void anUnknownSslModeIsRefused() throws Exception { - String[] typos = new String[] { "Require", "required", "requrie", "verify-full" }; + String[] typos = new String[] { "Require", "required", "requrie", "verify_full" }; for(int iter = 0 ; iter < typos.length ; iter++) { check("sslmode=" + typos[iter] + " is refused", "refused", sslModeRefused(typos[iter])); } - // THE THREE THIS RUNTIME IMPLEMENTS still open a connection -- there is + // THE MODES THIS RUNTIME IMPLEMENTS still open a connection -- there is // nothing listening on port 1, so a connection failure is what "accepted" // looks like here, and it is what says the refusals are about the mode. - String[] known = new String[] { "require", "prefer", "disable" }; + String[] known = new String[] { "require", "prefer", "disable", "verify-ca", + "verify-full", "VERIFY_IDENTITY" }; for(int iter = 0 ; iter < known.length ; iter++) { check("sslmode=" + known[iter] + " is accepted", "accepted", sslModeRefused(known[iter])); From dac2e3aa053c320f89caf254786b82bc57cd16bd Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Thu, 29 Oct 2026 09:59:09 +0200 Subject: [PATCH 107/112] PMD is zero-findings now: python-transpiler's five findings fixed or marked Co-Authored-By: Claude Opus 5.5 --- .../main/java/com/codename1/python/transpiler/ast/Node.java | 4 ++-- .../python/transpiler/codegen/JavaInteropGenerator.java | 2 +- .../com/codename1/python/transpiler/codegen/ShadowTypes.java | 2 +- .../com/codename1/python/transpiler/codegen/TypedEmit.java | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/ast/Node.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/ast/Node.java index f00ea1af595..94406cdc7d0 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/ast/Node.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/ast/Node.java @@ -46,7 +46,7 @@ public T at(int l, int c) { // Expressions // ===================================================================== - public abstract static class Expr extends Node { + public abstract static class Expr extends Node { //NOPMD AbstractClassWithoutAnyMethod - a category of node, never instantiated itself } public static final class Name extends Expr { @@ -358,7 +358,7 @@ public List names() { // Statements // ===================================================================== - public abstract static class Stmt extends Node { + public abstract static class Stmt extends Node { //NOPMD AbstractClassWithoutAnyMethod - a category of node, never instantiated itself } public static final class ExprStmt extends Stmt { diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/JavaInteropGenerator.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/JavaInteropGenerator.java index 58f222d6c42..cbd670f326d 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/JavaInteropGenerator.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/JavaInteropGenerator.java @@ -977,7 +977,7 @@ private static void walk(Object o, List out, Set seen) { continue; } try { - f.setAccessible(true); + f.setAccessible(true); //NOPMD AvoidAccessibilityAlteration - build-time walk of the transpiler's own AST, never device code walk(f.get(o), out, seen); } catch (IllegalAccessException e) { throw new IllegalStateException(e); diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/ShadowTypes.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/ShadowTypes.java index 4f6fb86afee..69d310a8c62 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/ShadowTypes.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/ShadowTypes.java @@ -108,7 +108,7 @@ private void walk(Object o) { continue; } try { - f.setAccessible(true); + f.setAccessible(true); //NOPMD AvoidAccessibilityAlteration - build-time walk of the transpiler's own AST, never device code walk(f.get(o)); } catch (IllegalAccessException e) { throw new IllegalStateException(e); diff --git a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/TypedEmit.java b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/TypedEmit.java index d0714a497c8..442a7260e7f 100644 --- a/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/TypedEmit.java +++ b/maven/python-transpiler/src/main/java/com/codename1/python/transpiler/codegen/TypedEmit.java @@ -295,7 +295,7 @@ private boolean pureNode(Object o) { } else if (o instanceof Assign) { for (Expr t : ((Assign) o).targets) { if (!localTarget(t) && !(initSelf != null && t instanceof Attribute - && ((Attribute) t).value instanceof Name && ((Name) ((Attribute) t).value).id.equals(initSelf))) { + && ((Attribute) t).value instanceof Name && initSelf.equals(((Name) ((Attribute) t).value).id))) { return false; } } From 994a49b8978caff87af556fa7cc3a14b1a58c63e Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Thu, 29 Oct 2026 11:10:30 +0200 Subject: [PATCH 108/112] Android: loadTrueTypeFontFromData answers null for bytes that are not a font Typeface.createFromFile falls back to the default typeface instead of failing, so TrueTypeFontFromDataTest's non-SFNT case got a font back. Co-Authored-By: Claude Opus 5.5 --- .../impl/android/AndroidImplementation.java | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/Ports/Android/src/com/codename1/impl/android/AndroidImplementation.java b/Ports/Android/src/com/codename1/impl/android/AndroidImplementation.java index a8d7ac54c5c..60962071b7f 100644 --- a/Ports/Android/src/com/codename1/impl/android/AndroidImplementation.java +++ b/Ports/Android/src/com/codename1/impl/android/AndroidImplementation.java @@ -2935,6 +2935,13 @@ public Object loadTrueTypeFontFromData(String fontName, byte[] data) { // the memory based Typeface.Builder needs API 26. The file is named by a // hash of the content, so the same font is written once however many // times it is loaded, and two fonts can never overwrite each other. + // The contract for data that is not a font is null, but createFromFile + // answers the default typeface for a file it cannot parse rather than + // failing, so bytes without an SFNT header are rejected up front and a + // DEFAULT answer for bytes with one is treated as a failure too. + if (!isSfntHeader(data)) { + return null; + } try { byte[] digest = MessageDigest.getInstance("SHA-256").digest(data); StringBuilder name = new StringBuilder("font_"); @@ -2961,7 +2968,7 @@ public Object loadTrueTypeFontFromData(String fontName, byte[] data) { } } Typeface t = Typeface.createFromFile(file); - if (t == null) { + if (t == null || t == Typeface.DEFAULT) { return null; } CodenameOneTextPaint newPaint = new CodenameOneTextPaint(t); @@ -2976,6 +2983,18 @@ public Object loadTrueTypeFontFromData(String fontName, byte[] data) { } } + /// True when data starts with a TrueType (0x00010000 or 'true'), OpenType + /// CFF ('OTTO') or collection ('ttcf') tag. + private static boolean isSfntHeader(byte[] data) { + if (data == null || data.length < 12) { + return false; + } + int tag = ((data[0] & 0xff) << 24) | ((data[1] & 0xff) << 16) + | ((data[2] & 0xff) << 8) | (data[3] & 0xff); + return tag == 0x00010000 || tag == 0x74727565 || tag == 0x4f54544f + || tag == 0x74746366; + } + public static class NativeFont { int face; int style; From 9ac01d91efed3c7a0c5a2f46091d057fb8c16797 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Thu, 29 Oct 2026 14:38:24 +0200 Subject: [PATCH 109/112] Python: str case and classes from CPython's tables; native conformance in one binary The native conformance job never finished: one -O3 build per program is nine hours of runner time. It now builds every program into one binary (a Java driver runs each as __main__ behind a marker line), at -O1 with the objects compiled in parallel: about three minutes. Running it found that str.lower/upper/isupper/isalpha/... answered from java.lang.Character, which on a device is the CLDC class (ASCII, at most Latin-1): non-ASCII text did not fold and idna rejected nothing. The runtime now carries CPython 3.13's own BMP tables (generated by tools/gen_unicode_tables.py), so every target answers as CPython does; str.istitle was missing and is added. unicode_case holds it. Also: the native build links core's io/gzip (zlib needs it); numbers.py is renamed fast_paths.py so it no longer shadows the stdlib module when programs share a root; compile-dist passes sources through a response file; vm-tests installs JavaAPI before the SQLAlchemy step that needs it. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/parparvm-tests.yml | 4 + .../codename1/python/runtime/TypeMethods.java | 124 ++++++++++------- .../com/codename1/python/runtime/Unicode.java | 129 ++++++++++++++++++ .../python/runtime/UnicodeTables.java | 55 ++++++++ .../tools/gen_unicode_tables.py | 127 +++++++++++++++++ .../{numbers.out => fast_paths.out} | 0 .../conformance/{numbers.py => fast_paths.py} | 0 .../resources/conformance/unicode_case.out | 21 +++ .../resources/conformance/unicode_case.py | 10 ++ scripts/python-bench/build_parparvm.sh | 18 ++- scripts/python-bench/conformance_parparvm.sh | 103 ++++++++++++-- vm/selfhost/compile-dist.sh | 21 ++- 12 files changed, 543 insertions(+), 69 deletions(-) create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/Unicode.java create mode 100644 maven/python-runtime/src/main/java/com/codename1/python/runtime/UnicodeTables.java create mode 100644 maven/python-runtime/tools/gen_unicode_tables.py rename maven/python-transpiler/src/test/resources/conformance/{numbers.out => fast_paths.out} (100%) rename maven/python-transpiler/src/test/resources/conformance/{numbers.py => fast_paths.py} (100%) create mode 100644 maven/python-transpiler/src/test/resources/conformance/unicode_case.out create mode 100644 maven/python-transpiler/src/test/resources/conformance/unicode_case.py diff --git a/.github/workflows/parparvm-tests.yml b/.github/workflows/parparvm-tests.yml index 977917c3957..faab7c7d648 100644 --- a/.github/workflows/parparvm-tests.yml +++ b/.github/workflows/parparvm-tests.yml @@ -490,6 +490,10 @@ jobs: (cd maven && bash "$GITHUB_WORKSPACE/scripts/ci/retry.sh" mvn -B -pl python-runtime,python-transpiler \ install -DskipTests -Dspotbugs.skip=true -Dpmd.skip=true -Dcheckstyle.skip=true \ -Dmaven.javadoc.skip=true -Dmaven.repo.local="$GITHUB_WORKSPACE/.m2-repo") + # The test transpiles against JavaAPI from the same repository, and + # the step that installs it for the native samples runs after this. + bash "$GITHUB_WORKSPACE/scripts/ci/retry.sh" mvn -B -f vm/pom.xml -pl JavaAPI install -DskipTests \ + -Dmaven.repo.local="$GITHUB_WORKSPACE/.m2-repo" bash scripts/python-backend/sqlalchemy-realdb-test.sh "$GITHUB_WORKSPACE/.m2-repo" env: JAVA_HOME: ${{ env.JDK_17_HOME }} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/TypeMethods.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/TypeMethods.java index 602e6dad60b..acc8897fb06 100644 --- a/maven/python-runtime/src/main/java/com/codename1/python/runtime/TypeMethods.java +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/TypeMethods.java @@ -113,27 +113,30 @@ static String mapCase(String s, int mode) { boolean prevCased = false; for (int i = 0; i < s.length(); i++) { char c = s.charAt(i); - char r; switch (mode) { case 0: - r = Character.toLowerCase(c); + Unicode.LOWER_MAP.append(sb, c); break; case 1: - r = Character.toUpperCase(c); + Unicode.UPPER_MAP.append(sb, c); break; case 2: - r = Character.isUpperCase(c) ? Character.toLowerCase(c) : Character.toUpperCase(c); + if (Unicode.is(c, Unicode.UPPER)) { + Unicode.LOWER_MAP.append(sb, c); + } else if (Unicode.is(c, Unicode.LOWER)) { + Unicode.UPPER_MAP.append(sb, c); + } else { + sb.append(c); + } + break; + case 3: + (prevCased ? Unicode.LOWER_MAP : Unicode.TITLE_MAP).append(sb, c); + prevCased = Unicode.is(c, Unicode.CASED); break; default: - r = prevCased ? Character.toLowerCase(c) : Character.toUpperCase(c); - prevCased = Character.isLetter(c); + Unicode.FOLD_MAP.append(sb, c); break; } - if (mode == 1 && c == '\u00df') { - sb.append("SS"); - continue; - } - sb.append(r); } return sb.toString(); } @@ -225,7 +228,7 @@ private static boolean isSpace(char c) { } return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\u000b' || c == '\u000c' || c == '\u001c' || c == '\u001d' || c == '\u001e' || c == '\u001f' || c == '\u0085' - || Character.isWhitespace(c) || c == '\u00a0'; + || Unicode.is(c, Unicode.SPACE); } private static String strip(String v, Object chars, boolean left, boolean right) { @@ -398,41 +401,58 @@ private static Object pad(Object[] a, int mode) { return Py.str(sb.toString()); } - private static boolean all(String v, int kind) { + /// True when `v` is not empty and every char has one of the `mask` bits. + private static boolean all(String v, int mask) { if (v.length() == 0) { return false; } - boolean cased = false; + for (int i = 0; i < v.length(); i++) { + if (!Unicode.is(v.charAt(i), mask)) { + return false; + } + } + return true; + } + + /// `istitle`: an upper or titlecase char only after an uncased one, a + /// lowercase char only after a cased one, and at least one cased char. + private static boolean isTitle(String v) { + boolean prevCased = false; + boolean seen = false; for (int i = 0; i < v.length(); i++) { char c = v.charAt(i); - boolean ok; - switch (kind) { - case 0: - ok = Character.isDigit(c); - break; - case 1: - ok = Character.isLetter(c); - break; - case 2: - ok = Character.isLetterOrDigit(c); - break; - case 3: - ok = isSpace(c); - break; - case 4: - ok = !Character.isLowerCase(c); - cased |= Character.isUpperCase(c); - break; - default: - ok = !Character.isUpperCase(c); - cased |= Character.isLowerCase(c); - break; + boolean lower = Unicode.is(c, Unicode.LOWER); + if (Unicode.is(c, Unicode.CASED) && !lower) { + if (prevCased) { + return false; + } + prevCased = true; + seen = true; + } else if (lower) { + if (!prevCased) { + return false; + } + seen = true; + } else { + prevCased = false; } - if (!ok) { + } + return seen; + } + + /// `isupper`/`islower`: at least one char has `want` and none has + /// `against`; a titlecase letter is cased but neither, so it fails both. + private static boolean cased(String v, int want, int against) { + boolean seen = false; + for (int i = 0; i < v.length(); i++) { + char c = v.charAt(i); + if (Unicode.is(c, against) + || Unicode.is(c, Unicode.CASED) && !Unicode.is(c, Unicode.UPPER | Unicode.LOWER)) { return false; } + seen |= Unicode.is(c, want); } - return kind < 4 || cased; + return seen; } /// `sep.join(items)`; a list is read in place, without an iterator. @@ -583,7 +603,7 @@ private static void installStr() { }); def(t, "upper", a -> Py.str(mapCase(s(a[0]), 1))); def(t, "lower", a -> Py.str(mapCase(s(a[0]), 0))); - def(t, "casefold", a -> Py.str(mapCase(s(a[0]), 0))); + def(t, "casefold", a -> Py.str(mapCase(s(a[0]), 4))); def(t, "swapcase", a -> Py.str(mapCase(s(a[0]), 2))); def(t, "title", a -> Py.str(mapCase(s(a[0]), 3))); def(t, "capitalize", a -> { @@ -591,16 +611,20 @@ private static void installStr() { if (v.length() == 0) { return a[0]; } - return Py.str(Character.toUpperCase(v.charAt(0)) + mapCase(v.substring(1), 0)); + StringBuilder first = new StringBuilder(); + Unicode.TITLE_MAP.append(first, v.charAt(0)); + return Py.str(first.append(mapCase(v.substring(1), 0)).toString()); }); - def(t, "isdigit", a -> Py.bool(all(s(a[0]), 0))); - def(t, "isdecimal", a -> Py.bool(all(s(a[0]), 0))); - def(t, "isnumeric", a -> Py.bool(all(s(a[0]), 0))); - def(t, "isalpha", a -> Py.bool(all(s(a[0]), 1))); - def(t, "isalnum", a -> Py.bool(all(s(a[0]), 2))); - def(t, "isspace", a -> Py.bool(all(s(a[0]), 3))); - def(t, "isupper", a -> Py.bool(all(s(a[0]), 4))); - def(t, "islower", a -> Py.bool(all(s(a[0]), 5))); + def(t, "isdigit", a -> Py.bool(all(s(a[0]), Unicode.DIGIT))); + def(t, "isdecimal", a -> Py.bool(all(s(a[0]), Unicode.DECIMAL))); + def(t, "isnumeric", a -> Py.bool(all(s(a[0]), Unicode.NUMERIC))); + def(t, "isalpha", a -> Py.bool(all(s(a[0]), Unicode.ALPHA))); + def(t, "isalnum", a -> Py.bool(all(s(a[0]), + Unicode.ALPHA | Unicode.DECIMAL | Unicode.DIGIT | Unicode.NUMERIC))); + def(t, "isspace", a -> Py.bool(all(s(a[0]), Unicode.SPACE))); + def(t, "isupper", a -> Py.bool(cased(s(a[0]), Unicode.UPPER, Unicode.LOWER))); + def(t, "islower", a -> Py.bool(cased(s(a[0]), Unicode.LOWER, Unicode.UPPER))); + def(t, "istitle", a -> Py.bool(isTitle(s(a[0])))); def(t, "isascii", a -> { String v = s(a[0]); for (int i = 0; i < v.length(); i++) { @@ -612,11 +636,11 @@ private static void installStr() { }); def(t, "isidentifier", a -> { String v = s(a[0]); - if (v.length() == 0 || !(Character.isLetter(v.charAt(0)) || v.charAt(0) == '_')) { + if (v.length() == 0 || !(Unicode.is(v.charAt(0), Unicode.ALPHA) || v.charAt(0) == '_')) { return Py.FALSE; } for (int i = 1; i < v.length(); i++) { - if (!(Character.isLetterOrDigit(v.charAt(i)) || v.charAt(i) == '_')) { + if (!(Unicode.is(v.charAt(i), Unicode.ALPHA | Unicode.DECIMAL) || v.charAt(i) == '_')) { return Py.FALSE; } } diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/Unicode.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Unicode.java new file mode 100644 index 00000000000..48591f28af5 --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/Unicode.java @@ -0,0 +1,129 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +/// Python's character properties and case mappings, from [UnicodeTables]. +/// +/// The str methods answer from here rather than from `java.lang.Character`, +/// which on a device is the CLDC class: it knows ASCII and at most Latin-1, +/// so lowering a U with diaeresis left it unchanged there while the JVM, +/// asking the JDK, was right. The tables are CPython's own, so every target answers as +/// the CPython the conformance suite records against. Only the Basic +/// Multilingual Plane is covered; a surrogate has no properties and maps to +/// itself. +final class Unicode { + static final int UPPER = 1; + static final int LOWER = 2; + static final int ALPHA = 4; + static final int DECIMAL = 8; + static final int DIGIT = 16; + static final int NUMERIC = 32; + static final int SPACE = 64; + static final int CASED = 128; + + private static final byte[] FLAGS = new byte[0x10000]; + static final Mapping LOWER_MAP = new Mapping(UnicodeTables.LOWER); + static final Mapping UPPER_MAP = new Mapping(UnicodeTables.UPPER); + static final Mapping TITLE_MAP = new Mapping(UnicodeTables.TITLE); + static final Mapping FOLD_MAP = new Mapping(UnicodeTables.FOLD); + + static { + String runs = join(UnicodeTables.FLAGS); + for (int i = 0; i < runs.length(); i += 2) { + int start = runs.charAt(i); + int end = i + 2 < runs.length() ? runs.charAt(i + 2) : 0x10000; + byte f = (byte) runs.charAt(i + 1); + for (int c = start; c < end; c++) { + FLAGS[c] = f; + } + } + } + + private Unicode() { + } + + /// True when `c` has any of the property bits in `mask`. + static boolean is(char c, int mask) { + return (FLAGS[c] & mask) != 0; + } + + private static String join(String[] parts) { + StringBuilder sb = new StringBuilder(); + for (String p : parts) { + sb.append(p); + } + return sb.toString(); + } + + /// One case mapping: the chars that change, sorted, and what each becomes + /// (more than one char for some: the sharp s upper-cases to "SS"). + static final class Mapping { + private final char[] keys; + private final String[] values; + + Mapping(String[] table) { + String t = join(table); + int n = 0; + for (int i = 0; i < t.length(); i += 2 + t.charAt(i + 1)) { + n++; + } + keys = new char[n]; + values = new String[n]; + int k = 0; + for (int i = 0; i < t.length(); i += 2 + t.charAt(i + 1)) { + keys[k] = t.charAt(i); + values[k] = t.substring(i + 2, i + 2 + t.charAt(i + 1)); + k++; + } + } + + /// What `c` maps to, or null when it maps to itself. + String get(char c) { + int lo = 0; + int hi = keys.length - 1; + while (lo <= hi) { + int mid = (lo + hi) >>> 1; + char m = keys[mid]; + if (m < c) { + lo = mid + 1; + } else if (m > c) { + hi = mid - 1; + } else { + return values[mid]; + } + } + return null; + } + + /// Appends what `c` maps to. + void append(StringBuilder sb, char c) { + String m = get(c); + if (m == null) { + sb.append(c); + } else { + sb.append(m); + } + } + } +} diff --git a/maven/python-runtime/src/main/java/com/codename1/python/runtime/UnicodeTables.java b/maven/python-runtime/src/main/java/com/codename1/python/runtime/UnicodeTables.java new file mode 100644 index 00000000000..0e22b5c36ad --- /dev/null +++ b/maven/python-runtime/src/main/java/com/codename1/python/runtime/UnicodeTables.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.python.runtime; + +/// Generated by `maven/python-runtime/tools/gen_unicode_tables.py` from +/// the Unicode 15.1.0 data of CPython 3.13; do not edit by hand. +/// +/// `FLAGS` is runs of (first char, property bits) over the Basic +/// Multilingual Plane; each mapping table is entries of (char, length, +/// mapped chars), in char order. [Unicode] decodes them. +final class UnicodeTables { + private UnicodeTables() { + } + + static final String[] FLAGS = { + "\000\000\011@\016\000\034@!\00008:\000A\u0085[\000a\u0086{\000\u0085@\u0086\000\u00a0@\u00a1\000\u00aa\u0086\u00ab\000\u00b20\u00b4\000\u00b5\u0086\u00b6\000\u00b90\u00ba\u0086\u00bb\000\u00bc \u00bf\000\u00c0\u0085\u00d7\000\u00d8\u0085\u00df\u0086\u00f7\000\u00f8\u0086\u0100\u0085\u0101\u0086\u0102\u0085\u0103\u0086\u0104\u0085\u0105\u0086\u0106\u0085\u0107\u0086\u0108\u0085\u0109\u0086\u010a\u0085\u010b\u0086\u010c\u0085\u010d\u0086\u010e\u0085\u010f\u0086\u0110\u0085\u0111\u0086\u0112\u0085\u0113\u0086\u0114\u0085\u0115\u0086\u0116\u0085\u0117\u0086\u0118\u0085\u0119\u0086\u011a\u0085\u011b\u0086\u011c\u0085\u011d\u0086\u011e\u0085\u011f\u0086\u0120\u0085\u0121\u0086\u0122\u0085\u0123\u0086\u0124\u0085\u0125\u0086\u0126\u0085\u0127\u0086\u0128\u0085\u0129\u0086\u012a\u0085\u012b\u0086\u012c\u0085\u012d\u0086\u012e\u0085\u012f\u0086\u0130\u0085\u0131\u0086\u0132\u0085\u0133\u0086\u0134\u0085\u0135\u0086\u0136\u0085\u0137\u0086\u0139\u0085\u013a\u0086\u013b\u0085\u013c\u0086\u013d\u0085\u013e\u0086\u013f\u0085\u0140\u0086\u0141\u0085\u0142\u0086\u0143\u0085\u0144\u0086\u0145\u0085\u0146\u0086\u0147\u0085\u0148\u0086\u014a\u0085\u014b\u0086\u014c\u0085\u014d\u0086\u014e\u0085\u014f\u0086\u0150\u0085\u0151\u0086\u0152\u0085\u0153\u0086\u0154\u0085\u0155\u0086\u0156\u0085\u0157\u0086\u0158\u0085\u0159\u0086\u015a\u0085\u015b\u0086\u015c\u0085\u015d\u0086\u015e\u0085\u015f\u0086\u0160\u0085\u0161\u0086\u0162\u0085\u0163\u0086\u0164\u0085\u0165\u0086\u0166\u0085\u0167\u0086\u0168\u0085\u0169\u0086\u016a\u0085\u016b\u0086\u016c\u0085\u016d\u0086\u016e\u0085\u016f\u0086\u0170\u0085\u0171\u0086\u0172\u0085\u0173\u0086\u0174\u0085\u0175\u0086\u0176\u0085\u0177\u0086\u0178\u0085\u017a\u0086\u017b\u0085\u017c\u0086\u017d\u0085\u017e\u0086\u0181\u0085\u0183\u0086\u0184\u0085\u0185\u0086\u0186\u0085\u0188\u0086\u0189\u0085\u018c\u0086\u018e\u0085\u0192\u0086\u0193\u0085\u0195\u0086\u0196\u0085\u0199\u0086\u019c\u0085\u019e\u0086\u019f\u0085\u01a1\u0086\u01a2\u0085\u01a3\u0086\u01a4\u0085\u01a5\u0086\u01a6\u0085\u01a8\u0086\u01a9\u0085\u01aa\u0086\u01ac\u0085\u01ad\u0086\u01ae\u0085\u01b0\u0086\u01b1\u0085\u01b4\u0086\u01b5\u0085\u01b6\u0086\u01b7\u0085\u01b9\u0086\u01bb\004\u01bc\u0085\u01bd\u0086\u01c0\004\u01c4\u0085\u01c5\u0084\u01c6\u0086\u01c7\u0085\u01c8\u0084\u01c9\u0086\u01ca\u0085\u01cb\u0084\u01cc\u0086\u01cd\u0085\u01ce\u0086\u01cf\u0085\u01d0\u0086\u01d1\u0085\u01d2\u0086\u01d3\u0085\u01d4\u0086\u01d5\u0085\u01d6\u0086\u01d7\u0085\u01d8\u0086\u01d9\u0085\u01da\u0086\u01db\u0085\u01dc\u0086\u01de\u0085\u01df\u0086\u01e0\u0085\u01e1\u0086\u01e2\u0085\u01e3\u0086\u01e4\u0085\u01e5\u0086\u01e6\u0085\u01e7\u0086\u01e8\u0085\u01e9\u0086\u01ea\u0085\u01eb\u0086\u01ec\u0085\u01ed\u0086\u01ee\u0085\u01ef\u0086\u01f1\u0085\u01f2\u0084\u01f3\u0086\u01f4\u0085\u01f5\u0086\u01f6\u0085\u01f9\u0086\u01fa\u0085\u01fb\u0086\u01fc\u0085\u01fd\u0086\u01fe\u0085\u01ff\u0086\u0200\u0085\u0201\u0086\u0202\u0085\u0203\u0086\u0204\u0085\u0205\u0086\u0206\u0085\u0207\u0086\u0208\u0085\u0209\u0086\u020a\u0085\u020b\u0086\u020c\u0085\u020d\u0086\u020e\u0085\u020f\u0086\u0210\u0085\u0211\u0086\u0212\u0085\u0213\u0086\u0214\u0085\u0215\u0086\u0216\u0085\u0217\u0086\u0218\u0085\u0219\u0086\u021a\u0085\u021b\u0086\u021c\u0085\u021d\u0086\u021e\u0085\u021f\u0086\u0220\u0085\u0221\u0086\u0222\u0085\u0223\u0086\u0224\u0085\u0225\u0086\u0226\u0085\u0227\u0086\u0228\u0085\u0229\u0086\u022a\u0085\u022b\u0086\u022c\u0085\u022d\u0086\u022e\u0085\u022f\u0086\u0230\u0085\u0231\u0086\u0232\u0085\u0233\u0086\u023a\u0085\u023c\u0086\u023d\u0085\u023f\u0086\u0241\u0085\u0242\u0086\u0243\u0085\u0247\u0086\u0248\u0085\u0249\u0086\u024a\u0085\u024b\u0086\u024c\u0085\u024d\u0086\u024e\u0085\u024f\u0086\u0294\004\u0295\u0086\u02b9\004\u02c0\u0086\u02c2\000\u02c6\004\u02d2\000\u02e0\u0086\u02e5\000\u02ec\004\u02ed\000\u02ee\004\u02ef\000\u0345\u0082\u0346\000\u0370\u0085\u0371\u0086\u0372\u0085\u0373\u0086\u0374\004\u0375\000\u0376\u0085\u0377\u0086\u0378\000\u037a\u0086\u037e\000\u037f\u0085\u0380\000\u0386\u0085\u0387\000\u0388\u0085\u038b\000\u038c\u0085\u038d\000\u038e\u0085\u0390\u0086\u0391\u0085\u03a2\000\u03a3\u0085\u03ac\u0086\u03cf\u0085\u03d0\u0086\u03d2\u0085\u03d5\u0086\u03d8\u0085\u03d9\u0086\u03da\u0085\u03db\u0086\u03dc\u0085\u03dd\u0086\u03de\u0085\u03df\u0086\u03e0\u0085\u03e1\u0086\u03e2\u0085\u03e3\u0086\u03e4\u0085\u03e5\u0086\u03e6\u0085\u03e7\u0086\u03e8\u0085\u03e9\u0086\u03ea\u0085\u03eb\u0086\u03ec\u0085\u03ed\u0086\u03ee\u0085\u03ef\u0086\u03f4\u0085\u03f5\u0086\u03f6\000\u03f7\u0085\u03f8\u0086\u03f9\u0085\u03fb\u0086\u03fd\u0085\u0430\u0086\u0460\u0085\u0461\u0086\u0462\u0085\u0463\u0086\u0464\u0085\u0465\u0086\u0466\u0085\u0467\u0086\u0468\u0085\u0469\u0086\u046a\u0085\u046b\u0086\u046c\u0085\u046d\u0086\u046e\u0085\u046f\u0086\u0470\u0085\u0471\u0086\u0472\u0085\u0473\u0086\u0474\u0085\u0475\u0086\u0476\u0085\u0477\u0086\u0478\u0085\u0479\u0086\u047a\u0085\u047b\u0086\u047c\u0085\u047d\u0086\u047e\u0085\u047f\u0086\u0480\u0085\u0481\u0086\u0482\000\u048a\u0085\u048b\u0086\u048c\u0085\u048d\u0086\u048e\u0085\u048f\u0086\u0490\u0085\u0491\u0086\u0492\u0085\u0493\u0086\u0494\u0085\u0495\u0086\u0496\u0085\u0497\u0086\u0498\u0085\u0499\u0086\u049a\u0085\u049b\u0086\u049c\u0085\u049d\u0086\u049e\u0085\u049f\u0086\u04a0\u0085\u04a1\u0086\u04a2\u0085\u04a3\u0086\u04a4\u0085\u04a5\u0086\u04a6\u0085\u04a7\u0086\u04a8\u0085\u04a9\u0086\u04aa\u0085\u04ab\u0086\u04ac\u0085\u04ad\u0086\u04ae\u0085\u04af\u0086\u04b0\u0085\u04b1\u0086\u04b2\u0085\u04b3\u0086\u04b4\u0085\u04b5\u0086\u04b6\u0085\u04b7\u0086\u04b8\u0085\u04b9\u0086\u04ba\u0085\u04bb\u0086\u04bc\u0085\u04bd\u0086\u04be\u0085\u04bf\u0086\u04c0\u0085\u04c2\u0086\u04c3\u0085\u04c4\u0086\u04c5\u0085\u04c6\u0086\u04c7\u0085\u04c8\u0086\u04c9\u0085\u04ca\u0086\u04cb\u0085\u04cc\u0086\u04cd\u0085\u04ce\u0086\u04d0\u0085\u04d1\u0086\u04d2\u0085\u04d3\u0086\u04d4\u0085\u04d5\u0086\u04d6\u0085\u04d7\u0086\u04d8\u0085\u04d9\u0086\u04da\u0085\u04db\u0086\u04dc\u0085\u04dd\u0086\u04de\u0085\u04df\u0086\u04e0\u0085\u04e1\u0086\u04e2\u0085\u04e3\u0086\u04e4\u0085\u04e5\u0086\u04e6\u0085\u04e7\u0086\u04e8\u0085\u04e9\u0086\u04ea\u0085\u04eb\u0086\u04ec\u0085\u04ed\u0086\u04ee\u0085\u04ef\u0086\u04f0\u0085\u04f1\u0086\u04f2\u0085\u04f3\u0086\u04f4\u0085\u04f5\u0086\u04f6\u0085\u04f7\u0086\u04f8\u0085\u04f9\u0086\u04fa\u0085\u04fb\u0086\u04fc\u0085\u04fd\u0086\u04fe\u0085\u04ff\u0086\u0500\u0085\u0501\u0086\u0502\u0085\u0503\u0086\u0504\u0085\u0505\u0086\u0506\u0085\u0507\u0086\u0508\u0085\u0509\u0086\u050a\u0085\u050b\u0086\u050c\u0085\u050d\u0086\u050e\u0085\u050f\u0086\u0510\u0085\u0511\u0086\u0512\u0085\u0513\u0086\u0514\u0085\u0515\u0086\u0516\u0085\u0517\u0086\u0518\u0085\u0519\u0086\u051a\u0085\u051b\u0086\u051c\u0085\u051d\u0086\u051e\u0085\u051f\u0086\u0520\u0085\u0521\u0086\u0522\u0085\u0523\u0086\u0524\u0085\u0525\u0086\u0526\u0085\u0527\u0086\u0528\u0085\u0529\u0086\u052a\u0085\u052b\u0086\u052c\u0085\u052d\u0086\u052e\u0085\u052f\u0086\u0530\000\u0531\u0085\u0557\000\u0559\004\u055a\000\u0560\u0086\u0589\000\u05d0\004\u05eb\000\u05ef\004\u05f3\000\u0620\004\u064b\000\u06608\u066a\000\u066e\004\u0670\000\u0671\004\u06d4\000\u06d5\004\u06d6\000\u06e5\004\u06e7\000\u06ee\004\u06f08\u06fa\004\u06fd\000\u06ff\004\u0700\000\u0710\004\u0711\000\u0712\004\u0730\000\u074d\004\u07a6\000\u07b1\004\u07b2\000\u07c08\u07ca\004\u07eb\000\u07f4\004\u07f6\000\u07fa\004\u07fb\000\u0800\004\u0816\000\u081a\004\u081b\000\u0824\004\u0825\000\u0828\004\u0829\000\u0840\004\u0859\000\u0860\004\u086b\000\u0870\004\u0888\000\u0889\004\u088f\000\u08a0\004\u08ca\000\u0904\004\u093a\000\u093d\004\u093e\000\u0950\004\u0951\000\u0958\004\u0962\000\u09668\u0970\000\u0971\004\u0981\000\u0985\004\u098d\000\u098f\004\u0991\000\u0993\004\u09a9\000\u09aa\004\u09b1\000\u09b2\004\u09b3\000\u09b6\004\u09ba\000\u09bd\004\u09be\000\u09ce\004\u09cf\000\u09dc\004\u09de\000\u09df\004\u09e2\000\u09e68\u09f0\004\u09f2\000\u09f4 \u09fa\000\u09fc\004\u09fd\000\u0a05\004\u0a0b\000\u0a0f\004\u0a11\000\u0a13\004\u0a29\000\u0a2a\004\u0a31\000\u0a32\004\u0a34\000\u0a35\004\u0a37\000\u0a38\004\u0a3a\000\u0a59\004\u0a5d\000\u0a5e\004\u0a5f\000\u0a668\u0a70\000\u0a72\004\u0a75\000\u0a85\004\u0a8e\000\u0a8f\004\u0a92\000\u0a93\004\u0aa9\000\u0aaa\004\u0ab1\000\u0ab2\004\u0ab4\000\u0ab5\004\u0aba\000\u0abd\004\u0abe\000\u0ad0\004\u0ad1\000\u0ae0\004\u0ae2\000\u0ae68\u0af0\000\u0af9\004\u0afa\000\u0b05\004\u0b0d\000\u0b0f\004\u0b11\000\u0b13\004\u0b29\000\u0b2a\004\u0b31\000\u0b32\004\u0b34\000\u0b35\004\u0b3a\000\u0b3d\004\u0b3e\000\u0b5c\004\u0b5e\000\u0b5f\004\u0b62\000\u0b668\u0b70\000\u0b71\004\u0b72 \u0b78\000\u0b83\004\u0b84\000\u0b85\004\u0b8b\000\u0b8e\004\u0b91\000\u0b92\004\u0b96\000\u0b99\004\u0b9b\000\u0b9c\004\u0b9d\000\u0b9e\004\u0ba0\000\u0ba3\004\u0ba5\000\u0ba8\004\u0bab\000\u0bae\004\u0bba\000\u0bd0\004\u0bd1\000\u0be68\u0bf0 \u0bf3\000\u0c05\004\u0c0d\000\u0c0e\004\u0c11\000\u0c12\004\u0c29\000\u0c2a\004\u0c3a\000\u0c3d\004\u0c3e\000\u0c58\004\u0c5b\000\u0c5d\004\u0c5e\000\u0c60\004\u0c62\000\u0c668\u0c70\000\u0c78 \u0c7f\000\u0c80\004\u0c81\000\u0c85\004\u0c8d\000\u0c8e\004\u0c91\000\u0c92\004\u0ca9\000\u0caa\004\u0cb4\000\u0cb5\004\u0cba\000\u0cbd\004\u0cbe\000\u0cdd\004\u0cdf\000\u0ce0\004\u0ce2\000\u0ce68\u0cf0\000\u0cf1\004\u0cf3\000\u0d04\004\u0d0d\000\u0d0e\004\u0d11\000\u0d12\004\u0d3b\000\u0d3d\004\u0d3e\000\u0d4e\004\u0d4f\000\u0d54\004\u0d57\000\u0d58 \u0d5f\004\u0d62\000\u0d668\u0d70 \u0d79\000\u0d7a\004\u0d80\000\u0d85\004\u0d97\000\u0d9a\004\u0db2\000\u0db3\004\u0dbc\000\u0dbd\004\u0dbe\000\u0dc0\004\u0dc7\000\u0de68\u0df0\000\u0e01\004\u0e31\000\u0e32\004\u0e34\000\u0e40\004\u0e47\000\u0e508\u0e5a\000\u0e81\004\u0e83\000\u0e84\004\u0e85\000\u0e86\004\u0e8b\000\u0e8c\004\u0ea4\000\u0ea5\004\u0ea6\000\u0ea7\004\u0eb1\000\u0eb2\004\u0eb4\000\u0ebd\004\u0ebe\000\u0ec0\004\u0ec5\000\u0ec6\004\u0ec7\000\u0ed08\u0eda\000\u0edc\004\u0ee0\000\u0f00\004\u0f01\000\u0f208\u0f2a \u0f34\000\u0f40\004\u0f48\000\u0f49\004\u0f6d\000\u0f88\004\u0f8d\000\u1000\004\u102b\000\u103f\004\u10408\u104a\000\u1050\004\u1056\000\u105a\004\u105e\000\u1061\004\u1062\000\u1065\004\u1067\000\u106e\004\u1071\000\u1075\004\u1082\000\u108e\004\u108f\000\u10908\u109a\000\u10a0\u0085\u10c6\000\u10c7\u0085\u10c8\000\u10cd\u0085\u10ce\000\u10d0\u0086\u10fb\000\u10fc\u0086\u1100\004\u1249\000\u124a\004\u124e\000\u1250\004\u1257\000\u1258\004\u1259\000\u125a\004\u125e\000\u1260\004\u1289\000\u128a\004\u128e\000\u1290\004\u12b1\000\u12b2\004\u12b6\000\u12b8\004\u12bf\000\u12c0\004\u12c1\000\u12c2\004\u12c6\000\u12c8\004\u12d7\000\u12d8\004\u1311\000\u1312\004\u1316\000\u1318\004\u135b\000\u13690\u1372 \u137d\000\u1380\004\u1390\000\u13a0\u0085\u13f6\000\u13f8\u0086\u13fe\000\u1401\004\u166d\000\u166f\004\u1680@\u1681\004\u169b\000\u16a0\004\u16eb\000\u16ee \u16f1\004\u16f9\000\u1700\004\u1712\000\u171f\004\u1732\000\u1740\004\u1752\000\u1760\004\u176d\000\u176e\004\u1771\000\u1780\004\u17b4\000\u17d7\004\u17d8\000\u17dc\004\u17dd\000\u17e08\u17ea\000\u17f0 \u17fa\000\u18108\u181a\000\u1820\004\u1879\000\u1880\004\u1885\000\u1887\004\u18a9\000\u18aa\004\u18ab\000\u18b0\004\u18f6\000\u1900\004\u191f\000\u19468\u1950\004\u196e\000\u1970\004\u1975\000\u1980\004\u19ac\000\u19b0\004\u19ca\000\u19d08\u19da0\u19db\000\u1a00\004\u1a17\000\u1a20\004\u1a55\000\u1a808\u1a8a\000\u1a908\u1a9a\000\u1aa7\004\u1aa8\000\u1b05\004\u1b34\000\u1b45\004\u1b4d\000\u1b508\u1b5a\000\u1b83\004\u1ba1\000\u1bae\004\u1bb08\u1bba\004\u1be6\000\u1c00\004\u1c24\000\u1c408\u1c4a\000\u1c4d\004\u1c508\u1c5a\004\u1c7e\000\u1c80\u0086\u1c89\000\u1c90\u0085\u1cbb\000\u1cbd\u0085\u1cc0\000\u1ce9\004\u1ced\000\u1cee\004\u1cf4\000\u1cf5\004\u1cf7\000\u1cfa\004\u1cfb\000\u1d00\u0086\u1dc0\000\u1e00\u0085\u1e01\u0086\u1e02\u0085\u1e03\u0086\u1e04\u0085\u1e05\u0086\u1e06\u0085\u1e07\u0086\u1e08\u0085\u1e09\u0086\u1e0a\u0085\u1e0b\u0086\u1e0c\u0085\u1e0d\u0086\u1e0e\u0085\u1e0f\u0086\u1e10\u0085\u1e11\u0086\u1e12\u0085\u1e13\u0086\u1e14\u0085\u1e15\u0086\u1e16\u0085\u1e17\u0086\u1e18\u0085\u1e19\u0086\u1e1a\u0085\u1e1b\u0086\u1e1c\u0085\u1e1d\u0086\u1e1e\u0085\u1e1f\u0086\u1e20\u0085\u1e21\u0086\u1e22\u0085\u1e23\u0086\u1e24\u0085\u1e25\u0086\u1e26\u0085\u1e27\u0086\u1e28\u0085\u1e29\u0086\u1e2a\u0085\u1e2b\u0086\u1e2c\u0085\u1e2d\u0086\u1e2e\u0085\u1e2f\u0086\u1e30\u0085\u1e31\u0086\u1e32\u0085\u1e33\u0086\u1e34\u0085\u1e35\u0086\u1e36\u0085\u1e37\u0086\u1e38\u0085\u1e39\u0086\u1e3a\u0085\u1e3b\u0086\u1e3c\u0085\u1e3d\u0086\u1e3e\u0085\u1e3f\u0086\u1e40\u0085\u1e41\u0086\u1e42\u0085\u1e43\u0086\u1e44\u0085\u1e45\u0086\u1e46\u0085\u1e47\u0086\u1e48\u0085\u1e49\u0086\u1e4a\u0085\u1e4b\u0086\u1e4c\u0085\u1e4d\u0086\u1e4e\u0085\u1e4f\u0086\u1e50\u0085\u1e51\u0086\u1e52\u0085\u1e53\u0086\u1e54\u0085\u1e55\u0086\u1e56\u0085\u1e57\u0086\u1e58\u0085\u1e59\u0086\u1e5a\u0085\u1e5b\u0086\u1e5c\u0085\u1e5d\u0086\u1e5e\u0085\u1e5f\u0086\u1e60\u0085\u1e61\u0086\u1e62\u0085\u1e63\u0086\u1e64\u0085\u1e65\u0086\u1e66\u0085\u1e67\u0086\u1e68\u0085\u1e69\u0086\u1e6a\u0085\u1e6b\u0086\u1e6c\u0085\u1e6d\u0086\u1e6e\u0085\u1e6f\u0086\u1e70\u0085\u1e71\u0086\u1e72\u0085\u1e73\u0086\u1e74\u0085\u1e75\u0086\u1e76\u0085\u1e77\u0086\u1e78\u0085\u1e79\u0086\u1e7a\u0085\u1e7b\u0086\u1e7c\u0085\u1e7d\u0086\u1e7e\u0085\u1e7f\u0086\u1e80\u0085\u1e81\u0086\u1e82\u0085\u1e83\u0086\u1e84\u0085\u1e85\u0086\u1e86\u0085\u1e87\u0086\u1e88\u0085\u1e89\u0086\u1e8a\u0085\u1e8b\u0086\u1e8c\u0085\u1e8d\u0086\u1e8e\u0085\u1e8f\u0086\u1e90\u0085\u1e91\u0086\u1e92\u0085\u1e93\u0086\u1e94\u0085\u1e95\u0086\u1e9e\u0085\u1e9f\u0086\u1ea0\u0085\u1ea1\u0086\u1ea2\u0085\u1ea3\u0086\u1ea4\u0085\u1ea5\u0086\u1ea6\u0085\u1ea7\u0086\u1ea8\u0085\u1ea9\u0086\u1eaa\u0085\u1eab\u0086\u1eac\u0085\u1ead\u0086\u1eae\u0085\u1eaf\u0086\u1eb0\u0085\u1eb1\u0086\u1eb2\u0085\u1eb3\u0086\u1eb4\u0085\u1eb5\u0086\u1eb6\u0085\u1eb7\u0086\u1eb8\u0085\u1eb9\u0086\u1eba\u0085\u1ebb\u0086\u1ebc\u0085\u1ebd\u0086\u1ebe\u0085\u1ebf\u0086\u1ec0\u0085\u1ec1\u0086\u1ec2\u0085\u1ec3\u0086\u1ec4\u0085\u1ec5\u0086\u1ec6\u0085\u1ec7\u0086\u1ec8\u0085\u1ec9\u0086\u1eca\u0085\u1ecb\u0086\u1ecc\u0085\u1ecd\u0086\u1ece\u0085\u1ecf\u0086\u1ed0\u0085\u1ed1\u0086\u1ed2\u0085\u1ed3\u0086\u1ed4\u0085\u1ed5\u0086\u1ed6\u0085\u1ed7\u0086\u1ed8\u0085\u1ed9\u0086\u1eda\u0085\u1edb\u0086\u1edc\u0085\u1edd\u0086\u1ede\u0085\u1edf\u0086\u1ee0\u0085\u1ee1\u0086\u1ee2\u0085\u1ee3\u0086\u1ee4\u0085\u1ee5\u0086\u1ee6\u0085\u1ee7\u0086\u1ee8\u0085\u1ee9\u0086\u1eea\u0085\u1eeb\u0086\u1eec\u0085\u1eed\u0086\u1eee\u0085\u1eef\u0086\u1ef0\u0085\u1ef1\u0086\u1ef2\u0085\u1ef3\u0086\u1ef4\u0085\u1ef5\u0086\u1ef6\u0085\u1ef7\u0086\u1ef8\u0085\u1ef9\u0086\u1efa\u0085\u1efb\u0086\u1efc\u0085\u1efd\u0086\u1efe\u0085\u1eff\u0086\u1f08\u0085\u1f10\u0086\u1f16\000\u1f18\u0085\u1f1e\000\u1f20\u0086\u1f28\u0085\u1f30\u0086\u1f38\u0085\u1f40\u0086\u1f46\000\u1f48\u0085\u1f4e\000\u1f50\u0086\u1f58\000\u1f59\u0085\u1f5a\000\u1f5b\u0085\u1f5c\000\u1f5d\u0085\u1f5e\000\u1f5f\u0085\u1f60\u0086\u1f68\u0085\u1f70\u0086\u1f7e\000\u1f80\u0086\u1f88\u0084\u1f90\u0086\u1f98\u0084\u1fa0\u0086\u1fa8\u0084\u1fb0\u0086\u1fb5\000\u1fb6\u0086\u1fb8\u0085\u1fbc\u0084\u1fbd\000\u1fbe\u0086\u1fbf\000\u1fc2\u0086\u1fc5\000\u1fc6\u0086\u1fc8\u0085\u1fcc\u0084\u1fcd\000\u1fd0\u0086\u1fd4\000\u1fd6\u0086\u1fd8\u0085\u1fdc\000\u1fe0\u0086\u1fe8\u0085\u1fed\000\u1ff2\u0086\u1ff5\000\u1ff6\u0086\u1ff8\u0085\u1ffc\u0084\u1ffd\000\u2000@\u200b\000\u2028@\u202a\000\u202f@\u2030\000\u205f@\u2060\000\u20700\u2071\u0086\u2072\000\u20740\u207a\000\u207f\u0086\u20800\u208a\000\u2090\u0086\u209d\000\u2102\u0085\u2103\000\u2107\u0085\u2108\000\u210a\u0086\u210b\u0085\u210e\u0086\u2110\u0085\u2113\u0086\u2114\000\u2115\u0085\u2116\000\u2119\u0085\u211e\000\u2124\u0085\u2125\000\u2126\u0085\u2127\000\u2128\u0085\u2129\000\u212a\u0085\u212e\000\u212f\u0086\u2130\u0085\u2134\u0086\u2135\004\u2139\u0086\u213a\000\u213c\u0086\u213e\u0085\u2140\000\u2145\u0085\u2146\u0086\u214a\000\u214e\u0086\u214f\000\u2150 \u2160\u00a1\u2170\u00a2\u2180 \u2183\u0085\u2184\u0086\u2185 \u218a\000\u24600\u2469 \u24740\u247d \u24880\u2491 \u249c\000\u24b6\u0081\u24d0\u0082\u24ea0\u24eb \u24f50\u24fe \u24ff0\u2500\000\u27760\u277f \u27800\u2789 \u278a0\u2793 \u2794\000\u2c00\u0085\u2c30\u0086\u2c60\u0085\u2c61\u0086\u2c62\u0085\u2c65\u0086\u2c67\u0085\u2c68\u0086\u2c69\u0085\u2c6a\u0086\u2c6b\u0085\u2c6c\u0086\u2c6d\u0085\u2c71\u0086\u2c72\u0085\u2c73\u0086\u2c75\u0085\u2c76\u0086\u2c7e\u0085\u2c81\u0086\u2c82\u0085\u2c83\u0086\u2c84\u0085\u2c85\u0086\u2c86\u0085\u2c87\u0086\u2c88\u0085\u2c89\u0086\u2c8a\u0085\u2c8b\u0086\u2c8c\u0085\u2c8d\u0086\u2c8e\u0085\u2c8f\u0086\u2c90\u0085\u2c91\u0086\u2c92\u0085\u2c93\u0086\u2c94\u0085\u2c95\u0086\u2c96\u0085\u2c97\u0086\u2c98\u0085\u2c99\u0086\u2c9a\u0085\u2c9b\u0086\u2c9c\u0085\u2c9d\u0086\u2c9e\u0085\u2c9f\u0086\u2ca0\u0085\u2ca1\u0086\u2ca2\u0085\u2ca3\u0086\u2ca4\u0085\u2ca5\u0086\u2ca6\u0085\u2ca7\u0086\u2ca8\u0085\u2ca9\u0086\u2caa\u0085\u2cab\u0086\u2cac\u0085\u2cad\u0086\u2cae\u0085\u2caf\u0086\u2cb0\u0085\u2cb1\u0086\u2cb2\u0085\u2cb3\u0086\u2cb4\u0085\u2cb5\u0086\u2cb6\u0085\u2cb7\u0086\u2cb8\u0085\u2cb9\u0086\u2cba\u0085\u2cbb\u0086\u2cbc\u0085\u2cbd\u0086\u2cbe\u0085\u2cbf\u0086\u2cc0\u0085\u2cc1\u0086\u2cc2\u0085\u2cc3\u0086\u2cc4\u0085\u2cc5\u0086\u2cc6\u0085\u2cc7\u0086\u2cc8\u0085\u2cc9\u0086\u2cca\u0085\u2ccb\u0086\u2ccc\u0085\u2ccd\u0086\u2cce\u0085\u2ccf\u0086\u2cd0\u0085\u2cd1\u0086\u2cd2\u0085\u2cd3\u0086\u2cd4\u0085\u2cd5\u0086\u2cd6\u0085\u2cd7\u0086\u2cd8\u0085\u2cd9\u0086\u2cda\u0085\u2cdb\u0086\u2cdc\u0085\u2cdd\u0086\u2cde\u0085\u2cdf\u0086\u2ce0\u0085\u2ce1\u0086\u2ce2\u0085\u2ce3\u0086\u2ce5\000\u2ceb\u0085\u2cec\u0086\u2ced\u0085\u2cee\u0086\u2cef\000\u2cf2\u0085\u2cf3\u0086\u2cf4\000\u2cfd \u2cfe\000\u2d00\u0086\u2d26\000\u2d27\u0086\u2d28\000\u2d2d\u0086\u2d2e\000\u2d30\004\u2d68\000\u2d6f\004\u2d70\000\u2d80\004\u2d97\000\u2da0\004\u2da7\000\u2da8\004\u2daf\000\u2db0\004\u2db7\000\u2db8\004\u2dbf\000\u2dc0\004\u2dc7\000\u2dc8\004\u2dcf\000\u2dd0\004\u2dd7\000\u2dd8\004\u2ddf\000\u2e2f\004\u2e30\000\u3000@\u3001\000\u3005\004\u3007 \u3008\000\u3021 \u302a\000\u3031\004\u3036\000\u3038 \u303b\004\u303d\000\u3041\004\u3097\000\u309d\004\u30a0\000\u30a1\004\u30fb\000\u30fc\004\u3100\000\u3105\004\u3130\000\u3131\004\u318f\000\u3192 \u3196\000\u31a0\004\u31c0\000\u31f0\004\u3200\000\u3220 \u322a\000\u3248 \u3250\000\u3251 \u3260\000\u3280 \u328a\000\u32b1 \u32c0\000\u3400\004\u3405$\u3406\004\u3483$\u3484\004\u382a$\u382b\004\u3b4d$\u3b4e\004\u4dc0\000\u4e00$\u4e01\004\u4e03$\u4e04\004\u4e07$\u4e08\004\u4e09$\u4e0a\004\u4e24$\u4e25\004\u4e5d$\u4e5e\004\u4e8c$\u4e8d\004\u4e94$\u4e95\004\u4e96$\u4e97\004\u4eac$\u4ead\004\u4ebf$\u4ec1\004\u4edf$\u4ee0\004\u4ee8$\u4ee9\004\u4f0d$\u4f0e\004\u4f70$\u4f71\004\u4fe9$\u4fea\004\u5006$\u5007\004\u5104$\u5105\004\u5146$\u5147\004\u5169$\u516a\004\u516b$\u516c\004\u516d$\u516e\004\u5341$\u5342\004\u5343$\u5346\004\u534c$\u534d\004\u53c1$\u53c5\004\u56db$\u56dc\004\u58f1$\u58f2\004\u58f9$\u58fa\004\u5e7a$\u5e7b\004\u5efe$\u5f00\004\u5f0c$\u5f0f\004\u5f10$\u5f11\004\u62d0$\u62d1\004\u62fe$\u62ff\004\u634c$\u634d\004\u67d2$\u67d3\004\u6d1e$\u6d1f\004\u6f06$\u6f07\004\u7396$\u7397\004\u767e$\u767f\004\u7695$\u7696\004\u79ed$\u79ee\004\u8086$\u8087\004\u842c$\u842d\004\u8cae$\u8caf\004\u8cb3$\u8cb4\004\u8d30$\u8d31\004\u920e$\u920f\004\u94a9$\u94aa\004\u9621$\u9622\004\u9646$\u9647\004\u964c$\u964d\004\u9678$\u9679\004\u96f6$\u96f7\004\ua48d\000\ua4d0\004\ua4fe\000\ua500\004\ua60d\000\ua610\004\ua6208\ua62a\004\ua62c\000\ua640\u0085\ua641\u0086\ua642\u0085\ua643\u0086\ua644\u0085\ua645\u0086\ua646\u0085\ua647\u0086\ua648\u0085\ua649\u0086\ua64a\u0085\ua64b\u0086\ua64c\u0085\ua64d\u0086\ua64e\u0085\ua64f\u0086\ua650\u0085\ua651\u0086\ua652\u0085\ua653\u0086\ua654\u0085\ua655\u0086\ua656\u0085\ua657\u0086\ua658\u0085\ua659\u0086\ua65a\u0085\ua65b\u0086\ua65c\u0085\ua65d\u0086\ua65e\u0085\ua65f\u0086\ua660\u0085\ua661\u0086\ua662\u0085\ua663\u0086\ua664\u0085\ua665\u0086\ua666\u0085\ua667\u0086\ua668\u0085\ua669\u0086\ua66a\u0085\ua66b\u0086\ua66c\u0085\ua66d\u0086\ua66e\004\ua66f\000\ua67f\004\ua680\u0085\ua681\u0086\ua682\u0085\ua683\u0086\ua684\u0085\ua685\u0086\ua686\u0085\ua687\u0086\ua688\u0085\ua689\u0086\ua68a\u0085\ua68b\u0086\ua68c\u0085\ua68d\u0086\ua68e\u0085\ua68f\u0086\ua690\u0085\ua691\u0086\ua692\u0085\ua693\u0086\ua694\u0085\ua695\u0086\ua696\u0085\ua697\u0086\ua698\u0085\ua699\u0086\ua69a\u0085\ua69b\u0086\ua69e\000\ua6a0\004\ua6e6 \ua6f0\000\ua717\004\ua720\000\ua722\u0085\ua723\u0086\ua724\u0085\ua725\u0086\ua726\u0085\ua727\u0086\ua728\u0085\ua729\u0086\ua72a\u0085\ua72b\u0086\ua72c\u0085\ua72d\u0086\ua72e\u0085\ua72f\u0086\ua732\u0085\ua733\u0086\ua734\u0085\ua735\u0086\ua736\u0085\ua737\u0086\ua738\u0085\ua739\u0086\ua73a\u0085\ua73b\u0086\ua73c\u0085\ua73d\u0086\ua73e\u0085\ua73f\u0086\ua740\u0085\ua741\u0086\ua742\u0085\ua743\u0086\ua744\u0085\ua745\u0086\ua746\u0085\ua747\u0086\ua748\u0085\ua749\u0086\ua74a\u0085\ua74b\u0086\ua74c\u0085\ua74d\u0086\ua74e\u0085\ua74f\u0086\ua750\u0085\ua751\u0086\ua752\u0085\ua753\u0086\ua754\u0085\ua755\u0086\ua756\u0085\ua757\u0086\ua758\u0085\ua759\u0086\ua75a\u0085\ua75b\u0086\ua75c\u0085\ua75d\u0086\ua75e\u0085\ua75f\u0086\ua760\u0085\ua761\u0086\ua762\u0085\ua763\u0086\ua764\u0085\ua765\u0086\ua766\u0085\ua767\u0086\ua768\u0085\ua769\u0086\ua76a\u0085\ua76b\u0086\ua76c\u0085\ua76d\u0086\ua76e\u0085\ua76f\u0086\ua779\u0085\ua77a\u0086\ua77b\u0085\ua77c\u0086\ua77d\u0085\ua77f\u0086\ua780\u0085\ua781\u0086\ua782\u0085\ua783\u0086\ua784\u0085\ua785\u0086\ua786\u0085\ua787\u0086\ua788\004\ua789\000\ua78b\u0085\ua78c\u0086\ua78d\u0085\ua78e\u0086\ua78f\004\ua790\u0085\ua791\u0086\ua792\u0085\ua793\u0086\ua796\u0085\ua797\u0086\ua798\u0085\ua799\u0086\ua79a\u0085\ua79b\u0086\ua79c\u0085\ua79d\u0086\ua79e\u0085\ua79f\u0086\ua7a0\u0085\ua7a1\u0086\ua7a2\u0085\ua7a3\u0086\ua7a4\u0085\ua7a5\u0086\ua7a6\u0085\ua7a7\u0086\ua7a8\u0085\ua7a9\u0086\ua7aa\u0085\ua7af\u0086\ua7b0\u0085\ua7b5\u0086\ua7b6\u0085\ua7b7\u0086\ua7b8\u0085\ua7b9\u0086\ua7ba\u0085\ua7bb\u0086\ua7bc\u0085\ua7bd\u0086\ua7be\u0085\ua7bf\u0086\ua7c0\u0085\ua7c1\u0086\ua7c2\u0085\ua7c3\u0086\ua7c4\u0085\ua7c8\u0086\ua7c9\u0085\ua7ca\u0086\ua7cb\000\ua7d0\u0085\ua7d1\u0086\ua7d2\000\ua7d3\u0086\ua7d4\000\ua7d5\u0086\ua7d6\u0085\ua7d7\u0086\ua7d8\u0085\ua7d9\u0086\ua7da\000\ua7f2\u0086\ua7f5\u0085\ua7f6\u0086\ua7f7\004\ua7f8\u0086\ua7fb\004\ua802\000\ua803\004\ua806\000\ua807\004\ua80b\000\ua80c\004\ua823\000\ua830 \ua836\000\ua840\004\ua874\000\ua882\004\ua8b4\000\ua8d08\ua8da\000\ua8f2\004\ua8f8\000\ua8fb\004\ua8fc\000\ua8fd\004\ua8ff\000\ua9008\ua90a\004\ua926\000\ua930\004\ua947\000\ua960\004\ua97d\000\ua984\004\ua9b3\000\ua9cf\004\ua9d08\ua9da\000\ua9e0\004\ua9e5\000\ua9e6\004\ua9f08\ua9fa\004\ua9ff\000\uaa00\004\uaa29\000\uaa40\004\uaa43\000\uaa44\004\uaa4c\000\uaa508\uaa5a\000\uaa60\004\uaa77\000\uaa7a\004\uaa7b\000\uaa7e\004\uaab0\000\uaab1\004\uaab2\000\uaab5\004\uaab7\000\uaab9\004\uaabe\000\uaac0\004\uaac1\000\uaac2\004\uaac3\000\uaadb\004\uaade\000\uaae0\004\uaaeb\000\uaaf2\004\uaaf5\000\uab01\004\uab07\000\uab09\004\uab0f\000\uab11\004\uab17\000\uab20\004\uab27\000\uab28\004\uab2f\000\uab30\u0086\uab5b\000\uab5c\u0086\uab6a\000\uab70\u0086\uabc0\004\uabe3\000\uabf08\uabfa\000\uac00\004\ud7a4\000\ud7b0\004\ud7c7\000\ud7cb\004\ud7fc\000\uf900\004\uf96b$\uf96c\004\uf973$\uf974\004\uf978$\uf979\004\uf9b2$\uf9b3\004\uf9d1$\uf9d2\004\uf9d3$\uf9d4\004\uf9fd$\uf9fe\004\ufa6e\000\ufa70\004\ufada\000\ufb00\u0086\ufb07\000\ufb13\u0086\ufb18\000\ufb1d\004\ufb1e\000\ufb1f\004\ufb29\000\ufb2a\004\ufb37\000\ufb38\004\ufb3d\000\ufb3e\004\ufb3f\000\ufb40\004\ufb42\000\ufb43\004\ufb45\000\ufb46\004\ufbb2\000\ufbd3\004\ufd3e\000\ufd50\004\ufd90\000\ufd92\004\ufdc8\000\ufdf0\004\ufdfc\000\ufe70\004\ufe75\000\ufe76\004\ufefd\000\uff108\uff1a\000\uff21\u0085\uff3b\000\uff41\u0086\uff5b\000\uff66\004\uffbf\000\uffc2\004\uffc8\000\uffca\004\uffd0\000\uffd2\004\uffd8\000\uffda\004\uffdd\000", + }; + + static final String[] LOWER = { + "A\001aB\001bC\001cD\001dE\001eF\001fG\001gH\001hI\001iJ\001jK\001kL\001lM\001mN\001nO\001oP\001pQ\001qR\001rS\001sT\001tU\001uV\001vW\001wX\001xY\001yZ\001z\u00c0\001\u00e0\u00c1\001\u00e1\u00c2\001\u00e2\u00c3\001\u00e3\u00c4\001\u00e4\u00c5\001\u00e5\u00c6\001\u00e6\u00c7\001\u00e7\u00c8\001\u00e8\u00c9\001\u00e9\u00ca\001\u00ea\u00cb\001\u00eb\u00cc\001\u00ec\u00cd\001\u00ed\u00ce\001\u00ee\u00cf\001\u00ef\u00d0\001\u00f0\u00d1\001\u00f1\u00d2\001\u00f2\u00d3\001\u00f3\u00d4\001\u00f4\u00d5\001\u00f5\u00d6\001\u00f6\u00d8\001\u00f8\u00d9\001\u00f9\u00da\001\u00fa\u00db\001\u00fb\u00dc\001\u00fc\u00dd\001\u00fd\u00de\001\u00fe\u0100\001\u0101\u0102\001\u0103\u0104\001\u0105\u0106\001\u0107\u0108\001\u0109\u010a\001\u010b\u010c\001\u010d\u010e\001\u010f\u0110\001\u0111\u0112\001\u0113\u0114\001\u0115\u0116\001\u0117\u0118\001\u0119\u011a\001\u011b\u011c\001\u011d\u011e\001\u011f\u0120\001\u0121\u0122\001\u0123\u0124\001\u0125\u0126\001\u0127\u0128\001\u0129\u012a\001\u012b\u012c\001\u012d\u012e\001\u012f\u0130\002i\u0307\u0132\001\u0133\u0134\001\u0135\u0136\001\u0137\u0139\001\u013a\u013b\001\u013c\u013d\001\u013e\u013f\001\u0140\u0141\001\u0142\u0143\001\u0144\u0145\001\u0146\u0147\001\u0148\u014a\001\u014b\u014c\001\u014d\u014e\001\u014f\u0150\001\u0151\u0152\001\u0153\u0154\001\u0155\u0156\001\u0157\u0158\001\u0159\u015a\001\u015b\u015c\001\u015d\u015e\001\u015f\u0160\001\u0161\u0162\001\u0163\u0164\001\u0165\u0166\001\u0167\u0168\001\u0169\u016a\001\u016b\u016c\001\u016d\u016e\001\u016f\u0170\001\u0171\u0172\001\u0173\u0174\001\u0175\u0176\001\u0177\u0178\001\u00ff\u0179\001\u017a\u017b\001\u017c\u017d\001\u017e\u0181\001\u0253\u0182\001\u0183\u0184\001\u0185\u0186\001\u0254\u0187\001\u0188\u0189\001\u0256\u018a\001\u0257\u018b\001\u018c\u018e\001\u01dd\u018f\001\u0259\u0190\001\u025b\u0191\001\u0192\u0193\001\u0260\u0194\001\u0263\u0196\001\u0269\u0197\001\u0268\u0198\001\u0199\u019c\001\u026f\u019d\001\u0272\u019f\001\u0275\u01a0\001\u01a1\u01a2\001\u01a3\u01a4\001\u01a5\u01a6\001\u0280\u01a7\001\u01a8\u01a9\001\u0283\u01ac\001\u01ad\u01ae\001\u0288\u01af\001\u01b0\u01b1\001\u028a\u01b2\001\u028b\u01b3\001\u01b4\u01b5\001\u01b6\u01b7\001\u0292\u01b8\001\u01b9\u01bc\001\u01bd\u01c4\001\u01c6\u01c5\001\u01c6\u01c7\001\u01c9\u01c8\001\u01c9\u01ca\001\u01cc\u01cb\001\u01cc\u01cd\001\u01ce\u01cf\001\u01d0\u01d1\001\u01d2\u01d3\001\u01d4\u01d5\001\u01d6\u01d7\001\u01d8\u01d9\001\u01da\u01db\001\u01dc\u01de\001\u01df\u01e0\001\u01e1\u01e2\001\u01e3\u01e4\001\u01e5\u01e6\001\u01e7\u01e8\001\u01e9\u01ea\001\u01eb\u01ec\001\u01ed\u01ee\001\u01ef\u01f1\001\u01f3\u01f2\001\u01f3\u01f4\001\u01f5\u01f6\001\u0195\u01f7\001\u01bf\u01f8\001\u01f9\u01fa\001\u01fb\u01fc\001\u01fd\u01fe\001\u01ff\u0200\001\u0201\u0202\001\u0203\u0204\001\u0205\u0206\001\u0207\u0208\001\u0209\u020a\001\u020b\u020c\001\u020d\u020e\001\u020f\u0210\001\u0211\u0212\001\u0213\u0214\001\u0215\u0216\001\u0217\u0218\001\u0219\u021a\001\u021b\u021c\001\u021d\u021e\001\u021f\u0220\001\u019e\u0222\001\u0223\u0224\001\u0225\u0226\001\u0227\u0228\001\u0229\u022a\001\u022b\u022c\001\u022d\u022e\001\u022f\u0230\001\u0231\u0232\001\u0233\u023a\001\u2c65\u023b\001\u023c\u023d\001\u019a\u023e\001\u2c66\u0241\001\u0242\u0243\001\u0180\u0244\001\u0289\u0245\001\u028c\u0246\001\u0247\u0248\001\u0249\u024a\001\u024b\u024c\001\u024d\u024e\001\u024f\u0370\001\u0371\u0372\001\u0373\u0376\001\u0377\u037f\001\u03f3\u0386\001\u03ac\u0388\001\u03ad\u0389\001\u03ae\u038a\001\u03af\u038c\001\u03cc\u038e\001\u03cd\u038f\001\u03ce\u0391\001\u03b1\u0392\001\u03b2\u0393\001\u03b3\u0394\001\u03b4\u0395\001\u03b5\u0396\001\u03b6\u0397\001\u03b7\u0398\001\u03b8\u0399\001\u03b9\u039a\001\u03ba\u039b\001\u03bb\u039c\001\u03bc\u039d\001\u03bd\u039e\001\u03be\u039f\001\u03bf\u03a0\001\u03c0\u03a1\001\u03c1\u03a3\001\u03c3\u03a4\001\u03c4\u03a5\001\u03c5\u03a6\001\u03c6\u03a7\001\u03c7\u03a8\001\u03c8\u03a9\001\u03c9\u03aa\001\u03ca\u03ab\001\u03cb\u03cf\001\u03d7\u03d8\001\u03d9\u03da\001\u03db\u03dc\001\u03dd\u03de\001\u03df\u03e0\001\u03e1\u03e2\001\u03e3\u03e4\001\u03e5\u03e6\001\u03e7\u03e8\001\u03e9\u03ea\001\u03eb\u03ec\001\u03ed\u03ee\001\u03ef\u03f4\001\u03b8\u03f7\001\u03f8\u03f9\001\u03f2\u03fa\001\u03fb\u03fd\001\u037b\u03fe\001\u037c\u03ff\001\u037d\u0400\001\u0450\u0401\001\u0451\u0402\001\u0452\u0403\001\u0453\u0404\001\u0454\u0405\001\u0455\u0406\001\u0456\u0407\001\u0457\u0408\001\u0458\u0409\001\u0459\u040a\001\u045a\u040b\001\u045b\u040c\001\u045c\u040d\001\u045d\u040e\001\u045e\u040f\001\u045f\u0410\001\u0430\u0411\001\u0431\u0412\001\u0432\u0413\001\u0433\u0414\001\u0434\u0415\001\u0435\u0416\001\u0436\u0417\001\u0437\u0418\001\u0438\u0419\001\u0439\u041a\001\u043a\u041b\001\u043b\u041c\001\u043c\u041d\001\u043d\u041e\001\u043e\u041f\001\u043f\u0420\001\u0440\u0421\001\u0441\u0422\001\u0442\u0423\001\u0443\u0424\001\u0444\u0425\001\u0445\u0426\001\u0446\u0427\001\u0447\u0428\001\u0448\u0429\001\u0449\u042a\001\u044a\u042b\001\u044b\u042c\001\u044c\u042d\001\u044d\u042e\001\u044e\u042f\001\u044f\u0460\001\u0461\u0462\001\u0463\u0464\001\u0465\u0466\001\u0467\u0468\001\u0469\u046a\001\u046b\u046c\001\u046d\u046e\001\u046f\u0470\001\u0471\u0472\001\u0473\u0474\001\u0475\u0476\001\u0477\u0478\001\u0479\u047a\001\u047b\u047c\001\u047d\u047e\001\u047f\u0480\001\u0481\u048a\001\u048b\u048c\001\u048d\u048e\001\u048f\u0490\001\u0491\u0492\001\u0493\u0494\001\u0495\u0496\001\u0497\u0498\001\u0499\u049a\001\u049b\u049c\001\u049d\u049e\001\u049f\u04a0\001\u04a1\u04a2\001\u04a3\u04a4\001\u04a5\u04a6\001\u04a7\u04a8\001\u04a9\u04aa\001\u04ab\u04ac\001\u04ad\u04ae\001\u04af\u04b0\001\u04b1\u04b2\001\u04b3\u04b4\001\u04b5\u04b6\001\u04b7\u04b8\001\u04b9\u04ba\001\u04bb\u04bc\001\u04bd\u04be\001\u04bf\u04c0\001\u04cf\u04c1\001\u04c2\u04c3\001\u04c4\u04c5\001\u04c6\u04c7\001\u04c8\u04c9\001\u04ca\u04cb\001\u04cc\u04cd\001\u04ce\u04d0\001\u04d1\u04d2\001\u04d3\u04d4\001\u04d5\u04d6\001\u04d7\u04d8\001\u04d9\u04da\001\u04db\u04dc\001\u04dd\u04de\001\u04df\u04e0\001\u04e1\u04e2\001\u04e3\u04e4\001\u04e5\u04e6\001\u04e7\u04e8\001\u04e9\u04ea\001\u04eb\u04ec\001\u04ed\u04ee\001\u04ef\u04f0\001\u04f1\u04f2\001\u04f3\u04f4\001\u04f5\u04f6\001\u04f7\u04f8\001\u04f9\u04fa\001\u04fb\u04fc\001\u04fd\u04fe\001\u04ff\u0500\001\u0501\u0502\001\u0503\u0504\001\u0505\u0506\001\u0507\u0508\001\u0509\u050a\001\u050b\u050c\001\u050d\u050e\001\u050f\u0510\001\u0511\u0512\001\u0513\u0514\001\u0515\u0516\001\u0517\u0518\001\u0519\u051a\001\u051b\u051c\001\u051d\u051e\001\u051f\u0520\001\u0521\u0522\001\u0523\u0524\001\u0525\u0526\001\u0527\u0528\001\u0529\u052a\001\u052b\u052c\001\u052d\u052e\001\u052f\u0531\001\u0561\u0532\001\u0562\u0533\001\u0563\u0534\001\u0564\u0535\001\u0565\u0536\001\u0566\u0537\001\u0567\u0538\001\u0568\u0539\001\u0569\u053a\001\u056a\u053b\001\u056b\u053c\001\u056c\u053d\001\u056d\u053e\001\u056e\u053f\001\u056f\u0540\001\u0570\u0541\001\u0571\u0542\001\u0572\u0543\001\u0573\u0544\001\u0574\u0545\001\u0575\u0546\001\u0576\u0547\001\u0577\u0548\001\u0578\u0549\001\u0579\u054a\001\u057a\u054b\001\u057b\u054c\001\u057c\u054d\001\u057d\u054e\001\u057e\u054f\001\u057f\u0550\001\u0580\u0551\001\u0581\u0552\001\u0582\u0553\001\u0583\u0554\001\u0584\u0555\001\u0585\u0556\001\u0586\u10a0\001\u2d00\u10a1\001\u2d01\u10a2\001\u2d02\u10a3\001\u2d03\u10a4\001\u2d04\u10a5\001\u2d05\u10a6\001\u2d06\u10a7\001\u2d07\u10a8\001\u2d08\u10a9\001\u2d09\u10aa\001\u2d0a\u10ab\001\u2d0b\u10ac\001\u2d0c\u10ad\001\u2d0d\u10ae\001\u2d0e\u10af\001\u2d0f\u10b0\001\u2d10\u10b1\001\u2d11\u10b2\001\u2d12\u10b3\001\u2d13\u10b4\001\u2d14\u10b5\001\u2d15\u10b6\001\u2d16\u10b7\001\u2d17\u10b8\001\u2d18\u10b9\001\u2d19\u10ba\001\u2d1a\u10bb\001\u2d1b\u10bc\001\u2d1c\u10bd\001\u2d1d\u10be\001\u2d1e\u10bf\001\u2d1f\u10c0\001\u2d20\u10c1\001\u2d21\u10c2\001\u2d22\u10c3\001\u2d23\u10c4\001\u2d24\u10c5\001\u2d25\u10c7\001\u2d27\u10cd\001\u2d2d\u13a0\001\uab70\u13a1\001\uab71\u13a2\001\uab72\u13a3\001\uab73\u13a4\001\uab74\u13a5\001\uab75\u13a6\001\uab76\u13a7\001\uab77\u13a8\001\uab78\u13a9\001\uab79\u13aa\001\uab7a\u13ab\001\uab7b\u13ac\001\uab7c\u13ad\001\uab7d\u13ae\001\uab7e\u13af\001\uab7f\u13b0\001\uab80\u13b1\001\uab81\u13b2\001\uab82\u13b3\001\uab83\u13b4\001\uab84\u13b5\001\uab85\u13b6\001\uab86\u13b7\001\uab87\u13b8\001\uab88\u13b9\001\uab89\u13ba\001\uab8a\u13bb\001\uab8b\u13bc\001\uab8c\u13bd\001\uab8d\u13be\001\uab8e\u13bf\001\uab8f\u13c0\001\uab90\u13c1\001\uab91\u13c2\001\uab92\u13c3\001\uab93\u13c4\001\uab94\u13c5\001\uab95\u13c6\001\uab96\u13c7\001\uab97\u13c8\001\uab98\u13c9\001\uab99\u13ca\001\uab9a\u13cb\001\uab9b\u13cc\001\uab9c\u13cd\001\uab9d\u13ce\001\uab9e\u13cf\001\uab9f\u13d0\001\uaba0\u13d1\001\uaba1\u13d2\001\uaba2\u13d3\001\uaba3\u13d4\001\uaba4\u13d5\001\uaba5\u13d6\001\uaba6\u13d7\001\uaba7\u13d8\001\uaba8\u13d9\001\uaba9\u13da\001\uabaa\u13db\001\uabab\u13dc\001\uabac\u13dd\001\uabad\u13de\001\uabae\u13df\001\uabaf\u13e0\001\uabb0\u13e1\001\uabb1\u13e2\001\uabb2\u13e3\001\uabb3\u13e4\001\uabb4\u13e5\001\uabb5\u13e6\001\uabb6\u13e7\001\uabb7\u13e8\001\uabb8\u13e9\001\uabb9\u13ea\001\uabba\u13eb\001\uabbb\u13ec\001\uabbc\u13ed\001\uabbd\u13ee\001\uabbe\u13ef\001\uabbf\u13f0\001\u13f8\u13f1\001\u13f9\u13f2\001\u13fa\u13f3\001\u13fb\u13f4\001\u13fc\u13f5\001\u13fd\u1c90\001\u10d0\u1c91\001\u10d1\u1c92\001\u10d2\u1c93\001\u10d3\u1c94\001\u10d4\u1c95\001\u10d5\u1c96\001\u10d6\u1c97\001\u10d7\u1c98\001\u10d8\u1c99\001\u10d9\u1c9a\001\u10da\u1c9b\001\u10db\u1c9c\001\u10dc\u1c9d\001\u10dd\u1c9e\001\u10de\u1c9f\001\u10df\u1ca0\001\u10e0\u1ca1\001\u10e1\u1ca2\001\u10e2\u1ca3\001\u10e3\u1ca4\001\u10e4\u1ca5\001\u10e5\u1ca6\001\u10e6\u1ca7\001\u10e7\u1ca8\001\u10e8\u1ca9\001\u10e9\u1caa\001\u10ea\u1cab\001\u10eb\u1cac\001\u10ec\u1cad\001\u10ed\u1cae\001\u10ee\u1caf\001\u10ef\u1cb0\001\u10f0\u1cb1\001\u10f1\u1cb2\001\u10f2\u1cb3\001\u10f3\u1cb4\001\u10f4\u1cb5\001\u10f5\u1cb6\001\u10f6\u1cb7\001\u10f7\u1cb8\001\u10f8\u1cb9\001\u10f9\u1cba\001\u10fa\u1cbd\001\u10fd\u1cbe\001\u10fe\u1cbf\001\u10ff\u1e00\001\u1e01\u1e02\001\u1e03\u1e04\001\u1e05\u1e06\001\u1e07\u1e08\001\u1e09\u1e0a\001\u1e0b\u1e0c\001\u1e0d\u1e0e\001\u1e0f\u1e10\001\u1e11\u1e12\001\u1e13\u1e14\001\u1e15\u1e16\001\u1e17\u1e18\001\u1e19\u1e1a\001\u1e1b\u1e1c\001\u1e1d\u1e1e\001\u1e1f\u1e20\001\u1e21\u1e22\001\u1e23\u1e24\001\u1e25\u1e26\001\u1e27\u1e28\001\u1e29\u1e2a\001\u1e2b\u1e2c\001\u1e2d\u1e2e\001\u1e2f\u1e30\001\u1e31\u1e32\001\u1e33\u1e34\001\u1e35\u1e36\001\u1e37\u1e38\001\u1e39\u1e3a\001\u1e3b\u1e3c\001\u1e3d\u1e3e\001\u1e3f\u1e40\001\u1e41\u1e42\001\u1e43\u1e44\001\u1e45\u1e46\001\u1e47\u1e48\001\u1e49\u1e4a\001\u1e4b\u1e4c\001\u1e4d\u1e4e\001\u1e4f\u1e50\001\u1e51\u1e52\001\u1e53\u1e54\001\u1e55\u1e56\001\u1e57\u1e58\001\u1e59\u1e5a\001\u1e5b\u1e5c\001\u1e5d\u1e5e\001\u1e5f\u1e60\001\u1e61\u1e62\001\u1e63\u1e64\001\u1e65\u1e66\001\u1e67\u1e68\001\u1e69\u1e6a\001\u1e6b\u1e6c\001\u1e6d\u1e6e\001\u1e6f\u1e70\001\u1e71\u1e72\001\u1e73\u1e74\001\u1e75\u1e76\001\u1e77\u1e78\001\u1e79\u1e7a\001\u1e7b\u1e7c\001\u1e7d\u1e7e\001\u1e7f\u1e80\001\u1e81\u1e82\001\u1e83\u1e84\001\u1e85\u1e86\001\u1e87\u1e88\001\u1e89\u1e8a\001\u1e8b\u1e8c\001\u1e8d\u1e8e\001\u1e8f\u1e90\001\u1e91\u1e92\001\u1e93\u1e94\001\u1e95\u1e9e\001\u00df\u1ea0\001\u1ea1\u1ea2\001\u1ea3\u1ea4\001\u1ea5\u1ea6\001\u1ea7\u1ea8\001\u1ea9\u1eaa\001\u1eab\u1eac\001\u1ead\u1eae\001\u1eaf\u1eb0\001\u1eb1\u1eb2\001\u1eb3\u1eb4\001\u1eb5\u1eb6\001\u1eb7\u1eb8\001\u1eb9\u1eba\001\u1ebb\u1ebc\001\u1ebd\u1ebe\001\u1ebf\u1ec0\001\u1ec1\u1ec2\001\u1ec3\u1ec4\001\u1ec5\u1ec6\001\u1ec7\u1ec8\001\u1ec9\u1eca\001\u1ecb\u1ecc\001\u1ecd\u1ece\001\u1ecf\u1ed0\001\u1ed1\u1ed2\001\u1ed3\u1ed4\001\u1ed5\u1ed6\001\u1ed7\u1ed8\001\u1ed9\u1eda\001\u1edb\u1edc\001\u1edd\u1ede\001\u1edf\u1ee0\001\u1ee1\u1ee2\001\u1ee3\u1ee4\001\u1ee5\u1ee6\001\u1ee7\u1ee8\001\u1ee9\u1eea\001\u1eeb\u1eec\001\u1eed\u1eee\001\u1eef\u1ef0\001\u1ef1\u1ef2\001\u1ef3\u1ef4\001\u1ef5\u1ef6\001\u1ef7\u1ef8\001\u1ef9\u1efa\001\u1efb\u1efc\001\u1efd\u1efe\001\u1eff\u1f08\001\u1f00\u1f09\001\u1f01\u1f0a\001\u1f02\u1f0b\001\u1f03\u1f0c\001\u1f04\u1f0d\001\u1f05\u1f0e\001\u1f06\u1f0f\001\u1f07\u1f18\001\u1f10\u1f19\001\u1f11\u1f1a\001\u1f12\u1f1b\001\u1f13\u1f1c\001\u1f14\u1f1d\001\u1f15\u1f28\001\u1f20\u1f29\001\u1f21\u1f2a\001\u1f22\u1f2b\001\u1f23\u1f2c\001\u1f24\u1f2d\001\u1f25\u1f2e\001\u1f26\u1f2f\001\u1f27\u1f38\001\u1f30\u1f39\001\u1f31\u1f3a\001\u1f32\u1f3b\001\u1f33\u1f3c\001\u1f34\u1f3d\001\u1f35\u1f3e\001\u1f36\u1f3f\001\u1f37\u1f48\001\u1f40\u1f49\001\u1f41\u1f4a\001\u1f42\u1f4b\001\u1f43\u1f4c\001\u1f44\u1f4d\001\u1f45\u1f59\001\u1f51\u1f5b\001\u1f53\u1f5d\001\u1f55\u1f5f\001\u1f57\u1f68\001\u1f60\u1f69\001\u1f61\u1f6a\001\u1f62\u1f6b\001\u1f63\u1f6c\001\u1f64\u1f6d\001\u1f65\u1f6e\001\u1f66\u1f6f\001\u1f67\u1f88\001\u1f80\u1f89\001\u1f81\u1f8a\001\u1f82\u1f8b\001\u1f83\u1f8c\001\u1f84\u1f8d\001\u1f85\u1f8e\001\u1f86\u1f8f\001\u1f87\u1f98\001\u1f90\u1f99\001\u1f91\u1f9a\001\u1f92\u1f9b\001\u1f93\u1f9c\001\u1f94\u1f9d\001\u1f95\u1f9e\001\u1f96\u1f9f\001\u1f97\u1fa8\001\u1fa0\u1fa9\001\u1fa1\u1faa\001\u1fa2\u1fab\001\u1fa3\u1fac\001\u1fa4\u1fad\001\u1fa5\u1fae\001\u1fa6\u1faf\001\u1fa7\u1fb8\001\u1fb0\u1fb9\001\u1fb1\u1fba\001\u1f70\u1fbb\001\u1f71\u1fbc\001\u1fb3\u1fc8\001\u1f72\u1fc9\001\u1f73\u1fca\001\u1f74\u1fcb\001\u1f75\u1fcc\001\u1fc3\u1fd8\001\u1fd0\u1fd9\001\u1fd1\u1fda\001\u1f76\u1fdb\001\u1f77\u1fe8\001\u1fe0\u1fe9\001\u1fe1\u1fea\001\u1f7a\u1feb\001\u1f7b\u1fec\001\u1fe5\u1ff8\001\u1f78\u1ff9\001\u1f79\u1ffa\001\u1f7c\u1ffb\001\u1f7d\u1ffc\001\u1ff3\u2126\001\u03c9\u212a\001k\u212b\001\u00e5\u2132\001\u214e\u2160\001\u2170\u2161\001\u2171\u2162\001\u2172\u2163\001\u2173\u2164\001\u2174\u2165\001\u2175\u2166\001\u2176\u2167\001\u2177\u2168\001\u2178\u2169\001\u2179\u216a\001\u217a\u216b\001\u217b\u216c\001\u217c\u216d\001\u217d\u216e\001\u217e\u216f\001\u217f\u2183\001\u2184\u24b6\001\u24d0\u24b7\001\u24d1\u24b8\001\u24d2\u24b9\001\u24d3\u24ba\001\u24d4\u24bb\001\u24d5\u24bc\001\u24d6\u24bd\001\u24d7\u24be\001\u24d8\u24bf\001\u24d9\u24c0\001\u24da\u24c1\001\u24db\u24c2\001\u24dc\u24c3\001\u24dd\u24c4\001\u24de\u24c5\001\u24df\u24c6\001\u24e0\u24c7\001\u24e1\u24c8\001\u24e2\u24c9\001\u24e3\u24ca\001\u24e4\u24cb\001\u24e5\u24cc\001\u24e6\u24cd\001\u24e7\u24ce\001\u24e8\u24cf\001\u24e9\u2c00\001\u2c30\u2c01\001\u2c31\u2c02\001\u2c32\u2c03\001\u2c33\u2c04\001\u2c34\u2c05\001\u2c35\u2c06\001\u2c36\u2c07\001\u2c37\u2c08\001\u2c38\u2c09\001\u2c39\u2c0a\001\u2c3a\u2c0b\001\u2c3b\u2c0c\001\u2c3c\u2c0d\001\u2c3d\u2c0e\001\u2c3e\u2c0f\001\u2c3f\u2c10\001\u2c40\u2c11\001\u2c41\u2c12\001\u2c42\u2c13\001\u2c43\u2c14\001\u2c44\u2c15\001\u2c45\u2c16\001\u2c46\u2c17\001\u2c47\u2c18\001\u2c48\u2c19\001\u2c49\u2c1a\001\u2c4a\u2c1b\001\u2c4b\u2c1c\001\u2c4c\u2c1d\001\u2c4d\u2c1e\001\u2c4e\u2c1f\001\u2c4f\u2c20\001\u2c50\u2c21\001\u2c51\u2c22\001\u2c52\u2c23\001\u2c53\u2c24\001\u2c54\u2c25\001\u2c55\u2c26\001\u2c56\u2c27\001\u2c57\u2c28\001\u2c58\u2c29\001\u2c59\u2c2a\001\u2c5a\u2c2b\001\u2c5b\u2c2c\001\u2c5c\u2c2d\001\u2c5d\u2c2e\001\u2c5e\u2c2f\001\u2c5f\u2c60\001\u2c61\u2c62\001\u026b\u2c63\001\u1d7d\u2c64\001\u027d\u2c67\001\u2c68\u2c69\001\u2c6a\u2c6b\001\u2c6c\u2c6d\001\u0251\u2c6e\001\u0271\u2c6f\001\u0250\u2c70\001\u0252\u2c72\001\u2c73\u2c75\001\u2c76\u2c7e\001\u023f\u2c7f\001\u0240\u2c80\001\u2c81\u2c82\001\u2c83\u2c84\001\u2c85\u2c86\001\u2c87\u2c88\001\u2c89\u2c8a\001\u2c8b\u2c8c\001\u2c8d\u2c8e\001\u2c8f\u2c90\001\u2c91\u2c92\001\u2c93\u2c94\001\u2c95\u2c96\001\u2c97\u2c98\001\u2c99\u2c9a\001\u2c9b\u2c9c\001\u2c9d\u2c9e\001\u2c9f\u2ca0\001\u2ca1\u2ca2\001\u2ca3\u2ca4\001\u2ca5\u2ca6\001\u2ca7\u2ca8\001\u2ca9\u2caa\001\u2cab\u2cac\001\u2cad\u2cae\001\u2caf\u2cb0\001\u2cb1\u2cb2\001\u2cb3\u2cb4\001\u2cb5\u2cb6\001\u2cb7\u2cb8\001\u2cb9\u2cba\001\u2cbb\u2cbc\001\u2cbd\u2cbe\001\u2cbf\u2cc0\001\u2cc1\u2cc2\001\u2cc3\u2cc4\001\u2cc5\u2cc6\001\u2cc7\u2cc8\001\u2cc9\u2cca\001\u2ccb\u2ccc\001\u2ccd\u2cce\001\u2ccf\u2cd0\001\u2cd1\u2cd2\001\u2cd3\u2cd4\001\u2cd5\u2cd6\001\u2cd7\u2cd8\001\u2cd9\u2cda\001\u2cdb\u2cdc\001\u2cdd\u2cde\001\u2cdf\u2ce0\001\u2ce1\u2ce2\001\u2ce3\u2ceb\001\u2cec\u2ced\001\u2cee\u2cf2\001\u2cf3\ua640\001\ua641\ua642\001\ua643\ua644\001\ua645\ua646\001\ua647\ua648\001\ua649\ua64a\001\ua64b\ua64c\001\ua64d\ua64e\001\ua64f\ua650\001\ua651\ua652\001\ua653\ua654\001\ua655\ua656\001\ua657\ua658\001\ua659\ua65a\001\ua65b\ua65c\001\ua65d\ua65e\001\ua65f\ua660\001\ua661\ua662\001\ua663\ua664\001\ua665\ua666\001\ua667\ua668\001\ua669\ua66a\001\ua66b\ua66c\001\ua66d\ua680\001\ua681\ua682\001\ua683\ua684\001\ua685\ua686\001\ua687\ua688\001\ua689\ua68a\001\ua68b\ua68c\001\ua68d\ua68e\001\ua68f\ua690\001\ua691\ua692\001\ua693\ua694\001\ua695\ua696\001\ua697\ua698\001\ua699\ua69a\001\ua69b\ua722\001\ua723\ua724\001\ua725\ua726\001\ua727\ua728\001\ua729\ua72a\001\ua72b\ua72c\001\ua72d\ua72e\001\ua72f\ua732\001\ua733\ua734\001\ua735\ua736\001\ua737\ua738\001\ua739\ua73a\001\ua73b\ua73c\001\ua73d\ua73e\001\ua73f\ua740\001\ua741\ua742\001\ua743\ua744\001\ua745\ua746\001\ua747\ua748\001\ua749\ua74a\001\ua74b\ua74c\001\ua74d\ua74e\001\ua74f\ua750\001\ua751\ua752\001\ua753\ua754\001\ua755\ua756\001\ua757\ua758\001\ua759\ua75a\001\ua75b\ua75c\001\ua75d\ua75e\001\ua75f\ua760\001\ua761\ua762\001\ua763\ua764\001\ua765\ua766\001\ua767\ua768\001\ua769\ua76a\001\ua76b\ua76c\001\ua76d\ua76e\001\ua76f\ua779\001\ua77a\ua77b\001\ua77c\ua77d\001\u1d79\ua77e\001\ua77f\ua780\001\ua781\ua782\001\ua783\ua784\001\ua785\ua786\001\ua787\ua78b\001\ua78c\ua78d\001\u0265\ua790\001\ua791\ua792\001\ua793\ua796\001\ua797\ua798\001\ua799\ua79a\001\ua79b\ua79c\001\ua79d\ua79e\001\ua79f\ua7a0\001\ua7a1\ua7a2\001\ua7a3\ua7a4\001\ua7a5\ua7a6\001\ua7a7\ua7a8\001\ua7a9\ua7aa\001\u0266\ua7ab\001\u025c\ua7ac\001\u0261\ua7ad\001\u026c\ua7ae\001\u026a\ua7b0\001\u029e\ua7b1\001\u0287\ua7b2\001\u029d\ua7b3\001\uab53\ua7b4\001\ua7b5\ua7b6\001\ua7b7\ua7b8\001\ua7b9\ua7ba\001\ua7bb\ua7bc\001\ua7bd\ua7be\001\ua7bf\ua7c0\001\ua7c1\ua7c2\001\ua7c3\ua7c4\001\ua794\ua7c5\001\u0282\ua7c6\001\u1d8e\ua7c7\001\ua7c8\ua7c9\001\ua7ca\ua7d0\001\ua7d1\ua7d6\001\ua7d7\ua7d8\001\ua7d9\ua7f5\001\ua7f6\uff21\001\uff41\uff22\001\uff42\uff23\001\uff43\uff24\001\uff44\uff25\001\uff45\uff26\001\uff46\uff27\001\uff47\uff28\001\uff48\uff29\001\uff49\uff2a\001\uff4a\uff2b\001\uff4b\uff2c\001\uff4c\uff2d\001\uff4d\uff2e\001\uff4e\uff2f\001\uff4f\uff30\001\uff50\uff31\001\uff51\uff32\001\uff52\uff33\001\uff53\uff34\001\uff54\uff35\001\uff55\uff36\001\uff56\uff37\001\uff57\uff38\001\uff58\uff39\001\uff59\uff3a\001\uff5a", + }; + + static final String[] UPPER = { + "a\001Ab\001Bc\001Cd\001De\001Ef\001Fg\001Gh\001Hi\001Ij\001Jk\001Kl\001Lm\001Mn\001No\001Op\001Pq\001Qr\001Rs\001St\001Tu\001Uv\001Vw\001Wx\001Xy\001Yz\001Z\u00b5\001\u039c\u00df\002SS\u00e0\001\u00c0\u00e1\001\u00c1\u00e2\001\u00c2\u00e3\001\u00c3\u00e4\001\u00c4\u00e5\001\u00c5\u00e6\001\u00c6\u00e7\001\u00c7\u00e8\001\u00c8\u00e9\001\u00c9\u00ea\001\u00ca\u00eb\001\u00cb\u00ec\001\u00cc\u00ed\001\u00cd\u00ee\001\u00ce\u00ef\001\u00cf\u00f0\001\u00d0\u00f1\001\u00d1\u00f2\001\u00d2\u00f3\001\u00d3\u00f4\001\u00d4\u00f5\001\u00d5\u00f6\001\u00d6\u00f8\001\u00d8\u00f9\001\u00d9\u00fa\001\u00da\u00fb\001\u00db\u00fc\001\u00dc\u00fd\001\u00dd\u00fe\001\u00de\u00ff\001\u0178\u0101\001\u0100\u0103\001\u0102\u0105\001\u0104\u0107\001\u0106\u0109\001\u0108\u010b\001\u010a\u010d\001\u010c\u010f\001\u010e\u0111\001\u0110\u0113\001\u0112\u0115\001\u0114\u0117\001\u0116\u0119\001\u0118\u011b\001\u011a\u011d\001\u011c\u011f\001\u011e\u0121\001\u0120\u0123\001\u0122\u0125\001\u0124\u0127\001\u0126\u0129\001\u0128\u012b\001\u012a\u012d\001\u012c\u012f\001\u012e\u0131\001I\u0133\001\u0132\u0135\001\u0134\u0137\001\u0136\u013a\001\u0139\u013c\001\u013b\u013e\001\u013d\u0140\001\u013f\u0142\001\u0141\u0144\001\u0143\u0146\001\u0145\u0148\001\u0147\u0149\002\u02bcN\u014b\001\u014a\u014d\001\u014c\u014f\001\u014e\u0151\001\u0150\u0153\001\u0152\u0155\001\u0154\u0157\001\u0156\u0159\001\u0158\u015b\001\u015a\u015d\001\u015c\u015f\001\u015e\u0161\001\u0160\u0163\001\u0162\u0165\001\u0164\u0167\001\u0166\u0169\001\u0168\u016b\001\u016a\u016d\001\u016c\u016f\001\u016e\u0171\001\u0170\u0173\001\u0172\u0175\001\u0174\u0177\001\u0176\u017a\001\u0179\u017c\001\u017b\u017e\001\u017d\u017f\001S\u0180\001\u0243\u0183\001\u0182\u0185\001\u0184\u0188\001\u0187\u018c\001\u018b\u0192\001\u0191\u0195\001\u01f6\u0199\001\u0198\u019a\001\u023d\u019e\001\u0220\u01a1\001\u01a0\u01a3\001\u01a2\u01a5\001\u01a4\u01a8\001\u01a7\u01ad\001\u01ac\u01b0\001\u01af\u01b4\001\u01b3\u01b6\001\u01b5\u01b9\001\u01b8\u01bd\001\u01bc\u01bf\001\u01f7\u01c5\001\u01c4\u01c6\001\u01c4\u01c8\001\u01c7\u01c9\001\u01c7\u01cb\001\u01ca\u01cc\001\u01ca\u01ce\001\u01cd\u01d0\001\u01cf\u01d2\001\u01d1\u01d4\001\u01d3\u01d6\001\u01d5\u01d8\001\u01d7\u01da\001\u01d9\u01dc\001\u01db\u01dd\001\u018e\u01df\001\u01de\u01e1\001\u01e0\u01e3\001\u01e2\u01e5\001\u01e4\u01e7\001\u01e6\u01e9\001\u01e8\u01eb\001\u01ea\u01ed\001\u01ec\u01ef\001\u01ee\u01f0\002J\u030c\u01f2\001\u01f1\u01f3\001\u01f1\u01f5\001\u01f4\u01f9\001\u01f8\u01fb\001\u01fa\u01fd\001\u01fc\u01ff\001\u01fe\u0201\001\u0200\u0203\001\u0202\u0205\001\u0204\u0207\001\u0206\u0209\001\u0208\u020b\001\u020a\u020d\001\u020c\u020f\001\u020e\u0211\001\u0210\u0213\001\u0212\u0215\001\u0214\u0217\001\u0216\u0219\001\u0218\u021b\001\u021a\u021d\001\u021c\u021f\001\u021e\u0223\001\u0222\u0225\001\u0224\u0227\001\u0226\u0229\001\u0228\u022b\001\u022a\u022d\001\u022c\u022f\001\u022e\u0231\001\u0230\u0233\001\u0232\u023c\001\u023b\u023f\001\u2c7e\u0240\001\u2c7f\u0242\001\u0241\u0247\001\u0246\u0249\001\u0248\u024b\001\u024a\u024d\001\u024c\u024f\001\u024e\u0250\001\u2c6f\u0251\001\u2c6d\u0252\001\u2c70\u0253\001\u0181\u0254\001\u0186\u0256\001\u0189\u0257\001\u018a\u0259\001\u018f\u025b\001\u0190\u025c\001\ua7ab\u0260\001\u0193\u0261\001\ua7ac\u0263\001\u0194\u0265\001\ua78d\u0266\001\ua7aa\u0268\001\u0197\u0269\001\u0196\u026a\001\ua7ae\u026b\001\u2c62\u026c\001\ua7ad\u026f\001\u019c\u0271\001\u2c6e\u0272\001\u019d\u0275\001\u019f\u027d\001\u2c64\u0280\001\u01a6\u0282\001\ua7c5\u0283\001\u01a9\u0287\001\ua7b1\u0288\001\u01ae\u0289\001\u0244\u028a\001\u01b1\u028b\001\u01b2\u028c\001\u0245\u0292\001\u01b7\u029d\001\ua7b2\u029e\001\ua7b0\u0345\001\u0399\u0371\001\u0370\u0373\001\u0372\u0377\001\u0376\u037b\001\u03fd\u037c\001\u03fe\u037d\001\u03ff\u0390\003\u0399\u0308\u0301\u03ac\001\u0386\u03ad\001\u0388\u03ae\001\u0389\u03af\001\u038a\u03b0\003\u03a5\u0308\u0301\u03b1\001\u0391\u03b2\001\u0392\u03b3\001\u0393\u03b4\001\u0394\u03b5\001\u0395\u03b6\001\u0396\u03b7\001\u0397\u03b8\001\u0398\u03b9\001\u0399\u03ba\001\u039a\u03bb\001\u039b\u03bc\001\u039c\u03bd\001\u039d\u03be\001\u039e\u03bf\001\u039f\u03c0\001\u03a0\u03c1\001\u03a1\u03c2\001\u03a3\u03c3\001\u03a3\u03c4\001\u03a4\u03c5\001\u03a5\u03c6\001\u03a6\u03c7\001\u03a7\u03c8\001\u03a8\u03c9\001\u03a9\u03ca\001\u03aa\u03cb\001\u03ab\u03cc\001\u038c\u03cd\001\u038e\u03ce\001\u038f\u03d0\001\u0392\u03d1\001\u0398\u03d5\001\u03a6\u03d6\001\u03a0\u03d7\001\u03cf\u03d9\001\u03d8\u03db\001\u03da\u03dd\001\u03dc\u03df\001\u03de\u03e1\001\u03e0\u03e3\001\u03e2\u03e5\001\u03e4\u03e7\001\u03e6\u03e9\001\u03e8\u03eb\001\u03ea\u03ed\001\u03ec\u03ef\001\u03ee\u03f0\001\u039a\u03f1\001\u03a1\u03f2\001\u03f9\u03f3\001\u037f\u03f5\001\u0395\u03f8\001\u03f7\u03fb\001\u03fa\u0430\001\u0410\u0431\001\u0411\u0432\001\u0412\u0433\001\u0413\u0434\001\u0414\u0435\001\u0415\u0436\001\u0416\u0437\001\u0417\u0438\001\u0418\u0439\001\u0419\u043a\001\u041a\u043b\001\u041b\u043c\001\u041c\u043d\001\u041d\u043e\001\u041e\u043f\001\u041f\u0440\001\u0420\u0441\001\u0421\u0442\001\u0422\u0443\001\u0423\u0444\001\u0424\u0445\001\u0425\u0446\001\u0426\u0447\001\u0427\u0448\001\u0428\u0449\001\u0429\u044a\001\u042a\u044b\001\u042b\u044c\001\u042c\u044d\001\u042d\u044e\001\u042e\u044f\001\u042f\u0450\001\u0400\u0451\001\u0401\u0452\001\u0402\u0453\001\u0403\u0454\001\u0404\u0455\001\u0405\u0456\001\u0406\u0457\001\u0407\u0458\001\u0408\u0459\001\u0409\u045a\001\u040a\u045b\001\u040b\u045c\001\u040c\u045d\001\u040d\u045e\001\u040e\u045f\001\u040f\u0461\001\u0460\u0463\001\u0462\u0465\001\u0464\u0467\001\u0466\u0469\001\u0468\u046b\001\u046a\u046d\001\u046c\u046f\001\u046e\u0471\001\u0470\u0473\001\u0472\u0475\001\u0474\u0477\001\u0476\u0479\001\u0478\u047b\001\u047a\u047d\001\u047c\u047f\001\u047e\u0481\001\u0480\u048b\001\u048a\u048d\001\u048c\u048f\001\u048e\u0491\001\u0490\u0493\001\u0492\u0495\001\u0494\u0497\001\u0496\u0499\001\u0498\u049b\001\u049a\u049d\001\u049c\u049f\001\u049e\u04a1\001\u04a0\u04a3\001\u04a2\u04a5\001\u04a4\u04a7\001\u04a6\u04a9\001\u04a8\u04ab\001\u04aa\u04ad\001\u04ac\u04af\001\u04ae\u04b1\001\u04b0\u04b3\001\u04b2\u04b5\001\u04b4\u04b7\001\u04b6\u04b9\001\u04b8\u04bb\001\u04ba\u04bd\001\u04bc\u04bf\001\u04be\u04c2\001\u04c1\u04c4\001\u04c3\u04c6\001\u04c5\u04c8\001\u04c7\u04ca\001\u04c9\u04cc\001\u04cb\u04ce\001\u04cd\u04cf\001\u04c0\u04d1\001\u04d0\u04d3\001\u04d2\u04d5\001\u04d4\u04d7\001\u04d6\u04d9\001\u04d8\u04db\001\u04da\u04dd\001\u04dc\u04df\001\u04de\u04e1\001\u04e0\u04e3\001\u04e2\u04e5\001\u04e4\u04e7\001\u04e6\u04e9\001\u04e8\u04eb\001\u04ea\u04ed\001\u04ec\u04ef\001\u04ee\u04f1\001\u04f0\u04f3\001\u04f2\u04f5\001\u04f4\u04f7\001\u04f6\u04f9\001\u04f8\u04fb\001\u04fa\u04fd\001\u04fc\u04ff\001\u04fe\u0501\001\u0500\u0503\001\u0502\u0505\001\u0504\u0507\001\u0506\u0509\001\u0508\u050b\001\u050a\u050d\001\u050c\u050f\001\u050e\u0511\001\u0510\u0513\001\u0512\u0515\001\u0514\u0517\001\u0516\u0519\001\u0518\u051b\001\u051a\u051d\001\u051c\u051f\001\u051e\u0521\001\u0520\u0523\001\u0522\u0525\001\u0524\u0527\001\u0526\u0529\001\u0528\u052b\001\u052a\u052d\001\u052c\u052f\001\u052e\u0561\001\u0531\u0562\001\u0532\u0563\001\u0533\u0564\001\u0534\u0565\001\u0535\u0566\001\u0536\u0567\001\u0537\u0568\001\u0538\u0569\001\u0539\u056a\001\u053a\u056b\001\u053b\u056c\001\u053c\u056d\001\u053d\u056e\001\u053e\u056f\001\u053f\u0570\001\u0540\u0571\001\u0541\u0572\001\u0542\u0573\001\u0543\u0574\001\u0544\u0575\001\u0545\u0576\001\u0546\u0577\001\u0547\u0578\001\u0548\u0579\001\u0549\u057a\001\u054a\u057b\001\u054b\u057c\001\u054c\u057d\001\u054d\u057e\001\u054e\u057f\001\u054f\u0580\001\u0550\u0581\001\u0551\u0582\001\u0552\u0583\001\u0553\u0584\001\u0554\u0585\001\u0555\u0586\001\u0556\u0587\002\u0535\u0552\u10d0\001\u1c90\u10d1\001\u1c91\u10d2\001\u1c92\u10d3\001\u1c93\u10d4\001\u1c94\u10d5\001\u1c95\u10d6\001\u1c96\u10d7\001\u1c97\u10d8\001\u1c98\u10d9\001\u1c99\u10da\001\u1c9a\u10db\001\u1c9b\u10dc\001\u1c9c\u10dd\001\u1c9d\u10de\001\u1c9e\u10df\001\u1c9f\u10e0\001\u1ca0\u10e1\001\u1ca1\u10e2\001\u1ca2\u10e3\001\u1ca3\u10e4\001\u1ca4\u10e5\001\u1ca5\u10e6\001\u1ca6\u10e7\001\u1ca7\u10e8\001\u1ca8\u10e9\001\u1ca9\u10ea\001\u1caa\u10eb\001\u1cab\u10ec\001\u1cac\u10ed\001\u1cad\u10ee\001\u1cae\u10ef\001\u1caf\u10f0\001\u1cb0\u10f1\001\u1cb1\u10f2\001\u1cb2\u10f3\001\u1cb3\u10f4\001\u1cb4\u10f5\001\u1cb5\u10f6\001\u1cb6\u10f7\001\u1cb7\u10f8\001\u1cb8\u10f9\001\u1cb9\u10fa\001\u1cba\u10fd\001\u1cbd\u10fe\001\u1cbe\u10ff\001\u1cbf\u13f8\001\u13f0\u13f9\001\u13f1\u13fa\001\u13f2\u13fb\001\u13f3\u13fc\001\u13f4\u13fd\001\u13f5\u1c80\001\u0412\u1c81\001\u0414\u1c82\001\u041e\u1c83\001\u0421\u1c84\001\u0422\u1c85\001\u0422\u1c86\001\u042a\u1c87\001\u0462\u1c88\001\ua64a\u1d79\001\ua77d\u1d7d\001\u2c63\u1d8e\001\ua7c6\u1e01\001\u1e00\u1e03\001\u1e02\u1e05\001\u1e04\u1e07\001\u1e06\u1e09\001\u1e08\u1e0b\001\u1e0a\u1e0d\001\u1e0c\u1e0f\001\u1e0e\u1e11\001\u1e10\u1e13\001\u1e12\u1e15\001\u1e14\u1e17\001\u1e16\u1e19\001\u1e18\u1e1b\001\u1e1a\u1e1d\001\u1e1c\u1e1f\001\u1e1e\u1e21\001\u1e20\u1e23\001\u1e22\u1e25\001\u1e24\u1e27\001\u1e26\u1e29\001\u1e28\u1e2b\001\u1e2a\u1e2d\001\u1e2c\u1e2f\001\u1e2e\u1e31\001\u1e30\u1e33\001\u1e32\u1e35\001\u1e34\u1e37\001\u1e36\u1e39\001\u1e38\u1e3b\001\u1e3a\u1e3d\001\u1e3c\u1e3f\001\u1e3e\u1e41\001\u1e40\u1e43\001\u1e42\u1e45\001\u1e44\u1e47\001\u1e46\u1e49\001\u1e48\u1e4b\001\u1e4a\u1e4d\001\u1e4c\u1e4f\001\u1e4e\u1e51\001\u1e50\u1e53\001\u1e52\u1e55\001\u1e54\u1e57\001\u1e56\u1e59\001\u1e58\u1e5b\001\u1e5a\u1e5d\001\u1e5c\u1e5f\001\u1e5e\u1e61\001\u1e60\u1e63\001\u1e62\u1e65\001\u1e64\u1e67\001\u1e66\u1e69\001\u1e68\u1e6b\001\u1e6a\u1e6d\001\u1e6c\u1e6f\001\u1e6e\u1e71\001\u1e70\u1e73\001\u1e72\u1e75\001\u1e74\u1e77\001\u1e76\u1e79\001\u1e78\u1e7b\001\u1e7a\u1e7d\001\u1e7c\u1e7f\001\u1e7e\u1e81\001\u1e80\u1e83\001\u1e82\u1e85\001\u1e84\u1e87\001\u1e86\u1e89\001\u1e88\u1e8b\001\u1e8a\u1e8d\001\u1e8c\u1e8f\001\u1e8e\u1e91\001\u1e90\u1e93\001\u1e92\u1e95\001\u1e94\u1e96\002H\u0331\u1e97\002T\u0308\u1e98\002W\u030a\u1e99\002Y\u030a\u1e9a\002A\u02be\u1e9b\001\u1e60\u1ea1\001\u1ea0\u1ea3\001\u1ea2\u1ea5\001\u1ea4\u1ea7\001\u1ea6\u1ea9\001\u1ea8\u1eab\001\u1eaa\u1ead\001\u1eac\u1eaf\001\u1eae\u1eb1\001\u1eb0\u1eb3\001\u1eb2\u1eb5\001\u1eb4\u1eb7\001\u1eb6\u1eb9\001\u1eb8\u1ebb\001\u1eba\u1ebd\001\u1ebc\u1ebf\001\u1ebe\u1ec1\001\u1ec0\u1ec3\001\u1ec2\u1ec5\001\u1ec4\u1ec7\001\u1ec6\u1ec9\001\u1ec8\u1ecb\001\u1eca\u1ecd\001\u1ecc\u1ecf\001\u1ece\u1ed1\001\u1ed0\u1ed3\001\u1ed2\u1ed5\001\u1ed4\u1ed7\001\u1ed6\u1ed9\001\u1ed8\u1edb\001\u1eda\u1edd\001\u1edc\u1edf\001\u1ede\u1ee1\001\u1ee0\u1ee3\001\u1ee2\u1ee5\001\u1ee4\u1ee7\001\u1ee6\u1ee9\001\u1ee8\u1eeb\001\u1eea\u1eed\001\u1eec\u1eef\001\u1eee\u1ef1\001\u1ef0\u1ef3\001\u1ef2\u1ef5\001\u1ef4\u1ef7\001\u1ef6\u1ef9\001\u1ef8\u1efb\001\u1efa\u1efd\001\u1efc\u1eff\001\u1efe\u1f00\001\u1f08\u1f01\001\u1f09\u1f02\001\u1f0a\u1f03\001\u1f0b\u1f04\001\u1f0c\u1f05\001\u1f0d\u1f06\001\u1f0e\u1f07\001\u1f0f\u1f10\001\u1f18\u1f11\001\u1f19\u1f12\001\u1f1a\u1f13\001\u1f1b\u1f14\001\u1f1c\u1f15\001\u1f1d\u1f20\001\u1f28\u1f21\001\u1f29\u1f22\001\u1f2a\u1f23\001\u1f2b\u1f24\001\u1f2c\u1f25\001\u1f2d\u1f26\001\u1f2e\u1f27\001\u1f2f\u1f30\001\u1f38\u1f31\001\u1f39\u1f32\001\u1f3a\u1f33\001\u1f3b\u1f34\001\u1f3c\u1f35\001\u1f3d\u1f36\001\u1f3e\u1f37\001\u1f3f\u1f40\001\u1f48\u1f41\001\u1f49\u1f42\001\u1f4a\u1f43\001\u1f4b\u1f44\001\u1f4c\u1f45\001\u1f4d\u1f50\002\u03a5\u0313\u1f51\001\u1f59\u1f52\003\u03a5\u0313\u0300\u1f53\001\u1f5b\u1f54\003\u03a5\u0313\u0301\u1f55\001\u1f5d\u1f56\003\u03a5\u0313\u0342\u1f57\001\u1f5f\u1f60\001\u1f68\u1f61\001\u1f69\u1f62\001\u1f6a\u1f63\001\u1f6b\u1f64\001\u1f6c\u1f65\001\u1f6d\u1f66\001\u1f6e\u1f67\001\u1f6f\u1f70\001\u1fba\u1f71\001\u1fbb\u1f72\001\u1fc8\u1f73\001\u1fc9\u1f74\001\u1fca\u1f75\001\u1fcb\u1f76\001\u1fda\u1f77\001\u1fdb\u1f78\001\u1ff8\u1f79\001\u1ff9\u1f7a\001\u1fea\u1f7b\001\u1feb\u1f7c\001\u1ffa\u1f7d\001\u1ffb\u1f80\002\u1f08\u0399\u1f81\002\u1f09\u0399\u1f82\002\u1f0a\u0399\u1f83\002\u1f0b\u0399\u1f84\002\u1f0c\u0399\u1f85\002\u1f0d\u0399\u1f86\002\u1f0e\u0399\u1f87\002\u1f0f\u0399\u1f88\002\u1f08\u0399\u1f89\002\u1f09\u0399\u1f8a\002\u1f0a\u0399\u1f8b\002\u1f0b\u0399\u1f8c\002\u1f0c\u0399\u1f8d\002\u1f0d\u0399\u1f8e\002\u1f0e\u0399\u1f8f\002\u1f0f\u0399\u1f90\002\u1f28\u0399\u1f91\002\u1f29\u0399\u1f92\002\u1f2a\u0399\u1f93\002\u1f2b\u0399\u1f94\002\u1f2c\u0399\u1f95\002\u1f2d\u0399\u1f96\002\u1f2e\u0399\u1f97\002\u1f2f\u0399\u1f98\002\u1f28\u0399\u1f99\002\u1f29\u0399\u1f9a\002\u1f2a\u0399\u1f9b\002\u1f2b\u0399\u1f9c\002\u1f2c\u0399\u1f9d\002\u1f2d\u0399\u1f9e\002\u1f2e\u0399\u1f9f\002\u1f2f\u0399\u1fa0\002\u1f68\u0399\u1fa1\002\u1f69\u0399\u1fa2\002\u1f6a\u0399\u1fa3\002\u1f6b\u0399\u1fa4\002\u1f6c\u0399\u1fa5\002\u1f6d\u0399\u1fa6\002\u1f6e\u0399\u1fa7\002\u1f6f\u0399\u1fa8\002\u1f68\u0399\u1fa9\002\u1f69\u0399\u1faa\002\u1f6a\u0399\u1fab\002\u1f6b\u0399\u1fac\002\u1f6c\u0399\u1fad\002\u1f6d\u0399\u1fae\002\u1f6e\u0399\u1faf\002\u1f6f\u0399\u1fb0\001\u1fb8\u1fb1\001\u1fb9\u1fb2\002\u1fba\u0399\u1fb3\002\u0391\u0399\u1fb4\002\u0386\u0399\u1fb6\002\u0391\u0342\u1fb7\003\u0391\u0342\u0399\u1fbc\002\u0391\u0399\u1fbe\001\u0399\u1fc2\002\u1fca\u0399\u1fc3\002\u0397\u0399\u1fc4\002\u0389\u0399\u1fc6\002\u0397\u0342\u1fc7\003\u0397\u0342\u0399\u1fcc\002\u0397\u0399\u1fd0\001\u1fd8\u1fd1\001\u1fd9\u1fd2\003\u0399\u0308\u0300\u1fd3\003\u0399\u0308\u0301\u1fd6\002\u0399\u0342\u1fd7\003\u0399\u0308\u0342\u1fe0\001\u1fe8\u1fe1\001\u1fe9\u1fe2\003\u03a5\u0308\u0300\u1fe3\003\u03a5\u0308\u0301\u1fe4\002\u03a1\u0313\u1fe5\001\u1fec\u1fe6\002\u03a5\u0342\u1fe7\003\u03a5\u0308\u0342\u1ff2\002\u1ffa\u0399\u1ff3\002\u03a9\u0399\u1ff4\002\u038f\u0399\u1ff6\002\u03a9\u0342\u1ff7\003\u03a9\u0342\u0399\u1ffc\002\u03a9\u0399\u214e\001\u2132\u2170\001\u2160\u2171\001\u2161\u2172\001\u2162\u2173\001\u2163\u2174\001\u2164\u2175\001\u2165\u2176\001\u2166\u2177\001\u2167\u2178\001\u2168\u2179\001\u2169\u217a\001\u216a\u217b\001\u216b\u217c\001\u216c\u217d\001\u216d\u217e\001\u216e\u217f\001\u216f\u2184\001\u2183\u24d0\001\u24b6\u24d1\001\u24b7\u24d2\001\u24b8\u24d3\001\u24b9\u24d4\001\u24ba\u24d5\001\u24bb\u24d6\001\u24bc\u24d7\001\u24bd\u24d8\001\u24be\u24d9\001\u24bf\u24da\001\u24c0\u24db\001\u24c1\u24dc\001\u24c2\u24dd\001\u24c3\u24de\001\u24c4\u24df\001\u24c5\u24e0\001\u24c6\u24e1\001\u24c7\u24e2\001\u24c8\u24e3\001\u24c9\u24e4\001\u24ca\u24e5\001\u24cb\u24e6\001\u24cc\u24e7\001\u24cd\u24e8\001\u24ce\u24e9\001\u24cf\u2c30\001\u2c00\u2c31\001\u2c01\u2c32\001\u2c02\u2c33\001\u2c03\u2c34\001\u2c04\u2c35\001\u2c05\u2c36\001\u2c06\u2c37\001\u2c07\u2c38\001\u2c08\u2c39\001\u2c09\u2c3a\001\u2c0a\u2c3b\001\u2c0b\u2c3c\001\u2c0c\u2c3d\001\u2c0d\u2c3e\001\u2c0e\u2c3f\001\u2c0f\u2c40\001\u2c10\u2c41\001\u2c11\u2c42\001\u2c12\u2c43\001\u2c13\u2c44\001\u2c14\u2c45\001\u2c15\u2c46\001\u2c16\u2c47\001\u2c17\u2c48\001\u2c18\u2c49\001\u2c19\u2c4a\001\u2c1a\u2c4b\001\u2c1b\u2c4c\001\u2c1c\u2c4d\001\u2c1d\u2c4e\001\u2c1e\u2c4f\001\u2c1f\u2c50\001\u2c20\u2c51\001\u2c21\u2c52\001\u2c22\u2c53\001\u2c23\u2c54\001\u2c24\u2c55\001\u2c25\u2c56\001\u2c26\u2c57\001\u2c27\u2c58\001\u2c28\u2c59\001\u2c29\u2c5a\001\u2c2a\u2c5b\001\u2c2b\u2c5c\001\u2c2c\u2c5d\001\u2c2d\u2c5e\001\u2c2e\u2c5f\001\u2c2f\u2c61\001\u2c60\u2c65\001\u023a\u2c66\001\u023e\u2c68\001\u2c67\u2c6a\001\u2c69\u2c6c\001\u2c6b\u2c73\001\u2c72\u2c76\001\u2c75\u2c81\001\u2c80\u2c83\001\u2c82\u2c85\001\u2c84\u2c87\001\u2c86\u2c89\001\u2c88\u2c8b\001\u2c8a\u2c8d\001\u2c8c\u2c8f\001\u2c8e\u2c91\001\u2c90\u2c93\001\u2c92\u2c95\001\u2c94\u2c97\001\u2c96\u2c99\001\u2c98\u2c9b\001\u2c9a\u2c9d\001\u2c9c\u2c9f\001\u2c9e\u2ca1\001\u2ca0\u2ca3\001\u2ca2\u2ca5\001\u2ca4\u2ca7\001\u2ca6\u2ca9\001\u2ca8\u2cab\001\u2caa\u2cad\001\u2cac\u2caf\001\u2cae\u2cb1\001\u2cb0\u2cb3\001\u2cb2\u2cb5\001\u2cb4\u2cb7\001\u2cb6\u2cb9\001\u2cb8\u2cbb\001\u2cba\u2cbd\001\u2cbc\u2cbf\001\u2cbe\u2cc1\001\u2cc0\u2cc3\001\u2cc2\u2cc5\001\u2cc4\u2cc7\001\u2cc6\u2cc9\001\u2cc8\u2ccb\001\u2cca\u2ccd\001\u2ccc\u2ccf\001\u2cce\u2cd1\001\u2cd0\u2cd3\001\u2cd2\u2cd5\001\u2cd4\u2cd7\001\u2cd6\u2cd9\001\u2cd8\u2cdb\001\u2cda\u2cdd\001\u2cdc\u2cdf\001\u2cde\u2ce1\001\u2ce0\u2ce3\001\u2ce2\u2cec\001\u2ceb\u2cee\001\u2ced\u2cf3\001\u2cf2\u2d00\001\u10a0\u2d01\001\u10a1\u2d02\001\u10a2\u2d03\001\u10a3\u2d04\001\u10a4\u2d05\001\u10a5\u2d06\001\u10a6\u2d07\001\u10a7\u2d08\001\u10a8\u2d09\001\u10a9\u2d0a\001\u10aa\u2d0b\001\u10ab\u2d0c\001\u10ac\u2d0d\001\u10ad\u2d0e\001\u10ae\u2d0f\001\u10af\u2d10\001\u10b0\u2d11\001\u10b1\u2d12\001\u10b2\u2d13\001\u10b3\u2d14\001\u10b4\u2d15\001\u10b5\u2d16\001\u10b6\u2d17\001\u10b7\u2d18\001\u10b8\u2d19\001\u10b9\u2d1a\001\u10ba\u2d1b\001\u10bb\u2d1c\001\u10bc\u2d1d\001\u10bd\u2d1e\001\u10be\u2d1f\001\u10bf\u2d20\001\u10c0\u2d21\001\u10c1\u2d22\001\u10c2\u2d23\001\u10c3\u2d24\001\u10c4\u2d25\001\u10c5\u2d27\001\u10c7\u2d2d\001\u10cd\ua641\001\ua640\ua643\001\ua642\ua645\001\ua644\ua647\001\ua646\ua649\001\ua648\ua64b\001\ua64a\ua64d\001\ua64c\ua64f\001\ua64e\ua651\001\ua650\ua653\001\ua652\ua655\001\ua654\ua657\001\ua656\ua659\001\ua658\ua65b\001\ua65a\ua65d\001\ua65c\ua65f\001\ua65e\ua661\001\ua660\ua663\001\ua662\ua665\001\ua664\ua667\001\ua666\ua669\001\ua668\ua66b\001\ua66a\ua66d\001\ua66c\ua681\001\ua680\ua683\001\ua682\ua685\001\ua684\ua687\001\ua686\ua689\001\ua688\ua68b\001\ua68a\ua68d\001\ua68c\ua68f\001\ua68e\ua691\001\ua690\ua693\001\ua692\ua695\001\ua694\ua697\001\ua696\ua699\001\ua698\ua69b\001\ua69a\ua723\001\ua722\ua725\001\ua724\ua727\001\ua726\ua729\001\ua728\ua72b\001\ua72a\ua72d\001\ua72c\ua72f\001\ua72e\ua733\001\ua732\ua735\001\ua734\ua737\001\ua736\ua739\001\ua738\ua73b\001\ua73a\ua73d\001\ua73c\ua73f\001\ua73e\ua741\001\ua740\ua743\001\ua742\ua745\001\ua744\ua747\001\ua746\ua749\001\ua748\ua74b\001\ua74a\ua74d\001\ua74c\ua74f\001\ua74e\ua751\001\ua750\ua753\001\ua752\ua755\001\ua754\ua757\001\ua756\ua759\001\ua758\ua75b\001\ua75a\ua75d\001\ua75c\ua75f\001\ua75e\ua761\001\ua760\ua763\001\ua762\ua765\001\ua764\ua767\001\ua766\ua769\001\ua768\ua76b\001\ua76a\ua76d\001\ua76c\ua76f\001\ua76e\ua77a\001\ua779\ua77c\001\ua77b\ua77f\001\ua77e\ua781\001\ua780\ua783\001\ua782\ua785\001\ua784\ua787\001\ua786\ua78c\001\ua78b\ua791\001\ua790\ua793\001\ua792\ua794\001\ua7c4\ua797\001\ua796\ua799\001\ua798\ua79b\001\ua79a\ua79d\001\ua79c\ua79f\001\ua79e\ua7a1\001\ua7a0\ua7a3\001\ua7a2\ua7a5\001\ua7a4\ua7a7\001\ua7a6\ua7a9\001\ua7a8\ua7b5\001\ua7b4\ua7b7\001\ua7b6\ua7b9\001\ua7b8\ua7bb\001\ua7ba\ua7bd\001\ua7bc\ua7bf\001\ua7be\ua7c1\001\ua7c0\ua7c3\001\ua7c2\ua7c8\001\ua7c7\ua7ca\001\ua7c9\ua7d1\001\ua7d0\ua7d7\001\ua7d6\ua7d9\001\ua7d8\ua7f6\001\ua7f5\uab53\001\ua7b3\uab70\001\u13a0\uab71\001\u13a1\uab72\001\u13a2\uab73\001\u13a3\uab74\001\u13a4\uab75\001\u13a5\uab76\001\u13a6\uab77\001\u13a7\uab78\001\u13a8\uab79\001\u13a9\uab7a\001\u13aa\uab7b\001\u13ab\uab7c\001\u13ac\uab7d\001\u13ad\uab7e\001\u13ae\uab7f\001\u13af\uab80\001\u13b0\uab81\001\u13b1\uab82\001\u13b2\uab83\001\u13b3\uab84\001\u13b4\uab85\001\u13b5\uab86\001\u13b6\uab87\001\u13b7\uab88\001\u13b8\uab89\001\u13b9\uab8a\001\u13ba\uab8b\001\u13bb\uab8c\001\u13bc\uab8d\001\u13bd\uab8e\001\u13be\uab8f\001\u13bf\uab90\001\u13c0\uab91\001\u13c1\uab92\001\u13c2\uab93\001\u13c3\uab94\001\u13c4\uab95\001\u13c5\uab96\001\u13c6\uab97\001\u13c7\uab98\001\u13c8\uab99\001\u13c9\uab9a\001\u13ca\uab9b\001\u13cb\uab9c\001\u13cc\uab9d\001\u13cd\uab9e\001\u13ce\uab9f\001\u13cf\uaba0\001\u13d0\uaba1\001\u13d1\uaba2\001\u13d2\uaba3\001\u13d3\uaba4\001\u13d4\uaba5\001\u13d5\uaba6\001\u13d6\uaba7\001\u13d7\uaba8\001\u13d8\uaba9\001\u13d9\uabaa\001\u13da\uabab\001\u13db\uabac\001\u13dc\uabad\001\u13dd\uabae\001\u13de\uabaf\001\u13df\uabb0\001\u13e0\uabb1\001\u13e1\uabb2\001\u13e2\uabb3\001\u13e3\uabb4\001\u13e4\uabb5\001\u13e5\uabb6\001\u13e6\uabb7\001\u13e7\uabb8\001\u13e8\uabb9\001\u13e9\uabba\001\u13ea\uabbb\001\u13eb\uabbc\001\u13ec\uabbd\001\u13ed\uabbe\001\u13ee\uabbf\001\u13ef\ufb00\002FF\ufb01\002FI\ufb02\002FL\ufb03\003FFI\ufb04\003FFL\ufb05\002ST\ufb06\002ST\ufb13\002\u0544\u0546\ufb14\002\u0544\u0535\ufb15\002\u0544\u053b\ufb16\002\u054e\u0546\ufb17\002\u0544\u053d\uff41\001\uff21\uff42\001\uff22\uff43\001\uff23\uff44\001\uff24\uff45\001\uff25\uff46\001\uff26\uff47\001\uff27\uff48\001\uff28\uff49\001\uff29\uff4a\001\uff2a\uff4b\001\uff2b\uff4c\001\uff2c\uff4d\001\uff2d\uff4e\001\uff2e\uff4f\001\uff2f\uff50\001\uff30\uff51\001\uff31\uff52\001\uff32\uff53\001\uff33\uff54\001\uff34\uff55\001\uff35\uff56\001\uff36\uff57\001\uff37\uff58\001\uff38\uff59\001\uff39\uff5a\001\uff3a", + }; + + static final String[] TITLE = { + "a\001Ab\001Bc\001Cd\001De\001Ef\001Fg\001Gh\001Hi\001Ij\001Jk\001Kl\001Lm\001Mn\001No\001Op\001Pq\001Qr\001Rs\001St\001Tu\001Uv\001Vw\001Wx\001Xy\001Yz\001Z\u00b5\001\u039c\u00df\002Ss\u00e0\001\u00c0\u00e1\001\u00c1\u00e2\001\u00c2\u00e3\001\u00c3\u00e4\001\u00c4\u00e5\001\u00c5\u00e6\001\u00c6\u00e7\001\u00c7\u00e8\001\u00c8\u00e9\001\u00c9\u00ea\001\u00ca\u00eb\001\u00cb\u00ec\001\u00cc\u00ed\001\u00cd\u00ee\001\u00ce\u00ef\001\u00cf\u00f0\001\u00d0\u00f1\001\u00d1\u00f2\001\u00d2\u00f3\001\u00d3\u00f4\001\u00d4\u00f5\001\u00d5\u00f6\001\u00d6\u00f8\001\u00d8\u00f9\001\u00d9\u00fa\001\u00da\u00fb\001\u00db\u00fc\001\u00dc\u00fd\001\u00dd\u00fe\001\u00de\u00ff\001\u0178\u0101\001\u0100\u0103\001\u0102\u0105\001\u0104\u0107\001\u0106\u0109\001\u0108\u010b\001\u010a\u010d\001\u010c\u010f\001\u010e\u0111\001\u0110\u0113\001\u0112\u0115\001\u0114\u0117\001\u0116\u0119\001\u0118\u011b\001\u011a\u011d\001\u011c\u011f\001\u011e\u0121\001\u0120\u0123\001\u0122\u0125\001\u0124\u0127\001\u0126\u0129\001\u0128\u012b\001\u012a\u012d\001\u012c\u012f\001\u012e\u0131\001I\u0133\001\u0132\u0135\001\u0134\u0137\001\u0136\u013a\001\u0139\u013c\001\u013b\u013e\001\u013d\u0140\001\u013f\u0142\001\u0141\u0144\001\u0143\u0146\001\u0145\u0148\001\u0147\u0149\002\u02bcN\u014b\001\u014a\u014d\001\u014c\u014f\001\u014e\u0151\001\u0150\u0153\001\u0152\u0155\001\u0154\u0157\001\u0156\u0159\001\u0158\u015b\001\u015a\u015d\001\u015c\u015f\001\u015e\u0161\001\u0160\u0163\001\u0162\u0165\001\u0164\u0167\001\u0166\u0169\001\u0168\u016b\001\u016a\u016d\001\u016c\u016f\001\u016e\u0171\001\u0170\u0173\001\u0172\u0175\001\u0174\u0177\001\u0176\u017a\001\u0179\u017c\001\u017b\u017e\001\u017d\u017f\001S\u0180\001\u0243\u0183\001\u0182\u0185\001\u0184\u0188\001\u0187\u018c\001\u018b\u0192\001\u0191\u0195\001\u01f6\u0199\001\u0198\u019a\001\u023d\u019e\001\u0220\u01a1\001\u01a0\u01a3\001\u01a2\u01a5\001\u01a4\u01a8\001\u01a7\u01ad\001\u01ac\u01b0\001\u01af\u01b4\001\u01b3\u01b6\001\u01b5\u01b9\001\u01b8\u01bd\001\u01bc\u01bf\001\u01f7\u01c4\001\u01c5\u01c6\001\u01c5\u01c7\001\u01c8\u01c9\001\u01c8\u01ca\001\u01cb\u01cc\001\u01cb\u01ce\001\u01cd\u01d0\001\u01cf\u01d2\001\u01d1\u01d4\001\u01d3\u01d6\001\u01d5\u01d8\001\u01d7\u01da\001\u01d9\u01dc\001\u01db\u01dd\001\u018e\u01df\001\u01de\u01e1\001\u01e0\u01e3\001\u01e2\u01e5\001\u01e4\u01e7\001\u01e6\u01e9\001\u01e8\u01eb\001\u01ea\u01ed\001\u01ec\u01ef\001\u01ee\u01f0\002J\u030c\u01f1\001\u01f2\u01f3\001\u01f2\u01f5\001\u01f4\u01f9\001\u01f8\u01fb\001\u01fa\u01fd\001\u01fc\u01ff\001\u01fe\u0201\001\u0200\u0203\001\u0202\u0205\001\u0204\u0207\001\u0206\u0209\001\u0208\u020b\001\u020a\u020d\001\u020c\u020f\001\u020e\u0211\001\u0210\u0213\001\u0212\u0215\001\u0214\u0217\001\u0216\u0219\001\u0218\u021b\001\u021a\u021d\001\u021c\u021f\001\u021e\u0223\001\u0222\u0225\001\u0224\u0227\001\u0226\u0229\001\u0228\u022b\001\u022a\u022d\001\u022c\u022f\001\u022e\u0231\001\u0230\u0233\001\u0232\u023c\001\u023b\u023f\001\u2c7e\u0240\001\u2c7f\u0242\001\u0241\u0247\001\u0246\u0249\001\u0248\u024b\001\u024a\u024d\001\u024c\u024f\001\u024e\u0250\001\u2c6f\u0251\001\u2c6d\u0252\001\u2c70\u0253\001\u0181\u0254\001\u0186\u0256\001\u0189\u0257\001\u018a\u0259\001\u018f\u025b\001\u0190\u025c\001\ua7ab\u0260\001\u0193\u0261\001\ua7ac\u0263\001\u0194\u0265\001\ua78d\u0266\001\ua7aa\u0268\001\u0197\u0269\001\u0196\u026a\001\ua7ae\u026b\001\u2c62\u026c\001\ua7ad\u026f\001\u019c\u0271\001\u2c6e\u0272\001\u019d\u0275\001\u019f\u027d\001\u2c64\u0280\001\u01a6\u0282\001\ua7c5\u0283\001\u01a9\u0287\001\ua7b1\u0288\001\u01ae\u0289\001\u0244\u028a\001\u01b1\u028b\001\u01b2\u028c\001\u0245\u0292\001\u01b7\u029d\001\ua7b2\u029e\001\ua7b0\u0345\001\u0399\u0371\001\u0370\u0373\001\u0372\u0377\001\u0376\u037b\001\u03fd\u037c\001\u03fe\u037d\001\u03ff\u0390\003\u0399\u0308\u0301\u03ac\001\u0386\u03ad\001\u0388\u03ae\001\u0389\u03af\001\u038a\u03b0\003\u03a5\u0308\u0301\u03b1\001\u0391\u03b2\001\u0392\u03b3\001\u0393\u03b4\001\u0394\u03b5\001\u0395\u03b6\001\u0396\u03b7\001\u0397\u03b8\001\u0398\u03b9\001\u0399\u03ba\001\u039a\u03bb\001\u039b\u03bc\001\u039c\u03bd\001\u039d\u03be\001\u039e\u03bf\001\u039f\u03c0\001\u03a0\u03c1\001\u03a1\u03c2\001\u03a3\u03c3\001\u03a3\u03c4\001\u03a4\u03c5\001\u03a5\u03c6\001\u03a6\u03c7\001\u03a7\u03c8\001\u03a8\u03c9\001\u03a9\u03ca\001\u03aa\u03cb\001\u03ab\u03cc\001\u038c\u03cd\001\u038e\u03ce\001\u038f\u03d0\001\u0392\u03d1\001\u0398\u03d5\001\u03a6\u03d6\001\u03a0\u03d7\001\u03cf\u03d9\001\u03d8\u03db\001\u03da\u03dd\001\u03dc\u03df\001\u03de\u03e1\001\u03e0\u03e3\001\u03e2\u03e5\001\u03e4\u03e7\001\u03e6\u03e9\001\u03e8\u03eb\001\u03ea\u03ed\001\u03ec\u03ef\001\u03ee\u03f0\001\u039a\u03f1\001\u03a1\u03f2\001\u03f9\u03f3\001\u037f\u03f5\001\u0395\u03f8\001\u03f7\u03fb\001\u03fa\u0430\001\u0410\u0431\001\u0411\u0432\001\u0412\u0433\001\u0413\u0434\001\u0414\u0435\001\u0415\u0436\001\u0416\u0437\001\u0417\u0438\001\u0418\u0439\001\u0419\u043a\001\u041a\u043b\001\u041b\u043c\001\u041c\u043d\001\u041d\u043e\001\u041e\u043f\001\u041f\u0440\001\u0420\u0441\001\u0421\u0442\001\u0422\u0443\001\u0423\u0444\001\u0424\u0445\001\u0425\u0446\001\u0426\u0447\001\u0427\u0448\001\u0428\u0449\001\u0429\u044a\001\u042a\u044b\001\u042b\u044c\001\u042c\u044d\001\u042d\u044e\001\u042e\u044f\001\u042f\u0450\001\u0400\u0451\001\u0401\u0452\001\u0402\u0453\001\u0403\u0454\001\u0404\u0455\001\u0405\u0456\001\u0406\u0457\001\u0407\u0458\001\u0408\u0459\001\u0409\u045a\001\u040a\u045b\001\u040b\u045c\001\u040c\u045d\001\u040d\u045e\001\u040e\u045f\001\u040f\u0461\001\u0460\u0463\001\u0462\u0465\001\u0464\u0467\001\u0466\u0469\001\u0468\u046b\001\u046a\u046d\001\u046c\u046f\001\u046e\u0471\001\u0470\u0473\001\u0472\u0475\001\u0474\u0477\001\u0476\u0479\001\u0478\u047b\001\u047a\u047d\001\u047c\u047f\001\u047e\u0481\001\u0480\u048b\001\u048a\u048d\001\u048c\u048f\001\u048e\u0491\001\u0490\u0493\001\u0492\u0495\001\u0494\u0497\001\u0496\u0499\001\u0498\u049b\001\u049a\u049d\001\u049c\u049f\001\u049e\u04a1\001\u04a0\u04a3\001\u04a2\u04a5\001\u04a4\u04a7\001\u04a6\u04a9\001\u04a8\u04ab\001\u04aa\u04ad\001\u04ac\u04af\001\u04ae\u04b1\001\u04b0\u04b3\001\u04b2\u04b5\001\u04b4\u04b7\001\u04b6\u04b9\001\u04b8\u04bb\001\u04ba\u04bd\001\u04bc\u04bf\001\u04be\u04c2\001\u04c1\u04c4\001\u04c3\u04c6\001\u04c5\u04c8\001\u04c7\u04ca\001\u04c9\u04cc\001\u04cb\u04ce\001\u04cd\u04cf\001\u04c0\u04d1\001\u04d0\u04d3\001\u04d2\u04d5\001\u04d4\u04d7\001\u04d6\u04d9\001\u04d8\u04db\001\u04da\u04dd\001\u04dc\u04df\001\u04de\u04e1\001\u04e0\u04e3\001\u04e2\u04e5\001\u04e4\u04e7\001\u04e6\u04e9\001\u04e8\u04eb\001\u04ea\u04ed\001\u04ec\u04ef\001\u04ee\u04f1\001\u04f0\u04f3\001\u04f2\u04f5\001\u04f4\u04f7\001\u04f6\u04f9\001\u04f8\u04fb\001\u04fa\u04fd\001\u04fc\u04ff\001\u04fe\u0501\001\u0500\u0503\001\u0502\u0505\001\u0504\u0507\001\u0506\u0509\001\u0508\u050b\001\u050a\u050d\001\u050c\u050f\001\u050e\u0511\001\u0510\u0513\001\u0512\u0515\001\u0514\u0517\001\u0516\u0519\001\u0518\u051b\001\u051a\u051d\001\u051c\u051f\001\u051e\u0521\001\u0520\u0523\001\u0522\u0525\001\u0524\u0527\001\u0526\u0529\001\u0528\u052b\001\u052a\u052d\001\u052c\u052f\001\u052e\u0561\001\u0531\u0562\001\u0532\u0563\001\u0533\u0564\001\u0534\u0565\001\u0535\u0566\001\u0536\u0567\001\u0537\u0568\001\u0538\u0569\001\u0539\u056a\001\u053a\u056b\001\u053b\u056c\001\u053c\u056d\001\u053d\u056e\001\u053e\u056f\001\u053f\u0570\001\u0540\u0571\001\u0541\u0572\001\u0542\u0573\001\u0543\u0574\001\u0544\u0575\001\u0545\u0576\001\u0546\u0577\001\u0547\u0578\001\u0548\u0579\001\u0549\u057a\001\u054a\u057b\001\u054b\u057c\001\u054c\u057d\001\u054d\u057e\001\u054e\u057f\001\u054f\u0580\001\u0550\u0581\001\u0551\u0582\001\u0552\u0583\001\u0553\u0584\001\u0554\u0585\001\u0555\u0586\001\u0556\u0587\002\u0535\u0582\u13f8\001\u13f0\u13f9\001\u13f1\u13fa\001\u13f2\u13fb\001\u13f3\u13fc\001\u13f4\u13fd\001\u13f5\u1c80\001\u0412\u1c81\001\u0414\u1c82\001\u041e\u1c83\001\u0421\u1c84\001\u0422\u1c85\001\u0422\u1c86\001\u042a\u1c87\001\u0462\u1c88\001\ua64a\u1d79\001\ua77d\u1d7d\001\u2c63\u1d8e\001\ua7c6\u1e01\001\u1e00\u1e03\001\u1e02\u1e05\001\u1e04\u1e07\001\u1e06\u1e09\001\u1e08\u1e0b\001\u1e0a\u1e0d\001\u1e0c\u1e0f\001\u1e0e\u1e11\001\u1e10\u1e13\001\u1e12\u1e15\001\u1e14\u1e17\001\u1e16\u1e19\001\u1e18\u1e1b\001\u1e1a\u1e1d\001\u1e1c\u1e1f\001\u1e1e\u1e21\001\u1e20\u1e23\001\u1e22\u1e25\001\u1e24\u1e27\001\u1e26\u1e29\001\u1e28\u1e2b\001\u1e2a\u1e2d\001\u1e2c\u1e2f\001\u1e2e\u1e31\001\u1e30\u1e33\001\u1e32\u1e35\001\u1e34\u1e37\001\u1e36\u1e39\001\u1e38\u1e3b\001\u1e3a\u1e3d\001\u1e3c\u1e3f\001\u1e3e\u1e41\001\u1e40\u1e43\001\u1e42\u1e45\001\u1e44\u1e47\001\u1e46\u1e49\001\u1e48\u1e4b\001\u1e4a\u1e4d\001\u1e4c\u1e4f\001\u1e4e\u1e51\001\u1e50\u1e53\001\u1e52\u1e55\001\u1e54\u1e57\001\u1e56\u1e59\001\u1e58\u1e5b\001\u1e5a\u1e5d\001\u1e5c\u1e5f\001\u1e5e\u1e61\001\u1e60\u1e63\001\u1e62\u1e65\001\u1e64\u1e67\001\u1e66\u1e69\001\u1e68\u1e6b\001\u1e6a\u1e6d\001\u1e6c\u1e6f\001\u1e6e\u1e71\001\u1e70\u1e73\001\u1e72\u1e75\001\u1e74\u1e77\001\u1e76\u1e79\001\u1e78\u1e7b\001\u1e7a\u1e7d\001\u1e7c\u1e7f\001\u1e7e\u1e81\001\u1e80\u1e83\001\u1e82\u1e85\001\u1e84\u1e87\001\u1e86\u1e89\001\u1e88\u1e8b\001\u1e8a\u1e8d\001\u1e8c\u1e8f\001\u1e8e\u1e91\001\u1e90\u1e93\001\u1e92\u1e95\001\u1e94\u1e96\002H\u0331\u1e97\002T\u0308\u1e98\002W\u030a\u1e99\002Y\u030a\u1e9a\002A\u02be\u1e9b\001\u1e60\u1ea1\001\u1ea0\u1ea3\001\u1ea2\u1ea5\001\u1ea4\u1ea7\001\u1ea6\u1ea9\001\u1ea8\u1eab\001\u1eaa\u1ead\001\u1eac\u1eaf\001\u1eae\u1eb1\001\u1eb0\u1eb3\001\u1eb2\u1eb5\001\u1eb4\u1eb7\001\u1eb6\u1eb9\001\u1eb8\u1ebb\001\u1eba\u1ebd\001\u1ebc\u1ebf\001\u1ebe\u1ec1\001\u1ec0\u1ec3\001\u1ec2\u1ec5\001\u1ec4\u1ec7\001\u1ec6\u1ec9\001\u1ec8\u1ecb\001\u1eca\u1ecd\001\u1ecc\u1ecf\001\u1ece\u1ed1\001\u1ed0\u1ed3\001\u1ed2\u1ed5\001\u1ed4\u1ed7\001\u1ed6\u1ed9\001\u1ed8\u1edb\001\u1eda\u1edd\001\u1edc\u1edf\001\u1ede\u1ee1\001\u1ee0\u1ee3\001\u1ee2\u1ee5\001\u1ee4\u1ee7\001\u1ee6\u1ee9\001\u1ee8\u1eeb\001\u1eea\u1eed\001\u1eec\u1eef\001\u1eee\u1ef1\001\u1ef0\u1ef3\001\u1ef2\u1ef5\001\u1ef4\u1ef7\001\u1ef6\u1ef9\001\u1ef8\u1efb\001\u1efa\u1efd\001\u1efc\u1eff\001\u1efe\u1f00\001\u1f08\u1f01\001\u1f09\u1f02\001\u1f0a\u1f03\001\u1f0b\u1f04\001\u1f0c\u1f05\001\u1f0d\u1f06\001\u1f0e\u1f07\001\u1f0f\u1f10\001\u1f18\u1f11\001\u1f19\u1f12\001\u1f1a\u1f13\001\u1f1b\u1f14\001\u1f1c\u1f15\001\u1f1d\u1f20\001\u1f28\u1f21\001\u1f29\u1f22\001\u1f2a\u1f23\001\u1f2b\u1f24\001\u1f2c\u1f25\001\u1f2d\u1f26\001\u1f2e\u1f27\001\u1f2f\u1f30\001\u1f38\u1f31\001\u1f39\u1f32\001\u1f3a\u1f33\001\u1f3b\u1f34\001\u1f3c\u1f35\001\u1f3d\u1f36\001\u1f3e\u1f37\001\u1f3f\u1f40\001\u1f48\u1f41\001\u1f49\u1f42\001\u1f4a\u1f43\001\u1f4b\u1f44\001\u1f4c\u1f45\001\u1f4d\u1f50\002\u03a5\u0313\u1f51\001\u1f59\u1f52\003\u03a5\u0313\u0300\u1f53\001\u1f5b\u1f54\003\u03a5\u0313\u0301\u1f55\001\u1f5d\u1f56\003\u03a5\u0313\u0342\u1f57\001\u1f5f\u1f60\001\u1f68\u1f61\001\u1f69\u1f62\001\u1f6a\u1f63\001\u1f6b\u1f64\001\u1f6c\u1f65\001\u1f6d\u1f66\001\u1f6e\u1f67\001\u1f6f\u1f70\001\u1fba\u1f71\001\u1fbb\u1f72\001\u1fc8\u1f73\001\u1fc9\u1f74\001\u1fca\u1f75\001\u1fcb\u1f76\001\u1fda\u1f77\001\u1fdb\u1f78\001\u1ff8\u1f79\001\u1ff9\u1f7a\001\u1fea\u1f7b\001\u1feb\u1f7c\001\u1ffa\u1f7d\001\u1ffb\u1f80\001\u1f88\u1f81\001\u1f89\u1f82\001\u1f8a\u1f83\001\u1f8b\u1f84\001\u1f8c\u1f85\001\u1f8d\u1f86\001\u1f8e\u1f87\001\u1f8f\u1f90\001\u1f98\u1f91\001\u1f99\u1f92\001\u1f9a\u1f93\001\u1f9b\u1f94\001\u1f9c\u1f95\001\u1f9d\u1f96\001\u1f9e\u1f97\001\u1f9f\u1fa0\001\u1fa8\u1fa1\001\u1fa9\u1fa2\001\u1faa\u1fa3\001\u1fab\u1fa4\001\u1fac\u1fa5\001\u1fad\u1fa6\001\u1fae\u1fa7\001\u1faf\u1fb0\001\u1fb8\u1fb1\001\u1fb9\u1fb2\002\u1fba\u0345\u1fb3\001\u1fbc\u1fb4\002\u0386\u0345\u1fb6\002\u0391\u0342\u1fb7\003\u0391\u0342\u0345\u1fbe\001\u0399\u1fc2\002\u1fca\u0345\u1fc3\001\u1fcc\u1fc4\002\u0389\u0345\u1fc6\002\u0397\u0342\u1fc7\003\u0397\u0342\u0345\u1fd0\001\u1fd8\u1fd1\001\u1fd9\u1fd2\003\u0399\u0308\u0300\u1fd3\003\u0399\u0308\u0301\u1fd6\002\u0399\u0342\u1fd7\003\u0399\u0308\u0342\u1fe0\001\u1fe8\u1fe1\001\u1fe9\u1fe2\003\u03a5\u0308\u0300\u1fe3\003\u03a5\u0308\u0301\u1fe4\002\u03a1\u0313\u1fe5\001\u1fec\u1fe6\002\u03a5\u0342\u1fe7\003\u03a5\u0308\u0342\u1ff2\002\u1ffa\u0345\u1ff3\001\u1ffc\u1ff4\002\u038f\u0345\u1ff6\002\u03a9\u0342\u1ff7\003\u03a9\u0342\u0345\u214e\001\u2132\u2170\001\u2160\u2171\001\u2161\u2172\001\u2162\u2173\001\u2163\u2174\001\u2164\u2175\001\u2165\u2176\001\u2166\u2177\001\u2167\u2178\001\u2168\u2179\001\u2169\u217a\001\u216a\u217b\001\u216b\u217c\001\u216c\u217d\001\u216d\u217e\001\u216e\u217f\001\u216f\u2184\001\u2183\u24d0\001\u24b6\u24d1\001\u24b7\u24d2\001\u24b8\u24d3\001\u24b9\u24d4\001\u24ba\u24d5\001\u24bb\u24d6\001\u24bc\u24d7\001\u24bd\u24d8\001\u24be\u24d9\001\u24bf\u24da\001\u24c0\u24db\001\u24c1\u24dc\001\u24c2\u24dd\001\u24c3\u24de\001\u24c4\u24df\001\u24c5\u24e0\001\u24c6\u24e1\001\u24c7\u24e2\001\u24c8\u24e3\001\u24c9\u24e4\001\u24ca\u24e5\001\u24cb\u24e6\001\u24cc\u24e7\001\u24cd\u24e8\001\u24ce\u24e9\001\u24cf\u2c30\001\u2c00\u2c31\001\u2c01\u2c32\001\u2c02\u2c33\001\u2c03\u2c34\001\u2c04\u2c35\001\u2c05\u2c36\001\u2c06\u2c37\001\u2c07\u2c38\001\u2c08\u2c39\001\u2c09\u2c3a\001\u2c0a\u2c3b\001\u2c0b\u2c3c\001\u2c0c\u2c3d\001\u2c0d\u2c3e\001\u2c0e\u2c3f\001\u2c0f\u2c40\001\u2c10\u2c41\001\u2c11\u2c42\001\u2c12\u2c43\001\u2c13\u2c44\001\u2c14\u2c45\001\u2c15\u2c46\001\u2c16\u2c47\001\u2c17\u2c48\001\u2c18\u2c49\001\u2c19\u2c4a\001\u2c1a\u2c4b\001\u2c1b\u2c4c\001\u2c1c\u2c4d\001\u2c1d\u2c4e\001\u2c1e\u2c4f\001\u2c1f\u2c50\001\u2c20\u2c51\001\u2c21\u2c52\001\u2c22\u2c53\001\u2c23\u2c54\001\u2c24\u2c55\001\u2c25\u2c56\001\u2c26\u2c57\001\u2c27\u2c58\001\u2c28\u2c59\001\u2c29\u2c5a\001\u2c2a\u2c5b\001\u2c2b\u2c5c\001\u2c2c\u2c5d\001\u2c2d\u2c5e\001\u2c2e\u2c5f\001\u2c2f\u2c61\001\u2c60\u2c65\001\u023a\u2c66\001\u023e\u2c68\001\u2c67\u2c6a\001\u2c69\u2c6c\001\u2c6b\u2c73\001\u2c72\u2c76\001\u2c75\u2c81\001\u2c80\u2c83\001\u2c82\u2c85\001\u2c84\u2c87\001\u2c86\u2c89\001\u2c88\u2c8b\001\u2c8a\u2c8d\001\u2c8c\u2c8f\001\u2c8e\u2c91\001\u2c90\u2c93\001\u2c92\u2c95\001\u2c94\u2c97\001\u2c96\u2c99\001\u2c98\u2c9b\001\u2c9a\u2c9d\001\u2c9c\u2c9f\001\u2c9e\u2ca1\001\u2ca0\u2ca3\001\u2ca2\u2ca5\001\u2ca4\u2ca7\001\u2ca6\u2ca9\001\u2ca8\u2cab\001\u2caa\u2cad\001\u2cac\u2caf\001\u2cae\u2cb1\001\u2cb0\u2cb3\001\u2cb2\u2cb5\001\u2cb4\u2cb7\001\u2cb6\u2cb9\001\u2cb8\u2cbb\001\u2cba\u2cbd\001\u2cbc\u2cbf\001\u2cbe\u2cc1\001\u2cc0\u2cc3\001\u2cc2\u2cc5\001\u2cc4\u2cc7\001\u2cc6\u2cc9\001\u2cc8\u2ccb\001\u2cca\u2ccd\001\u2ccc\u2ccf\001\u2cce\u2cd1\001\u2cd0\u2cd3\001\u2cd2\u2cd5\001\u2cd4\u2cd7\001\u2cd6\u2cd9\001\u2cd8\u2cdb\001\u2cda\u2cdd\001\u2cdc\u2cdf\001\u2cde\u2ce1\001\u2ce0\u2ce3\001\u2ce2\u2cec\001\u2ceb\u2cee\001\u2ced\u2cf3\001\u2cf2\u2d00\001\u10a0\u2d01\001\u10a1\u2d02\001\u10a2\u2d03\001\u10a3\u2d04\001\u10a4\u2d05\001\u10a5\u2d06\001\u10a6\u2d07\001\u10a7\u2d08\001\u10a8\u2d09\001\u10a9\u2d0a\001\u10aa\u2d0b\001\u10ab\u2d0c\001\u10ac\u2d0d\001\u10ad\u2d0e\001\u10ae\u2d0f\001\u10af\u2d10\001\u10b0\u2d11\001\u10b1\u2d12\001\u10b2\u2d13\001\u10b3\u2d14\001\u10b4\u2d15\001\u10b5\u2d16\001\u10b6\u2d17\001\u10b7\u2d18\001\u10b8\u2d19\001\u10b9\u2d1a\001\u10ba\u2d1b\001\u10bb\u2d1c\001\u10bc\u2d1d\001\u10bd\u2d1e\001\u10be\u2d1f\001\u10bf\u2d20\001\u10c0\u2d21\001\u10c1\u2d22\001\u10c2\u2d23\001\u10c3\u2d24\001\u10c4\u2d25\001\u10c5\u2d27\001\u10c7\u2d2d\001\u10cd\ua641\001\ua640\ua643\001\ua642\ua645\001\ua644\ua647\001\ua646\ua649\001\ua648\ua64b\001\ua64a\ua64d\001\ua64c\ua64f\001\ua64e\ua651\001\ua650\ua653\001\ua652\ua655\001\ua654\ua657\001\ua656\ua659\001\ua658\ua65b\001\ua65a\ua65d\001\ua65c\ua65f\001\ua65e\ua661\001\ua660\ua663\001\ua662\ua665\001\ua664\ua667\001\ua666\ua669\001\ua668\ua66b\001\ua66a\ua66d\001\ua66c\ua681\001\ua680\ua683\001\ua682\ua685\001\ua684\ua687\001\ua686\ua689\001\ua688\ua68b\001\ua68a\ua68d\001\ua68c\ua68f\001\ua68e\ua691\001\ua690\ua693\001\ua692\ua695\001\ua694\ua697\001\ua696\ua699\001\ua698\ua69b\001\ua69a\ua723\001\ua722\ua725\001\ua724\ua727\001\ua726\ua729\001\ua728\ua72b\001\ua72a\ua72d\001\ua72c\ua72f\001\ua72e\ua733\001\ua732\ua735\001\ua734\ua737\001\ua736\ua739\001\ua738\ua73b\001\ua73a\ua73d\001\ua73c\ua73f\001\ua73e\ua741\001\ua740\ua743\001\ua742\ua745\001\ua744\ua747\001\ua746\ua749\001\ua748\ua74b\001\ua74a\ua74d\001\ua74c\ua74f\001\ua74e\ua751\001\ua750\ua753\001\ua752\ua755\001\ua754\ua757\001\ua756\ua759\001\ua758\ua75b\001\ua75a\ua75d\001\ua75c\ua75f\001\ua75e\ua761\001\ua760\ua763\001\ua762\ua765\001\ua764\ua767\001\ua766\ua769\001\ua768\ua76b\001\ua76a\ua76d\001\ua76c\ua76f\001\ua76e\ua77a\001\ua779\ua77c\001\ua77b\ua77f\001\ua77e\ua781\001\ua780\ua783\001\ua782\ua785\001\ua784\ua787\001\ua786\ua78c\001\ua78b\ua791\001\ua790\ua793\001\ua792\ua794\001\ua7c4\ua797\001\ua796\ua799\001\ua798\ua79b\001\ua79a\ua79d\001\ua79c\ua79f\001\ua79e\ua7a1\001\ua7a0\ua7a3\001\ua7a2\ua7a5\001\ua7a4\ua7a7\001\ua7a6\ua7a9\001\ua7a8\ua7b5\001\ua7b4\ua7b7\001\ua7b6\ua7b9\001\ua7b8\ua7bb\001\ua7ba\ua7bd\001\ua7bc\ua7bf\001\ua7be\ua7c1\001\ua7c0\ua7c3\001\ua7c2\ua7c8\001\ua7c7\ua7ca\001\ua7c9\ua7d1\001\ua7d0\ua7d7\001\ua7d6\ua7d9\001\ua7d8\ua7f6\001\ua7f5\uab53\001\ua7b3\uab70\001\u13a0\uab71\001\u13a1\uab72\001\u13a2\uab73\001\u13a3\uab74\001\u13a4\uab75\001\u13a5\uab76\001\u13a6\uab77\001\u13a7\uab78\001\u13a8\uab79\001\u13a9\uab7a\001\u13aa\uab7b\001\u13ab\uab7c\001\u13ac\uab7d\001\u13ad\uab7e\001\u13ae\uab7f\001\u13af\uab80\001\u13b0\uab81\001\u13b1\uab82\001\u13b2\uab83\001\u13b3\uab84\001\u13b4\uab85\001\u13b5\uab86\001\u13b6\uab87\001\u13b7\uab88\001\u13b8\uab89\001\u13b9\uab8a\001\u13ba\uab8b\001\u13bb\uab8c\001\u13bc\uab8d\001\u13bd\uab8e\001\u13be\uab8f\001\u13bf\uab90\001\u13c0\uab91\001\u13c1\uab92\001\u13c2\uab93\001\u13c3\uab94\001\u13c4\uab95\001\u13c5\uab96\001\u13c6\uab97\001\u13c7\uab98\001\u13c8\uab99\001\u13c9\uab9a\001\u13ca\uab9b\001\u13cb\uab9c\001\u13cc\uab9d\001\u13cd\uab9e\001\u13ce\uab9f\001\u13cf\uaba0\001\u13d0\uaba1\001\u13d1\uaba2\001\u13d2\uaba3\001\u13d3\uaba4\001\u13d4\uaba5\001\u13d5\uaba6\001\u13d6\uaba7\001\u13d7\uaba8\001\u13d8\uaba9\001\u13d9\uabaa\001\u13da\uabab\001\u13db\uabac\001\u13dc\uabad\001\u13dd\uabae\001\u13de\uabaf\001\u13df\uabb0\001\u13e0\uabb1\001\u13e1\uabb2\001\u13e2\uabb3\001\u13e3\uabb4\001\u13e4\uabb5\001\u13e5\uabb6\001\u13e6\uabb7\001\u13e7\uabb8\001\u13e8\uabb9\001\u13e9\uabba\001\u13ea\uabbb\001\u13eb\uabbc\001\u13ec\uabbd\001\u13ed\uabbe\001\u13ee\uabbf\001\u13ef\ufb00\002Ff\ufb01\002Fi\ufb02\002Fl\ufb03\003Ffi\ufb04\003Ffl\ufb05\002St\ufb06\002St\ufb13\002\u0544\u0576\ufb14\002\u0544\u0565\ufb15\002\u0544\u056b\ufb16\002\u054e\u0576\ufb17\002\u0544\u056d\uff41\001\uff21\uff42\001\uff22\uff43\001\uff23\uff44\001\uff24\uff45\001\uff25\uff46\001\uff26\uff47\001\uff27\uff48\001\uff28\uff49\001\uff29\uff4a\001\uff2a\uff4b\001\uff2b\uff4c\001\uff2c\uff4d\001\uff2d\uff4e\001\uff2e\uff4f\001\uff2f\uff50\001\uff30\uff51\001\uff31\uff52\001\uff32\uff53\001\uff33\uff54\001\uff34\uff55\001\uff35\uff56\001\uff36\uff57\001\uff37\uff58\001\uff38\uff59\001\uff39\uff5a\001\uff3a", + }; + + static final String[] FOLD = { + "A\001aB\001bC\001cD\001dE\001eF\001fG\001gH\001hI\001iJ\001jK\001kL\001lM\001mN\001nO\001oP\001pQ\001qR\001rS\001sT\001tU\001uV\001vW\001wX\001xY\001yZ\001z\u00b5\001\u03bc\u00c0\001\u00e0\u00c1\001\u00e1\u00c2\001\u00e2\u00c3\001\u00e3\u00c4\001\u00e4\u00c5\001\u00e5\u00c6\001\u00e6\u00c7\001\u00e7\u00c8\001\u00e8\u00c9\001\u00e9\u00ca\001\u00ea\u00cb\001\u00eb\u00cc\001\u00ec\u00cd\001\u00ed\u00ce\001\u00ee\u00cf\001\u00ef\u00d0\001\u00f0\u00d1\001\u00f1\u00d2\001\u00f2\u00d3\001\u00f3\u00d4\001\u00f4\u00d5\001\u00f5\u00d6\001\u00f6\u00d8\001\u00f8\u00d9\001\u00f9\u00da\001\u00fa\u00db\001\u00fb\u00dc\001\u00fc\u00dd\001\u00fd\u00de\001\u00fe\u00df\002ss\u0100\001\u0101\u0102\001\u0103\u0104\001\u0105\u0106\001\u0107\u0108\001\u0109\u010a\001\u010b\u010c\001\u010d\u010e\001\u010f\u0110\001\u0111\u0112\001\u0113\u0114\001\u0115\u0116\001\u0117\u0118\001\u0119\u011a\001\u011b\u011c\001\u011d\u011e\001\u011f\u0120\001\u0121\u0122\001\u0123\u0124\001\u0125\u0126\001\u0127\u0128\001\u0129\u012a\001\u012b\u012c\001\u012d\u012e\001\u012f\u0130\002i\u0307\u0132\001\u0133\u0134\001\u0135\u0136\001\u0137\u0139\001\u013a\u013b\001\u013c\u013d\001\u013e\u013f\001\u0140\u0141\001\u0142\u0143\001\u0144\u0145\001\u0146\u0147\001\u0148\u0149\002\u02bcn\u014a\001\u014b\u014c\001\u014d\u014e\001\u014f\u0150\001\u0151\u0152\001\u0153\u0154\001\u0155\u0156\001\u0157\u0158\001\u0159\u015a\001\u015b\u015c\001\u015d\u015e\001\u015f\u0160\001\u0161\u0162\001\u0163\u0164\001\u0165\u0166\001\u0167\u0168\001\u0169\u016a\001\u016b\u016c\001\u016d\u016e\001\u016f\u0170\001\u0171\u0172\001\u0173\u0174\001\u0175\u0176\001\u0177\u0178\001\u00ff\u0179\001\u017a\u017b\001\u017c\u017d\001\u017e\u017f\001s\u0181\001\u0253\u0182\001\u0183\u0184\001\u0185\u0186\001\u0254\u0187\001\u0188\u0189\001\u0256\u018a\001\u0257\u018b\001\u018c\u018e\001\u01dd\u018f\001\u0259\u0190\001\u025b\u0191\001\u0192\u0193\001\u0260\u0194\001\u0263\u0196\001\u0269\u0197\001\u0268\u0198\001\u0199\u019c\001\u026f\u019d\001\u0272\u019f\001\u0275\u01a0\001\u01a1\u01a2\001\u01a3\u01a4\001\u01a5\u01a6\001\u0280\u01a7\001\u01a8\u01a9\001\u0283\u01ac\001\u01ad\u01ae\001\u0288\u01af\001\u01b0\u01b1\001\u028a\u01b2\001\u028b\u01b3\001\u01b4\u01b5\001\u01b6\u01b7\001\u0292\u01b8\001\u01b9\u01bc\001\u01bd\u01c4\001\u01c6\u01c5\001\u01c6\u01c7\001\u01c9\u01c8\001\u01c9\u01ca\001\u01cc\u01cb\001\u01cc\u01cd\001\u01ce\u01cf\001\u01d0\u01d1\001\u01d2\u01d3\001\u01d4\u01d5\001\u01d6\u01d7\001\u01d8\u01d9\001\u01da\u01db\001\u01dc\u01de\001\u01df\u01e0\001\u01e1\u01e2\001\u01e3\u01e4\001\u01e5\u01e6\001\u01e7\u01e8\001\u01e9\u01ea\001\u01eb\u01ec\001\u01ed\u01ee\001\u01ef\u01f0\002j\u030c\u01f1\001\u01f3\u01f2\001\u01f3\u01f4\001\u01f5\u01f6\001\u0195\u01f7\001\u01bf\u01f8\001\u01f9\u01fa\001\u01fb\u01fc\001\u01fd\u01fe\001\u01ff\u0200\001\u0201\u0202\001\u0203\u0204\001\u0205\u0206\001\u0207\u0208\001\u0209\u020a\001\u020b\u020c\001\u020d\u020e\001\u020f\u0210\001\u0211\u0212\001\u0213\u0214\001\u0215\u0216\001\u0217\u0218\001\u0219\u021a\001\u021b\u021c\001\u021d\u021e\001\u021f\u0220\001\u019e\u0222\001\u0223\u0224\001\u0225\u0226\001\u0227\u0228\001\u0229\u022a\001\u022b\u022c\001\u022d\u022e\001\u022f\u0230\001\u0231\u0232\001\u0233\u023a\001\u2c65\u023b\001\u023c\u023d\001\u019a\u023e\001\u2c66\u0241\001\u0242\u0243\001\u0180\u0244\001\u0289\u0245\001\u028c\u0246\001\u0247\u0248\001\u0249\u024a\001\u024b\u024c\001\u024d\u024e\001\u024f\u0345\001\u03b9\u0370\001\u0371\u0372\001\u0373\u0376\001\u0377\u037f\001\u03f3\u0386\001\u03ac\u0388\001\u03ad\u0389\001\u03ae\u038a\001\u03af\u038c\001\u03cc\u038e\001\u03cd\u038f\001\u03ce\u0390\003\u03b9\u0308\u0301\u0391\001\u03b1\u0392\001\u03b2\u0393\001\u03b3\u0394\001\u03b4\u0395\001\u03b5\u0396\001\u03b6\u0397\001\u03b7\u0398\001\u03b8\u0399\001\u03b9\u039a\001\u03ba\u039b\001\u03bb\u039c\001\u03bc\u039d\001\u03bd\u039e\001\u03be\u039f\001\u03bf\u03a0\001\u03c0\u03a1\001\u03c1\u03a3\001\u03c3\u03a4\001\u03c4\u03a5\001\u03c5\u03a6\001\u03c6\u03a7\001\u03c7\u03a8\001\u03c8\u03a9\001\u03c9\u03aa\001\u03ca\u03ab\001\u03cb\u03b0\003\u03c5\u0308\u0301\u03c2\001\u03c3\u03cf\001\u03d7\u03d0\001\u03b2\u03d1\001\u03b8\u03d5\001\u03c6\u03d6\001\u03c0\u03d8\001\u03d9\u03da\001\u03db\u03dc\001\u03dd\u03de\001\u03df\u03e0\001\u03e1\u03e2\001\u03e3\u03e4\001\u03e5\u03e6\001\u03e7\u03e8\001\u03e9\u03ea\001\u03eb\u03ec\001\u03ed\u03ee\001\u03ef\u03f0\001\u03ba\u03f1\001\u03c1\u03f4\001\u03b8\u03f5\001\u03b5\u03f7\001\u03f8\u03f9\001\u03f2\u03fa\001\u03fb\u03fd\001\u037b\u03fe\001\u037c\u03ff\001\u037d\u0400\001\u0450\u0401\001\u0451\u0402\001\u0452\u0403\001\u0453\u0404\001\u0454\u0405\001\u0455\u0406\001\u0456\u0407\001\u0457\u0408\001\u0458\u0409\001\u0459\u040a\001\u045a\u040b\001\u045b\u040c\001\u045c\u040d\001\u045d\u040e\001\u045e\u040f\001\u045f\u0410\001\u0430\u0411\001\u0431\u0412\001\u0432\u0413\001\u0433\u0414\001\u0434\u0415\001\u0435\u0416\001\u0436\u0417\001\u0437\u0418\001\u0438\u0419\001\u0439\u041a\001\u043a\u041b\001\u043b\u041c\001\u043c\u041d\001\u043d\u041e\001\u043e\u041f\001\u043f\u0420\001\u0440\u0421\001\u0441\u0422\001\u0442\u0423\001\u0443\u0424\001\u0444\u0425\001\u0445\u0426\001\u0446\u0427\001\u0447\u0428\001\u0448\u0429\001\u0449\u042a\001\u044a\u042b\001\u044b\u042c\001\u044c\u042d\001\u044d\u042e\001\u044e\u042f\001\u044f\u0460\001\u0461\u0462\001\u0463\u0464\001\u0465\u0466\001\u0467\u0468\001\u0469\u046a\001\u046b\u046c\001\u046d\u046e\001\u046f\u0470\001\u0471\u0472\001\u0473\u0474\001\u0475\u0476\001\u0477\u0478\001\u0479\u047a\001\u047b\u047c\001\u047d\u047e\001\u047f\u0480\001\u0481\u048a\001\u048b\u048c\001\u048d\u048e\001\u048f\u0490\001\u0491\u0492\001\u0493\u0494\001\u0495\u0496\001\u0497\u0498\001\u0499\u049a\001\u049b\u049c\001\u049d\u049e\001\u049f\u04a0\001\u04a1\u04a2\001\u04a3\u04a4\001\u04a5\u04a6\001\u04a7\u04a8\001\u04a9\u04aa\001\u04ab\u04ac\001\u04ad\u04ae\001\u04af\u04b0\001\u04b1\u04b2\001\u04b3\u04b4\001\u04b5\u04b6\001\u04b7\u04b8\001\u04b9\u04ba\001\u04bb\u04bc\001\u04bd\u04be\001\u04bf\u04c0\001\u04cf\u04c1\001\u04c2\u04c3\001\u04c4\u04c5\001\u04c6\u04c7\001\u04c8\u04c9\001\u04ca\u04cb\001\u04cc\u04cd\001\u04ce\u04d0\001\u04d1\u04d2\001\u04d3\u04d4\001\u04d5\u04d6\001\u04d7\u04d8\001\u04d9\u04da\001\u04db\u04dc\001\u04dd\u04de\001\u04df\u04e0\001\u04e1\u04e2\001\u04e3\u04e4\001\u04e5\u04e6\001\u04e7\u04e8\001\u04e9\u04ea\001\u04eb\u04ec\001\u04ed\u04ee\001\u04ef\u04f0\001\u04f1\u04f2\001\u04f3\u04f4\001\u04f5\u04f6\001\u04f7\u04f8\001\u04f9\u04fa\001\u04fb\u04fc\001\u04fd\u04fe\001\u04ff\u0500\001\u0501\u0502\001\u0503\u0504\001\u0505\u0506\001\u0507\u0508\001\u0509\u050a\001\u050b\u050c\001\u050d\u050e\001\u050f\u0510\001\u0511\u0512\001\u0513\u0514\001\u0515\u0516\001\u0517\u0518\001\u0519\u051a\001\u051b\u051c\001\u051d\u051e\001\u051f\u0520\001\u0521\u0522\001\u0523\u0524\001\u0525\u0526\001\u0527\u0528\001\u0529\u052a\001\u052b\u052c\001\u052d\u052e\001\u052f\u0531\001\u0561\u0532\001\u0562\u0533\001\u0563\u0534\001\u0564\u0535\001\u0565\u0536\001\u0566\u0537\001\u0567\u0538\001\u0568\u0539\001\u0569\u053a\001\u056a\u053b\001\u056b\u053c\001\u056c\u053d\001\u056d\u053e\001\u056e\u053f\001\u056f\u0540\001\u0570\u0541\001\u0571\u0542\001\u0572\u0543\001\u0573\u0544\001\u0574\u0545\001\u0575\u0546\001\u0576\u0547\001\u0577\u0548\001\u0578\u0549\001\u0579\u054a\001\u057a\u054b\001\u057b\u054c\001\u057c\u054d\001\u057d\u054e\001\u057e\u054f\001\u057f\u0550\001\u0580\u0551\001\u0581\u0552\001\u0582\u0553\001\u0583\u0554\001\u0584\u0555\001\u0585\u0556\001\u0586\u0587\002\u0565\u0582\u10a0\001\u2d00\u10a1\001\u2d01\u10a2\001\u2d02\u10a3\001\u2d03\u10a4\001\u2d04\u10a5\001\u2d05\u10a6\001\u2d06\u10a7\001\u2d07\u10a8\001\u2d08\u10a9\001\u2d09\u10aa\001\u2d0a\u10ab\001\u2d0b\u10ac\001\u2d0c\u10ad\001\u2d0d\u10ae\001\u2d0e\u10af\001\u2d0f\u10b0\001\u2d10\u10b1\001\u2d11\u10b2\001\u2d12\u10b3\001\u2d13\u10b4\001\u2d14\u10b5\001\u2d15\u10b6\001\u2d16\u10b7\001\u2d17\u10b8\001\u2d18\u10b9\001\u2d19\u10ba\001\u2d1a\u10bb\001\u2d1b\u10bc\001\u2d1c\u10bd\001\u2d1d\u10be\001\u2d1e\u10bf\001\u2d1f\u10c0\001\u2d20\u10c1\001\u2d21\u10c2\001\u2d22\u10c3\001\u2d23\u10c4\001\u2d24\u10c5\001\u2d25\u10c7\001\u2d27\u10cd\001\u2d2d\u13f8\001\u13f0\u13f9\001\u13f1\u13fa\001\u13f2\u13fb\001\u13f3\u13fc\001\u13f4\u13fd\001\u13f5\u1c80\001\u0432\u1c81\001\u0434\u1c82\001\u043e\u1c83\001\u0441\u1c84\001\u0442\u1c85\001\u0442\u1c86\001\u044a\u1c87\001\u0463\u1c88\001\ua64b\u1c90\001\u10d0\u1c91\001\u10d1\u1c92\001\u10d2\u1c93\001\u10d3\u1c94\001\u10d4\u1c95\001\u10d5\u1c96\001\u10d6\u1c97\001\u10d7\u1c98\001\u10d8\u1c99\001\u10d9\u1c9a\001\u10da\u1c9b\001\u10db\u1c9c\001\u10dc\u1c9d\001\u10dd\u1c9e\001\u10de\u1c9f\001\u10df\u1ca0\001\u10e0\u1ca1\001\u10e1\u1ca2\001\u10e2\u1ca3\001\u10e3\u1ca4\001\u10e4\u1ca5\001\u10e5\u1ca6\001\u10e6\u1ca7\001\u10e7\u1ca8\001\u10e8\u1ca9\001\u10e9\u1caa\001\u10ea\u1cab\001\u10eb\u1cac\001\u10ec\u1cad\001\u10ed\u1cae\001\u10ee\u1caf\001\u10ef\u1cb0\001\u10f0\u1cb1\001\u10f1\u1cb2\001\u10f2\u1cb3\001\u10f3\u1cb4\001\u10f4\u1cb5\001\u10f5\u1cb6\001\u10f6\u1cb7\001\u10f7\u1cb8\001\u10f8\u1cb9\001\u10f9\u1cba\001\u10fa\u1cbd\001\u10fd\u1cbe\001\u10fe\u1cbf\001\u10ff\u1e00\001\u1e01\u1e02\001\u1e03\u1e04\001\u1e05\u1e06\001\u1e07\u1e08\001\u1e09\u1e0a\001\u1e0b\u1e0c\001\u1e0d\u1e0e\001\u1e0f\u1e10\001\u1e11\u1e12\001\u1e13\u1e14\001\u1e15\u1e16\001\u1e17\u1e18\001\u1e19\u1e1a\001\u1e1b\u1e1c\001\u1e1d\u1e1e\001\u1e1f\u1e20\001\u1e21\u1e22\001\u1e23\u1e24\001\u1e25\u1e26\001\u1e27\u1e28\001\u1e29\u1e2a\001\u1e2b\u1e2c\001\u1e2d\u1e2e\001\u1e2f\u1e30\001\u1e31\u1e32\001\u1e33\u1e34\001\u1e35\u1e36\001\u1e37\u1e38\001\u1e39\u1e3a\001\u1e3b\u1e3c\001\u1e3d\u1e3e\001\u1e3f\u1e40\001\u1e41\u1e42\001\u1e43\u1e44\001\u1e45\u1e46\001\u1e47\u1e48\001\u1e49\u1e4a\001\u1e4b\u1e4c\001\u1e4d\u1e4e\001\u1e4f\u1e50\001\u1e51\u1e52\001\u1e53\u1e54\001\u1e55\u1e56\001\u1e57\u1e58\001\u1e59\u1e5a\001\u1e5b\u1e5c\001\u1e5d\u1e5e\001\u1e5f\u1e60\001\u1e61\u1e62\001\u1e63\u1e64\001\u1e65\u1e66\001\u1e67\u1e68\001\u1e69\u1e6a\001\u1e6b\u1e6c\001\u1e6d\u1e6e\001\u1e6f\u1e70\001\u1e71\u1e72\001\u1e73\u1e74\001\u1e75\u1e76\001\u1e77\u1e78\001\u1e79\u1e7a\001\u1e7b\u1e7c\001\u1e7d\u1e7e\001\u1e7f\u1e80\001\u1e81\u1e82\001\u1e83\u1e84\001\u1e85\u1e86\001\u1e87\u1e88\001\u1e89\u1e8a\001\u1e8b\u1e8c\001\u1e8d\u1e8e\001\u1e8f\u1e90\001\u1e91\u1e92\001\u1e93\u1e94\001\u1e95\u1e96\002h\u0331\u1e97\002t\u0308\u1e98\002w\u030a\u1e99\002y\u030a\u1e9a\002a\u02be\u1e9b\001\u1e61\u1e9e\002ss\u1ea0\001\u1ea1\u1ea2\001\u1ea3\u1ea4\001\u1ea5\u1ea6\001\u1ea7\u1ea8\001\u1ea9\u1eaa\001\u1eab\u1eac\001\u1ead\u1eae\001\u1eaf\u1eb0\001\u1eb1\u1eb2\001\u1eb3\u1eb4\001\u1eb5\u1eb6\001\u1eb7\u1eb8\001\u1eb9\u1eba\001\u1ebb\u1ebc\001\u1ebd\u1ebe\001\u1ebf\u1ec0\001\u1ec1\u1ec2\001\u1ec3\u1ec4\001\u1ec5\u1ec6\001\u1ec7\u1ec8\001\u1ec9\u1eca\001\u1ecb\u1ecc\001\u1ecd\u1ece\001\u1ecf\u1ed0\001\u1ed1\u1ed2\001\u1ed3\u1ed4\001\u1ed5\u1ed6\001\u1ed7\u1ed8\001\u1ed9\u1eda\001\u1edb\u1edc\001\u1edd\u1ede\001\u1edf\u1ee0\001\u1ee1\u1ee2\001\u1ee3\u1ee4\001\u1ee5\u1ee6\001\u1ee7\u1ee8\001\u1ee9\u1eea\001\u1eeb\u1eec\001\u1eed\u1eee\001\u1eef\u1ef0\001\u1ef1\u1ef2\001\u1ef3\u1ef4\001\u1ef5\u1ef6\001\u1ef7\u1ef8\001\u1ef9\u1efa\001\u1efb\u1efc\001\u1efd\u1efe\001\u1eff\u1f08\001\u1f00\u1f09\001\u1f01\u1f0a\001\u1f02\u1f0b\001\u1f03\u1f0c\001\u1f04\u1f0d\001\u1f05\u1f0e\001\u1f06\u1f0f\001\u1f07\u1f18\001\u1f10\u1f19\001\u1f11\u1f1a\001\u1f12\u1f1b\001\u1f13\u1f1c\001\u1f14\u1f1d\001\u1f15\u1f28\001\u1f20\u1f29\001\u1f21\u1f2a\001\u1f22\u1f2b\001\u1f23\u1f2c\001\u1f24\u1f2d\001\u1f25\u1f2e\001\u1f26\u1f2f\001\u1f27\u1f38\001\u1f30\u1f39\001\u1f31\u1f3a\001\u1f32\u1f3b\001\u1f33\u1f3c\001\u1f34\u1f3d\001\u1f35\u1f3e\001\u1f36\u1f3f\001\u1f37\u1f48\001\u1f40\u1f49\001\u1f41\u1f4a\001\u1f42\u1f4b\001\u1f43\u1f4c\001\u1f44\u1f4d\001\u1f45\u1f50\002\u03c5\u0313\u1f52\003\u03c5\u0313\u0300\u1f54\003\u03c5\u0313\u0301\u1f56\003\u03c5\u0313\u0342\u1f59\001\u1f51\u1f5b\001\u1f53\u1f5d\001\u1f55\u1f5f\001\u1f57\u1f68\001\u1f60\u1f69\001\u1f61\u1f6a\001\u1f62\u1f6b\001\u1f63\u1f6c\001\u1f64\u1f6d\001\u1f65\u1f6e\001\u1f66\u1f6f\001\u1f67\u1f80\002\u1f00\u03b9\u1f81\002\u1f01\u03b9\u1f82\002\u1f02\u03b9\u1f83\002\u1f03\u03b9\u1f84\002\u1f04\u03b9\u1f85\002\u1f05\u03b9\u1f86\002\u1f06\u03b9\u1f87\002\u1f07\u03b9\u1f88\002\u1f00\u03b9\u1f89\002\u1f01\u03b9\u1f8a\002\u1f02\u03b9\u1f8b\002\u1f03\u03b9\u1f8c\002\u1f04\u03b9\u1f8d\002\u1f05\u03b9\u1f8e\002\u1f06\u03b9\u1f8f\002\u1f07\u03b9\u1f90\002\u1f20\u03b9\u1f91\002\u1f21\u03b9\u1f92\002\u1f22\u03b9\u1f93\002\u1f23\u03b9\u1f94\002\u1f24\u03b9\u1f95\002\u1f25\u03b9\u1f96\002\u1f26\u03b9\u1f97\002\u1f27\u03b9\u1f98\002\u1f20\u03b9\u1f99\002\u1f21\u03b9\u1f9a\002\u1f22\u03b9\u1f9b\002\u1f23\u03b9\u1f9c\002\u1f24\u03b9\u1f9d\002\u1f25\u03b9\u1f9e\002\u1f26\u03b9\u1f9f\002\u1f27\u03b9\u1fa0\002\u1f60\u03b9\u1fa1\002\u1f61\u03b9\u1fa2\002\u1f62\u03b9\u1fa3\002\u1f63\u03b9\u1fa4\002\u1f64\u03b9\u1fa5\002\u1f65\u03b9\u1fa6\002\u1f66\u03b9\u1fa7\002\u1f67\u03b9\u1fa8\002\u1f60\u03b9\u1fa9\002\u1f61\u03b9\u1faa\002\u1f62\u03b9\u1fab\002\u1f63\u03b9\u1fac\002\u1f64\u03b9\u1fad\002\u1f65\u03b9\u1fae\002\u1f66\u03b9\u1faf\002\u1f67\u03b9\u1fb2\002\u1f70\u03b9\u1fb3\002\u03b1\u03b9\u1fb4\002\u03ac\u03b9\u1fb6\002\u03b1\u0342\u1fb7\003\u03b1\u0342\u03b9\u1fb8\001\u1fb0\u1fb9\001\u1fb1\u1fba\001\u1f70\u1fbb\001\u1f71\u1fbc\002\u03b1\u03b9\u1fbe\001\u03b9\u1fc2\002\u1f74\u03b9\u1fc3\002\u03b7\u03b9\u1fc4\002\u03ae\u03b9\u1fc6\002\u03b7\u0342\u1fc7\003\u03b7\u0342\u03b9\u1fc8\001\u1f72\u1fc9\001\u1f73\u1fca\001\u1f74\u1fcb\001\u1f75\u1fcc\002\u03b7\u03b9\u1fd2\003\u03b9\u0308\u0300\u1fd3\003\u03b9\u0308\u0301\u1fd6\002\u03b9\u0342\u1fd7\003\u03b9\u0308\u0342\u1fd8\001\u1fd0\u1fd9\001\u1fd1\u1fda\001\u1f76\u1fdb\001\u1f77\u1fe2\003\u03c5\u0308\u0300\u1fe3\003\u03c5\u0308\u0301\u1fe4\002\u03c1\u0313\u1fe6\002\u03c5\u0342\u1fe7\003\u03c5\u0308\u0342\u1fe8\001\u1fe0\u1fe9\001\u1fe1\u1fea\001\u1f7a\u1feb\001\u1f7b\u1fec\001\u1fe5\u1ff2\002\u1f7c\u03b9\u1ff3\002\u03c9\u03b9\u1ff4\002\u03ce\u03b9\u1ff6\002\u03c9\u0342\u1ff7\003\u03c9\u0342\u03b9\u1ff8\001\u1f78\u1ff9\001\u1f79\u1ffa\001\u1f7c\u1ffb\001\u1f7d\u1ffc\002\u03c9\u03b9\u2126\001\u03c9\u212a\001k\u212b\001\u00e5\u2132\001\u214e\u2160\001\u2170\u2161\001\u2171\u2162\001\u2172\u2163\001\u2173\u2164\001\u2174\u2165\001\u2175\u2166\001\u2176\u2167\001\u2177\u2168\001\u2178\u2169\001\u2179\u216a\001\u217a\u216b\001\u217b\u216c\001\u217c\u216d\001\u217d\u216e\001\u217e\u216f\001\u217f\u2183\001\u2184\u24b6\001\u24d0\u24b7\001\u24d1\u24b8\001\u24d2\u24b9\001\u24d3\u24ba\001\u24d4\u24bb\001\u24d5\u24bc\001\u24d6\u24bd\001\u24d7\u24be\001\u24d8\u24bf\001\u24d9\u24c0\001\u24da\u24c1\001\u24db\u24c2\001\u24dc\u24c3\001\u24dd\u24c4\001\u24de\u24c5\001\u24df\u24c6\001\u24e0\u24c7\001\u24e1\u24c8\001\u24e2\u24c9\001\u24e3\u24ca\001\u24e4\u24cb\001\u24e5\u24cc\001\u24e6\u24cd\001\u24e7\u24ce\001\u24e8\u24cf\001\u24e9\u2c00\001\u2c30\u2c01\001\u2c31\u2c02\001\u2c32\u2c03\001\u2c33\u2c04\001\u2c34\u2c05\001\u2c35\u2c06\001\u2c36\u2c07\001\u2c37\u2c08\001\u2c38\u2c09\001\u2c39\u2c0a\001\u2c3a\u2c0b\001\u2c3b\u2c0c\001\u2c3c\u2c0d\001\u2c3d\u2c0e\001\u2c3e\u2c0f\001\u2c3f\u2c10\001\u2c40\u2c11\001\u2c41\u2c12\001\u2c42\u2c13\001\u2c43\u2c14\001\u2c44\u2c15\001\u2c45\u2c16\001\u2c46\u2c17\001\u2c47\u2c18\001\u2c48\u2c19\001\u2c49\u2c1a\001\u2c4a\u2c1b\001\u2c4b\u2c1c\001\u2c4c\u2c1d\001\u2c4d\u2c1e\001\u2c4e\u2c1f\001\u2c4f\u2c20\001\u2c50\u2c21\001\u2c51\u2c22\001\u2c52\u2c23\001\u2c53\u2c24\001\u2c54\u2c25\001\u2c55\u2c26\001\u2c56\u2c27\001\u2c57\u2c28\001\u2c58\u2c29\001\u2c59\u2c2a\001\u2c5a\u2c2b\001\u2c5b\u2c2c\001\u2c5c\u2c2d\001\u2c5d\u2c2e\001\u2c5e\u2c2f\001\u2c5f\u2c60\001\u2c61\u2c62\001\u026b\u2c63\001\u1d7d\u2c64\001\u027d\u2c67\001\u2c68\u2c69\001\u2c6a\u2c6b\001\u2c6c\u2c6d\001\u0251\u2c6e\001\u0271\u2c6f\001\u0250\u2c70\001\u0252\u2c72\001\u2c73\u2c75\001\u2c76\u2c7e\001\u023f\u2c7f\001\u0240\u2c80\001\u2c81\u2c82\001\u2c83\u2c84\001\u2c85\u2c86\001\u2c87\u2c88\001\u2c89\u2c8a\001\u2c8b\u2c8c\001\u2c8d\u2c8e\001\u2c8f\u2c90\001\u2c91\u2c92\001\u2c93\u2c94\001\u2c95\u2c96\001\u2c97\u2c98\001\u2c99\u2c9a\001\u2c9b\u2c9c\001\u2c9d\u2c9e\001\u2c9f\u2ca0\001\u2ca1\u2ca2\001\u2ca3\u2ca4\001\u2ca5\u2ca6\001\u2ca7\u2ca8\001\u2ca9\u2caa\001\u2cab\u2cac\001\u2cad\u2cae\001\u2caf\u2cb0\001\u2cb1\u2cb2\001\u2cb3\u2cb4\001\u2cb5\u2cb6\001\u2cb7\u2cb8\001\u2cb9\u2cba\001\u2cbb\u2cbc\001\u2cbd\u2cbe\001\u2cbf\u2cc0\001\u2cc1\u2cc2\001\u2cc3\u2cc4\001\u2cc5\u2cc6\001\u2cc7\u2cc8\001\u2cc9\u2cca\001\u2ccb\u2ccc\001\u2ccd\u2cce\001\u2ccf\u2cd0\001\u2cd1\u2cd2\001\u2cd3\u2cd4\001\u2cd5\u2cd6\001\u2cd7\u2cd8\001\u2cd9\u2cda\001\u2cdb\u2cdc\001\u2cdd\u2cde\001\u2cdf\u2ce0\001\u2ce1\u2ce2\001\u2ce3\u2ceb\001\u2cec\u2ced\001\u2cee\u2cf2\001\u2cf3\ua640\001\ua641\ua642\001\ua643\ua644\001\ua645\ua646\001\ua647\ua648\001\ua649\ua64a\001\ua64b\ua64c\001\ua64d\ua64e\001\ua64f\ua650\001\ua651\ua652\001\ua653\ua654\001\ua655\ua656\001\ua657\ua658\001\ua659\ua65a\001\ua65b\ua65c\001\ua65d\ua65e\001\ua65f\ua660\001\ua661\ua662\001\ua663\ua664\001\ua665\ua666\001\ua667\ua668\001\ua669\ua66a\001\ua66b\ua66c\001\ua66d\ua680\001\ua681\ua682\001\ua683\ua684\001\ua685\ua686\001\ua687\ua688\001\ua689\ua68a\001\ua68b\ua68c\001\ua68d\ua68e\001\ua68f\ua690\001\ua691\ua692\001\ua693\ua694\001\ua695\ua696\001\ua697\ua698\001\ua699\ua69a\001\ua69b\ua722\001\ua723\ua724\001\ua725\ua726\001\ua727\ua728\001\ua729\ua72a\001\ua72b\ua72c\001\ua72d\ua72e\001\ua72f\ua732\001\ua733\ua734\001\ua735\ua736\001\ua737\ua738\001\ua739\ua73a\001\ua73b\ua73c\001\ua73d\ua73e\001\ua73f\ua740\001\ua741\ua742\001\ua743\ua744\001\ua745\ua746\001\ua747\ua748\001\ua749\ua74a\001\ua74b\ua74c\001\ua74d\ua74e\001\ua74f\ua750\001\ua751\ua752\001\ua753\ua754\001\ua755\ua756\001\ua757\ua758\001\ua759\ua75a\001\ua75b\ua75c\001\ua75d\ua75e\001\ua75f\ua760\001\ua761\ua762\001\ua763\ua764\001\ua765\ua766\001\ua767\ua768\001\ua769\ua76a\001\ua76b\ua76c\001\ua76d\ua76e\001\ua76f\ua779\001\ua77a\ua77b\001\ua77c\ua77d\001\u1d79\ua77e\001\ua77f\ua780\001\ua781\ua782\001\ua783\ua784\001\ua785\ua786\001\ua787\ua78b\001\ua78c\ua78d\001\u0265\ua790\001\ua791\ua792\001\ua793\ua796\001\ua797\ua798\001\ua799\ua79a\001\ua79b\ua79c\001\ua79d\ua79e\001\ua79f\ua7a0\001\ua7a1\ua7a2\001\ua7a3\ua7a4\001\ua7a5\ua7a6\001\ua7a7\ua7a8\001\ua7a9\ua7aa\001\u0266\ua7ab\001\u025c\ua7ac\001\u0261\ua7ad\001\u026c\ua7ae\001\u026a\ua7b0\001\u029e\ua7b1\001\u0287\ua7b2\001\u029d\ua7b3\001\uab53\ua7b4\001\ua7b5\ua7b6\001\ua7b7\ua7b8\001\ua7b9\ua7ba\001\ua7bb\ua7bc\001\ua7bd\ua7be\001\ua7bf\ua7c0\001\ua7c1\ua7c2\001\ua7c3\ua7c4\001\ua794\ua7c5\001\u0282\ua7c6\001\u1d8e\ua7c7\001\ua7c8\ua7c9\001\ua7ca\ua7d0\001\ua7d1\ua7d6\001\ua7d7\ua7d8\001\ua7d9\ua7f5\001\ua7f6\uab70\001\u13a0\uab71\001\u13a1\uab72\001\u13a2\uab73\001\u13a3\uab74\001\u13a4\uab75\001\u13a5\uab76\001\u13a6\uab77\001\u13a7\uab78\001\u13a8\uab79\001\u13a9\uab7a\001\u13aa\uab7b\001\u13ab\uab7c\001\u13ac\uab7d\001\u13ad\uab7e\001\u13ae\uab7f\001\u13af\uab80\001\u13b0\uab81\001\u13b1\uab82\001\u13b2\uab83\001\u13b3\uab84\001\u13b4\uab85\001\u13b5\uab86\001\u13b6\uab87\001\u13b7\uab88\001\u13b8\uab89\001\u13b9\uab8a\001\u13ba\uab8b\001\u13bb\uab8c\001\u13bc\uab8d\001\u13bd\uab8e\001\u13be\uab8f\001\u13bf\uab90\001\u13c0\uab91\001\u13c1\uab92\001\u13c2\uab93\001\u13c3\uab94\001\u13c4\uab95\001\u13c5\uab96\001\u13c6\uab97\001\u13c7\uab98\001\u13c8\uab99\001\u13c9\uab9a\001\u13ca\uab9b\001\u13cb\uab9c\001\u13cc\uab9d\001\u13cd\uab9e\001\u13ce\uab9f\001\u13cf\uaba0\001\u13d0\uaba1\001\u13d1\uaba2\001\u13d2\uaba3\001\u13d3\uaba4\001\u13d4\uaba5\001\u13d5\uaba6\001\u13d6\uaba7\001\u13d7\uaba8\001\u13d8\uaba9\001\u13d9\uabaa\001\u13da\uabab\001\u13db\uabac\001\u13dc\uabad\001\u13dd\uabae\001\u13de\uabaf\001\u13df\uabb0\001\u13e0\uabb1\001\u13e1\uabb2\001\u13e2\uabb3\001\u13e3\uabb4\001\u13e4\uabb5\001\u13e5\uabb6\001\u13e6\uabb7\001\u13e7\uabb8\001\u13e8\uabb9\001\u13e9\uabba\001\u13ea\uabbb\001\u13eb\uabbc\001\u13ec\uabbd\001\u13ed\uabbe\001\u13ee\uabbf\001\u13ef\ufb00\002ff\ufb01\002fi\ufb02\002fl\ufb03\003ffi\ufb04\003ffl\ufb05\002st\ufb06\002st\ufb13\002\u0574\u0576\ufb14\002\u0574\u0565\ufb15\002\u0574\u056b\ufb16\002\u057e\u0576\ufb17\002\u0574\u056d\uff21\001\uff41\uff22\001\uff42\uff23\001\uff43\uff24\001\uff44\uff25\001\uff45\uff26\001\uff46\uff27\001\uff47\uff28\001\uff48\uff29\001\uff49\uff2a\001\uff4a\uff2b\001\uff4b\uff2c\001\uff4c\uff2d\001\uff4d\uff2e\001\uff4e\uff2f\001\uff4f\uff30\001\uff50\uff31\001\uff51\uff32\001\uff52\uff33\001\uff53\uff34\001\uff54\uff35\001\uff55\uff36\001\uff56\uff37\001\uff57\uff38\001\uff58\uff39\001\uff59\uff3a\001\uff5a", + }; +} diff --git a/maven/python-runtime/tools/gen_unicode_tables.py b/maven/python-runtime/tools/gen_unicode_tables.py new file mode 100644 index 00000000000..b06b1f8aa10 --- /dev/null +++ b/maven/python-runtime/tools/gen_unicode_tables.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 +# Generates UnicodeTables.java: the character properties and case mappings +# Python's str methods answer from, for the Basic Multilingual Plane, taken +# from the CPython that the conformance suite records against. +# +# python3.13 maven/python-runtime/tools/gen_unicode_tables.py +# +# The runtime cannot ask java.lang.Character: on a device that class is the +# CLDC one, which knows ASCII and at most Latin-1, so "Ü".lower() and +# "Α".isupper() answered wrongly there while the JVM run passed. +import os +import sys +import unicodedata + +if sys.version_info[:2] != (3, 13): + sys.exit("run with Python 3.13 (the version the conformance .out files record)") + +HERE = os.path.dirname(os.path.abspath(__file__)) +OUT = os.path.join(HERE, "..", "src", "main", "java", "com", "codename1", "python", + "runtime", "UnicodeTables.java") + +UPPER, LOWER, ALPHA, DECIMAL, DIGIT, NUMERIC, SPACE, CASED = (1 << i for i in range(8)) + + +def flags(c): + f = 0 + if c.isupper(): + f |= UPPER + if c.islower(): + f |= LOWER + if c.isalpha(): + f |= ALPHA + if c.isdecimal(): + f |= DECIMAL + if c.isdigit(): + f |= DIGIT + if c.isnumeric(): + f |= NUMERIC + if c.isspace(): + f |= SPACE + if c.lower() != c or c.upper() != c or c.title() != c or c.isupper() or c.islower() \ + or unicodedata.category(c) == "Lt": + f |= CASED + return f + + +def lit(s): + out = [] + for ch in s: + o = ord(ch) + if ch == '"': + out.append('\\"') + elif ch == '\\': + out.append('\\\\') + elif 32 <= o < 127: + out.append(ch) + elif o == 10: + out.append('\\n') + elif o == 13: + out.append('\\r') + else: + # Octal for the rest of ASCII: a backslash-u escape of a line + # break or quote is replaced before javac lexes the literal. + out.append('\\%03o' % o if o < 128 else '\\u%04x' % o) + return '"' + ''.join(out) + '"' + + +def chunks(s, n=8000): + return [s[i:i + n] for i in range(0, len(s), n)] or [""] + + +# Runs of equal flags: start, flags. +runs = [] +prev = None +for cp in range(0x10000): + f = 0 if 0xD800 <= cp <= 0xDFFF else flags(chr(cp)) + if f != prev: + runs.append((cp, f)) + prev = f +flag_str = "".join(chr(s) + chr(f) for s, f in runs) + + +def mapping(fn): + parts = [] + for cp in range(0x10000): + if 0xD800 <= cp <= 0xDFFF: + continue + c = chr(cp) + m = fn(c) + if m != c: + parts.append(chr(cp) + chr(len(m)) + m) + return "".join(parts) + + +tables = { + "FLAGS": flag_str, + "LOWER": mapping(str.lower), + "UPPER": mapping(str.upper), + "TITLE": mapping(str.title), + "FOLD": mapping(str.casefold), +} + +with open(os.path.join(HERE, "..", "src", "main", "java", "com", "codename1", "python", + "runtime", "PyStr.java"), encoding="ascii") as f: + header = f.read().split("package ", 1)[0] + +lines = [header.rstrip("\n"), "package com.codename1.python.runtime;", "", + "/// Generated by `maven/python-runtime/tools/gen_unicode_tables.py` from", + "/// the Unicode %s data of CPython %d.%d; do not edit by hand." + % (unicodedata.unidata_version, sys.version_info[0], sys.version_info[1]), + "///", + "/// `FLAGS` is runs of (first char, property bits) over the Basic", + "/// Multilingual Plane; each mapping table is entries of (char, length,", + "/// mapped chars), in char order. [Unicode] decodes them.", + "final class UnicodeTables {", + " private UnicodeTables() {", + " }", ""] +for name, data in tables.items(): + lines.append(" static final String[] %s = {" % name) + for c in chunks(data): + lines.append(" " + lit(c) + ",") + lines.append(" };") + lines.append("") +lines[-1] = "}" +with open(OUT, "w", encoding="ascii", newline="\n") as f: + f.write("\n".join(lines) + "\n") +print("wrote", OUT, {k: len(v) for k, v in tables.items()}) diff --git a/maven/python-transpiler/src/test/resources/conformance/numbers.out b/maven/python-transpiler/src/test/resources/conformance/fast_paths.out similarity index 100% rename from maven/python-transpiler/src/test/resources/conformance/numbers.out rename to maven/python-transpiler/src/test/resources/conformance/fast_paths.out diff --git a/maven/python-transpiler/src/test/resources/conformance/numbers.py b/maven/python-transpiler/src/test/resources/conformance/fast_paths.py similarity index 100% rename from maven/python-transpiler/src/test/resources/conformance/numbers.py rename to maven/python-transpiler/src/test/resources/conformance/fast_paths.py diff --git a/maven/python-transpiler/src/test/resources/conformance/unicode_case.out b/maven/python-transpiler/src/test/resources/conformance/unicode_case.out new file mode 100644 index 00000000000..81fead439fd --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/unicode_case.out @@ -0,0 +1,21 @@ +'m\xfcnchen' 'M\xdcNCHEN' 'm\xfcnchen' 'M\xfcnchen' 'm\xfcnchen' 'M\xfcnchen' +True False True False True +'stra\xdfe' 'STRASSE' 'strasse' 'Stra\xdfe' 'STRASSE' 'Stra\xdfe' +False True True False True +'\u03b1\u03b2\u03b3 \u03b1\u03b2\u03b3' '\u0391\u0392\u0393 \u0391\u0392\u0393' '\u03b1\u03b2\u03b3 \u03b1\u03b2\u03b3' '\u0391\u03b2\u03b3 \u0391\u03b2\u03b3' '\u03b1\u03b2\u03b3 \u0391\u0392\u0393' '\u0391\u03b2\u03b3 \u03b1\u03b2\u03b3' +False False False False False +'\u043f\u0440\u0438\u0432\u0435\u0442' '\u041f\u0420\u0418\u0412\u0415\u0422' '\u043f\u0440\u0438\u0432\u0435\u0442' '\u041f\u0440\u0438\u0432\u0435\u0442' '\u043f\u0420\u0418\u0412\u0415\u0422' '\u041f\u0440\u0438\u0432\u0435\u0442' +False False True True True +'\u01c6ungla' '\u01c4UNGLA' '\u01c6ungla' '\u01c5ungla' '\u01c5UNGLA' '\u01c5ungla' +False False True True True +'i\u0307stanbul' '\u0130STANBUL' 'i\u0307stanbul' '\u0130stanbul' 'i\u0307STANBUL' '\u0130stanbul' +False False True True True +'\uff41\uff42\uff43' '\uff21\uff22\uff23' '\uff41\uff42\uff43' '\uff21\uff42\uff43' '\uff41\uff22\uff23' '\uff21\uff42\uff43' +False False True True True +'\u0663\u0664' True True True False True +'\xb2' True False True False True +'\u2167' False False True False True +'\xbd' False False True False True +'\u3000\u2003' False False False True False +'\xa0' False False False True False +'abc\u0661' False False False False True diff --git a/maven/python-transpiler/src/test/resources/conformance/unicode_case.py b/maven/python-transpiler/src/test/resources/conformance/unicode_case.py new file mode 100644 index 00000000000..b55c7764ff1 --- /dev/null +++ b/maven/python-transpiler/src/test/resources/conformance/unicode_case.py @@ -0,0 +1,10 @@ +# Case mapping and character classes beyond ASCII, which a device's +# java.lang.Character (CLDC: Latin-1 at most) cannot answer. +words = ["M\u00dcNCHEN", "stra\u00dfe", "\u0391\u0392\u0393 \u03b1\u03b2\u03b3", + "\u041f\u0440\u0438\u0432\u0435\u0442", "\u01c5ungla", "\u0130stanbul", "\uff21\uff42\uff43"] +for w in words: + print(ascii(w.lower()), ascii(w.upper()), ascii(w.casefold()), ascii(w.title()), + ascii(w.swapcase()), ascii(w.capitalize())) + print(w.isupper(), w.islower(), w.isalpha(), w.istitle(), w.isidentifier()) +for s in ["\u0663\u0664", "\u00b2", "\u2167", "\u00bd", "\u3000\u2003", "\u00a0", "abc\u0661"]: + print(ascii(s), s.isdigit(), s.isdecimal(), s.isnumeric(), s.isspace(), s.isalnum()) diff --git a/scripts/python-bench/build_parparvm.sh b/scripts/python-bench/build_parparvm.sh index 492e98e4d79..5b6140df54c 100755 --- a/scripts/python-bench/build_parparvm.sh +++ b/scripts/python-bench/build_parparvm.sh @@ -54,6 +54,11 @@ if [ -n "${PY_JAVA_SOURCES:-}" ]; then fi # Java imports resolve against JavaAPI, the class library the binary links. "$J17/bin/java" -cp "$TRANSPILER" com.codename1.python.transpiler.api.Main -cp "$JAVAAPI:$CORE$APPCP" "$W/src" "$W/gen" "$K" +# PY_DRIVER replaces the entry point: the conformance suite runs every +# program from one binary rather than building one each. +if [ -n "${PY_DRIVER:-}" ]; then + cp "$PY_DRIVER" "$W/gen/com/pybench/PyBench.java" +else cat > "$W/gen/com/pybench/PyBench.java" < /dev/null 2>&1; then - echo "BUILD FAILED $p"; failed=1; continue - fi - got="$HERE/target/$p/output.txt" - (cd "$HERE/target/$p/src" && "../$p" > "$got" 2>&1) || true - if diff -q "$CONF/$p.out" "$got" > /dev/null; then - echo "ok $p" - else - echo "DIFF $p"; diff "$CONF/$p.out" "$got" | head -20; failed=1 - fi -done -exit $failed +# java_interop* call the Codename One UI classes, pil_* draw through the +# port's images, and sqlite3_ and the SQLAlchemy programs (through its +# sqlite dialect) open core's Database: each needs a port to link. +[ -n "$progs" ] || progs=$(cd "$CONF" && ls *.py | sed 's/\.py$//' \ + | grep -vE '^(http_|java_interop|pil_|sqlalchemy_|guide_sqlalchemy$|sqlite3_$)') +K=conformance_all +STAGE="$HERE/target/$K-stage" +mkdir -p "$STAGE" +echo "pass" > "$STAGE/$K.py" +MARK="@@CN1CONF@@" +{ + echo "package com.pybench;" + echo + echo "public class PyBench {" + echo " private static void run(String name, Runnable r) {" + echo " System.out.print(\"\\n$MARK \" + name + \"\\n\");" + echo " com.codename1.python.runtime.Gil.enter();" + echo " try {" + echo " r.run();" + echo " } catch (com.codename1.python.runtime.PyException e) {" + echo " com.codename1.python.runtime.StdModules.exitStatus(e);" + echo " } finally {" + echo " com.codename1.python.runtime.StdModules.runExitHandlers();" + echo " com.codename1.python.runtime.Gil.exit();" + echo " }" + echo " }" + echo + echo " public static void main(String[] args) {" + echo " com.codename1.generated.python.PythonModules.registerAll();" + for p in $progs; do + echo " com.codename1.python.runtime.Py.setArgv(\"$p\", args);" + echo " run(\"$p\", () -> com.codename1.generated.python.py_$p.exec(\"__main__\"));" + done + echo " System.out.print(\"\\n$MARK end\\n\");" + echo " System.exit(0);" + echo " }" + echo "}" +} > "$STAGE/PyBench.java" +extra="" +for p in $progs; do extra="$extra $CONF/$p.py"; done +if ! PY_SOURCES="$STAGE" PY_EXTRA_FILES="$extra" PY_DRIVER="$STAGE/PyBench.java" PY_OPT=-O1 \ + CN1_COMPILE_JOBS="$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 4)" \ + "$HERE/build_parparvm.sh" "$K" > "$STAGE/build.log" 2>&1; then + tail -40 "$STAGE/build.log" + echo "BUILD FAILED" + exit 1 +fi +got="$HERE/target/$K/output.txt" +(cd "$HERE/target/$K/src" && "../$K" > "$got" 2>&1) || true +python3 - "$CONF" "$got" "$MARK" $progs <<'PY' +import difflib, sys +conf, got, mark, progs = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4:] +text = open(got, encoding="utf-8", errors="replace").read() +sections = {} +for chunk in text.split("\n" + mark + " ")[1:]: + name, _, body = chunk.partition("\n") + sections[name] = body +failed = False +if "end" not in sections: + # Whatever ran last crashed the process; it and everything after it + # are reported, rather than read as empty output. + print("the binary did not finish; last program started: %s" + % (list(sections)[-1] if sections else "none")) + failed = True +for p in progs: + want = open("%s/%s.out" % (conf, p), encoding="utf-8").read() + if p not in sections: + print("NOT RUN %s" % p) + failed = True + elif sections[p] == want: + print("ok %s" % p) + else: + print("DIFF %s" % p) + diff = difflib.unified_diff(want.splitlines(), sections[p].splitlines(), + "expected", "native", lineterm="") + for line in list(diff)[:20]: + print(line) + failed = True +sys.exit(1 if failed else 0) +PY diff --git a/vm/selfhost/compile-dist.sh b/vm/selfhost/compile-dist.sh index 20838f3278a..2f930a0f4d9 100644 --- a/vm/selfhost/compile-dist.sh +++ b/vm/selfhost/compile-dist.sh @@ -55,8 +55,27 @@ PYEOF asms=$(ls "$srcdir"/*.S 2>/dev/null || true) # -flto=thin for -O3 arrives through CN1_SELFHOST_CFLAGS, which the caller sets. local flags="$opt" + # The sources go through a response file: a large translation (every + # Python conformance program in one binary) is more file names than + # macOS's argument limit takes. + # CN1_COMPILE_JOBS compiles the objects in parallel and links them after, + # for a build without LTO; one clang over thousands of files is serial. + if [ -n "${CN1_COMPILE_JOBS:-}" ]; then + local objs="$dist/../objs" + mkdir -p "$objs" + find "$srcdir" -maxdepth 1 \( -name '*.c' -o -name '*.S' \) | CC="$CC" FLAGS="$flags -w -fwrapv -fno-strict-aliasing -fno-builtin-fmod -fno-builtin-fmodf -ffp-contract=off $CN1_SELFHOST_CFLAGS -I$srcdir" OBJS="$objs" \ + xargs -P "$CN1_COMPILE_JOBS" -I{} sh -c '$CC $FLAGS -c "$1" -o "$OBJS/$(basename "$1").o"' _ {} \ + 2> "$log" || { echo "COMPILE FAILED"; tail -40 "$log"; return 1; } + find "$objs" -name '*.o' | sed 's/.*/"&"/' > "$dist/../objs.rsp" + $CC $flags @"$dist/../objs.rsp" -lm -lpthread -o "$bin" 2>> "$log" \ + || { echo "LINK FAILED"; tail -40 "$log"; return 1; } + echo "built $bin" + return 0 + fi + local rsp="$dist/../sources.rsp" + find "$srcdir" -maxdepth 1 \( -name '*.c' -o -name '*.S' \) | sed 's/.*/"&"/' > "$rsp" $CC $flags -w -fwrapv -fno-strict-aliasing -fno-builtin-fmod -fno-builtin-fmodf -ffp-contract=off \ - $CN1_SELFHOST_CFLAGS -I"$srcdir" "$srcdir"/*.c $asms -lm -lpthread -o "$bin" \ + $CN1_SELFHOST_CFLAGS -I"$srcdir" @"$rsp" -lm -lpthread -o "$bin" \ 2> "$log" || { echo "COMPILE FAILED"; tail -40 "$log"; return 1; } if [ -n "${CN1_BUILD_SOURCE_SNAPSHOT:-}" ]; then CN1_BUILD_FLAGS="$flags -fwrapv -fno-strict-aliasing -fno-builtin-fmod -fno-builtin-fmodf -ffp-contract=off $CN1_SELFHOST_CFLAGS" \ From 3184a6a22fac532873056017caec78e5fb6f2ad0 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Fri, 30 Oct 2026 16:18:01 +0200 Subject: [PATCH 110/112] Python benchmark: pin the jvm job to ubuntu-24.04, where CPython 3.13.9 exists Co-Authored-By: Claude Opus 5.5 --- .github/workflows/python-bench.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-bench.yml b/.github/workflows/python-bench.yml index af004db85c4..79fc1b7398d 100644 --- a/.github/workflows/python-bench.yml +++ b/.github/workflows/python-bench.yml @@ -97,7 +97,9 @@ jobs: measure: name: jvm needs: framework - runs-on: ubuntu-latest + # Pinned: setup-python has no CPYTHON_VERSION build for ubuntu-latest + # (26.04), and the baselines are measured against that exact version. + runs-on: ubuntu-24.04 timeout-minutes: 30 steps: - uses: actions/checkout@v4 From efd6fae1869c86a83fde02a3f1c98ac90c1b4c78 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Fri, 30 Oct 2026 20:10:48 +0200 Subject: [PATCH 111/112] iOS font from data: accept Core Text's duplicated-name answer; parallel benchmark compile The iOS 27 simulator answers kCTFontManagerErrorDuplicatedName (305) for a font the bundle already registered, which the native read as refusal. The ParparVM benchmark step compiles each kernel in parallel (thin LTO on compile and link), six minutes a kernel serially ran it past its hour. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/python-bench.yml | 4 ++++ Ports/iOSPort/nativeSources/IOSNative.m | 5 ++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-bench.yml b/.github/workflows/python-bench.yml index 79fc1b7398d..baab8fd2f88 100644 --- a/.github/workflows/python-bench.yml +++ b/.github/workflows/python-bench.yml @@ -192,7 +192,11 @@ jobs: run: scripts/python-bench/conformance_parparvm.sh - name: Measure and gate + # Each kernel's clang pass compiled one file at a time, six minutes + # a kernel. In parallel, with -flto=thin on the compile and the link, + # the binary is the one a single clang invocation makes. run: | + export CN1_COMPILE_JOBS="$(nproc)" python3 scripts/python-bench/parparvm_table.py --json python-bench-parparvm.json --check \ | tee -a "$GITHUB_STEP_SUMMARY" diff --git a/Ports/iOSPort/nativeSources/IOSNative.m b/Ports/iOSPort/nativeSources/IOSNative.m index 38e47f20999..5aec42bdaef 100644 --- a/Ports/iOSPort/nativeSources/IOSNative.m +++ b/Ports/iOSPort/nativeSources/IOSNative.m @@ -14702,8 +14702,11 @@ JAVA_OBJECT com_codename1_impl_ios_IOSNative_registerFontData___byte_1ARRAY_R_ja if (psName != NULL) { CFErrorRef error = NULL; BOOL usable = CTFontManagerRegisterGraphicsFont(cgFont, &error) ? YES : NO; + // Core Text can report a font the bundle already registered as a duplicated + // name (305, seen on iOS 27) rather than already registered (105). if (!usable && error != NULL - && CFErrorGetCode(error) == kCTFontManagerErrorAlreadyRegistered) { + && (CFErrorGetCode(error) == kCTFontManagerErrorAlreadyRegistered + || CFErrorGetCode(error) == kCTFontManagerErrorDuplicatedName)) { usable = cn1RegisteredFontMatches(psName, cgFont); } if (!usable) { From 233c69a1104dba8f3634112823ee47621f45c6e6 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Fri, 30 Oct 2026 20:11:56 +0200 Subject: [PATCH 112/112] CI: the native Python backend end to end runs in a job of its own It took 50 minutes of vm-tests, which master already fills to 143 of its 150; the job timed out in the benchmark step that follows. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/parparvm-tests.yml | 80 ++++++++++++++++++++++------ 1 file changed, 64 insertions(+), 16 deletions(-) diff --git a/.github/workflows/parparvm-tests.yml b/.github/workflows/parparvm-tests.yml index e145646827f..40e5c169cea 100644 --- a/.github/workflows/parparvm-tests.yml +++ b/.github/workflows/parparvm-tests.yml @@ -535,22 +535,6 @@ jobs: CN1_SQLA_POSTGRES: postgres://cn1:cn1secret@127.0.0.1:5432/cn1?sslmode=disable CN1_SQLA_MYSQL: mysql://cn1:cn1secret@127.0.0.1:3306/cn1?sslmode=disable - # The Flask and FastAPI samples built NATIVELY (ParparVM, clang) and - # driven over sockets by the same client as the JVM run. The only place - # Python handlers run on the backend's virtual threads, so the only run - # of the GIL's scheduler path. python-transpiler and python-runtime were - # installed by the step above; core, backend and build-engine by the - # contract install; JavaAPI explicitly, since the JVM tests may not. - - name: Python backend end to end, native - run: | - bash "$GITHUB_WORKSPACE/scripts/ci/retry.sh" mvn -B -f vm/pom.xml -pl JavaAPI install -DskipTests \ - -Dmaven.repo.local="$GITHUB_WORKSPACE/.m2-repo" - bash scripts/python-backend/e2e-test.sh --native "$GITHUB_WORKSPACE/.m2-repo" - env: - JAVA_HOME: ${{ env.JDK_8_HOME }} - JDK_8_HOME: ${{ env.JDK_8_HOME }} - JAVA17_HOME: ${{ env.JDK_17_HOME }} - - name: Run ParparVM Benchmark working-directory: vm # Each benchmark already creates its own mutator/collector threads. Run @@ -750,6 +734,70 @@ jobs: if-no-files-found: warn retention-days: 14 + python-backend-native: + # The Flask and FastAPI samples built NATIVELY (ParparVM, clang) and + # driven over sockets by the same client as the JVM run. The only place + # Python handlers run on the backend's virtual threads, so the only run + # of the GIL's scheduler path. A job of its own: it takes most of an + # hour, and vm-tests already fills its limit without it. + runs-on: ubuntu-24.04 + timeout-minutes: 90 + steps: + - uses: actions/checkout@v6 + + - name: Install native build tools + run: | + bash scripts/ci/apt-get-update.sh + sudo apt-get install -y clang zlib1g-dev libcurl4-openssl-dev libssl-dev libnghttp2-dev + + - name: Set up JDK 8 + uses: actions/setup-java@v5 + with: + distribution: 'temurin' + java-version: '8' + cache: 'maven' + - name: Save JDK 8 Path + run: echo "JDK_8_HOME=$JAVA_HOME" >> $GITHUB_ENV + + - name: Set up JDK 17 + uses: actions/setup-java@v5 + with: + distribution: 'temurin' + java-version: '17' + - name: Save JDK 17 Path + run: echo "JDK_17_HOME=$JAVA_HOME" >> $GITHUB_ENV + + # Gates off for the reason the vm-tests install step gives. + - name: Install core, the backend and the plugin + working-directory: maven + run: | + bash "$GITHUB_WORKSPACE/scripts/ci/retry.sh" mvn -B -pl core,backend,backend-test,codenameone-maven-plugin \ + -am install -DskipTests -Plocal-dev-javase \ + -Dspotbugs.skip=true -Dpmd.skip=true -Dcheckstyle.skip=true \ + -Dmaven.javadoc.skip=true \ + -Dmaven.repo.local="$GITHUB_WORKSPACE/.m2-repo" + env: + JAVA_HOME: ${{ env.JDK_8_HOME }} + JDK_8_HOME: ${{ env.JDK_8_HOME }} + + # python-runtime compiles at release 17, hence JDK 17 for the install. + - name: Install the Python transpiler and runtime, and JavaAPI + run: | + (cd maven && bash "$GITHUB_WORKSPACE/scripts/ci/retry.sh" mvn -B -pl python-runtime,python-transpiler \ + install -DskipTests -Dspotbugs.skip=true -Dpmd.skip=true -Dcheckstyle.skip=true \ + -Dmaven.javadoc.skip=true -Dmaven.repo.local="$GITHUB_WORKSPACE/.m2-repo") + bash "$GITHUB_WORKSPACE/scripts/ci/retry.sh" mvn -B -f vm/pom.xml -pl JavaAPI install -DskipTests \ + -Dmaven.repo.local="$GITHUB_WORKSPACE/.m2-repo" + env: + JAVA_HOME: ${{ env.JDK_17_HOME }} + + - name: Python backend end to end, native + run: bash scripts/python-backend/e2e-test.sh --native "$GITHUB_WORKSPACE/.m2-repo" + env: + JAVA_HOME: ${{ env.JDK_8_HOME }} + JDK_8_HOME: ${{ env.JDK_8_HOME }} + JAVA17_HOME: ${{ env.JDK_17_HOME }} + backend-websocket-conformance: name: WebSocket conformance (Autobahn) runs-on: ubuntu-24.04