perf: add app.pack and record javac.pack from a realistic compile

AGENTS.md

@@ -33,7 +33,7 @@ Node, which is how the tests run.
 - The release form is an npm package of the built files only
   (`java-web.js`, `java-web-worker.js`, `node.js` (the Node entry), `cli.js` (the
   `java-web` command), `boot.pack`, the `jdk/*.jar` image files,
-  `jdk/javac.pack`, and OpenJDK's notices in `jdk/legal/`), versioned, with
+  `jdk/app.pack`, `jdk/javac.pack`, and OpenJDK's notices in `jdk/legal/`), versioned, with
   nothing built at install. A GitHub Actions workflow triggered by a release
   publishes it to npm; never publish by hand.
 

docs/browser.md

@@ -9,8 +9,9 @@ next to itself, or where `cheerpjInit({ runtime: "/path/to/dist/" })` says.
   image, so a page reads only the classes it needs. `jdk/image.jar` holds
   `java.base` and the files around the classes, each other module is in
   `jdk/<module>.jar`, and `jdk/ct.sym.jar` holds what `javac --release` reads.
-  `boot.pack` holds what a typical start reads, and `jdk/javac.pack` what javac
-  adds. A host that compresses ranged responses still works, at the cost of one
+  `boot.pack` holds what a bare start reads. `jdk/app.pack` holds what a
+  typical program reads beyond that, and `jdk/javac.pack` what javac adds; a
+  page fetches each the first time it reads a class in it. A host that compresses ranged responses still works, at the cost of one
   full download of each image file a page reads.
 - A host can leave out the `jdk/<module>.jar` of a module its pages never
   load, and `jdk/ct.sym.jar` when nothing compiles with `--release`. A class of

docs/development.md

@@ -10,8 +10,9 @@ at `build/hotspot`. The build links the selected modules into `build/jdk/lib/mod
 `lib/packages.txt` indexes packages for the boot class loader. `pnpm build`
 puts it, with the files around it, in `dist/jdk/image.jar` and, for each module
 but `java.base`, `dist/jdk/<module>.jar`, all read by byte range.
-It packs the byte ranges a typical start reads into `boot.pack`, and those javac
-adds into `jdk/javac.pack`. Without `localeData`, the VM leaves `jdk.localedata`
+It packs the byte ranges a bare start reads into `boot.pack`, those the programs
+in `scripts/packs` read beyond that into `jdk/app.pack` (with every `java.base`
+part javac reads), and the rest of what their compile reads into `jdk/javac.pack`. Without `localeData`, the VM leaves `jdk.localedata`
 out with `jdk.module.limitmods`, as `--limit-modules` does.
 
 ```sh

package.json

@@ -36,6 +36,7 @@
     "dist/node.d.ts",
     "dist/boot.pack",
     "dist/jdk/*.jar",
+    "dist/jdk/app.pack",
     "dist/jdk/javac.pack",
     "dist/jdk/legal/"
   ],

scripts/build-web.ts

@@ -103,24 +103,44 @@ class Recorder implements Backend {
     return d;
   }
 }
-const recorder = new Recorder(new NodeBackend(jdk));
-for (const main of ["Hello", "Primes", "Shapes"]) {
+// Runs main over the JDK through rec, with app mounted at /app.
+async function record(rec: Recorder, app: string, classpath: string[], main: string, args: string[] = []) {
   const vfs = new VFS();
   vfs.mount("/", new MemoryBackend());
-  vfs.mount("/jdk", recorder);
+  vfs.mount("/jdk", rec);
   vfs.mount("/tmp", new MemoryBackend());
-  vfs.mount("/app", new NodeBackend(dist));
+  vfs.mount("/app", new NodeBackend(app));
   const vm = new JVM({
-    fs: vfs, javaHome: "/jdk", classpath: ["/app/samples"], cwd: "/tmp",
+    fs: vfs, javaHome: "/jdk", classpath, cwd: "/tmp",
     props: { "java.io.tmpdir": "/tmp", "user.home": "/tmp" },
     stdout: () => {}, stderr: (b) => process.stderr.write(b),
   });
-  const code = await vm.run(LM_CLASS, main, main === "Primes" ? ["100"] : []);
+  const code = await vm.run(LM_CLASS, main, args);
   if (code !== 0) throw new Error(`${main} exited with ${code}`);
 }
+const recorder = new Recorder(new NodeBackend(jdk));
+for (const main of ["Hello", "Primes", "Shapes"]) await record(recorder, dist, ["/app/samples"], main, main === "Primes" ? ["100"] : []);
+
+// The programs scripts/packs holds, built into build/packs: a typical program
+// run from jars, and a compile with a jar classpath and an annotation processor.
+const packs = join(root, "scripts", "packs"), fixtures = join(root, "build", "packs");
+rmSync(fixtures, { recursive: true, force: true });
+const javacTo = (out: string, ...args: string[]) => execFileSync(hotspot("javac"), ["-d", join(fixtures, out), ...args]);
+javacTo("lib", join(packs, "lib", "Lib.java"));
+javacTo("proc", join(packs, "proc", "Generate.java"), join(packs, "proc", "GenerateProcessor.java"));
+mkdirSync(join(fixtures, "proc", "META-INF", "services"), { recursive: true });
+writeFileSync(join(fixtures, "proc", "META-INF", "services", "javax.annotation.processing.Processor"), "proc.GenerateProcessor\n");
+javacTo("app", "-cp", join(fixtures, "lib"), join(packs, "App.java"));
+cpSync(join(packs, "app.properties"), join(fixtures, "app", "app.properties"));
+javacTo("driver", join(packs, "Compile.java"));
+for (const d of ["lib", "proc", "app"]) execFileSync(hotspot("jar"), ["cf", join(fixtures, `${d}.jar`), "-C", join(fixtures, d), "."]);
+cpSync(join(packs, "src"), join(fixtures, "src"), { recursive: true });
+const appRecorder = new Recorder(new NodeBackend(jdk));
+await record(appRecorder, fixtures, ["/app/app.jar", "/app/lib.jar"], "App");
 
-// Record what a javax.tools compile reads on top of that.
+// Record what javac reads: the compile above, and the one the browser self test runs.
 const javacRecorder = new Recorder(new NodeBackend(jdk));
+await record(javacRecorder, fixtures, ["/app/driver"], "Compile", ["/app/lib.jar:/app/proc.jar", "/app/proc.jar", "/tmp", "/app/src/Model.java", "/app/src/Uses.java"]);
 {
   const vfs = new VFS();
   vfs.mount("/", new MemoryBackend());
@@ -139,7 +159,6 @@ const javacRecorder = new Recorder(new NodeBackend(jdk));
   if (r !== "true ") throw new Error(`javac in the VM failed: ${r}`);
   vm.halt(0);
 }
-const javacFiles = [...javacRecorder.seen].filter((p) => !recorder.seen.has(p)).sort();
 
 // Sorted [start, end) spans covering the given ones. Spans less than GAP apart
 // join, since a pack entry costs more than a few bytes of filler.
@@ -167,11 +186,26 @@ function subtract(spans: [number, number][], covered: [number, number][]): [numb
   }
   return out;
 }
+// boot.pack holds what a bare start reads. app.pack, fetched the first time a
+// program reads one of its parts, holds what the typical program reads beyond
+// that and every java.base part javac reads. javac.pack holds the rest of what
+// javac reads: the compiler's own modules.
+const linked = new Set(/^MODULES="(.*)"$/m.exec(readFileSync(join(jdk, "release"), "utf8"))![1].split(" "));
+const compiler = merge(moduleRuns(readFileSync(join(jdk, "lib", "modules")))
+  .filter(([m]) => m !== "java.base" && linked.has(m)).map(([, s, e]): [number, number] => [s, e]));
 const bootRanges = new Map([...recorder.ranges].map(([p, r]) => [p, merge(r)]));
-const javacRanges = new Map([...javacRecorder.ranges].map(([p, r]) => [p, subtract(merge(r), bootRanges.get(p) ?? [])]));
-const rangeEntries = (ranges: Map<string, [number, number][]>, src: Recorder) =>
-  [...ranges].flatMap(([p, spans]) => spans.map(([s, e]): [string, Uint8Array] => [`@${s}:${p}`, src.readRange(p, s, e - s)!]));
-const inJavacPack = new Set(javacFiles);
+const beyondBoot = (rec: Recorder) => new Map([...rec.ranges].map(([p, r]) => [p, subtract(merge(r), bootRanges.get(p) ?? [])]));
+const appRanges = beyondBoot(appRecorder), javacRanges = new Map<string, [number, number][]>();
+for (const [p, r] of beyondBoot(javacRecorder)) {
+  const own = p === "/lib/modules" ? subtract(r, subtract(r, compiler)) : [];
+  appRanges.set(p, merge([...(appRanges.get(p) ?? []), ...subtract(r, own)]));
+  javacRanges.set(p, own);
+}
+const jdkFiles = new NodeBackend(jdk);
+const rangeEntries = (ranges: Map<string, [number, number][]>) =>
+  [...ranges].flatMap(([p, spans]) => spans.map(([s, e]): [string, Uint8Array] => [`@${s}:${p}`, jdkFiles.readRange(p, s, e - s)!]));
+const appFiles = [...new Set([...appRecorder.seen, ...javacRecorder.seen])].filter((p) => !recorder.seen.has(p)).sort();
+const inAppPack = new Set(appFiles);
 
 // A jimage's resources as runs of one module's, [module, start, end) in image
 // order. Empty resources and module-info.class are left out: jlink puts the
@@ -213,7 +247,6 @@ function moduleRuns(image: Uint8Array): [string, number, number][] {
 // may never read go in their own files, which a host can leave out: each
 // module of lib/modules but java.base in jdk/<module>.jar, and lib/ct.sym
 // (javac --release) in jdk/ct.sym.jar.
-const linked = new Set(/^MODULES="(.*)"$/m.exec(readFileSync(join(jdk, "release"), "utf8"))![1].split(" "));
 const imageSizes = new Map<string, number>();
 // appends data to jdk/<name>.jar, returning where it starts there
 const put = (name: string, data: Uint8Array) => {
@@ -250,7 +283,7 @@ const walk = (dir: string) => {
       });
     } else {
       const image = rel === "/lib/ct.sym" ? "ct.sym" : "image";
-      lines.push(`f ${rel} ${st.size} ${image}:${put(image, readFileSync(p))}${inJavacPack.has(rel) ? " javac" : ""}`);
+      lines.push(`f ${rel} ${st.size} ${image}:${put(image, readFileSync(p))}${inAppPack.has(rel) ? " app" : ""}`);
     }
   }
 };
@@ -297,15 +330,17 @@ const lazySpans = (name: string, ranges: Map<string, [number, number][]>): [stri
   return [`@lazy:${name}:${p}`, data];
 });
 const boot = writePack(join(dist, "boot.pack"), [
-  ...rangeEntries(bootRanges, recorder),
+  ...rangeEntries(bootRanges),
+  ...lazySpans("app", appRanges),
   ...lazySpans("javac", javacRanges),
   ["/index.prefix", prefixCode(lines)],
   ...[...recorder.seen].sort().map((p): [string, Uint8Array] => [p, recorder.read(p)!]),
 ]);
-const javacPack = writePack(join(dist, "jdk", "javac.pack"), [
-  ...javacFiles.map((p): [string, Uint8Array] => [p, javacRecorder.read(p)!]),
-  ...rangeEntries(javacRanges, javacRecorder),
+const appPack = writePack(join(dist, "jdk", "app.pack"), [
+  ...appFiles.map((p): [string, Uint8Array] => [p, jdkFiles.read(p)!]),
+  ...rangeEntries(appRanges),
 ]);
+const javacPack = writePack(join(dist, "jdk", "javac.pack"), rangeEntries(javacRanges));
 const mb = (n: number) => `${(n / 2 ** 20).toFixed(0)} MB`;
 const others = [...imageSizes].filter(([n]) => n !== "image").reduce((n, [, size]) => n + size, 0);
-console.log(`dist/ ready: boot.pack ${boot}, jdk/javac.pack ${javacPack}, jdk/image.jar ${mb(imageSizes.get("image")!)}, ${imageSizes.size - 1} more images ${mb(others)}`);
+console.log(`dist/ ready: boot.pack ${boot}, jdk/app.pack ${appPack}, jdk/javac.pack ${javacPack}, jdk/image.jar ${mb(imageSizes.get("image")!)}, ${imageSizes.size - 1} more images ${mb(others)}`);

scripts/packs/App.java

@@ -0,0 +1,90 @@
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.math.RoundingMode;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Duration;
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Properties;
+import java.util.Random;
+import java.util.TreeMap;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import lib.Lib;
+
+// A typical program, run from a jar: what it reads beyond a bare start goes in app.pack.
+public class App {
+  record Point(int x, int y) {}
+
+  enum Kind { A, B, C }
+
+  public static void main(String[] args) throws Exception {
+    List<Point> points = IntStream.range(0, 50).mapToObj(i -> new Point(i, i * i)).collect(Collectors.toList());
+    Map<Boolean, Long> parts = points.stream().collect(Collectors.partitioningBy(p -> p.x() % 2 == 0, Collectors.counting()));
+    TreeMap<String, Integer> kinds = new TreeMap<>();
+    for (Kind k : Kind.values()) kinds.put(k.name(), k.ordinal());
+    String shape = switch (points.get(3)) {
+      case Point(int x, int y) when x > 1 -> "big " + y;
+      default -> "small";
+    };
+    BigDecimal price = new BigDecimal("1.10").multiply(BigDecimal.valueOf(3)).setScale(1, RoundingMode.HALF_UP);
+    BigInteger factorial = BigInteger.ONE;
+    for (int i = 1; i < 30; i++) factorial = factorial.multiply(BigInteger.valueOf(i));
+    Instant start = Instant.now();
+    String when = DateTimeFormatter.ISO_LOCAL_DATE_TIME.format(LocalDateTime.ofInstant(start, ZoneOffset.UTC))
+        + " " + Duration.ofMillis(1500) + " " + ZonedDateTime.now().getZone();
+
+    ExecutorService pool = Executors.newFixedThreadPool(2);
+    Future<Integer> sum = pool.submit(() -> points.stream().mapToInt(Point::y).sum());
+    CompletableFuture<String> upper = CompletableFuture.supplyAsync(() -> "async", pool).thenApply(String::toUpperCase);
+    AtomicInteger ticks = new AtomicInteger();
+    Thread worker = new Thread(() -> {
+      synchronized (ticks) {
+        ticks.incrementAndGet();
+      }
+    });
+    worker.start();
+    worker.join();
+    int total = sum.get();
+    String async = upper.get();
+    pool.shutdown();
+
+    Path file = Files.createTempFile("app", ".txt");
+    Files.writeString(file, shape + "\n" + price + "\n");
+    List<String> lines = Files.readAllLines(file);
+    Files.delete(file);
+
+    Properties props = new Properties();
+    try (var in = App.class.getResourceAsStream("/app.properties")) {
+      props.load(in);
+    }
+    System.getLogger("app").log(System.Logger.Level.DEBUG, "not shown");
+    List<Integer> shuffled = new ArrayList<>(List.of(1, 2, 3, 4, 5));
+    Collections.shuffle(shuffled, new Random(1));
+    boolean digits = Pattern.compile("\\d+").matcher("abc123").find();
+    String parsed;
+    try {
+      parsed = String.valueOf(Integer.parseInt("x"));
+    } catch (NumberFormatException e) {
+      parsed = e.getClass().getSimpleName() + " at " + e.getStackTrace()[0].getMethodName();
+    }
+    System.out.println(String.format("%s %s %s %s %d %s %d %s %s %s %s %b %s %s %s", parts, kinds, shape, price, factorial.bitLength(),
+        when.length() > 0, total, async, ticks, lines, shuffled, digits, parsed, props.getProperty("name"),
+        Optional.of(Lib.greet("app")).map(String::strip).orElse("")));
+  }
+}

scripts/packs/Compile.java

@@ -0,0 +1,25 @@
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Locale;
+import javax.tools.JavaCompiler;
+import javax.tools.StandardJavaFileManager;
+import javax.tools.StandardLocation;
+import javax.tools.ToolProvider;
+
+// Compile CLASSPATH PROCESSORPATH OUTDIR SOURCE...: a javax.tools compile with
+// a jar classpath and an annotation processor, for javac.pack.
+public class Compile {
+  public static void main(String[] args) throws Exception {
+    JavaCompiler javac = ToolProvider.getSystemJavaCompiler();
+    StandardJavaFileManager fm = javac.getStandardFileManager(null, Locale.ROOT, null);
+    fm.setLocationFromPaths(StandardLocation.CLASS_OUTPUT, List.of(Path.of(args[2])));
+    fm.setLocationFromPaths(StandardLocation.SOURCE_OUTPUT, List.of(Path.of(args[2])));
+    List<String> options = new ArrayList<>(List.of("-classpath", args[0], "-processorpath", args[1], "-g", "-XDstringConcat=inline"));
+    List<Path> sources = Arrays.stream(args, 3, args.length).map(Path::of).toList();
+    List<String> diags = new ArrayList<>();
+    boolean ok = javac.getTask(null, fm, d -> diags.add(d.toString()), options, null, fm.getJavaFileObjectsFromPaths(sources)).call();
+    if (!ok) throw new IllegalStateException(String.join("\n", diags));
+  }
+}

scripts/packs/app.properties

@@ -0,0 +1 @@
+name=app

scripts/packs/lib/Lib.java

@@ -0,0 +1,16 @@
+package lib;
+
+// A library the packed program and compile take from a jar.
+public final class Lib {
+  public interface Named {
+    String name();
+
+    default String label() {
+      return "<" + name() + ">";
+    }
+  }
+
+  public static String greet(String who) {
+    return "hello " + who + " ";
+  }
+}

scripts/packs/proc/Generate.java

@@ -0,0 +1,10 @@
+package proc;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+@Retention(RetentionPolicy.SOURCE)
+@Target(ElementType.TYPE)
+public @interface Generate {}

scripts/packs/proc/GenerateProcessor.java

@@ -0,0 +1,36 @@
+package proc;
+
+import java.io.Writer;
+import java.util.Set;
+import javax.annotation.processing.AbstractProcessor;
+import javax.annotation.processing.RoundEnvironment;
+import javax.annotation.processing.SupportedAnnotationTypes;
+import javax.lang.model.SourceVersion;
+import javax.lang.model.element.Element;
+import javax.lang.model.element.TypeElement;
+
+// Writes <Class>Info with the field names of each @Generate class.
+@SupportedAnnotationTypes("proc.Generate")
+public class GenerateProcessor extends AbstractProcessor {
+  @Override
+  public SourceVersion getSupportedSourceVersion() {
+    return SourceVersion.latestSupported();
+  }
+
+  @Override
+  public boolean process(Set<? extends TypeElement> annotations, RoundEnvironment round) {
+    for (Element e : round.getElementsAnnotatedWith(Generate.class)) {
+      String name = e.getSimpleName() + "Info";
+      StringBuilder fields = new StringBuilder();
+      for (Element member : e.getEnclosedElements()) {
+        if (member.getKind().isField()) fields.append('"').append(member.getSimpleName()).append("\", ");
+      }
+      try (Writer w = processingEnv.getFiler().createSourceFile(name, e).openWriter()) {
+        w.write("public final class " + name + " { public static final String[] FIELDS = { " + fields + "}; }\n");
+      } catch (Exception x) {
+        throw new RuntimeException(x);
+      }
+    }
+    return true;
+  }
+}

scripts/packs/src/Model.java

@@ -0,0 +1,38 @@
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import lib.Lib;
+import proc.Generate;
+
+@Generate
+public class Model implements Lib.Named {
+  private final String name;
+  private final List<Integer> values;
+
+  public Model(String name, List<Integer> values) {
+    this.name = name;
+    this.values = values;
+  }
+
+  public String name() {
+    return name;
+  }
+
+  public <K> Map<K, List<Integer>> group(Function<Integer, K> key) {
+    return values.stream().collect(Collectors.groupingBy(key));
+  }
+
+  sealed interface Shape permits Circle, Square {}
+
+  record Circle(double r) implements Shape {}
+
+  record Square(double side) implements Shape {}
+
+  static double area(Shape s) {
+    return switch (s) {
+      case Circle c -> Math.PI * c.r() * c.r();
+      case Square q -> q.side() * q.side();
+    };
+  }
+}