Run the nightly on A64 and pick the QEMU CPU

.github/workflows/nightly.yml

@@ -10,6 +10,11 @@
 #
 # The rucc binary is the pinned release, checked against its hash, as in kernel.yml. A manual run
 # can name a branch or commit of tamnd/rucc instead.
+#
+# A manual run can also name the row, the versions and the QEMU CPU. A64 runs on the arm runner,
+# which has no KVM, so its boots are TCG and are given longer, and its rucc is built from source,
+# from the pinned release's tag when no branch or commit is named, since the release tarball is
+# the x86_64 one.
 
 name: nightly
 
@@ -26,6 +31,18 @@ on:
         description: The configuration target
         required: false
         default: tinyconfig
+      row:
+        description: The row, X64 or A64
+        required: false
+        default: X64
+      versions:
+        description: The versions as a JSON list, or empty for the current set
+        required: false
+        default: ""
+      cpu:
+        description: The QEMU CPU to boot on, or empty for the row's own
+        required: false
+        default: ""
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event_name == 'workflow_dispatch' && github.run_id || github.ref }}
@@ -40,6 +57,8 @@ env:
   RUCC_VERSION: v0.24.8
   RUCC_SHA256: f37e42c301f6867087566f40e99d9be0960e8d99c035da632f8e28e3c05b3ed8
   CONFIG: ${{ inputs.config || 'tinyconfig' }}
+  ROW: ${{ inputs.row || 'X64' }}
+  CPU: ${{ inputs.cpu }}
 
 jobs:
   set:
@@ -57,16 +76,20 @@ jobs:
           with open("pins.toml", "rb") as f:
               pins = tomllib.load(f)["pin"]
           versions = [p["version"] for p in pins if "current" in p.get("sets", [])]
+          if os.environ["ASKED"]:
+              versions = json.loads(os.environ["ASKED"])
           with open(os.environ["GITHUB_OUTPUT"], "a") as out:
               out.write(f"versions={json.dumps(versions)}\n")
           with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as out:
-              out.write(f"The current set is {', '.join(versions)}.\n")
+              out.write(f"The versions are {', '.join(versions)}.\n")
+        env:
+          ASKED: ${{ inputs.versions }}
 
   test:
-    name: ${{ matrix.version }} X64
+    name: ${{ matrix.version }} ${{ inputs.row || 'X64' }}
     needs: set
-    runs-on: ubuntu-24.04
-    timeout-minutes: 300
+    runs-on: ${{ inputs.row == 'A64' && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }}
+    timeout-minutes: ${{ inputs.row == 'A64' && 355 || 300 }}
     strategy:
       fail-fast: false
       matrix:
@@ -78,26 +101,29 @@ jobs:
         run: |
           sudo apt-get update -q
           sudo apt-get install -qy --no-install-recommends gcc-14 flex bison bc libelf-dev libssl-dev \
-            cpio qemu-system-x86 busybox-static libcap-dev libnuma-dev rsync libaio-dev libacl1-dev \
+            cpio "$QEMU" busybox-static libcap-dev libnuma-dev rsync libaio-dev libacl1-dev \
             libkeyutils-dev
+        env:
+          QEMU: ${{ inputs.row == 'A64' && 'qemu-system-arm' || 'qemu-system-x86' }}
       - name: let the runner user open /dev/kvm
+        if: inputs.row != 'A64'
         run: |
           echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \
             | sudo tee /etc/udev/rules.d/99-kvm.rules
           sudo udevadm control --reload-rules
           sudo udevadm trigger --name-match=kvm || true
-      - name: rucc ${{ inputs.rucc }} from source
-        if: inputs.rucc != ''
+      - name: rucc ${{ inputs.rucc || env.RUCC_VERSION }} from source
+        if: inputs.rucc != '' || inputs.row == 'A64'
         run: |
           git clone -q https://github.com/tamnd/rucc "$RUNNER_TEMP/rucc-src"
           git -C "$RUNNER_TEMP/rucc-src" checkout -q "$REF"
           cargo build --release --manifest-path "$RUNNER_TEMP/rucc-src/Cargo.toml" -p rucc
           echo "$RUNNER_TEMP/rucc-src/target/release" >> "$GITHUB_PATH"
           echo "rucc built from $(git -C "$RUNNER_TEMP/rucc-src" rev-parse HEAD)" >> "$GITHUB_STEP_SUMMARY"
         env:
-          REF: ${{ inputs.rucc }}
+          REF: ${{ inputs.rucc || env.RUCC_VERSION }}
       - name: rucc ${{ env.RUCC_VERSION }}
-        if: inputs.rucc == ''
+        if: inputs.rucc == '' && inputs.row != 'A64'
         run: |
           name="rucc-$RUCC_VERSION-x86_64-unknown-linux-gnu"
           gh release download "$RUCC_VERSION" -R tamnd/rucc -p "$name.tar.gz" -D "$RUNNER_TEMP"
@@ -115,12 +141,12 @@ jobs:
           restore-keys: kernel-archives-${{ matrix.version }}-
       - name: rk build with gcc-14
         run: |
-          target/release/rk build ${{ matrix.version }} --row X64 --config "$CONFIG" --fragment test \
+          target/release/rk build ${{ matrix.version }} --row "$ROW" --config "$CONFIG" --fragment test \
             --cc gcc-14 --targets "bzImage modules" --out "$RUNNER_TEMP/gcc" --stack-usage
       - name: rk build with rucc
         continue-on-error: true
         run: |
-          target/release/rk build ${{ matrix.version }} --row X64 --config "$CONFIG" --fragment test \
+          target/release/rk build ${{ matrix.version }} --row "$ROW" --config "$CONFIG" --fragment test \
             --cc rucc --targets "bzImage modules" --out "$RUNNER_TEMP/rucc" --keep-going --stack-usage
       - name: rk selftests with gcc-14
         run: |
@@ -134,7 +160,7 @@ jobs:
       - uses: actions/cache@v6.1.0
         with:
           path: ${{ env.RK_CACHE }}/ltp
-          key: ltp-${{ hashFiles('ltp.toml') }}
+          key: ltp-${{ runner.arch }}-${{ hashFiles('ltp.toml') }}
       - name: rk ltp with gcc-14
         run: |
           target/release/rk ltp --build "$RUNNER_TEMP/gcc" --cc gcc-14 --out "$RUNNER_TEMP/ltp-gcc"
@@ -145,6 +171,8 @@ jobs:
       - name: rk test
         id: test
         run: |
+          boot=(--timeout "$TIMEOUT")
+          if [ -n "$CPU" ]; then boot+=(--cpu "$CPU"); fi
           rucc_userland=()
           if [ -f "$RUNNER_TEMP/selftests-rucc/selftests.json" ]; then
             rucc_userland+=(--rucc-selftests "$RUNNER_TEMP/selftests-rucc")
@@ -154,13 +182,19 @@ jobs:
           fi
           target/release/rk test --reference "$RUNNER_TEMP/gcc" --other "$RUNNER_TEMP/rucc" \
             --kinds boot,smoke,kunit,kselftest,ltp --selftests "$RUNNER_TEMP/selftests-gcc" \
-            --ltp "$RUNNER_TEMP/ltp-gcc" "${rucc_userland[@]}" --out "$RUNNER_TEMP/test"
+            --ltp "$RUNNER_TEMP/ltp-gcc" "${rucc_userland[@]}" "${boot[@]}" --out "$RUNNER_TEMP/test"
+        env:
+          TIMEOUT: ${{ inputs.row == 'A64' && 1500 || 600 }}
       - name: rk cross-modules
         if: always() && steps.test.outcome != 'skipped'
         continue-on-error: true
         run: |
+          boot=(--timeout "$TIMEOUT")
+          if [ -n "$CPU" ]; then boot+=(--cpu "$CPU"); fi
           target/release/rk cross-modules --reference "$RUNNER_TEMP/gcc" --other "$RUNNER_TEMP/rucc" \
-            --out "$RUNNER_TEMP/cross"
+            "${boot[@]}" --out "$RUNNER_TEMP/cross"
+        env:
+          TIMEOUT: ${{ inputs.row == 'A64' && 1500 || 600 }}
       - name: rk objtool-report
         if: always() && steps.test.outcome != 'skipped'
         run: |
@@ -183,7 +217,9 @@ jobs:
         run: |
           unit=$(python3 -c 'import json, sys; t = json.load(open(sys.argv[1])); print(next((u["name"] for u in t["units"] if u["graded"] and not u["passed"] and u.get("attribution", "kernel") == "kernel"), ""))' "$RUNNER_TEMP/test/test.json")
           if [ -z "$unit" ]; then echo "no kernel failure to bisect"; exit 0; fi
-          target/release/rk mixed --reference "$RUNNER_TEMP/gcc" --other "$RUNNER_TEMP/rucc" \
+          boot=()
+          if [ -n "$CPU" ]; then boot+=(--cpu "$CPU"); fi
+          target/release/rk mixed "${boot[@]}" --reference "$RUNNER_TEMP/gcc" --other "$RUNNER_TEMP/rucc" \
             --unit "$unit" --selftests "$RUNNER_TEMP/selftests-gcc" --ltp "$RUNNER_TEMP/ltp-gcc" \
             --out "$RUNNER_TEMP/mixed"
       - name: keep the record
@@ -208,7 +244,7 @@ jobs:
       - uses: actions/upload-artifact@v7.0.1
         if: always()
         with:
-          name: nightly-${{ matrix.version }}-X64
+          name: nightly-${{ matrix.version }}-${{ inputs.row || 'X64' }}
           path: record
           include-hidden-files: true
           retention-days: 30
@@ -227,9 +263,10 @@ jobs:
         shell: python3 {0}
         run: |
           import json, os, pathlib
-          lines = ["### nightly, X64", "", "| version | graded | failed | splats | verdict | objtool | frames |", "|---|---|---|---|---|---|---|"]
+          row = os.environ["ROW"]
+          lines = [f"### nightly, {row}", "", "| version | graded | failed | splats | verdict | objtool | frames |", "|---|---|---|---|---|---|---|"]
           for record in sorted(pathlib.Path("records").iterdir()):
-              version = record.name.removeprefix("nightly-").removesuffix("-X64")
+              version = record.name.removeprefix("nightly-").removesuffix(f"-{row}")
               path = record / "test" / "test.json"
               checks = []
               for name in ("objtool", "frames"):

README.md

@@ -66,7 +66,7 @@ The reference compilers live in era containers built from `provision/eras` and p
 
 `rk sections-diff`, `rk symvers-diff`, `rk vec-audit`, `rk modules-audit` and `rk objtool-report` look at what the compilers produced. `rk sections-diff` compares the section names of every object two builds share, the size and relocation count of each kernel table (exports, jump labels, alternatives, exception fixups, initcalls and the rest), the call-site lists per function, and the `.modinfo` and `__ksymtab_strings` strings. A difference that `sections-divergences.toml` explains, such as a constant pool only gcc builds or a WARN check only one compiler could prove dead, is listed apart with its reason and does not fail the run. It lists the first 60 differences of each kind, and every one with `--all`. `rk symvers-diff` compares `Module.symvers`, CRCs and namespaces included, and the global symbols of `System.map`. `rk vec-audit` decodes the x86-64 code of every unit built with `-mno-sse` or `-mgeneral-regs-only` and lists the vector and x87 instructions it finds, only those above the reference's count when given `--reference`. `rk modules-audit` fails on any module relocation the loader rejects, a wrong `vermagic` or an imported CRC that does not match `Module.symvers`. `rk objtool-report` buckets the objtool warnings of two build logs by message and function shape. `rk frames` compares the stack frame of every function in two builds made with `rk build --stack-usage`, which passes `KCFLAGS=-fstack-usage` so that each compiler writes a `.su` file next to every object. It fails on a function over `CONFIG_FRAME_WARN` that the reference keeps under it, and on a run time stack high water mark more than 10% above the reference's when both build directories have a `boot.log` from a kernel with `CONFIG_DEBUG_STACK_USAGE`. `rk sections-diff`, `rk vec-audit` and `rk frames` take `--save FILE` to keep what they read as JSON, and accept such a file in place of a build directory, which is how CI compares builds without moving their objects.
 
-The nightly workflow builds every pin in the current set on X64 with gcc-14 and with rucc, from `tinyconfig` and the test fragment, and runs `rk test` on boot, smoke and KUnit. When the rucc kernel fails a unit, `rk mixed` runs on the first one in the same job. The run's summary has a line per version, green or red, whether `rk objtool-report` and `rk frames` came out clean, and what `rk mixed` found. A manual run takes a branch or commit of tamnd/rucc and another configuration target.
+The nightly workflow builds every pin in the current set on X64 with gcc-14 and with rucc, from `tinyconfig` and the test fragment, and runs `rk test` on boot, smoke and KUnit. When the rucc kernel fails a unit, `rk mixed` runs on the first one in the same job. The run's summary has a line per version, green or red, whether `rk objtool-report` and `rk frames` came out clean, and what `rk mixed` found. A manual run takes a branch or commit of tamnd/rucc, another configuration target, the versions, the row and the QEMU CPU, so `row=A64 cpu=cortex-a57 config=debian-13-arm64` runs every test kind on arm64 under TCG on the arm runner.
 
 The mixed workflow is run by hand. It builds the rucc kernel with `-fenable-inject-fault=time64_to_tm`, checks that the KUnit case for that function fails, and passes only when `rk mixed` traces the failure back to the `inject-fault` pass in that function.
 

crates/rk/src/cli.rs

@@ -70,14 +70,23 @@ fn accepted(command: &str) -> Option<(bool, &'static [&'static str])> {
         "demands" => (false, &["builds", "limit"]),
         "boot" => (
             false,
-            &["build", "row", "initramfs", "busybox", "timeout", "append"],
+            &[
+                "build",
+                "row",
+                "cpu",
+                "initramfs",
+                "busybox",
+                "timeout",
+                "append",
+            ],
         ),
         "test" => (
             false,
             &[
                 "reference",
                 "other",
                 "row",
+                "cpu",
                 "kinds",
                 "busybox",
                 "rucc-busybox",
@@ -98,6 +107,7 @@ fn accepted(command: &str) -> Option<(bool, &'static [&'static str])> {
                 "reference",
                 "other",
                 "row",
+                "cpu",
                 "unit",
                 "busybox",
                 "selftests",
@@ -114,6 +124,7 @@ fn accepted(command: &str) -> Option<(bool, &'static [&'static str])> {
                 "reference",
                 "other",
                 "row",
+                "cpu",
                 "busybox",
                 "runs",
                 "timeout",
@@ -162,15 +173,15 @@ usage:
   rk objtool-report --reference DIR|LOG --other DIR|LOG
   rk frames --reference DIR|FILE [--other DIR|FILE] [--save FILE]
   rk syntax --build DIR [--cc rucc] [--allow FILE] [--jobs N]
-  rk boot --build DIR [--row X64] [--initramfs FILE | --busybox PATH] [--timeout 300]
-          [--append WORDS]
+  rk boot --build DIR [--row X64] [--cpu NAME] [--initramfs FILE | --busybox PATH]
+          [--timeout 300] [--append WORDS]
   rk test --reference DIR --other DIR [--kinds boot,smoke,kunit,kselftest,ltp] [--row X64]
-          [--runs 1] [--selftests DIR] [--rucc-selftests DIR] [--ltp DIR] [--rucc-ltp DIR]
-          [--busybox PATH] [--rucc-busybox PATH] [--timeout 600] [--out DIR]
-  rk mixed --reference DIR --other DIR --unit UNIT [--row X64] [--timeout 600] [--no-fuel]
-          [--selftests DIR] [--ltp DIR]
-  rk cross-modules --reference DIR --other DIR [--row X64] [--runs 1] [--timeout 600]
-          [--busybox PATH] [--out DIR]
+          [--cpu NAME] [--runs 1] [--selftests DIR] [--rucc-selftests DIR] [--ltp DIR]
+          [--rucc-ltp DIR] [--busybox PATH] [--rucc-busybox PATH] [--timeout 600] [--out DIR]
+  rk mixed --reference DIR --other DIR --unit UNIT [--row X64] [--cpu NAME] [--timeout 600]
+          [--no-fuel] [--selftests DIR] [--ltp DIR]
+  rk cross-modules --reference DIR --other DIR [--row X64] [--cpu NAME] [--runs 1]
+          [--timeout 600] [--busybox PATH] [--out DIR]
   rk selftests --build DIR [--cc COMPILER] [--collections timers,size] [--jobs N] [--out DIR]
   rk ltp --build DIR [--cc COMPILER] [--runtests syscalls,mm] [--jobs N] [--out DIR]
   rk initramfs --out FILE [--busybox PATH]

crates/rk/src/main.rs

@@ -722,6 +722,15 @@ fn write_initramfs(out: &std::path::Path, busybox: Option<&str>) -> Result<(), S
         .map_err(|e| format!("writing {}: {e}", out.display()))
 }
 
+/// The row, with the QEMU CPU `--cpu` names in place of its own, so that an arm64 kernel can be
+/// run on `cortex-a57` as well as on `max`.
+fn with_cpu(mut row: personas::Row, args: &Args) -> personas::Row {
+    if let Some(cpu) = args.get("cpu") {
+        row.cpu = cpu.to_string();
+    }
+    row
+}
+
 /// Boot a build under QEMU and run the smoke checks.
 fn boot_command(repo: &Repo, args: &Args) -> Result<ExitCode, String> {
     let build = std::path::PathBuf::from(args.get("build").ok_or("rk boot needs --build")?);
@@ -735,7 +744,7 @@ fn boot_command(repo: &Repo, args: &Args) -> Result<ExitCode, String> {
             .map_or_else(|| "X64".to_string(), |o| o.row),
     };
     let rows = personas::Rows::load(&repo.file("rows.toml"))?;
-    let row = rows.get(&row_name)?.clone();
+    let row = with_cpu(rows.get(&row_name)?.clone(), args);
     let initramfs = if let Some(path) = args.get("initramfs") {
         std::path::PathBuf::from(path)
     } else {
@@ -820,7 +829,7 @@ fn test_command(repo: &Repo, args: &Args) -> Result<ExitCode, String> {
         str::to_string,
     );
     let rows = personas::Rows::load(&repo.file("rows.toml"))?;
-    let row = rows.get(&row_name)?.clone();
+    let row = with_cpu(rows.get(&row_name)?.clone(), args);
     let number = |name: &str, default: u64| -> Result<u64, String> {
         args.get(name).map_or(Ok(default), |n| {
             n.parse()
@@ -1016,7 +1025,7 @@ fn cross_modules(repo: &Repo, args: &Args) -> Result<ExitCode, String> {
         str::to_string,
     );
     let rows = personas::Rows::load(&repo.file("rows.toml"))?;
-    let row = rows.get(&row_name)?.clone();
+    let row = with_cpu(rows.get(&row_name)?.clone(), args);
     let number = |name: &str, default: u64| -> Result<u64, String> {
         args.get(name).map_or(Ok(default), |n| {
             n.parse()
@@ -1070,7 +1079,10 @@ fn mixed_command(repo: &Repo, args: &Args) -> Result<ExitCode, String> {
         .to_string();
     let built = outcome_of(&other)?;
     let rows = personas::Rows::load(&repo.file("rows.toml"))?;
-    let row = rows.get(args.get("row").unwrap_or(&built.row))?.clone();
+    let row = with_cpu(
+        rows.get(args.get("row").unwrap_or(&built.row))?.clone(),
+        args,
+    );
     let number = |name: &str, default: u64| -> Result<u64, String> {
         args.get(name).map_or(Ok(default), |n| {
             n.parse()