diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index e7654d5..3a4ef03 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -10,6 +10,11 @@ # # The rucc binary is the pinned release, checked against its hash, as in kernel.yml. A manual run # can name a branch or commit of tamnd/rucc instead. +# +# A manual run can also name the row, the versions and the QEMU CPU. A64 runs on the arm runner, +# which has no KVM, so its boots are TCG and are given longer, and its rucc is built from source, +# from the pinned release's tag when no branch or commit is named, since the release tarball is +# the x86_64 one. name: nightly @@ -26,6 +31,18 @@ on: description: The configuration target required: false default: tinyconfig + row: + description: The row, X64 or A64 + required: false + default: X64 + versions: + description: The versions as a JSON list, or empty for the current set + required: false + default: "" + cpu: + description: The QEMU CPU to boot on, or empty for the row's own + required: false + default: "" concurrency: group: ${{ github.workflow }}-${{ github.event_name == 'workflow_dispatch' && github.run_id || github.ref }} @@ -40,6 +57,8 @@ env: RUCC_VERSION: v0.24.8 RUCC_SHA256: f37e42c301f6867087566f40e99d9be0960e8d99c035da632f8e28e3c05b3ed8 CONFIG: ${{ inputs.config || 'tinyconfig' }} + ROW: ${{ inputs.row || 'X64' }} + CPU: ${{ inputs.cpu }} jobs: set: @@ -57,16 +76,20 @@ jobs: with open("pins.toml", "rb") as f: pins = tomllib.load(f)["pin"] versions = [p["version"] for p in pins if "current" in p.get("sets", [])] + if os.environ["ASKED"]: + versions = json.loads(os.environ["ASKED"]) with open(os.environ["GITHUB_OUTPUT"], "a") as out: out.write(f"versions={json.dumps(versions)}\n") with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as out: - out.write(f"The current set is {', '.join(versions)}.\n") + out.write(f"The versions are {', '.join(versions)}.\n") + env: + ASKED: ${{ inputs.versions }} test: - name: ${{ matrix.version }} X64 + name: ${{ matrix.version }} ${{ inputs.row || 'X64' }} needs: set - runs-on: ubuntu-24.04 - timeout-minutes: 300 + runs-on: ${{ inputs.row == 'A64' && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} + timeout-minutes: ${{ inputs.row == 'A64' && 355 || 300 }} strategy: fail-fast: false matrix: @@ -78,16 +101,19 @@ jobs: run: | sudo apt-get update -q sudo apt-get install -qy --no-install-recommends gcc-14 flex bison bc libelf-dev libssl-dev \ - cpio qemu-system-x86 busybox-static libcap-dev libnuma-dev rsync libaio-dev libacl1-dev \ + cpio "$QEMU" busybox-static libcap-dev libnuma-dev rsync libaio-dev libacl1-dev \ libkeyutils-dev + env: + QEMU: ${{ inputs.row == 'A64' && 'qemu-system-arm' || 'qemu-system-x86' }} - name: let the runner user open /dev/kvm + if: inputs.row != 'A64' run: | echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \ | sudo tee /etc/udev/rules.d/99-kvm.rules sudo udevadm control --reload-rules sudo udevadm trigger --name-match=kvm || true - - name: rucc ${{ inputs.rucc }} from source - if: inputs.rucc != '' + - name: rucc ${{ inputs.rucc || env.RUCC_VERSION }} from source + if: inputs.rucc != '' || inputs.row == 'A64' run: | git clone -q https://github.com/tamnd/rucc "$RUNNER_TEMP/rucc-src" git -C "$RUNNER_TEMP/rucc-src" checkout -q "$REF" @@ -95,9 +121,9 @@ jobs: echo "$RUNNER_TEMP/rucc-src/target/release" >> "$GITHUB_PATH" echo "rucc built from $(git -C "$RUNNER_TEMP/rucc-src" rev-parse HEAD)" >> "$GITHUB_STEP_SUMMARY" env: - REF: ${{ inputs.rucc }} + REF: ${{ inputs.rucc || env.RUCC_VERSION }} - name: rucc ${{ env.RUCC_VERSION }} - if: inputs.rucc == '' + if: inputs.rucc == '' && inputs.row != 'A64' run: | name="rucc-$RUCC_VERSION-x86_64-unknown-linux-gnu" gh release download "$RUCC_VERSION" -R tamnd/rucc -p "$name.tar.gz" -D "$RUNNER_TEMP" @@ -115,12 +141,12 @@ jobs: restore-keys: kernel-archives-${{ matrix.version }}- - name: rk build with gcc-14 run: | - target/release/rk build ${{ matrix.version }} --row X64 --config "$CONFIG" --fragment test \ + target/release/rk build ${{ matrix.version }} --row "$ROW" --config "$CONFIG" --fragment test \ --cc gcc-14 --targets "bzImage modules" --out "$RUNNER_TEMP/gcc" --stack-usage - name: rk build with rucc continue-on-error: true run: | - target/release/rk build ${{ matrix.version }} --row X64 --config "$CONFIG" --fragment test \ + target/release/rk build ${{ matrix.version }} --row "$ROW" --config "$CONFIG" --fragment test \ --cc rucc --targets "bzImage modules" --out "$RUNNER_TEMP/rucc" --keep-going --stack-usage - name: rk selftests with gcc-14 run: | @@ -134,7 +160,7 @@ jobs: - uses: actions/cache@v6.1.0 with: path: ${{ env.RK_CACHE }}/ltp - key: ltp-${{ hashFiles('ltp.toml') }} + key: ltp-${{ runner.arch }}-${{ hashFiles('ltp.toml') }} - name: rk ltp with gcc-14 run: | target/release/rk ltp --build "$RUNNER_TEMP/gcc" --cc gcc-14 --out "$RUNNER_TEMP/ltp-gcc" @@ -145,6 +171,8 @@ jobs: - name: rk test id: test run: | + boot=(--timeout "$TIMEOUT") + if [ -n "$CPU" ]; then boot+=(--cpu "$CPU"); fi rucc_userland=() if [ -f "$RUNNER_TEMP/selftests-rucc/selftests.json" ]; then rucc_userland+=(--rucc-selftests "$RUNNER_TEMP/selftests-rucc") @@ -154,13 +182,19 @@ jobs: fi target/release/rk test --reference "$RUNNER_TEMP/gcc" --other "$RUNNER_TEMP/rucc" \ --kinds boot,smoke,kunit,kselftest,ltp --selftests "$RUNNER_TEMP/selftests-gcc" \ - --ltp "$RUNNER_TEMP/ltp-gcc" "${rucc_userland[@]}" --out "$RUNNER_TEMP/test" + --ltp "$RUNNER_TEMP/ltp-gcc" "${rucc_userland[@]}" "${boot[@]}" --out "$RUNNER_TEMP/test" + env: + TIMEOUT: ${{ inputs.row == 'A64' && 1500 || 600 }} - name: rk cross-modules if: always() && steps.test.outcome != 'skipped' continue-on-error: true run: | + boot=(--timeout "$TIMEOUT") + if [ -n "$CPU" ]; then boot+=(--cpu "$CPU"); fi target/release/rk cross-modules --reference "$RUNNER_TEMP/gcc" --other "$RUNNER_TEMP/rucc" \ - --out "$RUNNER_TEMP/cross" + "${boot[@]}" --out "$RUNNER_TEMP/cross" + env: + TIMEOUT: ${{ inputs.row == 'A64' && 1500 || 600 }} - name: rk objtool-report if: always() && steps.test.outcome != 'skipped' run: | @@ -183,7 +217,9 @@ jobs: run: | unit=$(python3 -c 'import json, sys; t = json.load(open(sys.argv[1])); print(next((u["name"] for u in t["units"] if u["graded"] and not u["passed"] and u.get("attribution", "kernel") == "kernel"), ""))' "$RUNNER_TEMP/test/test.json") if [ -z "$unit" ]; then echo "no kernel failure to bisect"; exit 0; fi - target/release/rk mixed --reference "$RUNNER_TEMP/gcc" --other "$RUNNER_TEMP/rucc" \ + boot=() + if [ -n "$CPU" ]; then boot+=(--cpu "$CPU"); fi + target/release/rk mixed "${boot[@]}" --reference "$RUNNER_TEMP/gcc" --other "$RUNNER_TEMP/rucc" \ --unit "$unit" --selftests "$RUNNER_TEMP/selftests-gcc" --ltp "$RUNNER_TEMP/ltp-gcc" \ --out "$RUNNER_TEMP/mixed" - name: keep the record @@ -208,7 +244,7 @@ jobs: - uses: actions/upload-artifact@v7.0.1 if: always() with: - name: nightly-${{ matrix.version }}-X64 + name: nightly-${{ matrix.version }}-${{ inputs.row || 'X64' }} path: record include-hidden-files: true retention-days: 30 @@ -227,9 +263,10 @@ jobs: shell: python3 {0} run: | import json, os, pathlib - lines = ["### nightly, X64", "", "| version | graded | failed | splats | verdict | objtool | frames |", "|---|---|---|---|---|---|---|"] + row = os.environ["ROW"] + lines = [f"### nightly, {row}", "", "| version | graded | failed | splats | verdict | objtool | frames |", "|---|---|---|---|---|---|---|"] for record in sorted(pathlib.Path("records").iterdir()): - version = record.name.removeprefix("nightly-").removesuffix("-X64") + version = record.name.removeprefix("nightly-").removesuffix(f"-{row}") path = record / "test" / "test.json" checks = [] for name in ("objtool", "frames"): diff --git a/README.md b/README.md index 0d1ab3a..2cbee10 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ The reference compilers live in era containers built from `provision/eras` and p `rk sections-diff`, `rk symvers-diff`, `rk vec-audit`, `rk modules-audit` and `rk objtool-report` look at what the compilers produced. `rk sections-diff` compares the section names of every object two builds share, the size and relocation count of each kernel table (exports, jump labels, alternatives, exception fixups, initcalls and the rest), the call-site lists per function, and the `.modinfo` and `__ksymtab_strings` strings. A difference that `sections-divergences.toml` explains, such as a constant pool only gcc builds or a WARN check only one compiler could prove dead, is listed apart with its reason and does not fail the run. It lists the first 60 differences of each kind, and every one with `--all`. `rk symvers-diff` compares `Module.symvers`, CRCs and namespaces included, and the global symbols of `System.map`. `rk vec-audit` decodes the x86-64 code of every unit built with `-mno-sse` or `-mgeneral-regs-only` and lists the vector and x87 instructions it finds, only those above the reference's count when given `--reference`. `rk modules-audit` fails on any module relocation the loader rejects, a wrong `vermagic` or an imported CRC that does not match `Module.symvers`. `rk objtool-report` buckets the objtool warnings of two build logs by message and function shape. `rk frames` compares the stack frame of every function in two builds made with `rk build --stack-usage`, which passes `KCFLAGS=-fstack-usage` so that each compiler writes a `.su` file next to every object. It fails on a function over `CONFIG_FRAME_WARN` that the reference keeps under it, and on a run time stack high water mark more than 10% above the reference's when both build directories have a `boot.log` from a kernel with `CONFIG_DEBUG_STACK_USAGE`. `rk sections-diff`, `rk vec-audit` and `rk frames` take `--save FILE` to keep what they read as JSON, and accept such a file in place of a build directory, which is how CI compares builds without moving their objects. -The nightly workflow builds every pin in the current set on X64 with gcc-14 and with rucc, from `tinyconfig` and the test fragment, and runs `rk test` on boot, smoke and KUnit. When the rucc kernel fails a unit, `rk mixed` runs on the first one in the same job. The run's summary has a line per version, green or red, whether `rk objtool-report` and `rk frames` came out clean, and what `rk mixed` found. A manual run takes a branch or commit of tamnd/rucc and another configuration target. +The nightly workflow builds every pin in the current set on X64 with gcc-14 and with rucc, from `tinyconfig` and the test fragment, and runs `rk test` on boot, smoke and KUnit. When the rucc kernel fails a unit, `rk mixed` runs on the first one in the same job. The run's summary has a line per version, green or red, whether `rk objtool-report` and `rk frames` came out clean, and what `rk mixed` found. A manual run takes a branch or commit of tamnd/rucc, another configuration target, the versions, the row and the QEMU CPU, so `row=A64 cpu=cortex-a57 config=debian-13-arm64` runs every test kind on arm64 under TCG on the arm runner. The mixed workflow is run by hand. It builds the rucc kernel with `-fenable-inject-fault=time64_to_tm`, checks that the KUnit case for that function fails, and passes only when `rk mixed` traces the failure back to the `inject-fault` pass in that function. diff --git a/crates/rk/src/cli.rs b/crates/rk/src/cli.rs index 2e57239..d9ebed3 100644 --- a/crates/rk/src/cli.rs +++ b/crates/rk/src/cli.rs @@ -70,7 +70,15 @@ fn accepted(command: &str) -> Option<(bool, &'static [&'static str])> { "demands" => (false, &["builds", "limit"]), "boot" => ( false, - &["build", "row", "initramfs", "busybox", "timeout", "append"], + &[ + "build", + "row", + "cpu", + "initramfs", + "busybox", + "timeout", + "append", + ], ), "test" => ( false, @@ -78,6 +86,7 @@ fn accepted(command: &str) -> Option<(bool, &'static [&'static str])> { "reference", "other", "row", + "cpu", "kinds", "busybox", "rucc-busybox", @@ -98,6 +107,7 @@ fn accepted(command: &str) -> Option<(bool, &'static [&'static str])> { "reference", "other", "row", + "cpu", "unit", "busybox", "selftests", @@ -114,6 +124,7 @@ fn accepted(command: &str) -> Option<(bool, &'static [&'static str])> { "reference", "other", "row", + "cpu", "busybox", "runs", "timeout", @@ -162,15 +173,15 @@ usage: rk objtool-report --reference DIR|LOG --other DIR|LOG rk frames --reference DIR|FILE [--other DIR|FILE] [--save FILE] rk syntax --build DIR [--cc rucc] [--allow FILE] [--jobs N] - rk boot --build DIR [--row X64] [--initramfs FILE | --busybox PATH] [--timeout 300] - [--append WORDS] + rk boot --build DIR [--row X64] [--cpu NAME] [--initramfs FILE | --busybox PATH] + [--timeout 300] [--append WORDS] rk test --reference DIR --other DIR [--kinds boot,smoke,kunit,kselftest,ltp] [--row X64] - [--runs 1] [--selftests DIR] [--rucc-selftests DIR] [--ltp DIR] [--rucc-ltp DIR] - [--busybox PATH] [--rucc-busybox PATH] [--timeout 600] [--out DIR] - rk mixed --reference DIR --other DIR --unit UNIT [--row X64] [--timeout 600] [--no-fuel] - [--selftests DIR] [--ltp DIR] - rk cross-modules --reference DIR --other DIR [--row X64] [--runs 1] [--timeout 600] - [--busybox PATH] [--out DIR] + [--cpu NAME] [--runs 1] [--selftests DIR] [--rucc-selftests DIR] [--ltp DIR] + [--rucc-ltp DIR] [--busybox PATH] [--rucc-busybox PATH] [--timeout 600] [--out DIR] + rk mixed --reference DIR --other DIR --unit UNIT [--row X64] [--cpu NAME] [--timeout 600] + [--no-fuel] [--selftests DIR] [--ltp DIR] + rk cross-modules --reference DIR --other DIR [--row X64] [--cpu NAME] [--runs 1] + [--timeout 600] [--busybox PATH] [--out DIR] rk selftests --build DIR [--cc COMPILER] [--collections timers,size] [--jobs N] [--out DIR] rk ltp --build DIR [--cc COMPILER] [--runtests syscalls,mm] [--jobs N] [--out DIR] rk initramfs --out FILE [--busybox PATH] diff --git a/crates/rk/src/main.rs b/crates/rk/src/main.rs index f837e93..bae2462 100644 --- a/crates/rk/src/main.rs +++ b/crates/rk/src/main.rs @@ -722,6 +722,15 @@ fn write_initramfs(out: &std::path::Path, busybox: Option<&str>) -> Result<(), S .map_err(|e| format!("writing {}: {e}", out.display())) } +/// The row, with the QEMU CPU `--cpu` names in place of its own, so that an arm64 kernel can be +/// run on `cortex-a57` as well as on `max`. +fn with_cpu(mut row: personas::Row, args: &Args) -> personas::Row { + if let Some(cpu) = args.get("cpu") { + row.cpu = cpu.to_string(); + } + row +} + /// Boot a build under QEMU and run the smoke checks. fn boot_command(repo: &Repo, args: &Args) -> Result { let build = std::path::PathBuf::from(args.get("build").ok_or("rk boot needs --build")?); @@ -735,7 +744,7 @@ fn boot_command(repo: &Repo, args: &Args) -> Result { .map_or_else(|| "X64".to_string(), |o| o.row), }; let rows = personas::Rows::load(&repo.file("rows.toml"))?; - let row = rows.get(&row_name)?.clone(); + let row = with_cpu(rows.get(&row_name)?.clone(), args); let initramfs = if let Some(path) = args.get("initramfs") { std::path::PathBuf::from(path) } else { @@ -820,7 +829,7 @@ fn test_command(repo: &Repo, args: &Args) -> Result { str::to_string, ); let rows = personas::Rows::load(&repo.file("rows.toml"))?; - let row = rows.get(&row_name)?.clone(); + let row = with_cpu(rows.get(&row_name)?.clone(), args); let number = |name: &str, default: u64| -> Result { args.get(name).map_or(Ok(default), |n| { n.parse() @@ -1016,7 +1025,7 @@ fn cross_modules(repo: &Repo, args: &Args) -> Result { str::to_string, ); let rows = personas::Rows::load(&repo.file("rows.toml"))?; - let row = rows.get(&row_name)?.clone(); + let row = with_cpu(rows.get(&row_name)?.clone(), args); let number = |name: &str, default: u64| -> Result { args.get(name).map_or(Ok(default), |n| { n.parse() @@ -1070,7 +1079,10 @@ fn mixed_command(repo: &Repo, args: &Args) -> Result { .to_string(); let built = outcome_of(&other)?; let rows = personas::Rows::load(&repo.file("rows.toml"))?; - let row = rows.get(args.get("row").unwrap_or(&built.row))?.clone(); + let row = with_cpu( + rows.get(args.get("row").unwrap_or(&built.row))?.clone(), + args, + ); let number = |name: &str, default: u64| -> Result { args.get(name).map_or(Ok(default), |n| { n.parse()