From ce40b708736b665f1e3f11d850b0bc068500ad56 Mon Sep 17 00:00:00 2001 From: Mykyta Yatsenko Date: Tue, 19 May 2026 12:32:22 -0700 Subject: [PATCH] run-vmtest: fail selftest runs that produce kernel splats A non-fatal splat (WARN, KASAN report, lockdep, hung task) leaves the VM running, so the selftest binary still exits 0 and CI goes green on a kernel that just hit a real bug. We have no signal for these today. Scan dmesg after the run and add a kernel_splats row to the existing exitstatus aggregator. What counts as a splat, and what is benign, is policy that differs per arch and per kernel, so it lives in $VMTEST_CONFIGS rather than in this action. A scan that cannot run fails the row rather than reporting clean: with no denylist there is no check, and a silently disabled detector is worse than none. Assisted-by: Claude:claude-opus-4-7 Signed-off-by: Mykyta Yatsenko --- .github/workflows/kernel-test.yml | 9 ++ ci/vmtest/configs/SPLAT_ALLOWLIST | 21 +++++ ci/vmtest/configs/SPLAT_DENYLIST | 24 ++++++ ci/vmtest/configs/run-vmtest.env | 7 ++ run-vmtest/README.md | 24 ++++++ run-vmtest/check-kernel-splats.sh | 82 +++++++++++++++++++ run-vmtest/run-bpf-selftests.sh | 5 +- run-vmtest/run-scx-selftests.sh | 2 + run-vmtest/run.sh | 2 +- .../cases/base_allowlist/dmesg.txt | 3 + .../cases/base_allowlist/expected-output.txt | 5 ++ .../cases/base_allowlist/expected-status.txt | 1 + .../cases/case_allowlist/allowlist.txt | 2 + .../cases/case_allowlist/dmesg.txt | 14 ++++ .../cases/case_allowlist/env | 1 + .../cases/case_allowlist/expected-output.txt | 5 ++ .../cases/case_allowlist/expected-status.txt | 1 + .../check_kernel_splats/cases/clean/dmesg.txt | 4 + .../cases/clean/expected-output.txt | 5 ++ .../cases/clean/expected-status.txt | 1 + .../check_kernel_splats/cases/kasan/dmesg.txt | 13 +++ .../cases/kasan/expected-output.txt | 18 ++++ .../cases/kasan/expected-status.txt | 1 + .../check_kernel_splats/cases/multi/dmesg.txt | 14 ++++ .../cases/multi/expected-output.txt | 19 +++++ .../cases/multi/expected-status.txt | 1 + .../cases/no_denylist/dmesg.txt | 14 ++++ .../check_kernel_splats/cases/no_denylist/env | 1 + .../cases/no_denylist/expected-output.txt | 3 + .../cases/no_denylist/expected-status.txt | 1 + .../check_kernel_splats/cases/warn/dmesg.txt | 14 ++++ .../cases/warn/expected-output.txt | 19 +++++ .../cases/warn/expected-status.txt | 1 + .../tests/check_kernel_splats/run-test.sh | 41 ++++++++++ 34 files changed, 376 insertions(+), 2 deletions(-) create mode 100644 ci/vmtest/configs/SPLAT_ALLOWLIST create mode 100644 ci/vmtest/configs/SPLAT_DENYLIST create mode 100755 run-vmtest/check-kernel-splats.sh create mode 100644 run-vmtest/tests/check_kernel_splats/cases/base_allowlist/dmesg.txt create mode 100644 run-vmtest/tests/check_kernel_splats/cases/base_allowlist/expected-output.txt create mode 100644 run-vmtest/tests/check_kernel_splats/cases/base_allowlist/expected-status.txt create mode 100644 run-vmtest/tests/check_kernel_splats/cases/case_allowlist/allowlist.txt create mode 100644 run-vmtest/tests/check_kernel_splats/cases/case_allowlist/dmesg.txt create mode 100644 run-vmtest/tests/check_kernel_splats/cases/case_allowlist/env create mode 100644 run-vmtest/tests/check_kernel_splats/cases/case_allowlist/expected-output.txt create mode 100644 run-vmtest/tests/check_kernel_splats/cases/case_allowlist/expected-status.txt create mode 100644 run-vmtest/tests/check_kernel_splats/cases/clean/dmesg.txt create mode 100644 run-vmtest/tests/check_kernel_splats/cases/clean/expected-output.txt create mode 100644 run-vmtest/tests/check_kernel_splats/cases/clean/expected-status.txt create mode 100644 run-vmtest/tests/check_kernel_splats/cases/kasan/dmesg.txt create mode 100644 run-vmtest/tests/check_kernel_splats/cases/kasan/expected-output.txt create mode 100644 run-vmtest/tests/check_kernel_splats/cases/kasan/expected-status.txt create mode 100644 run-vmtest/tests/check_kernel_splats/cases/multi/dmesg.txt create mode 100644 run-vmtest/tests/check_kernel_splats/cases/multi/expected-output.txt create mode 100644 run-vmtest/tests/check_kernel_splats/cases/multi/expected-status.txt create mode 100644 run-vmtest/tests/check_kernel_splats/cases/no_denylist/dmesg.txt create mode 100644 run-vmtest/tests/check_kernel_splats/cases/no_denylist/env create mode 100644 run-vmtest/tests/check_kernel_splats/cases/no_denylist/expected-output.txt create mode 100644 run-vmtest/tests/check_kernel_splats/cases/no_denylist/expected-status.txt create mode 100644 run-vmtest/tests/check_kernel_splats/cases/warn/dmesg.txt create mode 100644 run-vmtest/tests/check_kernel_splats/cases/warn/expected-output.txt create mode 100644 run-vmtest/tests/check_kernel_splats/cases/warn/expected-status.txt create mode 100755 run-vmtest/tests/check_kernel_splats/run-test.sh diff --git a/.github/workflows/kernel-test.yml b/.github/workflows/kernel-test.yml index a57dbc25..1ca32b56 100644 --- a/.github/workflows/kernel-test.yml +++ b/.github/workflows/kernel-test.yml @@ -96,3 +96,12 @@ jobs: name: tmon-logs-${{ inputs.arch }}-${{ inputs.toolchain_full }}-${{ inputs.test }} if-no-files-found: ignore path: /tmp/tmon_pcap/* + + # Written by check-kernel-splats.sh inside the VM, into the bind-mounted + # workspace. Needed to triage a splat, and to audit the scan itself. + - if: ${{ always() }} + uses: actions/upload-artifact@v4 + with: + name: kernel-log-${{ inputs.arch }}-${{ inputs.toolchain_full }}-${{ inputs.test }} + if-no-files-found: ignore + path: dmesg.txt diff --git a/ci/vmtest/configs/SPLAT_ALLOWLIST b/ci/vmtest/configs/SPLAT_ALLOWLIST new file mode 100644 index 00000000..0c23bfb3 --- /dev/null +++ b/ci/vmtest/configs/SPLAT_ALLOWLIST @@ -0,0 +1,21 @@ +# Kernel splats that check-kernel-splats.sh must not fail the run on. +# +# One extended regex per line, matched against the dmesg line. `#` comments +# and blank lines are ignored. +# +# Every entry is a hole in the scan, so give each one a reason and a link, and +# delete it once the fix reaches the CI kernel. An entry with no expiry is a +# bug that CI has agreed to stop reporting. +# +# The selftest config leaves unprivileged BPF on (CONFIG_BPF_UNPRIV_DEFAULT_OFF +# is not set in tools/testing/selftests/bpf/config), so the Spectre v2 code +# warns about it on every boot on x86_64, and whenever a test writes the sysctl +# on arm64. It states a config choice, not a kernel bug. +WARNING: Unprivileged eBPF is enabled + +# arm64 stack unwinder writes past the entry array while KASAN saves a free +# stack, so any kfree() under test_progs reports stack-out-of-bounds. Not a BPF +# bug; BPF only walks that path often. Fix posted 2026-08-12: +# https://lore.kernel.org/all/20260812-hello_world-v1-1-c3c2ddcb362d@meta.com/ +# Delete this entry once that fix reaches the CI kernel. +BUG: KASAN: stack-out-of-bounds in (stack_trace_consume_entry|filter_irq_stacks)\+ diff --git a/ci/vmtest/configs/SPLAT_DENYLIST b/ci/vmtest/configs/SPLAT_DENYLIST new file mode 100644 index 00000000..5f1477d1 --- /dev/null +++ b/ci/vmtest/configs/SPLAT_DENYLIST @@ -0,0 +1,24 @@ +# Kernel splat signatures for check-kernel-splats.sh. A dmesg line matching +# any of these fails the run. +# +# One extended regex per line. `#` comments and blank lines are ignored. This +# file is required: with no patterns there is no check, so the action fails the +# run rather than reporting a clean log. +# +# `^(\[[^]]*\] *)*` eats the timestamp and the CONFIG_PRINTK_CALLER field, so a +# pattern holds for `dmesg`, for `dmesg -t` and for both configs. + +# `BUG:` covers KASAN, KCSAN, KMSAN and KFENCE. `WARNING:` covers __warn(), and +# with it lockdep, refcount_t and list corruption, which all print through +# WARN(). An oops panics the VM (PANIC_ON_OOPS plus panic=-1), which fails the +# job on its own, so it needs no pattern here. +^(\[[^]]*\] *)*(BUG|WARNING|UBSAN|Oops)[: ] +^(\[[^]]*\] *)*kernel BUG at + +# kernel/watchdog.c sets pr_fmt, so a lockup line starts with `watchdog: `, +# not with `BUG:`. +^(\[[^]]*\] *)*watchdog: .*(soft lockup|hard LOCKUP) + +# kernel/rcu/tree.c sets pr_fmt to "rcu: ", which hides the INFO: from an +# anchored match. +^(\[[^]]*\] *)*(rcu: )?INFO: (task .* blocked for more than|[_a-z]+ (self-)?detected stall) diff --git a/ci/vmtest/configs/run-vmtest.env b/ci/vmtest/configs/run-vmtest.env index 0a4e0216..1606ae63 100644 --- a/ci/vmtest/configs/run-vmtest.env +++ b/ci/vmtest/configs/run-vmtest.env @@ -41,3 +41,10 @@ DENYLIST_FILES=( # Export pipe-separated strings, because bash doesn't support array export export SELFTESTS_BPF_ALLOWLIST_FILES=$(IFS="|"; echo "${ALLOWLIST_FILES[*]}") export SELFTESTS_BPF_DENYLIST_FILES=$(IFS="|"; echo "${DENYLIST_FILES[*]}") + +# Kernel splat matching for check-kernel-splats.sh. The denylist says what a +# splat is, the allowlist drops the matches that are benign for this CI. The +# action carries no patterns of its own, so a change to either costs one PR to +# this repo, and no libbpf/ci sync. +export SPLAT_DENYLIST_FILE="${VMTEST_CONFIGS}/SPLAT_DENYLIST" +export SPLAT_ALLOWLIST_FILE="${VMTEST_CONFIGS}/SPLAT_ALLOWLIST" diff --git a/run-vmtest/README.md b/run-vmtest/README.md index 2cf13a76..c489fb55 100644 --- a/run-vmtest/README.md +++ b/run-vmtest/README.md @@ -39,6 +39,30 @@ environment variables. * `kbuild-output` (default: `./kbuild-output`) - path to Linux Kernel binaries, aka `$KBUILD_OUTPUT` * `vmtest-release` - release version name of the vmtest tool +## Kernel splats + +`check-kernel-splats.sh` runs after the selftests and greps the kernel log for +splats that leave the VM alive: WARN, KASAN and friends, lockdep, RCU stalls, +hung tasks and lockups. A hit adds a `kernel_splats` row to `exitstatus`, which +turns the run red like any other failing test group. Anything fatal panics the +VM instead, and fails the job on its own. + +The action carries no patterns. What a splat is, and what is benign, is policy +that changes per arch and per kernel, so it lives with the configs. Two files +of extended regexes, one per line, `#` comments and blank lines ignored, named +by `run-vmtest.env` next to the allow and denylists: + +* `SPLAT_DENYLIST_FILE` - a matching dmesg line is a splat. **Required.** +* `SPLAT_ALLOWLIST_FILE` - a matching splat line is ignored. Optional. + +The denylist is required on purpose: with no patterns there is no check, so a +missing or empty file fails the run rather than reporting a clean log. + +See `ci/vmtest/configs/SPLAT_DENYLIST` for the set BPF CI uses. + +`dmesg.txt` is written to the output dir and uploaded as the `kernel-log-*` +artifact, so the full log is always one click away. + ## run-vmtest.env There are a couple of scripts, as well as code in the diff --git a/run-vmtest/check-kernel-splats.sh b/run-vmtest/check-kernel-splats.sh new file mode 100755 index 00000000..28ee5284 --- /dev/null +++ b/run-vmtest/check-kernel-splats.sh @@ -0,0 +1,82 @@ +#!/bin/bash +# Fail the selftest run if the kernel logged a splat. A WARN, a KASAN report, +# a lockdep report or a hung task leaves the VM running, so the test binary +# still exits 0 and the job goes green on a kernel that hit a real bug. +# +# What counts as a splat, and what is benign, is policy, so both live in the +# config repo rather than here: +# +# SPLAT_DENYLIST_FILE extended regexes; a matching dmesg line is a splat. +# Required: with no patterns there is no check, so a +# missing file fails the run instead of passing it. +# SPLAT_ALLOWLIST_FILE extended regexes; a matching splat line is ignored. +# Optional: no file means no exceptions. +# +# `#` comments and blank lines are ignored in both. See run-vmtest.env in the +# repo that owns $VMTEST_CONFIGS. +# +# The whole log is written to dmesg.txt, which the workflow uploads. +# +# $1 - log file to scan instead of running dmesg (used by the unit tests) + +set -euo pipefail + +source "$(cd "$(dirname "$0")" && pwd)/helpers.sh" + +STATUS_FILE=${STATUS_FILE:-/mnt/vmtest/exitstatus} +OUTPUT_DIR=${OUTPUT_DIR:-/mnt/vmtest} +SPLAT_DENYLIST_FILE=${SPLAT_DENYLIST_FILE:-} +SPLAT_ALLOWLIST_FILE=${SPLAT_ALLOWLIST_FILE:-} + +# Fail the row and stop. Also used when the scan cannot run: a check that is +# not working must not look like a clean log. +fail_check() { + echo "$1" + echo "kernel_splats:1" >> "${STATUS_FILE}" + foldable end kernel_splats + exit 0 +} + +# `grep -f` has no comment syntax, and to grep an empty line is a regex that +# matches every line, so drop both before either file can hide a splat. +read_patterns() { + grep -vE '^[[:space:]]*(#|$)' "$1" || [ $? = 1 ] +} + +log=${1:-} +if [ -z "${log}" ]; then + log="${OUTPUT_DIR}/dmesg.txt" + dmesg > "${log}" +fi + +foldable start kernel_splats "Checking kernel log for splats" + +[ -s "${SPLAT_DENYLIST_FILE}" ] || \ + fail_check "No splat denylist: SPLAT_DENYLIST_FILE=${SPLAT_DENYLIST_FILE:-}" +echo "Splat denylist: ${SPLAT_DENYLIST_FILE}" +deny=$(read_patterns "${SPLAT_DENYLIST_FILE}") +[ -n "${deny}" ] || fail_check "Splat denylist holds no patterns" + +# `|| [ $? = 1 ]` lets "no match" through, but a grep error trips set -e, so a +# scan that cannot run fails the job instead of reporting a clean log. Two +# greps must not share a pipeline: pipefail reports the rightmost non-zero +# status, so "no match" from one would mask an error from the other. +hits=$(grep -E -f <(printf '%s\n' "${deny}") "${log}" || [ $? = 1 ]) + +if [ -s "${SPLAT_ALLOWLIST_FILE}" ]; then + echo "Splat allowlist: ${SPLAT_ALLOWLIST_FILE}" + allow=$(read_patterns "${SPLAT_ALLOWLIST_FILE}") + [ -z "${allow}" ] || hits=$(printf '%s' "${hits}" \ + | grep -vE -f <(printf '%s\n' "${allow}") || [ $? = 1 ]) +fi +foldable end kernel_splats + +if [ -z "${hits}" ]; then + echo "No kernel splats detected" + echo "kernel_splats:0" >> "${STATUS_FILE}" + exit 0 +fi + +echo "kernel_splats:1" >> "${STATUS_FILE}" +grep -nC 30 -E -f <(printf '%s\n' "${deny}") "${log}" || true +echo "::error title=kernel_splats::kernel splat detected, see the dmesg.txt artifact" diff --git a/run-vmtest/run-bpf-selftests.sh b/run-vmtest/run-bpf-selftests.sh index 9941e5bd..c598354e 100755 --- a/run-vmtest/run-bpf-selftests.sh +++ b/run-vmtest/run-bpf-selftests.sh @@ -14,7 +14,8 @@ set -xeuo pipefail -source "$(cd "$(dirname "$0")" && pwd)/helpers.sh" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "${SCRIPT_DIR}/helpers.sh" ARCH=$(uname -m) @@ -176,3 +177,5 @@ else "${test_name}" done fi + +"${SCRIPT_DIR}/check-kernel-splats.sh" diff --git a/run-vmtest/run-scx-selftests.sh b/run-vmtest/run-scx-selftests.sh index 3ff110bc..e9436505 100755 --- a/run-vmtest/run-scx-selftests.sh +++ b/run-vmtest/run-scx-selftests.sh @@ -37,5 +37,7 @@ fi echo "selftests/sched_ext:$failed" >> "${STATUS_FILE}" +"$(cd "$(dirname "$0")" && pwd)/check-kernel-splats.sh" + exit 0 diff --git a/run-vmtest/run.sh b/run-vmtest/run.sh index d998e666..5396c212 100755 --- a/run-vmtest/run.sh +++ b/run-vmtest/run.sh @@ -96,7 +96,7 @@ cat > $VMTEST_TOML < +[ 12.345685] ? __warn+0x80/0xc0 +[ 12.345686] ? report_bug+0xa0/0xc0 +[ 12.345687] +[ 12.345688] ---[ end trace ]--- +[ 13.000000] test_progs: PASS diff --git a/run-vmtest/tests/check_kernel_splats/cases/case_allowlist/env b/run-vmtest/tests/check_kernel_splats/cases/case_allowlist/env new file mode 100644 index 00000000..1a6185b1 --- /dev/null +++ b/run-vmtest/tests/check_kernel_splats/cases/case_allowlist/env @@ -0,0 +1 @@ +export SPLAT_ALLOWLIST_FILE="${CASE_DIR}/allowlist.txt" diff --git a/run-vmtest/tests/check_kernel_splats/cases/case_allowlist/expected-output.txt b/run-vmtest/tests/check_kernel_splats/cases/case_allowlist/expected-output.txt new file mode 100644 index 00000000..cd5a0bb4 --- /dev/null +++ b/run-vmtest/tests/check_kernel_splats/cases/case_allowlist/expected-output.txt @@ -0,0 +1,5 @@ +::group::kernel_splats - Checking kernel log for splats +Splat denylist: $VMTEST_CONFIGS/SPLAT_DENYLIST +Splat allowlist: $CASE_DIR/allowlist.txt +::endgroup:: +No kernel splats detected diff --git a/run-vmtest/tests/check_kernel_splats/cases/case_allowlist/expected-status.txt b/run-vmtest/tests/check_kernel_splats/cases/case_allowlist/expected-status.txt new file mode 100644 index 00000000..391dcf85 --- /dev/null +++ b/run-vmtest/tests/check_kernel_splats/cases/case_allowlist/expected-status.txt @@ -0,0 +1 @@ +kernel_splats:0 diff --git a/run-vmtest/tests/check_kernel_splats/cases/clean/dmesg.txt b/run-vmtest/tests/check_kernel_splats/cases/clean/dmesg.txt new file mode 100644 index 00000000..f48a9ae4 --- /dev/null +++ b/run-vmtest/tests/check_kernel_splats/cases/clean/dmesg.txt @@ -0,0 +1,4 @@ +[ 0.000000] Linux version 6.5.0 +[ 0.001234] CPU: 0 +[ 0.002345] Memory: 4G +[ 1.000000] vmtest selftests complete diff --git a/run-vmtest/tests/check_kernel_splats/cases/clean/expected-output.txt b/run-vmtest/tests/check_kernel_splats/cases/clean/expected-output.txt new file mode 100644 index 00000000..e0d437e2 --- /dev/null +++ b/run-vmtest/tests/check_kernel_splats/cases/clean/expected-output.txt @@ -0,0 +1,5 @@ +::group::kernel_splats - Checking kernel log for splats +Splat denylist: $VMTEST_CONFIGS/SPLAT_DENYLIST +Splat allowlist: $VMTEST_CONFIGS/SPLAT_ALLOWLIST +::endgroup:: +No kernel splats detected diff --git a/run-vmtest/tests/check_kernel_splats/cases/clean/expected-status.txt b/run-vmtest/tests/check_kernel_splats/cases/clean/expected-status.txt new file mode 100644 index 00000000..391dcf85 --- /dev/null +++ b/run-vmtest/tests/check_kernel_splats/cases/clean/expected-status.txt @@ -0,0 +1 @@ +kernel_splats:0 diff --git a/run-vmtest/tests/check_kernel_splats/cases/kasan/dmesg.txt b/run-vmtest/tests/check_kernel_splats/cases/kasan/dmesg.txt new file mode 100644 index 00000000..ddda0fbd --- /dev/null +++ b/run-vmtest/tests/check_kernel_splats/cases/kasan/dmesg.txt @@ -0,0 +1,13 @@ +[ 0.000000] Linux version 6.5.0 +[ 20.000000] ================================================================== +[ 20.000001] BUG: KASAN: slab-use-after-free in foo+0x42/0x100 +[ 20.000002] Read of size 8 at addr ffff888100001234 by task test_progs/123 +[ 20.000003] +[ 20.000004] CPU: 0 PID: 123 Comm: test_progs +[ 20.000005] Call Trace: +[ 20.000006] +[ 20.000007] dump_stack_lvl+0x60/0x80 +[ 20.000008] print_report+0xc4/0x630 +[ 20.000009] kasan_report+0xb6/0xf0 +[ 20.000010] +[ 20.000011] ================================================================== diff --git a/run-vmtest/tests/check_kernel_splats/cases/kasan/expected-output.txt b/run-vmtest/tests/check_kernel_splats/cases/kasan/expected-output.txt new file mode 100644 index 00000000..0769d2f9 --- /dev/null +++ b/run-vmtest/tests/check_kernel_splats/cases/kasan/expected-output.txt @@ -0,0 +1,18 @@ +::group::kernel_splats - Checking kernel log for splats +Splat denylist: $VMTEST_CONFIGS/SPLAT_DENYLIST +Splat allowlist: $VMTEST_CONFIGS/SPLAT_ALLOWLIST +::endgroup:: +1-[ 0.000000] Linux version 6.5.0 +2-[ 20.000000] ================================================================== +3:[ 20.000001] BUG: KASAN: slab-use-after-free in foo+0x42/0x100 +4-[ 20.000002] Read of size 8 at addr ffff888100001234 by task test_progs/123 +5-[ 20.000003] +6-[ 20.000004] CPU: 0 PID: 123 Comm: test_progs +7-[ 20.000005] Call Trace: +8-[ 20.000006] +9-[ 20.000007] dump_stack_lvl+0x60/0x80 +10-[ 20.000008] print_report+0xc4/0x630 +11-[ 20.000009] kasan_report+0xb6/0xf0 +12-[ 20.000010] +13-[ 20.000011] ================================================================== +::error title=kernel_splats::kernel splat detected, see the dmesg.txt artifact diff --git a/run-vmtest/tests/check_kernel_splats/cases/kasan/expected-status.txt b/run-vmtest/tests/check_kernel_splats/cases/kasan/expected-status.txt new file mode 100644 index 00000000..e78f9fae --- /dev/null +++ b/run-vmtest/tests/check_kernel_splats/cases/kasan/expected-status.txt @@ -0,0 +1 @@ +kernel_splats:1 diff --git a/run-vmtest/tests/check_kernel_splats/cases/multi/dmesg.txt b/run-vmtest/tests/check_kernel_splats/cases/multi/dmesg.txt new file mode 100644 index 00000000..db60c35e --- /dev/null +++ b/run-vmtest/tests/check_kernel_splats/cases/multi/dmesg.txt @@ -0,0 +1,14 @@ +[ 0.100000] Linux version 6.5.0 +[ 10.000000] WARNING: CPU: 0 PID: 100 at kernel/sched/core.c:1 first+0x1/0x2 +[ 10.000001] Call Trace: +[ 10.000002] ? __warn+0x80/0xc0 +[ 10.000003] ---[ end trace ]--- +[ 10.500000] test_progs: PASS one +[ 11.000000] test_progs: running two +[ 11.000001] BUG: KASAN: slab-out-of-bounds in second+0x3/0x4 +[ 11.000002] Read of size 8 at addr ffff888000000000 by task test_progs/9 +[ 11.000003] ---[ end trace ]--- +[ 199.000000] test_progs: running three +[ 200.000000] watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [test_progs:9] +[ 200.000001] watchdog: BUG: soft lockup - CPU#1 stuck for 22s! [test_progs:9] +[ 201.000000] test_progs: PASS diff --git a/run-vmtest/tests/check_kernel_splats/cases/multi/expected-output.txt b/run-vmtest/tests/check_kernel_splats/cases/multi/expected-output.txt new file mode 100644 index 00000000..cffc0bd9 --- /dev/null +++ b/run-vmtest/tests/check_kernel_splats/cases/multi/expected-output.txt @@ -0,0 +1,19 @@ +::group::kernel_splats - Checking kernel log for splats +Splat denylist: $VMTEST_CONFIGS/SPLAT_DENYLIST +Splat allowlist: $VMTEST_CONFIGS/SPLAT_ALLOWLIST +::endgroup:: +1-[ 0.100000] Linux version 6.5.0 +2:[ 10.000000] WARNING: CPU: 0 PID: 100 at kernel/sched/core.c:1 first+0x1/0x2 +3-[ 10.000001] Call Trace: +4-[ 10.000002] ? __warn+0x80/0xc0 +5-[ 10.000003] ---[ end trace ]--- +6-[ 10.500000] test_progs: PASS one +7-[ 11.000000] test_progs: running two +8:[ 11.000001] BUG: KASAN: slab-out-of-bounds in second+0x3/0x4 +9-[ 11.000002] Read of size 8 at addr ffff888000000000 by task test_progs/9 +10-[ 11.000003] ---[ end trace ]--- +11-[ 199.000000] test_progs: running three +12:[ 200.000000] watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [test_progs:9] +13:[ 200.000001] watchdog: BUG: soft lockup - CPU#1 stuck for 22s! [test_progs:9] +14-[ 201.000000] test_progs: PASS +::error title=kernel_splats::kernel splat detected, see the dmesg.txt artifact diff --git a/run-vmtest/tests/check_kernel_splats/cases/multi/expected-status.txt b/run-vmtest/tests/check_kernel_splats/cases/multi/expected-status.txt new file mode 100644 index 00000000..e78f9fae --- /dev/null +++ b/run-vmtest/tests/check_kernel_splats/cases/multi/expected-status.txt @@ -0,0 +1 @@ +kernel_splats:1 diff --git a/run-vmtest/tests/check_kernel_splats/cases/no_denylist/dmesg.txt b/run-vmtest/tests/check_kernel_splats/cases/no_denylist/dmesg.txt new file mode 100644 index 00000000..6e6683da --- /dev/null +++ b/run-vmtest/tests/check_kernel_splats/cases/no_denylist/dmesg.txt @@ -0,0 +1,14 @@ +[ 0.000000] Linux version 6.5.0 +[ 0.001234] CPU: 0 +[ 12.345678] ------------[ cut here ]------------ +[ 12.345679] WARNING: CPU: 0 PID: 100 at kernel/sched/core.c:1234 do_thing+0x10/0x20 +[ 12.345680] Modules linked in: +[ 12.345681] CPU: 0 PID: 100 Comm: test_progs +[ 12.345682] RIP: 0010:do_thing+0x10/0x20 +[ 12.345683] Call Trace: +[ 12.345684] +[ 12.345685] ? __warn+0x80/0xc0 +[ 12.345686] ? report_bug+0xa0/0xc0 +[ 12.345687] +[ 12.345688] ---[ end trace ]--- +[ 13.000000] test_progs: PASS diff --git a/run-vmtest/tests/check_kernel_splats/cases/no_denylist/env b/run-vmtest/tests/check_kernel_splats/cases/no_denylist/env new file mode 100644 index 00000000..7fb12aae --- /dev/null +++ b/run-vmtest/tests/check_kernel_splats/cases/no_denylist/env @@ -0,0 +1 @@ +export SPLAT_DENYLIST_FILE=/nonexistent/SPLAT_DENYLIST diff --git a/run-vmtest/tests/check_kernel_splats/cases/no_denylist/expected-output.txt b/run-vmtest/tests/check_kernel_splats/cases/no_denylist/expected-output.txt new file mode 100644 index 00000000..a548f3e5 --- /dev/null +++ b/run-vmtest/tests/check_kernel_splats/cases/no_denylist/expected-output.txt @@ -0,0 +1,3 @@ +::group::kernel_splats - Checking kernel log for splats +No splat denylist: SPLAT_DENYLIST_FILE=/nonexistent/SPLAT_DENYLIST +::endgroup:: diff --git a/run-vmtest/tests/check_kernel_splats/cases/no_denylist/expected-status.txt b/run-vmtest/tests/check_kernel_splats/cases/no_denylist/expected-status.txt new file mode 100644 index 00000000..e78f9fae --- /dev/null +++ b/run-vmtest/tests/check_kernel_splats/cases/no_denylist/expected-status.txt @@ -0,0 +1 @@ +kernel_splats:1 diff --git a/run-vmtest/tests/check_kernel_splats/cases/warn/dmesg.txt b/run-vmtest/tests/check_kernel_splats/cases/warn/dmesg.txt new file mode 100644 index 00000000..6e6683da --- /dev/null +++ b/run-vmtest/tests/check_kernel_splats/cases/warn/dmesg.txt @@ -0,0 +1,14 @@ +[ 0.000000] Linux version 6.5.0 +[ 0.001234] CPU: 0 +[ 12.345678] ------------[ cut here ]------------ +[ 12.345679] WARNING: CPU: 0 PID: 100 at kernel/sched/core.c:1234 do_thing+0x10/0x20 +[ 12.345680] Modules linked in: +[ 12.345681] CPU: 0 PID: 100 Comm: test_progs +[ 12.345682] RIP: 0010:do_thing+0x10/0x20 +[ 12.345683] Call Trace: +[ 12.345684] +[ 12.345685] ? __warn+0x80/0xc0 +[ 12.345686] ? report_bug+0xa0/0xc0 +[ 12.345687] +[ 12.345688] ---[ end trace ]--- +[ 13.000000] test_progs: PASS diff --git a/run-vmtest/tests/check_kernel_splats/cases/warn/expected-output.txt b/run-vmtest/tests/check_kernel_splats/cases/warn/expected-output.txt new file mode 100644 index 00000000..2b4221ed --- /dev/null +++ b/run-vmtest/tests/check_kernel_splats/cases/warn/expected-output.txt @@ -0,0 +1,19 @@ +::group::kernel_splats - Checking kernel log for splats +Splat denylist: $VMTEST_CONFIGS/SPLAT_DENYLIST +Splat allowlist: $VMTEST_CONFIGS/SPLAT_ALLOWLIST +::endgroup:: +1-[ 0.000000] Linux version 6.5.0 +2-[ 0.001234] CPU: 0 +3-[ 12.345678] ------------[ cut here ]------------ +4:[ 12.345679] WARNING: CPU: 0 PID: 100 at kernel/sched/core.c:1234 do_thing+0x10/0x20 +5-[ 12.345680] Modules linked in: +6-[ 12.345681] CPU: 0 PID: 100 Comm: test_progs +7-[ 12.345682] RIP: 0010:do_thing+0x10/0x20 +8-[ 12.345683] Call Trace: +9-[ 12.345684] +10-[ 12.345685] ? __warn+0x80/0xc0 +11-[ 12.345686] ? report_bug+0xa0/0xc0 +12-[ 12.345687] +13-[ 12.345688] ---[ end trace ]--- +14-[ 13.000000] test_progs: PASS +::error title=kernel_splats::kernel splat detected, see the dmesg.txt artifact diff --git a/run-vmtest/tests/check_kernel_splats/cases/warn/expected-status.txt b/run-vmtest/tests/check_kernel_splats/cases/warn/expected-status.txt new file mode 100644 index 00000000..e78f9fae --- /dev/null +++ b/run-vmtest/tests/check_kernel_splats/cases/warn/expected-status.txt @@ -0,0 +1 @@ +kernel_splats:1 diff --git a/run-vmtest/tests/check_kernel_splats/run-test.sh b/run-vmtest/tests/check_kernel_splats/run-test.sh new file mode 100755 index 00000000..f3a5f798 --- /dev/null +++ b/run-vmtest/tests/check_kernel_splats/run-test.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +set -euo pipefail + +GITHUB_ACTION_PATH=$(realpath ../..) + +# A case runs in a subshell, so that a variable it exports through its `env` +# file cannot leak into the next case. +run_case() ( + local case_dir=$1 tmpdir=$2 + + CASE_DIR=$(realpath "${case_dir}") + export CASE_DIR OUTPUT_DIR="$tmpdir" STATUS_FILE="$tmpdir/exitstatus" + + # Default to the configs this repo carries, which is what a job gets from + # $VMTEST_CONFIGS. A case overrides either one through its `env` file. + CONFIGS=$(realpath ../../../ci/vmtest/configs) + export SPLAT_DENYLIST_FILE="${CONFIGS}/SPLAT_DENYLIST" + export SPLAT_ALLOWLIST_FILE="${CONFIGS}/SPLAT_ALLOWLIST" + if [ -f "${case_dir}/env" ]; then + # shellcheck source=/dev/null + source "${case_dir}/env" + fi + + # Strip colour, and fold the path that differs per checkout, so that + # expected-output.txt stays portable. + "$GITHUB_ACTION_PATH/check-kernel-splats.sh" "${case_dir}/dmesg.txt" 2>&1 \ + | sed -E -e $'s/\x1b\\[[0-9;]*m//g' -e "s|${CASE_DIR}|\$CASE_DIR|g" \ + -e "s|${CONFIGS}|\$VMTEST_CONFIGS|g" > "$tmpdir/output.txt" + + diff -u "${case_dir}/expected-output.txt" "$tmpdir/output.txt" + diff -u "${case_dir}/expected-status.txt" "$tmpdir/exitstatus" +) + +for case_dir in cases/*/; do + case_name=$(basename "${case_dir}") + tmpdir=$(mktemp -d) + run_case "${case_dir}" "${tmpdir}" + echo "case ${case_name} ok" + rm -rf "$tmpdir" +done