diff --git a/README.en.md b/README.en.md index a5a7a1d..ef2ba11 100644 --- a/README.en.md +++ b/README.en.md @@ -64,6 +64,8 @@ Network never changes the headline score. ABS also applies a component floor: be Compare results only when ABS version, profile, thread count, fio size, and `DIRECT` setting match. Only a full local result with `DIRECT=1` is directly comparable. +Scoring formulas, reference constants, applicability limits, and calibration caveats are documented in [docs/scoring.md](docs/scoring.md). + ## Common commands ```bash diff --git a/README.md b/README.md index 4c72fec..d33cf0c 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,8 @@ curl -fsSL https://askclaw.top/abs.sh | bash -s -- --lang en 比较机器时,应使用相同的 ABS 版本、测试模式、线程数、fio 大小和 `DIRECT` 设置。只有 `DIRECT=1` 的完整本地结果可直接比较。 +评分公式、参考常数、适用边界和校准局限见 [docs/scoring.md](docs/scoring.md)。 + ## 常用命令 ```bash diff --git a/abs.sh b/abs.sh index db330c0..07f4b00 100755 --- a/abs.sh +++ b/abs.sh @@ -13,6 +13,9 @@ T="${T:-$VCPU}" SIZE="${SIZE:-auto}" DIR="${DIR:-$PWD/.abs}" CPU_PRIME="${CPU_PRIME:-20000}" +MEM_BLOCK="${MEM_BLOCK:-auto}" +STRICT="${STRICT:-0}" +ABS_CMD_TIMEOUT="${ABS_CMD_TIMEOUT:-900}" DIRECT="${DIRECT:-1}" FIO_ENGINE_ENV="${FIO_ENGINE-}" FIO_ENGINE="${FIO_ENGINE:-libaio}" @@ -81,6 +84,7 @@ ABS v$VERSION — AskClaw VPS 跑分脚本 -z, --size SIZE fio 测试文件大小,如 512M、2G、8G -t, --threads N CPU/内存测试线程数 -n, --no-install 不安装缺失工具 + --strict 测试不完整时以非零状态退出 --lang zh|en 输出语言,默认 zh --net-info 查询 IPv4/IPv6 和外部 IP/ASN --no-net-info 不查询外部 IP/ASN(默认) @@ -118,6 +122,7 @@ Options: -z, --size SIZE fio test file size, e.g. 512M, 2G, 8G; default auto -t, --threads N CPU/memory threads; default detected vCPU ($T) -n, --no-install do not install missing packages + --strict exit non-zero when the score is incomplete --lang zh|en output language; default zh --net-info check IPv4/IPv6 and external IP/ASN --no-net-info skip external IP/ASN lookup (default) @@ -160,6 +165,7 @@ while [ "$#" -gt 0 ]; do shift; [ "$#" -gt 0 ] || die "缺少参数值:-t/--threads" "Missing value for -t/--threads" T="$1" ;; -n|--no-install) INSTALL=0 ;; + --strict) STRICT=1 ;; --lang) shift; [ "$#" -gt 0 ] || die "缺少参数值:--lang" "Missing value for --lang" ABS_LANG="$1" ;; @@ -201,6 +207,8 @@ case "$PROFILE" in esac is_pos_int() { [[ "${1:-}" =~ ^[1-9][0-9]*$ ]]; } + +is_num() { [[ "${1:-}" =~ ^[0-9]+(\.[0-9]+)?$ ]]; } valid_port_range() { local range="${1:-}" low high [[ "$range" =~ ^([0-9]{1,5})(-([0-9]{1,5}))?$ ]] || return 1 @@ -261,6 +269,12 @@ else JOBS=$(( T < 4 ? T : 4 )) fi if ! is_pos_int "$JOBS"; then die "JOBS 无效:$JOBS" "Invalid JOBS: $JOBS"; fi +if [ "$MEM_BLOCK" != "auto" ] && [[ ! "$MEM_BLOCK" =~ ^[0-9]+[KkMmGg]?$ ]]; then + die "MEM_BLOCK 无效:$MEM_BLOCK" "Invalid MEM_BLOCK: $MEM_BLOCK" +fi +if [ "$ABS_CMD_TIMEOUT" != "0" ] && ! is_pos_int "$ABS_CMD_TIMEOUT"; then + die "ABS_CMD_TIMEOUT 无效:$ABS_CMD_TIMEOUT" "Invalid ABS_CMD_TIMEOUT: $ABS_CMD_TIMEOUT" +fi mkdir -p -- "$DIR" if [ -n "$LOGDIR_OVERRIDE" ]; then @@ -272,7 +286,9 @@ fi RESULTS="$LOGDIR/results.tsv" JSON_RESULT="$LOGDIR/result.json" COMPONENTS_FILE="$LOGDIR/components.tsv" +KV="$LOGDIR/metrics.tsv" printf 'Metric\tResult\n' > "$RESULTS" +printf 'Metric\tValue\n' > "$KV" have() { command -v "$1" >/dev/null 2>&1; } @@ -651,6 +667,12 @@ add() { printf '%-48s %s\n' "$1" "$2" fi printf '%s\t%s\n' "$1" "$2" >> "$RESULTS" + # Optional third arg: the raw numeric value behind the display text. + # Scoring reads metrics.tsv first so it never has to re-parse prose + # (the class of bug where 'p95 1.00' scored as throughput). + if [ -n "${3:-}" ] && is_num "$3"; then + printf '%s\t%s\n' "$1" "$3" >> "$KV" + fi } add_note() { @@ -666,11 +688,29 @@ eps() { field 'events per second' "$1"; } p95() { field '95th percentile' "$1"; } memspeed() { awk -F'[()]' '/MiB transferred/ {print $2; exit}' "$1"; } +abs_timeout() { + # Portable timeout (no GNU coreutils dependency, e.g. stock macOS). + local secs="$1"; shift + if [ "$secs" = "0" ]; then + "$@" + return $? + fi + "$@" & + local pid=$! + ( sleep "$secs"; kill -TERM "$pid" 2>/dev/null ) & + local watcher=$! + local rc=0 + wait "$pid" 2>/dev/null || rc=$? + kill "$watcher" 2>/dev/null + wait "$watcher" 2>/dev/null + return "$rc" +} + run_log() { local name="$1" shift LAST_LOG="$LOGDIR/$name.log" - if "$@" >"$LAST_LOG" 2>&1; then + if abs_timeout "$ABS_CMD_TIMEOUT" "$@" >"$LAST_LOG" 2>&1; then return 0 fi return 1 @@ -725,7 +765,7 @@ fio_run() { args+=("$@") - if "$FIO_BIN" "${args[@]}" >"$LAST_JSON" 2>"$err"; then + if abs_timeout "$ABS_CMD_TIMEOUT" "$FIO_BIN" "${args[@]}" >"$LAST_JSON" 2>"$err"; then return 0 fi return 1 @@ -753,9 +793,12 @@ try: with open(path, 'r', encoding='utf-8', errors='replace') as f: data = json.load(f) except Exception: - print('0.00') - raise SystemExit + print('FAILED') + raise SystemExit(1) jobs = data.get('jobs') or [] +if not jobs: + print('FAILED') + raise SystemExit(1) def nums(path): out = [] @@ -801,12 +844,23 @@ elif metric == 'read_iops': val = sum_path(['read', 'iops']) elif metric == 'write_iops': val = sum_path(['write', 'iops']) +# NOTE: empty sums legitimately mean 'measured zero' (e.g. a pure-read job's +# write side), so keep 0.00 here; only unreadable/jobless JSON is FAILED. elif metric == 'read_p95_ms': val = max_p95('read') + if val == 0.0: + print('FAILED') + raise SystemExit(1) elif metric == 'write_p95_ms': val = max_p95('write') + if val == 0.0: + print('FAILED') + raise SystemExit(1) elif metric == 'mixed_p95_ms': val = max(max_p95('read'), max_p95('write')) + if val == 0.0: + print('FAILED') + raise SystemExit(1) elif metric == 'sync_p95_ms': val = sync_p95() if val is None: @@ -819,7 +873,7 @@ PY } abs_local_score() { - ABS_COMPONENTS_FILE="$COMPONENTS_FILE" python3 - "$RESULTS" "$T" "$DIRECT" <<'PY' + ABS_COMPONENTS_FILE="$COMPONENTS_FILE" python3 - "$RESULTS" "$T" "$DIRECT" "$KV" <<'PY' import csv, math, os, re, sys path = sys.argv[1] threads = float(sys.argv[2]) if len(sys.argv) > 2 else 1.0 @@ -834,7 +888,24 @@ except Exception: print('n/a') raise SystemExit +# Structured metrics captured at measurement time (metrics.tsv). These are +# authoritative for scoring; the display-text rows are only a fallback for +# callers that feed results.tsv directly. +kv = {} +try: + with open(sys.argv[4], newline='', encoding='utf-8', errors='replace') as f: + for row in csv.reader(f, delimiter='\t'): + if len(row) >= 2 and row[0] != 'Metric': + try: + kv.setdefault(row[0], []).append(float(row[1])) + except ValueError: + pass +except Exception: + pass + def first_num(text): + if isinstance(text, (int, float)): + return float(text) text = text or '' low = text.lower() if any(word in low for word in ('failed', 'skipped', 'invalid', 'not enough', 'not installed')): @@ -843,6 +914,9 @@ def first_num(text): return float(m.group(0)) if m else None def metric(prefix): + for k, vals in kv.items(): + if k.startswith(prefix): + return vals[0] for k, v in rows.items(): if k.startswith(prefix): return v @@ -911,7 +985,7 @@ network_score() { printf 'N/A - skipped (network excluded; local score is standalone)' return 0 fi - NETWORK_PROFILE_IN="$NETWORK_PROFILE" IPERF_SERVER_IN="$IPERF_SERVER" python3 - "$RESULTS" <<'PY' + NETWORK_PROFILE_IN="$NETWORK_PROFILE" IPERF_SERVER_IN="$IPERF_SERVER" python3 - "$RESULTS" "$KV" <<'PY' import csv, math, os, re, statistics, sys rows = {} try: @@ -923,7 +997,21 @@ except Exception: print('n/a') raise SystemExit +kv = {} +try: + with open(sys.argv[2], newline='', encoding='utf-8', errors='replace') as f: + for row in csv.reader(f, delimiter='\t'): + if len(row) >= 2 and row[0] != 'Metric': + try: + kv.setdefault(row[0], []).append(float(row[1])) + except ValueError: + pass +except Exception: + pass + def parse_num(v): + if isinstance(v, (int, float)): + return float(v) low = (v or '').lower() if any(w in low for w in ('failed', 'skipped', 'not installed', 'missing')): return None @@ -931,6 +1019,9 @@ def parse_num(v): return float(m.group(0)) if m else None def first_num(prefix): + for k, vals in kv.items(): + if k.startswith(prefix): + return vals[0] for k, v in rows.items(): if k.startswith(prefix): return parse_num(v) @@ -938,6 +1029,11 @@ def first_num(prefix): def nums(prefix): out = [] + for k, vals in kv.items(): + if k.startswith(prefix): + out.extend(vals) + if out: + return out for k, v in rows.items(): if k.startswith(prefix): n = parse_num(v) @@ -1017,7 +1113,7 @@ PY } abs_verdict() { - python3 - "$SCORE_TEXT" "$RESULTS" "$VM_TYPE" "$T" <<'PY' + python3 - "$SCORE_TEXT" "$RESULTS" "$VM_TYPE" "$T" "$KV" <<'PY' import csv, math, re, sys score_text, results, vm_type = sys.argv[1], sys.argv[2], (sys.argv[3] or '').lower() threads = float(sys.argv[4]) if len(sys.argv) > 4 else 1.0 @@ -1034,7 +1130,22 @@ try: except Exception: pass +kv = {} +try: + with open(sys.argv[5], newline='', encoding='utf-8', errors='replace') as f: + for row in csv.reader(f, delimiter='\t'): + if len(row) >= 2 and row[0] != 'Metric': + try: + kv.setdefault(row[0], []).append(float(row[1])) + except ValueError: + pass +except Exception: + pass + def num(prefix): + for k, vals in kv.items(): + if k.startswith(prefix): + return vals[0] for k, v in rows.items(): if k.startswith(prefix): mm = re.search(r'[0-9]+(?:\.[0-9]+)?', v) @@ -1190,21 +1301,24 @@ network_sanity() { local lat down up if lat="$(curl -fL -sS --max-time 8 -o /dev/null -w '%{time_starttransfer}' 'https://speed.cloudflare.com/__down?bytes=0' 2>"$LOGDIR/net-ttfb.err")" && awk -v x="$lat" 'BEGIN{exit !(x+0>0)}'; then - add "Network Cloudflare TTFB" "$(awk -v s="$lat" 'BEGIN{printf "%.2f ms", s*1000}')" + ttfb_ms="$(awk -v s="$lat" 'BEGIN{printf "%.2f", s*1000}')" + add "Network Cloudflare TTFB" "$ttfb_ms ms" "$ttfb_ms" else add "Network Cloudflare TTFB" "FAILED; see $LOGDIR/net-ttfb.err" fi if cloudflare_download; then down="$NETWORK_VALUE" - add "Network Cloudflare download" "$(awk -v b="$down" 'BEGIN{printf "%.2f Mbps", b*8/1000000}')" + down_mbps="$(awk -v b="$down" 'BEGIN{printf "%.2f", b*8/1000000}')" + add "Network Cloudflare download" "$down_mbps Mbps" "$down_mbps" else add "Network Cloudflare download" "$(network_failure_result "$LOGDIR/net-down.err")" fi if cloudflare_upload; then up="$NETWORK_VALUE" - add "Network Cloudflare upload" "$(awk -v b="$up" 'BEGIN{printf "%.2f Mbps", b*8/1000000}')" + up_mbps="$(awk -v b="$up" 'BEGIN{printf "%.2f", b*8/1000000}')" + add "Network Cloudflare upload" "$up_mbps Mbps" "$up_mbps" else add "Network Cloudflare upload" "$(network_failure_result "$LOGDIR/net-up.err")" fi @@ -1223,11 +1337,8 @@ pick_port() { } run_iperf_cmd() { - if have timeout; then - timeout 18 "$@" - else - "$@" - fi + # Portable watchdog (GNU timeout is absent on stock macOS). + abs_timeout 18 "$@" } parse_iperf_bps() { @@ -1257,7 +1368,8 @@ iperf_pair() { if run_iperf_cmd iperf3 -J -t "$IPERF_TIME" -P "$IPERF_PARALLEL" -c "$host" -p "$port" >"$json" 2>"$json.err"; then bps="$(parse_iperf_bps "$json" send)" if awk -v b="$bps" 'BEGIN{exit !(b+0>0)}'; then - add "Network iperf3 send $label" "$(awk -v b="$bps" 'BEGIN{printf "%.2f Mbps", b/1000000}')" + send_mbps="$(awk -v b="$bps" 'BEGIN{printf "%.2f", b/1000000}')" + add "Network iperf3 send $label" "$send_mbps Mbps" "$send_mbps" ok=1 break fi @@ -1272,7 +1384,8 @@ iperf_pair() { if run_iperf_cmd iperf3 -J -R -t "$IPERF_TIME" -P "$IPERF_PARALLEL" -c "$host" -p "$port" >"$json" 2>"$json.err"; then bps="$(parse_iperf_bps "$json" recv)" if awk -v b="$bps" 'BEGIN{exit !(b+0>0)}'; then - add "Network iperf3 recv $label" "$(awk -v b="$bps" 'BEGIN{printf "%.2f Mbps", b/1000000}')" + recv_mbps="$(awk -v b="$bps" 'BEGIN{printf "%.2f", b/1000000}')" + add "Network iperf3 recv $label" "$recv_mbps Mbps" "$recv_mbps" ok=1 break fi @@ -1330,17 +1443,17 @@ dd_fallback() { dd if=/dev/null of=/dev/null bs=1 count=0 conv=fsync >/dev/null 2>&1 || dd_sync_arg="" fi # shellcheck disable=SC2086 - if dd if=/dev/zero of="$dd_file" bs=64M count=4 oflag=direct $dd_sync_arg >"$write_log" 2>&1; then + if abs_timeout "$ABS_CMD_TIMEOUT" dd if=/dev/zero of="$dd_file" bs=64M count=4 oflag=direct $dd_sync_arg >"$write_log" 2>&1; then add "Disk fallback dd write" "$(awk 'END{print}' "$write_log") (not scored)" - elif dd if=/dev/zero of="$dd_file" bs=64M count=4 $dd_sync_arg >"$write_log" 2>&1; then + elif abs_timeout "$ABS_CMD_TIMEOUT" dd if=/dev/zero of="$dd_file" bs=64M count=4 $dd_sync_arg >"$write_log" 2>&1; then add "Disk fallback dd write" "$(awk 'END{print}' "$write_log") (buffered; not scored)" else add "Disk fallback dd write" "FAILED; see $write_log" fi if [ -f "$dd_file" ]; then - if dd if="$dd_file" of=/dev/null bs=64M iflag=direct >"$read_log" 2>&1; then + if abs_timeout "$ABS_CMD_TIMEOUT" dd if="$dd_file" of=/dev/null bs=64M iflag=direct >"$read_log" 2>&1; then add "Disk fallback dd read" "$(awk 'END{print}' "$read_log") (not scored)" - elif dd if="$dd_file" of=/dev/null bs=64M >"$read_log" 2>&1; then + elif abs_timeout "$ABS_CMD_TIMEOUT" dd if="$dd_file" of=/dev/null bs=64M >"$read_log" 2>&1; then add "Disk fallback dd read" "$(awk 'END{print}' "$read_log") (buffered; not scored)" else add "Disk fallback dd read" "FAILED; see $read_log" @@ -1363,6 +1476,9 @@ json_report() { ABS_NET_INFO="$NET_INFO" ABS_NETWORK="$NETWORK" ABS_NETWORK_PROFILE="$NETWORK_PROFILE" \ ABS_MISSING_TOOLS="$MISSING_AFTER_INSTALL" ABS_DIRECT="$DIRECT" ABS_FIO_ENGINE="$FIO_ENGINE" \ ABS_JOBS="$JOBS" ABS_DEPTH="$DEPTH" ABS_IPERF_SERVER="$IPERF_SERVER" ABS_IPERF_TIME="$IPERF_TIME" ABS_IPERF_PARALLEL="$IPERF_PARALLEL" \ + ABS_CPU_PRIME="$CPU_PRIME" ABS_MEM_BLOCK="$MEM_BLOCK" ABS_STRICT="$STRICT" \ + ABS_SYSBENCH_VER="$SYSBENCH_VER" ABS_FIO_VER="$FIO_VER" ABS_PYTHON_VER="$PYTHON_VER" \ + ABS_SCRIPT_SHA256="$SCRIPT_SHA256" ABS_SCORE_MODEL="abs-local-v1" \ ABS_SCORE="$SCORE_TEXT" ABS_LOCAL_SCORE="$LOCAL_SCORE_TEXT" ABS_NETWORK_SCORE="$NETWORK_SCORE_TEXT" ABS_VERDICT="$VERDICT_TEXT" \ python3 - "$RESULTS" <<'PY' import csv, json, os, re, shutil, sys @@ -1424,6 +1540,14 @@ out = { 'missing_tools': env('ABS_MISSING_TOOLS'), 'direct': env('ABS_DIRECT'), 'fio_engine': env('ABS_FIO_ENGINE'), 'pressure_jobs': env_int('ABS_JOBS'), 'pressure_depth': env_int('ABS_DEPTH'), 'iperf_server': env('ABS_IPERF_SERVER'), 'iperf_time': env_int('ABS_IPERF_TIME'), 'iperf_parallel': env_int('ABS_IPERF_PARALLEL'), + 'cpu_prime': env_int('ABS_CPU_PRIME'), 'mem_block': env('ABS_MEM_BLOCK'), 'strict': env('ABS_STRICT') == '1', + }, + 'provenance': { + 'score_model': env('ABS_SCORE_MODEL'), + 'script_sha256': env('ABS_SCRIPT_SHA256'), + 'tool_versions': { + 'sysbench': env('ABS_SYSBENCH_VER'), 'fio': env('ABS_FIO_VER'), 'python3': env('ABS_PYTHON_VER'), + }, }, 'score': { 'text': score_text, @@ -1506,6 +1630,16 @@ if [ "${SWAP_KIB:-0}" = "0" ]; then fi RAM_TEXT="$(human_kib "$RAM_KIB")" SWAP_TEXT="$(human_kib "$SWAP_KIB")" +if [ "$MEM_BLOCK" = "auto" ]; then + # sysbench memory with --memory-scope=local gives each thread its own + # working set sized to MEM_BLOCK, so the test leaves last-level cache + # instead of cycling one shared 1 MiB buffer. Adaptive default: + # min(256 MiB, RAM/4/threads), floor 32 MiB. + mem_block_mib=$(( ${RAM_KIB:-0} / 1024 / T / 4 )) + [ "$mem_block_mib" -gt 256 ] && mem_block_mib=256 + [ "$mem_block_mib" -lt 32 ] && mem_block_mib=32 + MEM_BLOCK="${mem_block_mib}M" +fi DISK_FREE_KB="$(df -Pk "$DIR" 2>/dev/null | awk 'NR==2 {print $4+0}' || echo 0)" DISK_TOTAL_KB="$(df -Pk "$DIR" 2>/dev/null | awk 'NR==2 {print $2+0}' || echo 0)" DISK_TEXT="$(human_kib "$DISK_FREE_KB") free / $(human_kib "$DISK_TOTAL_KB") total" @@ -1628,25 +1762,45 @@ fi if have sysbench; then if run_log cpu-1t sysbench cpu --threads=1 --time="$D" --events=0 --percentile=95 --cpu-max-prime="$CPU_PRIME" run; then - add "CPU single thread" "$(eps "$LAST_LOG") events/s, p95 $(p95 "$LAST_LOG") ms" + cpu_1t_eps="$(eps "$LAST_LOG")" + if is_num "$cpu_1t_eps"; then + add "CPU single thread" "$cpu_1t_eps events/s, p95 $(p95 "$LAST_LOG") ms" "$cpu_1t_eps" + else + add "CPU single thread" "FAILED; no events-per-second field; see $LAST_LOG" + fi else add "CPU single thread" "FAILED; see $LAST_LOG" fi if run_log cpu-all sysbench cpu --threads="$T" --time="$D" --events=0 --percentile=95 --cpu-max-prime="$CPU_PRIME" run; then - add "CPU all threads ($T)" "$(eps "$LAST_LOG") events/s, p95 $(p95 "$LAST_LOG") ms" + cpu_all_eps="$(eps "$LAST_LOG")" + if is_num "$cpu_all_eps"; then + add "CPU all threads ($T)" "$cpu_all_eps events/s, p95 $(p95 "$LAST_LOG") ms" "$cpu_all_eps" + else + add "CPU all threads ($T)" "FAILED; no events-per-second field; see $LAST_LOG" + fi else add "CPU all threads ($T)" "FAILED; see $LAST_LOG" fi - if run_log mem-read sysbench memory --threads="$T" --time="$D" --events=0 --percentile=95 --memory-block-size=1M --memory-total-size=100T --memory-oper=read --memory-access-mode=seq run; then - add "Memory read ($T threads)" "$(memspeed "$LAST_LOG")" + if run_log mem-read sysbench memory --threads="$T" --time="$D" --events=0 --percentile=95 --memory-scope=local --memory-block-size="$MEM_BLOCK" --memory-total-size=100T --memory-oper=read --memory-access-mode=seq run; then + mem_read="$(memspeed "$LAST_LOG")" + if [ -n "$mem_read" ]; then + add "Memory read ($T threads)" "$mem_read" "${mem_read%% *}" + else + add "Memory read" "FAILED; no throughput field; see $LAST_LOG" + fi else add "Memory read" "FAILED; see $LAST_LOG" fi - if run_log mem-write sysbench memory --threads="$T" --time="$D" --events=0 --percentile=95 --memory-block-size=1M --memory-total-size=100T --memory-oper=write --memory-access-mode=seq run; then - add "Memory write ($T threads)" "$(memspeed "$LAST_LOG")" + if run_log mem-write sysbench memory --threads="$T" --time="$D" --events=0 --percentile=95 --memory-scope=local --memory-block-size="$MEM_BLOCK" --memory-total-size=100T --memory-oper=write --memory-access-mode=seq run; then + mem_write="$(memspeed "$LAST_LOG")" + if [ -n "$mem_write" ]; then + add "Memory write ($T threads)" "$mem_write" "${mem_write%% *}" + else + add "Memory write" "FAILED; no throughput field; see $LAST_LOG" + fi else add "Memory write" "FAILED; see $LAST_LOG" fi @@ -1684,13 +1838,15 @@ if [ -n "$FIO_BIN" ] && have python3; then fi if fio_run fio-rand-read-q1 randread 4k 1 1 --randrepeat=0; then - add "Disk random read 4K QD1" "$(fio_metric read_iops) IOPS, $(fio_metric read_mib) MiB/s, p95 $(fio_metric read_p95_ms) ms" + qd1_read_iops="$(fio_metric read_iops)" + add "Disk random read 4K QD1" "$qd1_read_iops IOPS, $(fio_metric read_mib) MiB/s, p95 $(fio_metric read_p95_ms) ms" "$qd1_read_iops" else add "Disk random read 4K QD1" "FAILED; see $LOGDIR/fio-rand-read-q1.err" fi if fio_run fio-rand-write-q1 randwrite 4k 1 1 --randrepeat=0; then - add "Disk random write 4K QD1" "$(fio_metric write_iops) IOPS, $(fio_metric write_mib) MiB/s, p95 $(fio_metric write_p95_ms) ms" + qd1_write_iops="$(fio_metric write_iops)" + add "Disk random write 4K QD1" "$qd1_write_iops IOPS, $(fio_metric write_mib) MiB/s, p95 $(fio_metric read_p95_ms) ms" "$qd1_write_iops" else add "Disk random write 4K QD1" "FAILED; see $LOGDIR/fio-rand-write-q1.err" fi @@ -1714,7 +1870,8 @@ if [ -n "$FIO_BIN" ] && have python3; then fi if fio_run fio-fsync-write randwrite 4k 1 1 --fsync=1 --randrepeat=0; then - add "Disk durable write 4K fsync" "$(fio_metric write_iops) writes/s, write p95 $(fio_metric write_p95_ms) ms, sync p95 $(fio_metric sync_p95_ms) ms" + fsync_iops="$(fio_metric write_iops)" + add "Disk durable write 4K fsync" "$fsync_iops writes/s, write p95 $(fio_metric write_p95_ms) ms, sync p95 $(fio_metric sync_p95_ms) ms" "$fsync_iops" else add "Disk durable write 4K fsync" "FAILED; see $LOGDIR/fio-fsync-write.err" fi @@ -1743,6 +1900,9 @@ if have python3; then add "Network component" "$NETWORK_SCORE_TEXT" VERDICT_TEXT="$(abs_verdict)" add "ABS VERDICT" "$VERDICT_TEXT" + if [ "$STRICT" = "1" ] && [ "${VERDICT_TEXT#INCOMPLETE}" != "$VERDICT_TEXT" ]; then + FINAL_STATUS=1 + fi else SCORE_TEXT="n/a (python3 missing)" LOCAL_SCORE_TEXT="n/a (python3 missing)" @@ -1758,6 +1918,11 @@ add_note "Privacy note" "No result upload. Default network sanity uses Cloudflar ELAPSED_SECONDS=$(( $(date +%s) - TIME_START_EPOCH )) if have python3; then + SYSBENCH_VER="$(sysbench --version 2>/dev/null | awk 'NR==1{print $2}')" + if [ -n "$FIO_BIN" ]; then FIO_VER="$("$FIO_BIN" --version 2>/dev/null | sed 's/^fio-//')"; else FIO_VER=""; fi + PYTHON_VER="$(python3 --version 2>/dev/null | awk '{print $2}')" + SCRIPT_SHA256="$([ -f "$0" ] && { sha256sum "$0" 2>/dev/null || shasum -a 256 "$0" 2>/dev/null; } | awk 'NR==1{print $1}')" + [ -z "${SCRIPT_SHA256:-}" ] && SCRIPT_SHA256="n/a" if JSON_WRITTEN="$(json_report)"; then add_note "JSON result" "$JSON_WRITTEN" [ -n "$JSON_FILE" ] && add_note "JSON copy" "$JSON_FILE" diff --git a/docs/scoring.md b/docs/scoring.md new file mode 100644 index 0000000..a4b5245 --- /dev/null +++ b/docs/scoring.md @@ -0,0 +1,97 @@ +# ABS scoring model (`abs-local-v1`) + +This document records how ABS turns raw measurements into a local score and +verdict, where each constant came from, and the limits of that calibration. +It exists so the score can be audited, compared across script versions, and +re-derived from `result.json`. + +## What the score is — and is not + +The composite score is a **heuristic for spotting weak VPS plans**, not a +statistically calibrated benchmark. The reference constants and weights were +chosen by judgement so that a typical mid-range modern VPS lands near +component score ~1000 and total ~1200+. They have **not** been fit to a +labelled dataset of "good" and "bad" machines, and ABS publishes no error +analysis for them. Treat the absolute verdicts as informed defaults, and +prefer relative comparisons between machines run under identical settings. + +## Formulas + +Each component maps its measurements to 0–3000 via +`1000 * sqrt(value / reference)` (geometric mean when two inputs), then the +composite is the weighted sum. `result.json` reports the same numbers. + +| Component | Weight | Formula (references in units shown) | +|---|---|---| +| CPU | 40% | `1000 * sqrt((single_eps/400) * (per_thread_eps/400))` — sysbench `cpu`, prime `CPU_PRIME` (default 20000), events/s | +| Memory | 15% | `1000 * sqrt((read/30000 MiB/s) * (write/20000 MiB/s))` — sysbench `memory`, sequential | +| Disk | 30% | `1000 * sqrt((qd1_read/10000 IOPS) * (qd1_write/5000 IOPS))` — fio 4K random, queue depth 1, single job | +| fsync | 15% | `1000 * sqrt(fsync_writes/2000 per s)` — fio 4K `randwrite --fsync=1` | + +Component scores clamp at 3000 (`min(3000, ...)`). + +## Verdicts + +| Total score | Verdict | +|---|---| +| ≥ 1200 | KEEP — practical VPS profile looks acceptable | +| ≥ 800 | MAYBE — usable, but has notable weaknesses | +| < 800 | AVOID — weak practical VPS performance | +| any core test missing | INCOMPLETE — not comparable | + +Two overrides applied after the weighted sum, so a strong average cannot +hide a broken machine: + +- **Component floor**: weakest component < 500 caps the verdict at MAYBE; + < 250 forces AVOID. +- **FULL ≠ fast**: FULL only means every core test produced data. + +## Where the constants came from + +Order-of-magnitude anchors for "a healthy modern VPS", not measurements +from a reference fleet: + +- `400 events/s/thread` (sysbench cpu, prime 20000): typical modern dedicated + vCPU. Shared/burstable vCPUs often land far below. +- `30000/20000 MiB/s` (memory read/write): DRAM-class sequential bandwidth; + reachable by current server CPUs, far above cache-starved or throttled + guests. Memory runs with `--memory-scope=local` and a per-thread working + set (`MEM_BLOCK`, default `RAM/threads/4` clamped to 32–256 MiB) so the + metric reflects DRAM rather than a shared hot buffer. +- `10000/5000 IOPS` (4K QD1 read/write): mid-range NVMe; SATA and + network-backed storage are typically well under. +- `2000 writes/s` (fsync): durable-write rate where storage genuinely + persists data vs. where flushes are absorbed by volatile cache. +- `800/1200` verdict thresholds and the `0.40/0.15/0.30/0.15` weights: chosen + to weight CPU and 4K QD1 disk (what makes a VPS *feel* slow) above memory + bandwidth and fsync rate; not empirically optimized. + +## Comparability contract + +Two scores are comparable only when run with the same: script version or +`score_model`, test mode (`--quick`/`--full`), thread count, `fio` size, +`DIRECT` setting, `CPU_PRIME`, and `MEM_BLOCK`. `result.json` records all of +these under `mode` plus a `provenance` block (script SHA-256, +sysbench/fio/python3 versions, `score_model` id) — check them before +comparing JSON files. Only `DIRECT=1` FULL local results are directly +comparable; network results never enter the score. + +## Known limits + +- No calibration dataset: the constants are judgement calls. If you re-run + the same machine the score is stable, but cross-version comparisons need + matching `score_model`. +- Burstable/shared vCPUs and steal time can depress CPU far below the 400/s + anchor; that is a *feature* (it is what the user experiences) but means + scores fluctuate with host load. +- OpenVZ/LXC and some virtualized storage inflate write/fsync numbers + through host caches — ABS emits a `why` hint for these cases instead of + silently trusting the numbers. +- The network check is a single-CDN probe (or optional iperf3 regions) and + is informational only. + +## Changing the model + +Bump `score_model` in `result.json` whenever a formula, reference constant, +weight, or measured-input definition changes, and note it here. Constants +live inline in the scoring Python inside `abs.sh` (search for `abs_local_score`). diff --git a/tests/test_abs.py b/tests/test_abs.py index c87adb3..32cf2e5 100644 --- a/tests/test_abs.py +++ b/tests/test_abs.py @@ -269,7 +269,7 @@ def test_score_uses_local_only_and_network_is_reference(self) -> None: def test_verdict_reasons_match_verdict_code(self) -> None: match = re.search( - r"abs_verdict\(\) \{\n python3 - \"\$SCORE_TEXT\" \"\$RESULTS\" \"\$VM_TYPE\" \"\$T\" <<'PY'\n(.*?)\nPY\n\}", + r"abs_verdict\(\) \{\n python3 - \"\$SCORE_TEXT\" \"\$RESULTS\" \"\$VM_TYPE\" \"\$T\" \"\$KV\" <<'PY'\n(.*?)\nPY\n\}", self.source, re.S, ) @@ -335,7 +335,7 @@ def test_verdict_reasons_match_verdict_code(self) -> None: def test_component_floor_threshold_boundaries(self) -> None: match = re.search( - r"abs_verdict\(\) \{\n python3 - \"\$SCORE_TEXT\" \"\$RESULTS\" \"\$VM_TYPE\" \"\$T\" <<'PY'\n(.*?)\nPY\n\}", + r"abs_verdict\(\) \{\n python3 - \"\$SCORE_TEXT\" \"\$RESULTS\" \"\$VM_TYPE\" \"\$T\" \"\$KV\" <<'PY'\n(.*?)\nPY\n\}", self.source, re.S, ) @@ -591,6 +591,160 @@ def test_buffered_fio_cannot_produce_comparable_score(self) -> None: self.assertEqual(result["score"]["missing_components"], ["disk-buffered", "fsync-buffered"]) self.assertEqual(result["verdict"]["code"], "INCOMPLETE") + def test_missing_cpu_eps_cannot_score_from_p95_label(self) -> None: + with tempfile.TemporaryDirectory() as td: + root = Path(td) + bin_dir = root / "bin" + data_dir = root / "data" + log_dir = root / "logs" + bin_dir.mkdir() + data_dir.mkdir() + write_fake_tools(bin_dir) + # sysbench exits 0 but emits only a percentile line — no EPS field. + sysbench = bin_dir / "sysbench" + sysbench.write_text( + "#!/usr/bin/env bash\n" + "if [ \"${1:-}\" = \"--version\" ]; then echo 'sysbench 1.0.20'; exit 0; fi\n" + "case \" ${*} \" in\n" + " *\" memory \"*) echo '102400.00 MiB transferred (30000.00 MiB/sec)' ;;\n" + " *) echo '95th percentile: 1.00' ;;\n" + "esac\n" + ) + sysbench.chmod(0o755) + result_json = root / "result.json" + env = os.environ.copy() + env.update( + { + "PATH": f"{bin_dir}:{env['PATH']}", + "DIR": str(data_dir), + "LOGDIR": str(log_dir), + "INSTALL": "0", + "ABS_LANG": "en", + } + ) + proc = subprocess.run( + [str(SCRIPT), "--quick", "-n", "--no-network", "--json-file", str(result_json)], + env=env, + text=True, + capture_output=True, + timeout=60, + ) + self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) + result = json.loads(result_json.read_text(encoding="utf-8")) + self.assertEqual(result["score"]["status"], "partial") + self.assertFalse(result["score"]["comparable"]) + self.assertIn("cpu", result["score"]["missing_components"]) + metrics = {row["metric"]: row["result"] for row in result["results"]} + self.assertTrue(metrics["CPU single thread"].startswith("FAILED"), metrics["CPU single thread"]) + all_threads = next(v for k, v in metrics.items() if k.startswith("CPU all threads")) + self.assertTrue(all_threads.startswith("FAILED"), all_threads) + + def test_memory_tests_use_per_thread_scope(self) -> None: + self.assertIn("--memory-scope=local", self.source) + self.assertNotIn("--memory-block-size=1M", self.source) + self.assertIn('--memory-block-size="$MEM_BLOCK"', self.source) + + def test_scoring_prefers_metrics_tsv_over_display_text(self) -> None: + fn = extract_between(self.source, "abs_local_score() {", "\n}\n\nnetwork_score()") + "\n}\n" + with tempfile.TemporaryDirectory() as td: + root = Path(td) + results = root / "results.tsv" + kv = root / "metrics.tsv" + comp = root / "components.tsv" + # Display text claims inflated numbers; metrics.tsv holds the real + # measured values. The scorer must use metrics.tsv. + results.write_text( + "Metric\tResult\n" + "CPU single thread\t99999 events/s, p95 0.01 ms\n" + "CPU all threads (8)\t99999 events/s, p95 0.01 ms\n" + "Memory read (8 threads)\t99999 MiB/sec\n" + "Memory write (8 threads)\t99999 MiB/sec\n" + "Disk random read 4K QD1\t99999 IOPS\n" + "Disk random write 4K QD1\t99999 IOPS\n" + "Disk durable write 4K fsync\t99999 writes/s\n", + encoding="utf-8", + ) + kv.write_text( + "Metric\tValue\n" + "CPU single thread\t400\n" + "CPU all threads (8)\t3200\n" + "Memory read (8 threads)\t30000\n" + "Memory write (8 threads)\t20000\n" + "Disk random read 4K QD1\t10000\n" + "Disk random write 4K QD1\t5000\n" + "Disk durable write 4K fsync\t2000\n", + encoding="utf-8", + ) + env = os.environ.copy() + env.update( + { + "RESULTS": str(results), + "KV": str(kv), + "COMPONENTS_FILE": str(comp), + "T": "8", + "DIRECT": "1", + } + ) + proc = subprocess.run( + ["bash", "-c", fn + "\nabs_local_score"], + env=env, + text=True, + capture_output=True, + check=True, + ) + self.assertTrue(proc.stdout.startswith("FULL 1000"), proc.stdout) + + def test_malformed_fio_json_is_failed_not_zero(self) -> None: + metric = extract_between(self.source, "fio_metric() {", "\n}\n\nabs_local_score()") + "\n}\n" + with tempfile.NamedTemporaryFile("w", suffix=".json") as bad: + bad.write("{not json") + bad.flush() + proc = subprocess.run( + ["bash", "-c", metric + '\nfio_metric read_iops "$1"', "bash", bad.name], + text=True, + capture_output=True, + ) + self.assertEqual(proc.stdout.strip(), "FAILED") + self.assertNotEqual(proc.returncode, 0) + + def test_strict_mode_exits_nonzero_on_incomplete(self) -> None: + with tempfile.TemporaryDirectory() as td: + root = Path(td) + bin_dir = root / "bin" + data_dir = root / "data" + log_dir = root / "logs" + bin_dir.mkdir() + data_dir.mkdir() + write_fake_tools(bin_dir) + sysbench = bin_dir / "sysbench" + sysbench.write_text( + "#!/usr/bin/env bash\n" + "if [ \"${1:-}\" = \"--version\" ]; then echo 'sysbench 1.0.20'; exit 0; fi\n" + "exit 1\n" + ) + sysbench.chmod(0o755) + env = os.environ.copy() + env.update( + { + "PATH": f"{bin_dir}:{env['PATH']}", + "DIR": str(data_dir), + "LOGDIR": str(log_dir), + "INSTALL": "0", + "ABS_LANG": "en", + } + ) + lax = subprocess.run( + [str(SCRIPT), "--quick", "-n", "--no-network"], + env=env, text=True, capture_output=True, timeout=60, + ) + self.assertEqual(lax.returncode, 0) + strict = subprocess.run( + [str(SCRIPT), "--quick", "-n", "--no-network", "--strict"], + env=env, text=True, capture_output=True, timeout=60, + ) + self.assertEqual(strict.returncode, 1) + self.assertIn("INCOMPLETE", strict.stdout) + def test_json_copy_failure_returns_nonzero_without_false_success(self) -> None: with tempfile.TemporaryDirectory() as td: root = Path(td)