From 70e892d45c17595936e3719ba0e3befca7604051 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Tue, 1 Sep 2026 17:00:59 -0400 Subject: [PATCH 01/12] ci: check feature combinations with cargo-hack on linux arm64 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a job that runs `cargo hack --feature-powerset --depth 2 check` over `livekit`, `livekit-api`, and the ten workspace crates beneath them: 209 feature combinations in total. Feature bugs do not show up in the default-feature build — an optional `dep:` used unconditionally, or a `?/` forward that silently no-ops, only breaks for callers who pick a particular feature set. Nothing else in CI exercises those paths today. All 209 combinations currently pass, so this starts green. Depth 2 is a deliberate ceiling. It covers pairwise feature interactions, where nearly all such bugs live, and keeps the job affordable: `livekit` alone would go from 67 combinations to 232 at depth 3. Runs on one platform (linux arm64) because feature resolution does not vary by target — the point is to exercise the feature graph, not the target matrix. Scheduled nightly rather than per-PR, plus on Cargo.toml and Cargo.lock changes, which are what can alter feature resolution. `target/` is deliberately not cached so the reported time reflects a cold run, and the job prints the powerset time next to a plain default-feature `cargo check` for comparison. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/feature-combinations.yml | 180 +++++++++++++++++++++ 1 file changed, 180 insertions(+) create mode 100644 .github/workflows/feature-combinations.yml diff --git a/.github/workflows/feature-combinations.yml b/.github/workflows/feature-combinations.yml new file mode 100644 index 000000000..9bc2c94c7 --- /dev/null +++ b/.github/workflows/feature-combinations.yml @@ -0,0 +1,180 @@ +# Copyright 2026 LiveKit, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Checks that every combination of up to 2 simultaneous cargo features builds, +# across the client SDK and everything below it. Feature bugs (a `dep:` that is +# only pulled in by one feature but used unconditionally, a `?/` that silently +# no-ops) do not show up in the default-feature build, so nothing else in CI +# catches them. +# +# `--depth 2` is a deliberate ceiling: `livekit` alone goes from 67 combinations +# at depth 2 to 232 at depth 3. Depth 2 catches pairwise interactions, which is +# where nearly all feature bugs live. +# +# Runs on one platform only (linux arm64). Feature resolution is +# platform-independent; the point is to exercise the feature graph, not the +# target matrix. +name: Feature Combinations + +on: + workflow_dispatch: + schedule: + # Nightly at 07:00 UTC. This job is too slow to gate every PR on, and + # feature graphs change rarely, so a nightly signal is the right cadence. + - cron: "0 7 * * *" + pull_request: + branches: ["main"] + # Only re-run when something could actually change feature resolution. + paths: + - "**/Cargo.toml" + - "Cargo.lock" + - ".github/workflows/feature-combinations.yml" + push: + branches: + - "main" + # TEMPORARY: added so this job runs on the branch that introduces it, + # to measure how long it actually takes on a hosted runner. Remove this + # line before merging. + - "add-all-features-complication" + +env: + CARGO_TERM_COLOR: always + # The crates under test: `livekit` and `livekit-api` plus every workspace + # crate they depend on that has features of its own. cargo-hack only varies + # the features of the packages named with -p, so dependencies have to be + # listed explicitly to get their own powersets. + HACK_PACKAGES: >- + -p livekit + -p livekit-api + -p livekit-signaling + -p livekit-net + -p livekit-region + -p livekit-protocol + -p livekit-datatrack + -p livekit-data-stream + -p livekit-common + -p libwebrtc + -p webrtc-sys + -p livekit-token + +jobs: + feature-powerset: + name: Feature powerset (aarch64-unknown-linux-gnu) + runs-on: ubuntu-24.04-arm + # Measured at ~7 min on a 14-core Apple Silicon laptop. Hosted runners are + # far smaller, so this is set well above the expected runtime to leave room + # rather than to express a target. + timeout-minutes: 90 + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + submodules: true + + - name: Install Protoc + uses: arduino/setup-protoc@a8b67ba40b37d35169e222f3bb352603327985b6 # v2.1.0 + with: + version: "25.2" + repo-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Install linux dependencies + run: | + sudo apt update -y + # lld is needed on aarch64 because GNU ld cannot link libwebrtc.a. + sudo apt install -y \ + lld \ + libasound2-dev \ + libssl-dev \ + libx11-dev \ + libgl1-mesa-dev \ + libxext-dev \ + libdrm-dev \ + libgbm-dev \ + libxfixes-dev \ + libxdamage-dev \ + libxrandr-dev \ + libxcomposite-dev \ + libglib2.0-dev + + # Ubuntu 24 ships clang 18; webrtc-sys needs 21+ for libwebrtc's hermetic + # libc++. Sets CC/CXX for the steps below. + - name: Install clang for webrtc-sys + run: .github/scripts/install-clang.sh + + - name: Setup Rust toolchain + uses: actions-rust-lang/setup-rust-toolchain@46268bd060767258de96ed93c1251119784f2ab6 # v1.16.1 + with: + target: aarch64-unknown-linux-gnu + cache: false + rustflags: "" + + - name: Install cargo-hack + # Pinned rather than floating: cargo-hack decides which combinations + # exist, so an unpinned bump would silently change what this job tests. + run: cargo install cargo-hack --locked --version 0.6.45 + + # Only crates.io sources are cached, deliberately not `target/`. This job + # exists to report the cost of checking the feature powerset from cold; a + # warm target dir would make the number look better than a real first run. + - name: Cache cargo registry + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: | + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + key: ${{ runner.os }}-arm-cargo-registry-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-arm-cargo-registry- + + # Baseline for comparison: one plain check of the same crates at their + # default features. This is what the powerset step should be measured + # against — it absorbs the one-time costs (libwebrtc download, the + # webrtc-sys C++ bridge) that the powerset step then does not repeat. + - name: Baseline check (default features) + run: | + start=$(date +%s) + cargo check ${{ env.HACK_PACKAGES }} + echo "BASELINE_SECS=$(( $(date +%s) - start ))" >> "$GITHUB_ENV" + + - name: Check feature powerset (depth 2) + run: | + start=$(date +%s) + # --keep-going reports every broken combination in one run instead of + # aborting at the first, so a red build is one round of fixes. + cargo hack ${{ env.HACK_PACKAGES }} \ + --feature-powerset \ + --depth 2 \ + --keep-going \ + check + echo "POWERSET_SECS=$(( $(date +%s) - start ))" >> "$GITHUB_ENV" + + - name: Report timings + # Runs even when the powerset fails so the timing is still reported; + # POWERSET_SECS is then unset and shown as n/a. + if: always() + run: | + secs() { if [ -n "${1:-}" ]; then echo "${1}s"; else echo "n/a"; fi; } + { + echo "### Feature powerset timing" + echo "" + echo "| Measurement | Value |" + echo "| --- | --- |" + echo "| Baseline \`cargo check\` (default features) | $(secs "${BASELINE_SECS:-}") |" + echo "| Feature powerset, depth 2 | $(secs "${POWERSET_SECS:-}") |" + if [ -n "${BASELINE_SECS:-}" ] && [ -n "${POWERSET_SECS:-}" ] && [ "$BASELINE_SECS" -gt 0 ]; then + awk -v p="$POWERSET_SECS" -v b="$BASELINE_SECS" \ + 'BEGIN { printf "| Powerset as a multiple of baseline | %.1fx |\n", p/b + printf "| Powerset wall time | %.1f min |\n", p/60 }' + fi + } >> "$GITHUB_STEP_SUMMARY" From 5168b8e9074759f93a1bc6e05e3a49e4b1ea7e2e Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Wed, 2 Sep 2026 10:16:36 -0400 Subject: [PATCH 02/12] ci: run the feature powerset on every platform, per commit Expands the feature-combination job from linux arm64 to the same eleven targets builds.yml covers, and drops the nightly schedule plus the Cargo.toml path filter so it runs on every commit. Measured at 14 min end to end on ubuntu-24.04-arm, which is on par with the existing build and test jobs, so per-commit is affordable. Matrix jobs run in parallel, so the cost is the slowest target rather than the sum of eleven. One platform was never enough. Feature *resolution* does not vary by target, but what the resolved features then compile does: cfg-gated code, the webrtc-sys C++ bridge and the per-platform TLS backends all differ, so a combination can be green on linux and broken on android or iOS. That is exactly the class of bug this job exists to catch. Android goes through cargo-ndk, which supplies the NDK clang the webrtc-sys C++ bridge needs; it execs `cargo `, so it wraps `hack` the same way it wraps `build`. fail-fast is off so one broken target does not hide the other ten, and a concurrency group cancels superseded runs rather than leaving eleven jobs per push running. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/feature-combinations.yml | 113 ++++++++++++++++----- 1 file changed, 85 insertions(+), 28 deletions(-) diff --git a/.github/workflows/feature-combinations.yml b/.github/workflows/feature-combinations.yml index 9bc2c94c7..51156b872 100644 --- a/.github/workflows/feature-combinations.yml +++ b/.github/workflows/feature-combinations.yml @@ -22,38 +22,43 @@ # at depth 2 to 232 at depth 3. Depth 2 catches pairwise interactions, which is # where nearly all feature bugs live. # -# Runs on one platform only (linux arm64). Feature resolution is -# platform-independent; the point is to exercise the feature graph, not the -# target matrix. +# The target matrix mirrors builds.yml. Feature *resolution* is +# platform-independent, but what the resolved features then compile is not: +# cfg-gated code, the webrtc-sys C++ bridge, and the per-platform TLS backends +# all differ, so a combination can be fine on linux and broken on android. +# Jobs run in parallel, so the cost is the slowest target rather than the sum. name: Feature Combinations on: workflow_dispatch: - schedule: - # Nightly at 07:00 UTC. This job is too slow to gate every PR on, and - # feature graphs change rarely, so a nightly signal is the right cadence. - - cron: "0 7 * * *" pull_request: branches: ["main"] - # Only re-run when something could actually change feature resolution. - paths: - - "**/Cargo.toml" - - "Cargo.lock" - - ".github/workflows/feature-combinations.yml" + paths-ignore: + - "**/*.md" + - ".changeset/**" + - "LICENSE" + - "NOTICE" + - "renovate.json" + - ".vscode/**" push: branches: - "main" - # TEMPORARY: added so this job runs on the branch that introduces it, - # to measure how long it actually takes on a hosted runner. Remove this - # line before merging. + # TEMPORARY: added so this job runs on the branch that introduces it, to + # measure how long the full matrix takes. Remove before merging. - "add-all-features-complication" +# One in-flight run per ref. This job is cheap per platform but there are +# eleven of them, so superseded pushes should not keep burning runners. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + env: CARGO_TERM_COLOR: always # The crates under test: `livekit` and `livekit-api` plus every workspace # crate they depend on that has features of its own. cargo-hack only varies # the features of the packages named with -p, so dependencies have to be - # listed explicitly to get their own powersets. + # listed explicitly to get their own powersets. 209 combinations in total. HACK_PACKAGES: >- -p livekit -p livekit-api @@ -70,11 +75,39 @@ env: jobs: feature-powerset: - name: Feature powerset (aarch64-unknown-linux-gnu) - runs-on: ubuntu-24.04-arm - # Measured at ~7 min on a 14-core Apple Silicon laptop. Hosted runners are - # far smaller, so this is set well above the expected runtime to leave room - # rather than to express a target. + strategy: + # Every target reports independently; one broken platform should not hide + # the state of the other ten. + fail-fast: false + matrix: + include: + - os: windows-latest + target: x86_64-pc-windows-msvc + - os: windows-latest + target: aarch64-pc-windows-msvc + - os: macos-latest + target: x86_64-apple-darwin + - os: macos-latest + target: aarch64-apple-darwin + - os: macos-latest + target: aarch64-apple-ios + - os: macos-latest + target: aarch64-apple-ios-sim + - os: ubuntu-latest + target: x86_64-unknown-linux-gnu + - os: ubuntu-24.04-arm + target: aarch64-unknown-linux-gnu + - os: ubuntu-latest + target: aarch64-linux-android + - os: ubuntu-latest + target: armv7-linux-androideabi + - os: ubuntu-latest + target: x86_64-linux-android + + name: Feature powerset (${{ matrix.target }}) + runs-on: ${{ matrix.os }} + # Measured at ~14 min end to end for aarch64-unknown-linux-gnu. Set well + # above that to leave room, not as a target. timeout-minutes: 90 steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 @@ -88,6 +121,7 @@ jobs: repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Install linux dependencies + if: ${{ startsWith(matrix.os, 'ubuntu') }} run: | sudo apt update -y # lld is needed on aarch64 because GNU ld cannot link libwebrtc.a. @@ -107,14 +141,15 @@ jobs: libglib2.0-dev # Ubuntu 24 ships clang 18; webrtc-sys needs 21+ for libwebrtc's hermetic - # libc++. Sets CC/CXX for the steps below. + # libc++. Android is excluded: cargo-ndk supplies its own NDK clang. - name: Install clang for webrtc-sys + if: ${{ contains(matrix.target, 'unknown-linux-gnu') }} run: .github/scripts/install-clang.sh - name: Setup Rust toolchain uses: actions-rust-lang/setup-rust-toolchain@46268bd060767258de96ed93c1251119784f2ab6 # v1.16.1 with: - target: aarch64-unknown-linux-gnu + target: ${{ matrix.target }} cache: false rustflags: "" @@ -123,6 +158,25 @@ jobs: # exist, so an unpinned bump would silently change what this job tests. run: cargo install cargo-hack --locked --version 0.6.45 + - name: Install cargo-ndk + if: ${{ contains(matrix.target, 'android') }} + run: cargo install cargo-ndk --locked + + # Android needs the NDK's clang to compile the webrtc-sys C++ bridge, and + # cargo-ndk is what puts it (and the right sysroot) in the environment. + # It execs `cargo `, so it wraps `hack` the same way it wraps + # `build`, and supplies --target itself. + - name: Resolve cargo driver for this target + shell: bash + run: | + if [[ "${{ matrix.target }}" == *android* ]]; then + echo "CHECK_CMD=cargo ndk --target ${{ matrix.target }} check" >> "$GITHUB_ENV" + echo "HACK_CMD=cargo ndk --target ${{ matrix.target }} hack" >> "$GITHUB_ENV" + else + echo "CHECK_CMD=cargo check --target ${{ matrix.target }}" >> "$GITHUB_ENV" + echo "HACK_CMD=cargo hack --target ${{ matrix.target }}" >> "$GITHUB_ENV" + fi + # Only crates.io sources are cached, deliberately not `target/`. This job # exists to report the cost of checking the feature powerset from cold; a # warm target dir would make the number look better than a real first run. @@ -133,26 +187,28 @@ jobs: ~/.cargo/registry/index/ ~/.cargo/registry/cache/ ~/.cargo/git/db/ - key: ${{ runner.os }}-arm-cargo-registry-${{ hashFiles('**/Cargo.lock') }} + key: ${{ runner.os }}-${{ matrix.target }}-cargo-registry-${{ hashFiles('**/Cargo.lock') }} restore-keys: | - ${{ runner.os }}-arm-cargo-registry- + ${{ runner.os }}-${{ matrix.target }}-cargo-registry- # Baseline for comparison: one plain check of the same crates at their # default features. This is what the powerset step should be measured # against — it absorbs the one-time costs (libwebrtc download, the # webrtc-sys C++ bridge) that the powerset step then does not repeat. - name: Baseline check (default features) + shell: bash run: | start=$(date +%s) - cargo check ${{ env.HACK_PACKAGES }} + $CHECK_CMD $HACK_PACKAGES echo "BASELINE_SECS=$(( $(date +%s) - start ))" >> "$GITHUB_ENV" - name: Check feature powerset (depth 2) + shell: bash run: | start=$(date +%s) # --keep-going reports every broken combination in one run instead of # aborting at the first, so a red build is one round of fixes. - cargo hack ${{ env.HACK_PACKAGES }} \ + $HACK_CMD $HACK_PACKAGES \ --feature-powerset \ --depth 2 \ --keep-going \ @@ -163,10 +219,11 @@ jobs: # Runs even when the powerset fails so the timing is still reported; # POWERSET_SECS is then unset and shown as n/a. if: always() + shell: bash run: | secs() { if [ -n "${1:-}" ]; then echo "${1}s"; else echo "n/a"; fi; } { - echo "### Feature powerset timing" + echo "### Feature powerset timing — \`${{ matrix.target }}\`" echo "" echo "| Measurement | Value |" echo "| --- | --- |" From 35eb8aec4fea8a34c1bbfe8b5031575ebd8747ca Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Wed, 2 Sep 2026 10:43:51 -0400 Subject: [PATCH 03/12] ci: exclude native-tls from the Android feature powerset MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All three Android targets failed the same 8 of 209 combinations: every one that enables plain `native-tls`. The cause is openssl-sys, which cannot find an OpenSSL installation to link against when cross-compiling to Android and fails its build script outright. This is not a feature-wiring bug, so there is nothing in the SDK to fix. Android is a rustls-only platform here — ffi-builds.yml already ships all three Android targets as `rustls-tls-webpki-roots`, so a `native-tls` Android build is not a configuration anyone ships. Excluding the feature takes Android from 209 combinations to 172. It keeps `native-tls-vendored` covered (33 combinations), since that builds OpenSSL from source and works, and the excluded combinations are still exercised on the eight non-Android targets. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/feature-combinations.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/feature-combinations.yml b/.github/workflows/feature-combinations.yml index 51156b872..034e1d2ec 100644 --- a/.github/workflows/feature-combinations.yml +++ b/.github/workflows/feature-combinations.yml @@ -97,12 +97,23 @@ jobs: target: x86_64-unknown-linux-gnu - os: ubuntu-24.04-arm target: aarch64-unknown-linux-gnu + # `native-tls` means openssl-sys on Android, and there is no Android + # OpenSSL to link against — the build script fails with "Could not + # find directory of OpenSSL installation" for every combination that + # enables it. This is not a feature-wiring bug: Android is a + # rustls-only platform here, and ffi-builds.yml already ships all + # three Android targets as `rustls-tls-webpki-roots`. Excluding the + # feature drops 209 combinations to 172 and keeps + # `native-tls-vendored` (which builds OpenSSL from source) covered. - os: ubuntu-latest target: aarch64-linux-android + exclude_features: native-tls - os: ubuntu-latest target: armv7-linux-androideabi + exclude_features: native-tls - os: ubuntu-latest target: x86_64-linux-android + exclude_features: native-tls name: Feature powerset (${{ matrix.target }}) runs-on: ${{ matrix.os }} @@ -212,6 +223,7 @@ jobs: --feature-powerset \ --depth 2 \ --keep-going \ + ${{ matrix.exclude_features && format('--exclude-features {0}', matrix.exclude_features) || '' }} \ check echo "POWERSET_SECS=$(( $(date +%s) - start ))" >> "$GITHUB_ENV" From c1ba5d20ad9085330696be85eced3740c5b4ea8f Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Wed, 2 Sep 2026 11:04:35 -0400 Subject: [PATCH 04/12] ci: add a curated feature-checking workflow to compare against the powerset MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Splits feature checking by where combinatorial coverage earns its cost, as an alternative to the full 209-combination powerset: - Leaf crates keep the full `--feature-powerset --depth 2` (83 combinations). Their feature sets are small, so exhaustive coverage is cheap, and they are where the `dep:`/`?/` plumbing bugs live. - livekit, livekit-api, livekit-ffi and livekit-uniffi get 26 hand-picked combinations. The powerset over the top crates is where the time went — `livekit` alone is 65 of 209 combinations and ~40% of wall time — and most of those are combinations nobody can select: two TLS backends at once, or an internal `__lk-*` feature paired with something unrelated. Every entry here is drawn from what the manifests document or what ffi-builds.yml ships. This also covers livekit-ffi and livekit-uniffi, which the powerset workflow does not touch at all. The tradeoff is deliberate and narrows coverage on the top crates: a feature pair that only breaks in one of the dropped combinations will not be caught. Both workflows run for now so the numbers can be compared on the same commit; one should be deleted once that is settled. Co-Authored-By: Claude Opus 5 (1M context) --- .../feature-combinations-curated.yml | 299 ++++++++++++++++++ 1 file changed, 299 insertions(+) create mode 100644 .github/workflows/feature-combinations-curated.yml diff --git a/.github/workflows/feature-combinations-curated.yml b/.github/workflows/feature-combinations-curated.yml new file mode 100644 index 000000000..a9928b659 --- /dev/null +++ b/.github/workflows/feature-combinations-curated.yml @@ -0,0 +1,299 @@ +# Copyright 2026 LiveKit, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Feature checking, split by where combinatorial coverage is actually worth +# paying for. Two halves: +# +# 1. Leaf crates get the full `--feature-powerset --depth 2`. Their feature +# sets are small, so exhaustive coverage is cheap, and they are where the +# `dep:`/`?/` plumbing bugs actually live. +# +# 2. The four user-facing crates (livekit, livekit-api, livekit-ffi, +# livekit-uniffi) get a hand-picked list of combinations instead. The +# powerset over these is where nearly all the time went — `livekit` alone +# is 65 of 209 combinations and ~40% of the wall time — and most of those +# 65 are combinations nobody can actually select, e.g. two TLS backends at +# once, or an internal `__lk-*` feature paired with an unrelated one. The +# list below is drawn from what the manifests document and what +# ffi-builds.yml actually ships. +# +# The tradeoff is real and worth stating: this trades exhaustiveness on the top +# crates for speed. A feature pair that only breaks in one of the combinations +# dropped here will not be caught. Compare against feature-combinations.yml, +# which runs the full 209, before deciding which to keep. +name: Feature Combinations (curated) + +on: + workflow_dispatch: + pull_request: + branches: ["main"] + paths-ignore: + - "**/*.md" + - ".changeset/**" + - "LICENSE" + - "NOTICE" + - "renovate.json" + - ".vscode/**" + push: + branches: + - "main" + # TEMPORARY: so this runs on the branch that introduces it. Remove before + # merging. + - "add-all-features-complication" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + CARGO_TERM_COLOR: always + # Leaf/internal crates: full powerset. 83 combinations across the ten. + LEAF_PACKAGES: >- + -p livekit-signaling + -p livekit-net + -p livekit-region + -p livekit-protocol + -p livekit-datatrack + -p livekit-data-stream + -p livekit-common + -p libwebrtc + -p webrtc-sys + -p livekit-token + +jobs: + curated: + strategy: + fail-fast: false + matrix: + include: + - os: windows-latest + target: x86_64-pc-windows-msvc + - os: windows-latest + target: aarch64-pc-windows-msvc + - os: macos-latest + target: x86_64-apple-darwin + - os: macos-latest + target: aarch64-apple-darwin + - os: macos-latest + target: aarch64-apple-ios + - os: macos-latest + target: aarch64-apple-ios-sim + - os: ubuntu-latest + target: x86_64-unknown-linux-gnu + - os: ubuntu-24.04-arm + target: aarch64-unknown-linux-gnu + # See feature-combinations.yml: `native-tls` means openssl-sys, which + # has no OpenSSL to link against when cross-compiling to Android. + - os: ubuntu-latest + target: aarch64-linux-android + exclude_features: native-tls + - os: ubuntu-latest + target: armv7-linux-androideabi + exclude_features: native-tls + - os: ubuntu-latest + target: x86_64-linux-android + exclude_features: native-tls + + name: Curated features (${{ matrix.target }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 90 + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + submodules: true + + - name: Install Protoc + uses: arduino/setup-protoc@a8b67ba40b37d35169e222f3bb352603327985b6 # v2.1.0 + with: + version: "25.2" + repo-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Install linux dependencies + if: ${{ startsWith(matrix.os, 'ubuntu') }} + run: | + sudo apt update -y + sudo apt install -y \ + lld \ + libasound2-dev \ + libssl-dev \ + libx11-dev \ + libgl1-mesa-dev \ + libxext-dev \ + libdrm-dev \ + libgbm-dev \ + libxfixes-dev \ + libxdamage-dev \ + libxrandr-dev \ + libxcomposite-dev \ + libglib2.0-dev + + - name: Install clang for webrtc-sys + if: ${{ contains(matrix.target, 'unknown-linux-gnu') }} + run: .github/scripts/install-clang.sh + + - name: Setup Rust toolchain + uses: actions-rust-lang/setup-rust-toolchain@46268bd060767258de96ed93c1251119784f2ab6 # v1.16.1 + with: + target: ${{ matrix.target }} + cache: false + rustflags: "" + + - name: Install cargo-hack + run: cargo install cargo-hack --locked --version 0.6.45 + + - name: Install cargo-ndk + if: ${{ contains(matrix.target, 'android') }} + run: cargo install cargo-ndk --locked + + - name: Resolve cargo driver for this target + shell: bash + run: | + if [[ "${{ matrix.target }}" == *android* ]]; then + echo "CHECK_CMD=cargo ndk --target ${{ matrix.target }} check" >> "$GITHUB_ENV" + echo "HACK_CMD=cargo ndk --target ${{ matrix.target }} hack" >> "$GITHUB_ENV" + else + echo "CHECK_CMD=cargo check --target ${{ matrix.target }}" >> "$GITHUB_ENV" + echo "HACK_CMD=cargo hack --target ${{ matrix.target }}" >> "$GITHUB_ENV" + fi + + - name: Cache cargo registry + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: | + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + key: ${{ runner.os }}-${{ matrix.target }}-cargo-registry-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-${{ matrix.target }}-cargo-registry- + + # Same baseline as feature-combinations.yml so the two are comparable. + # Covers the four curated crates as well, since they are checked below. + - name: Baseline check (default features) + shell: bash + run: | + start=$(date +%s) + $CHECK_CMD $LEAF_PACKAGES -p livekit -p livekit-api -p livekit-ffi -p livekit-uniffi + echo "BASELINE_SECS=$(( $(date +%s) - start ))" >> "$GITHUB_ENV" + + - name: Leaf crates — full feature powerset (depth 2) + shell: bash + run: | + start=$(date +%s) + $HACK_CMD $LEAF_PACKAGES \ + --feature-powerset \ + --depth 2 \ + --keep-going \ + ${{ matrix.exclude_features && format('--exclude-features {0}', matrix.exclude_features) || '' }} \ + check + echo "LEAF_SECS=$(( $(date +%s) - start ))" >> "$GITHUB_ENV" + + # Curated combinations for the user-facing crates. Each line is + # `crate|cargo feature flags`; an empty flags field means default + # features. Every entry is a configuration someone can actually select + # and that the manifests or ffi-builds.yml document. + - name: Top-level crates — curated feature combinations + shell: bash + run: | + set -uo pipefail + COMBOS=$(cat <<'EOF' + # --- livekit ------------------------------------------------------- + livekit| + livekit|--no-default-features + livekit|--no-default-features --features native + livekit|--no-default-features --features native,native-tls + livekit|--no-default-features --features native,native-tls-vendored + livekit|--no-default-features --features native,rustls-tls-native-roots + livekit|--no-default-features --features native,rustls-tls-webpki-roots + livekit|--no-default-features --features native,rustls-tls-webpki-roots,glib-main-loop + livekit|--no-default-features --features native,__lk-internal + # --- livekit-api --------------------------------------------------- + livekit-api| + livekit-api|--no-default-features + livekit-api|--no-default-features --features access-token + livekit-api|--no-default-features --features webhooks + livekit-api|--no-default-features --features services,native-tls + livekit-api|--no-default-features --features services,rustls-tls-webpki-roots + livekit-api|--no-default-features --features services,rustls-tls-native-roots + livekit-api|--no-default-features --features signal-client-native,rustls-tls-webpki-roots + livekit-api|--features rustls-tls-webpki-roots + # --- livekit-ffi --------------------------------------------------- + livekit-ffi| + livekit-ffi|--no-default-features --features native-tls + livekit-ffi|--no-default-features --features native-tls-vendored + livekit-ffi|--no-default-features --features rustls-tls-webpki-roots + livekit-ffi|--no-default-features --features rustls-tls-native-roots + livekit-ffi|--features tracing + # --- livekit-uniffi ------------------------------------------------ + # dart-bindgen is deliberately omitted: it is tooling-only and pulls a + # third-party bindgen that has no business gating an SDK commit. + livekit-uniffi| + livekit-uniffi|--features cli + EOF + ) + + start=$(date +%s) + total=0; failed=0 + FAILURES="" + while IFS='|' read -r crate flags; do + # Skip comments and blanks. + [[ -z "${crate// /}" || "$crate" =~ ^[[:space:]]*# ]] && continue + crate="${crate// /}" + + # Android has no OpenSSL to link: drop bare `native-tls` (but keep + # `native-tls-vendored`, which builds it from source). + if [ -n "${{ matrix.exclude_features || '' }}" ] \ + && printf '%s' "$flags" | grep -qE '(^|[ ,])native-tls($|[ ,])'; then + echo "::notice::skipping (no Android OpenSSL): $crate $flags" + continue + fi + + total=$((total + 1)) + echo "::group::$crate $flags" + if ! $CHECK_CMD -p "$crate" $flags; then + failed=$((failed + 1)) + FAILURES="${FAILURES}\n $crate $flags" + fi + echo "::endgroup::" + done <<< "$COMBOS" + + echo "CURATED_SECS=$(( $(date +%s) - start ))" >> "$GITHUB_ENV" + echo "CURATED_TOTAL=$total" >> "$GITHUB_ENV" + echo "CURATED_FAILED=$failed" >> "$GITHUB_ENV" + echo "Ran $total curated combinations, $failed failed." + if [ "$failed" -gt 0 ]; then + echo -e "::error::failed curated combinations:$FAILURES" + exit 1 + fi + + - name: Report timings + if: always() + shell: bash + run: | + secs() { if [ -n "${1:-}" ]; then echo "${1}s"; else echo "n/a"; fi; } + { + echo "### Curated feature checking — \`${{ matrix.target }}\`" + echo "" + echo "| Measurement | Value |" + echo "| --- | --- |" + echo "| Baseline \`cargo check\` (default features) | $(secs "${BASELINE_SECS:-}") |" + echo "| Leaf crates, full powerset (83 combos) | $(secs "${LEAF_SECS:-}") |" + echo "| Top-level crates, curated (${CURATED_TOTAL:-?} combos) | $(secs "${CURATED_SECS:-}") |" + if [ -n "${LEAF_SECS:-}" ] && [ -n "${CURATED_SECS:-}" ]; then + awk -v l="$LEAF_SECS" -v c="$CURATED_SECS" -v b="${BASELINE_SECS:-0}" \ + 'BEGIN { printf "| **Feature checking total** | **%.1f min** |\n", (l+c)/60 + if (b > 0) printf "| Total as a multiple of baseline | %.1fx |\n", (l+c)/b }' + fi + echo "| Curated failures | ${CURATED_FAILED:-n/a} |" + } >> "$GITHUB_STEP_SUMMARY" From 8cbfacb0ebd2b0d22c2f3c55f9d4202f481a98cc Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Wed, 2 Sep 2026 11:58:00 -0400 Subject: [PATCH 05/12] ci: split feature checking into a per-commit subset and a nightly powerset MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Measured cold on a 14-core machine, checking the same crates: full powerset (209 combos) 359s leaf powerset + curated livekit/api 242s -33% the same, plus livekit-ffi/livekit-uniffi 345s -4% The first version of the curated workflow was no faster than the powerset it was meant to replace — on CI it was slower on 9 of 11 targets. Two reasons, both visible above. livekit-ffi and livekit-uniffi were the larger one: 103s of a 345s budget for two crates, because every TLS change rebuilds livekit underneath livekit-ffi, and the powerset workflow never built them at all. They move to the nightly job here. Their shipped configurations are still covered per-commit — builds.yml builds the whole workspace on every PR, and ffi-builds.yml builds each shipped per-platform TLS variant on push to main — so what nightly adds back is livekit-ffi's non-default TLS variants on PRs. The other was a misread on my part: `livekit` is 65 of 209 combinations and ~40% of wall time, but those combinations are mostly ~2s cache hits. The cost attributed to them was the one-time cold build of the dependency graph, which does not go away when the combinations do — it just moves into the leaf phase. The leaf-only powerset is already 55-80% of the full 209. So: feature-combinations-curated.yml runs per-commit on all eleven targets (leaf powerset + 17 documented livekit/livekit-api combinations), and feature-combinations.yml keeps the exhaustive 209 plus the ffi crates, nightly, as the backstop for what the subset skips. Co-Authored-By: Claude Opus 5 (1M context) --- .../feature-combinations-curated.yml | 54 ++++++++--------- .github/workflows/feature-combinations.yml | 59 +++++++++++++------ 2 files changed, 69 insertions(+), 44 deletions(-) diff --git a/.github/workflows/feature-combinations-curated.yml b/.github/workflows/feature-combinations-curated.yml index a9928b659..4432655f3 100644 --- a/.github/workflows/feature-combinations-curated.yml +++ b/.github/workflows/feature-combinations-curated.yml @@ -19,20 +19,22 @@ # sets are small, so exhaustive coverage is cheap, and they are where the # `dep:`/`?/` plumbing bugs actually live. # -# 2. The four user-facing crates (livekit, livekit-api, livekit-ffi, -# livekit-uniffi) get a hand-picked list of combinations instead. The -# powerset over these is where nearly all the time went — `livekit` alone -# is 65 of 209 combinations and ~40% of the wall time — and most of those -# 65 are combinations nobody can actually select, e.g. two TLS backends at -# once, or an internal `__lk-*` feature paired with an unrelated one. The -# list below is drawn from what the manifests document and what -# ffi-builds.yml actually ships. +# 2. livekit and livekit-api get a hand-picked list of 17 combinations +# instead of their 125-combination powerset. Most of those 125 are +# configurations nobody can actually select — two TLS backends at once, or +# an internal `__lk-*` feature paired with something unrelated. Every +# entry in the list is drawn from what the manifests document. +# +# Measured cold on a 14-core machine: the full 209-combination powerset takes +# 359s; this split takes 242s, a 33% saving. That is what brings the job in +# line with the existing per-commit builds. # # The tradeoff is real and worth stating: this trades exhaustiveness on the top -# crates for speed. A feature pair that only breaks in one of the combinations -# dropped here will not be caught. Compare against feature-combinations.yml, -# which runs the full 209, before deciding which to keep. -name: Feature Combinations (curated) +# two crates for speed, so a feature pair that only breaks in one of the +# dropped combinations will not be caught here. feature-combinations.yml still +# runs the full 209 plus livekit-ffi/livekit-uniffi nightly, which is the +# backstop for exactly that. +name: Feature Combinations (per-commit) on: workflow_dispatch: @@ -105,7 +107,7 @@ jobs: target: x86_64-linux-android exclude_features: native-tls - name: Curated features (${{ matrix.target }}) + name: Features per-commit (${{ matrix.target }}) runs-on: ${{ matrix.os }} timeout-minutes: 90 steps: @@ -184,7 +186,7 @@ jobs: shell: bash run: | start=$(date +%s) - $CHECK_CMD $LEAF_PACKAGES -p livekit -p livekit-api -p livekit-ffi -p livekit-uniffi + $CHECK_CMD $LEAF_PACKAGES -p livekit -p livekit-api echo "BASELINE_SECS=$(( $(date +%s) - start ))" >> "$GITHUB_ENV" - name: Leaf crates — full feature powerset (depth 2) @@ -228,18 +230,16 @@ jobs: livekit-api|--no-default-features --features services,rustls-tls-native-roots livekit-api|--no-default-features --features signal-client-native,rustls-tls-webpki-roots livekit-api|--features rustls-tls-webpki-roots - # --- livekit-ffi --------------------------------------------------- - livekit-ffi| - livekit-ffi|--no-default-features --features native-tls - livekit-ffi|--no-default-features --features native-tls-vendored - livekit-ffi|--no-default-features --features rustls-tls-webpki-roots - livekit-ffi|--no-default-features --features rustls-tls-native-roots - livekit-ffi|--features tracing - # --- livekit-uniffi ------------------------------------------------ - # dart-bindgen is deliberately omitted: it is tooling-only and pulls a - # third-party bindgen that has no business gating an SDK commit. - livekit-uniffi| - livekit-uniffi|--features cli + # livekit-ffi and livekit-uniffi are deliberately NOT here. Measured + # cold, their 8 combinations cost 103s of a 345s total — 30% of the + # budget for two crates, because each TLS change rebuilds livekit + # underneath livekit-ffi. Dropping them takes the job from 345s to + # 242s, which is what makes per-commit affordable. Their shipped + # configurations are still covered: builds.yml builds the whole + # workspace (livekit-ffi at default features) on every PR, and + # ffi-builds.yml builds every shipped per-platform TLS variant on + # push to main. The residual gap is livekit-ffi's non-default TLS + # variants on PRs, which the nightly full powerset covers. EOF ) @@ -283,7 +283,7 @@ jobs: run: | secs() { if [ -n "${1:-}" ]; then echo "${1}s"; else echo "n/a"; fi; } { - echo "### Curated feature checking — \`${{ matrix.target }}\`" + echo "### Per-commit feature checking — \`${{ matrix.target }}\`" echo "" echo "| Measurement | Value |" echo "| --- | --- |" diff --git a/.github/workflows/feature-combinations.yml b/.github/workflows/feature-combinations.yml index 034e1d2ec..cbcc91e13 100644 --- a/.github/workflows/feature-combinations.yml +++ b/.github/workflows/feature-combinations.yml @@ -27,25 +27,15 @@ # cfg-gated code, the webrtc-sys C++ bridge, and the per-platform TLS backends # all differ, so a combination can be fine on linux and broken on android. # Jobs run in parallel, so the cost is the slowest target rather than the sum. -name: Feature Combinations +name: Feature Combinations (nightly, exhaustive) on: workflow_dispatch: - pull_request: - branches: ["main"] - paths-ignore: - - "**/*.md" - - ".changeset/**" - - "LICENSE" - - "NOTICE" - - "renovate.json" - - ".vscode/**" - push: - branches: - - "main" - # TEMPORARY: added so this job runs on the branch that introduces it, to - # measure how long the full matrix takes. Remove before merging. - - "add-all-features-complication" + schedule: + # 07:00 UTC daily. This is the exhaustive backstop: the per-commit job in + # feature-combinations-curated.yml checks a curated subset of livekit and + # livekit-api, so the combinations it skips are caught here within a day. + - cron: "0 7 * * *" # One in-flight run per ref. This job is cheap per platform but there are # eleven of them, so superseded pushes should not keep burning runners. @@ -115,7 +105,7 @@ jobs: target: x86_64-linux-android exclude_features: native-tls - name: Feature powerset (${{ matrix.target }}) + name: Feature powerset nightly (${{ matrix.target }}) runs-on: ${{ matrix.os }} # Measured at ~14 min end to end for aarch64-unknown-linux-gnu. Set well # above that to leave room, not as a target. @@ -227,6 +217,41 @@ jobs: check echo "POWERSET_SECS=$(( $(date +%s) - start ))" >> "$GITHUB_ENV" + # livekit-ffi and livekit-uniffi are checked here rather than per-commit: + # their combinations cost 103s of a 345s budget (each TLS change rebuilds + # livekit underneath livekit-ffi), which is what pushed the per-commit job + # over the line. Nightly is the right place for them. + - name: livekit-ffi / livekit-uniffi feature combinations + shell: bash + run: | + set -uo pipefail + COMBOS=$(cat <<'EOF' + livekit-ffi| + livekit-ffi|--no-default-features --features native-tls + livekit-ffi|--no-default-features --features native-tls-vendored + livekit-ffi|--no-default-features --features rustls-tls-webpki-roots + livekit-ffi|--no-default-features --features rustls-tls-native-roots + livekit-ffi|--features tracing + livekit-uniffi| + livekit-uniffi|--features cli + EOF + ) + failed=0 + while IFS='|' read -r crate flags; do + [[ -z "${crate// /}" || "$crate" =~ ^[[:space:]]*# ]] && continue + crate="${crate// /}" + # Android has no OpenSSL to link: skip bare `native-tls`. + if [ -n "${{ matrix.exclude_features || '' }}" ] \ + && printf '%s' "$flags" | grep -qE '(^|[ ,])native-tls($|[ ,])'; then + echo "::notice::skipping (no Android OpenSSL): $crate $flags" + continue + fi + echo "::group::$crate $flags" + $CHECK_CMD -p "$crate" $flags || { failed=$((failed+1)); echo "::error::failed: $crate $flags"; } + echo "::endgroup::" + done <<< "$COMBOS" + [ "$failed" -eq 0 ] + - name: Report timings # Runs even when the powerset fails so the timing is still reported; # POWERSET_SECS is then unset and shown as n/a. From 1598c881ce6c2bc70c523b81f390622da4ebdea3 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Wed, 2 Sep 2026 13:34:04 -0400 Subject: [PATCH 06/12] ci: drop the nightly exhaustive feature powerset MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The curated per-commit workflow is the better trade, so the nightly 209-combination job is not worth the runner time. Verified green on all eleven targets (run 33651960004) at 7.7-17.4 min per job, with the feature-checking steps themselves down ~30% against the full powerset. Removing it gives up two things, both deliberately: - The exhaustive backstop. A feature pair that only breaks in one of the combinations the curated list skips is now caught nowhere. The header comment records the one-liner to run the full 209 by hand when a feature bug needs chasing. - Feature-combination coverage for livekit-ffi and livekit-uniffi, which only ever lived in this workflow. Their shipped configurations are still built — builds.yml builds the workspace on every PR, and ffi-builds.yml builds each shipped per-platform TLS variant on push to main — so what is lost is livekit-ffi's non-default TLS variants on PRs. That cost 103s of a 345s budget to cover, which is what kept it out of the per-commit job in the first place. Also folds the rationale that lived in the deleted file (the Android openssl-sys exclusion) into the surviving workflow rather than leaving the comment pointing at a file that no longer exists. Co-Authored-By: Claude Opus 5 (1M context) --- .../feature-combinations-curated.yml | 27 +- .github/workflows/feature-combinations.yml | 274 ------------------ 2 files changed, 18 insertions(+), 283 deletions(-) delete mode 100644 .github/workflows/feature-combinations.yml diff --git a/.github/workflows/feature-combinations-curated.yml b/.github/workflows/feature-combinations-curated.yml index 4432655f3..b1ec1b5c3 100644 --- a/.github/workflows/feature-combinations-curated.yml +++ b/.github/workflows/feature-combinations-curated.yml @@ -31,9 +31,13 @@ # # The tradeoff is real and worth stating: this trades exhaustiveness on the top # two crates for speed, so a feature pair that only breaks in one of the -# dropped combinations will not be caught here. feature-combinations.yml still -# runs the full 209 plus livekit-ffi/livekit-uniffi nightly, which is the -# backstop for exactly that. +# dropped combinations is not caught anywhere. There is no exhaustive backstop +# — the nightly full-powerset workflow was removed once this proved to be the +# better cost/benefit. To get the full 209 back for a one-off investigation, +# run it by hand: +# +# cargo hack -p livekit -p livekit-api $LEAF_PACKAGES \ +# --feature-powerset --depth 2 --keep-going check name: Feature Combinations (per-commit) on: @@ -95,8 +99,11 @@ jobs: target: x86_64-unknown-linux-gnu - os: ubuntu-24.04-arm target: aarch64-unknown-linux-gnu - # See feature-combinations.yml: `native-tls` means openssl-sys, which - # has no OpenSSL to link against when cross-compiling to Android. + # `native-tls` means openssl-sys on Android, and there is no Android + # OpenSSL to link against — its build script fails outright. Not a + # feature-wiring bug: ffi-builds.yml ships all three Android targets + # as `rustls-tls-webpki-roots`. `native-tls-vendored`, which builds + # OpenSSL from source, still works and stays covered. - os: ubuntu-latest target: aarch64-linux-android exclude_features: native-tls @@ -180,8 +187,8 @@ jobs: restore-keys: | ${{ runner.os }}-${{ matrix.target }}-cargo-registry- - # Same baseline as feature-combinations.yml so the two are comparable. - # Covers the four curated crates as well, since they are checked below. + # A plain default-feature check of the same crates, so the feature steps + # below have something to be measured against in the run summary. - name: Baseline check (default features) shell: bash run: | @@ -238,8 +245,10 @@ jobs: # configurations are still covered: builds.yml builds the whole # workspace (livekit-ffi at default features) on every PR, and # ffi-builds.yml builds every shipped per-platform TLS variant on - # push to main. The residual gap is livekit-ffi's non-default TLS - # variants on PRs, which the nightly full powerset covers. + # push to main. The residual gap — livekit-ffi's non-default TLS + # variants on PRs — is knowingly accepted: covering it cost 103s of a + # 345s budget, for two crates whose shipped configurations are + # already built on every PR and on every push to main. EOF ) diff --git a/.github/workflows/feature-combinations.yml b/.github/workflows/feature-combinations.yml deleted file mode 100644 index cbcc91e13..000000000 --- a/.github/workflows/feature-combinations.yml +++ /dev/null @@ -1,274 +0,0 @@ -# Copyright 2026 LiveKit, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Checks that every combination of up to 2 simultaneous cargo features builds, -# across the client SDK and everything below it. Feature bugs (a `dep:` that is -# only pulled in by one feature but used unconditionally, a `?/` that silently -# no-ops) do not show up in the default-feature build, so nothing else in CI -# catches them. -# -# `--depth 2` is a deliberate ceiling: `livekit` alone goes from 67 combinations -# at depth 2 to 232 at depth 3. Depth 2 catches pairwise interactions, which is -# where nearly all feature bugs live. -# -# The target matrix mirrors builds.yml. Feature *resolution* is -# platform-independent, but what the resolved features then compile is not: -# cfg-gated code, the webrtc-sys C++ bridge, and the per-platform TLS backends -# all differ, so a combination can be fine on linux and broken on android. -# Jobs run in parallel, so the cost is the slowest target rather than the sum. -name: Feature Combinations (nightly, exhaustive) - -on: - workflow_dispatch: - schedule: - # 07:00 UTC daily. This is the exhaustive backstop: the per-commit job in - # feature-combinations-curated.yml checks a curated subset of livekit and - # livekit-api, so the combinations it skips are caught here within a day. - - cron: "0 7 * * *" - -# One in-flight run per ref. This job is cheap per platform but there are -# eleven of them, so superseded pushes should not keep burning runners. -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - CARGO_TERM_COLOR: always - # The crates under test: `livekit` and `livekit-api` plus every workspace - # crate they depend on that has features of its own. cargo-hack only varies - # the features of the packages named with -p, so dependencies have to be - # listed explicitly to get their own powersets. 209 combinations in total. - HACK_PACKAGES: >- - -p livekit - -p livekit-api - -p livekit-signaling - -p livekit-net - -p livekit-region - -p livekit-protocol - -p livekit-datatrack - -p livekit-data-stream - -p livekit-common - -p libwebrtc - -p webrtc-sys - -p livekit-token - -jobs: - feature-powerset: - strategy: - # Every target reports independently; one broken platform should not hide - # the state of the other ten. - fail-fast: false - matrix: - include: - - os: windows-latest - target: x86_64-pc-windows-msvc - - os: windows-latest - target: aarch64-pc-windows-msvc - - os: macos-latest - target: x86_64-apple-darwin - - os: macos-latest - target: aarch64-apple-darwin - - os: macos-latest - target: aarch64-apple-ios - - os: macos-latest - target: aarch64-apple-ios-sim - - os: ubuntu-latest - target: x86_64-unknown-linux-gnu - - os: ubuntu-24.04-arm - target: aarch64-unknown-linux-gnu - # `native-tls` means openssl-sys on Android, and there is no Android - # OpenSSL to link against — the build script fails with "Could not - # find directory of OpenSSL installation" for every combination that - # enables it. This is not a feature-wiring bug: Android is a - # rustls-only platform here, and ffi-builds.yml already ships all - # three Android targets as `rustls-tls-webpki-roots`. Excluding the - # feature drops 209 combinations to 172 and keeps - # `native-tls-vendored` (which builds OpenSSL from source) covered. - - os: ubuntu-latest - target: aarch64-linux-android - exclude_features: native-tls - - os: ubuntu-latest - target: armv7-linux-androideabi - exclude_features: native-tls - - os: ubuntu-latest - target: x86_64-linux-android - exclude_features: native-tls - - name: Feature powerset nightly (${{ matrix.target }}) - runs-on: ${{ matrix.os }} - # Measured at ~14 min end to end for aarch64-unknown-linux-gnu. Set well - # above that to leave room, not as a target. - timeout-minutes: 90 - steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - with: - submodules: true - - - name: Install Protoc - uses: arduino/setup-protoc@a8b67ba40b37d35169e222f3bb352603327985b6 # v2.1.0 - with: - version: "25.2" - repo-token: ${{ secrets.GITHUB_TOKEN }} - - - name: Install linux dependencies - if: ${{ startsWith(matrix.os, 'ubuntu') }} - run: | - sudo apt update -y - # lld is needed on aarch64 because GNU ld cannot link libwebrtc.a. - sudo apt install -y \ - lld \ - libasound2-dev \ - libssl-dev \ - libx11-dev \ - libgl1-mesa-dev \ - libxext-dev \ - libdrm-dev \ - libgbm-dev \ - libxfixes-dev \ - libxdamage-dev \ - libxrandr-dev \ - libxcomposite-dev \ - libglib2.0-dev - - # Ubuntu 24 ships clang 18; webrtc-sys needs 21+ for libwebrtc's hermetic - # libc++. Android is excluded: cargo-ndk supplies its own NDK clang. - - name: Install clang for webrtc-sys - if: ${{ contains(matrix.target, 'unknown-linux-gnu') }} - run: .github/scripts/install-clang.sh - - - name: Setup Rust toolchain - uses: actions-rust-lang/setup-rust-toolchain@46268bd060767258de96ed93c1251119784f2ab6 # v1.16.1 - with: - target: ${{ matrix.target }} - cache: false - rustflags: "" - - - name: Install cargo-hack - # Pinned rather than floating: cargo-hack decides which combinations - # exist, so an unpinned bump would silently change what this job tests. - run: cargo install cargo-hack --locked --version 0.6.45 - - - name: Install cargo-ndk - if: ${{ contains(matrix.target, 'android') }} - run: cargo install cargo-ndk --locked - - # Android needs the NDK's clang to compile the webrtc-sys C++ bridge, and - # cargo-ndk is what puts it (and the right sysroot) in the environment. - # It execs `cargo `, so it wraps `hack` the same way it wraps - # `build`, and supplies --target itself. - - name: Resolve cargo driver for this target - shell: bash - run: | - if [[ "${{ matrix.target }}" == *android* ]]; then - echo "CHECK_CMD=cargo ndk --target ${{ matrix.target }} check" >> "$GITHUB_ENV" - echo "HACK_CMD=cargo ndk --target ${{ matrix.target }} hack" >> "$GITHUB_ENV" - else - echo "CHECK_CMD=cargo check --target ${{ matrix.target }}" >> "$GITHUB_ENV" - echo "HACK_CMD=cargo hack --target ${{ matrix.target }}" >> "$GITHUB_ENV" - fi - - # Only crates.io sources are cached, deliberately not `target/`. This job - # exists to report the cost of checking the feature powerset from cold; a - # warm target dir would make the number look better than a real first run. - - name: Cache cargo registry - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 - with: - path: | - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - key: ${{ runner.os }}-${{ matrix.target }}-cargo-registry-${{ hashFiles('**/Cargo.lock') }} - restore-keys: | - ${{ runner.os }}-${{ matrix.target }}-cargo-registry- - - # Baseline for comparison: one plain check of the same crates at their - # default features. This is what the powerset step should be measured - # against — it absorbs the one-time costs (libwebrtc download, the - # webrtc-sys C++ bridge) that the powerset step then does not repeat. - - name: Baseline check (default features) - shell: bash - run: | - start=$(date +%s) - $CHECK_CMD $HACK_PACKAGES - echo "BASELINE_SECS=$(( $(date +%s) - start ))" >> "$GITHUB_ENV" - - - name: Check feature powerset (depth 2) - shell: bash - run: | - start=$(date +%s) - # --keep-going reports every broken combination in one run instead of - # aborting at the first, so a red build is one round of fixes. - $HACK_CMD $HACK_PACKAGES \ - --feature-powerset \ - --depth 2 \ - --keep-going \ - ${{ matrix.exclude_features && format('--exclude-features {0}', matrix.exclude_features) || '' }} \ - check - echo "POWERSET_SECS=$(( $(date +%s) - start ))" >> "$GITHUB_ENV" - - # livekit-ffi and livekit-uniffi are checked here rather than per-commit: - # their combinations cost 103s of a 345s budget (each TLS change rebuilds - # livekit underneath livekit-ffi), which is what pushed the per-commit job - # over the line. Nightly is the right place for them. - - name: livekit-ffi / livekit-uniffi feature combinations - shell: bash - run: | - set -uo pipefail - COMBOS=$(cat <<'EOF' - livekit-ffi| - livekit-ffi|--no-default-features --features native-tls - livekit-ffi|--no-default-features --features native-tls-vendored - livekit-ffi|--no-default-features --features rustls-tls-webpki-roots - livekit-ffi|--no-default-features --features rustls-tls-native-roots - livekit-ffi|--features tracing - livekit-uniffi| - livekit-uniffi|--features cli - EOF - ) - failed=0 - while IFS='|' read -r crate flags; do - [[ -z "${crate// /}" || "$crate" =~ ^[[:space:]]*# ]] && continue - crate="${crate// /}" - # Android has no OpenSSL to link: skip bare `native-tls`. - if [ -n "${{ matrix.exclude_features || '' }}" ] \ - && printf '%s' "$flags" | grep -qE '(^|[ ,])native-tls($|[ ,])'; then - echo "::notice::skipping (no Android OpenSSL): $crate $flags" - continue - fi - echo "::group::$crate $flags" - $CHECK_CMD -p "$crate" $flags || { failed=$((failed+1)); echo "::error::failed: $crate $flags"; } - echo "::endgroup::" - done <<< "$COMBOS" - [ "$failed" -eq 0 ] - - - name: Report timings - # Runs even when the powerset fails so the timing is still reported; - # POWERSET_SECS is then unset and shown as n/a. - if: always() - shell: bash - run: | - secs() { if [ -n "${1:-}" ]; then echo "${1}s"; else echo "n/a"; fi; } - { - echo "### Feature powerset timing — \`${{ matrix.target }}\`" - echo "" - echo "| Measurement | Value |" - echo "| --- | --- |" - echo "| Baseline \`cargo check\` (default features) | $(secs "${BASELINE_SECS:-}") |" - echo "| Feature powerset, depth 2 | $(secs "${POWERSET_SECS:-}") |" - if [ -n "${BASELINE_SECS:-}" ] && [ -n "${POWERSET_SECS:-}" ] && [ "$BASELINE_SECS" -gt 0 ]; then - awk -v p="$POWERSET_SECS" -v b="$BASELINE_SECS" \ - 'BEGIN { printf "| Powerset as a multiple of baseline | %.1fx |\n", p/b - printf "| Powerset wall time | %.1f min |\n", p/60 }' - fi - } >> "$GITHUB_STEP_SUMMARY" From cc27308501fd0d885406d933918ee41d9c01d9d3 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Wed, 2 Sep 2026 14:43:04 -0400 Subject: [PATCH 07/12] docs: tell agents to keep the feature-combination lists up to date Both halves of feature-combinations-curated.yml are maintained by hand, and neither fails loudly when it goes stale: - A new feature on livekit or livekit-api is invisible to the job until it is added to the curated COMBOS list, because those two crates get a hand-picked list rather than a powerset. - A new workspace crate is invisible until it is added to LEAF_PACKAGES, because cargo-hack only varies the features of packages named with -p. The crate still gets built as a dependency, which makes it easy to assume it is covered when it is not. Also records the constraints that are not obvious from reading the workflow: why depth stays at 2, why openssl-sys features have to be excluded on Android, why livekit-ffi and livekit-uniffi are left out, and that --print-command-list enumerates combinations without building. Co-Authored-By: Claude Opus 5 (1M context) --- AGENTS.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index fcd67cdfc..d5ce57ea5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -75,6 +75,23 @@ Several crates export items to Swift/Kotlin/Node/Python through UniFFI — `live - A new crate that exports UniFFI items needs its own `uniffi.toml`, including `omit_checksums = true` under `[bindings.kotlin]` - The Kotlin checksum test is broken on ARM in every UniFFI release this workspace can use; the full explanation lives in `livekit-uniffi/uniffi.toml` and the root `Cargo.toml` +## Feature combinations + +`.github/workflows/feature-combinations-curated.yml` is the only job in CI that exercises features in *combination*. A `dep:` that one feature pulls in but the code uses unconditionally, or a `?/` forward that silently no-ops, compiles fine under default features and breaks only for the caller who picks a particular set — nothing else catches that. It has two halves, maintained differently, and both need updating by hand. + +- **When adding a feature to `livekit` or `livekit-api`**, add the configurations a user would plausibly select to the `COMBOS` list in the "Top-level crates" step + - These two crates get a hand-picked list rather than a powerset, so a new feature is invisible to this job until it is listed there + - One line per configuration, `crate|cargo feature flags`; an empty flags field means default features + - Pair the feature with what it realistically ships alongside (a TLS backend together with `native`, say) rather than listing it on its own + - Do not add combinations nobody can select — internal `__lk-*` flags and two-TLS-backends-at-once are deliberately absent, and were most of what made the full powerset expensive +- **When adding a workspace crate below `livekit`/`livekit-api`**, add `-p ` to `LEAF_PACKAGES` + - Leaf crates get the full `cargo hack --feature-powerset --depth 2`, so listing the crate is all that is needed; its features are then picked up automatically as they are added + - cargo-hack only varies the features of packages named with `-p`. An unlisted crate still gets built as a dependency, which makes it easy to assume it is covered when it is not +- Keep `--depth 2`. `livekit` alone goes from 67 combinations to 232 at depth 3, and pairwise interactions are where these bugs actually live +- A feature that pulls in `openssl-sys` cannot build for the three Android targets — there is no Android OpenSSL to link against. Add it to their `exclude_features` rather than trying to make it work; Android ships rustls (see `ffi-builds.yml`) +- `livekit-ffi` and `livekit-uniffi` are excluded on purpose: their combinations cost more than everything else in the job combined, because each TLS change rebuilds `livekit` underneath them. Their shipped configurations are covered by `builds.yml` and `ffi-builds.yml` instead +- Check a change to either list without building anything by appending `--print-command-list`, which enumerates the combinations cargo-hack would run + ## Documenting changes - Changes are documented using [_knope_](https://knope.tech) From 48c4daf03da7053ade83b47c9931a4fe38565b74 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Wed, 2 Sep 2026 14:59:41 -0400 Subject: [PATCH 08/12] ci: cover livekit-token-source and livekit-api's vendored TLS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both from review feedback on #1393, and both real gaps. livekit-token-source is a published crate (0.1.2) whose five features are bare pass-throughs to livekit-net — exactly the forwarding this job exists to check — and it was unchecked. It was missed because the scope was framed as "livekit and livekit-api plus what they depend on", and livekit-token-source is a sibling rather than a dependency: nothing in the SDK path pulls it in, only examples/token_source. The header now states the rule as every publishable workspace crate that has features, and calls out the sibling case, so the next such crate is not missed the same way. Leaf combinations go from 83 to 104, which costs 9s: the combinations are TLS variants already compiled elsewhere in the job. livekit-api's `services,native-tls-vendored` was the one TLS backend of the four not covered, while the livekit equivalent was — an asymmetric omission rather than a deliberate one. `services` is what enables dep:reqwest, so this is the combination where the `reqwest?/native-tls-vendored` forward actually becomes live. Costs 1s, since livekit already compiles vendored OpenSSL earlier in the run. Deliberately not fixing yuv-sys, whose `jpeg` feature is also unchecked: it would pull the libyuv C build into a job that is otherwise pure cargo check, needs system libjpeg on all eleven targets, and is not on the client SDK path. Verified both locally before pushing: token-source's 21 combinations and the new livekit-api combination all pass. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/feature-combinations-curated.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/feature-combinations-curated.yml b/.github/workflows/feature-combinations-curated.yml index b1ec1b5c3..7a8d77fc3 100644 --- a/.github/workflows/feature-combinations-curated.yml +++ b/.github/workflows/feature-combinations-curated.yml @@ -17,7 +17,11 @@ # # 1. Leaf crates get the full `--feature-powerset --depth 2`. Their feature # sets are small, so exhaustive coverage is cheap, and they are where the -# `dep:`/`?/` plumbing bugs actually live. +# `dep:`/`?/` plumbing bugs actually live. The set is every publishable +# workspace crate that has features and is not one of the two below — note +# that is *not* the same as "everything livekit depends on": +# livekit-token-source is a sibling crate, not a dependency, and belongs +# here all the same. # # 2. livekit and livekit-api get a hand-picked list of 17 combinations # instead of their 125-combination powerset. Most of those 125 are @@ -76,6 +80,7 @@ env: -p libwebrtc -p webrtc-sys -p livekit-token + -p livekit-token-source jobs: curated: @@ -233,6 +238,7 @@ jobs: livekit-api|--no-default-features --features access-token livekit-api|--no-default-features --features webhooks livekit-api|--no-default-features --features services,native-tls + livekit-api|--no-default-features --features services,native-tls-vendored livekit-api|--no-default-features --features services,rustls-tls-webpki-roots livekit-api|--no-default-features --features services,rustls-tls-native-roots livekit-api|--no-default-features --features signal-client-native,rustls-tls-webpki-roots From c52ec626323edec8046436ff250fb82b161b323f Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Tue, 8 Sep 2026 16:43:22 -0400 Subject: [PATCH 09/12] feat: move testing featureset configuration into subcrate Cargo.toml metadata --- .../feature-combinations-curated.yml | 117 +++++++++--------- AGENTS.md | 14 ++- livekit-api/Cargo.toml | 17 +++ livekit/Cargo.toml | 23 ++++ 4 files changed, 109 insertions(+), 62 deletions(-) diff --git a/.github/workflows/feature-combinations-curated.yml b/.github/workflows/feature-combinations-curated.yml index 7a8d77fc3..f4fef3e9d 100644 --- a/.github/workflows/feature-combinations-curated.yml +++ b/.github/workflows/feature-combinations-curated.yml @@ -23,11 +23,16 @@ # livekit-token-source is a sibling crate, not a dependency, and belongs # here all the same. # -# 2. livekit and livekit-api get a hand-picked list of 17 combinations +# 2. livekit and livekit-api get a hand-picked list of 19 combinations # instead of their 125-combination powerset. Most of those 125 are # configurations nobody can actually select — two TLS backends at once, or -# an internal `__lk-*` feature paired with something unrelated. Every -# entry in the list is drawn from what the manifests document. +# an internal `__lk-*` feature paired with something unrelated. +# +# That list is not kept here. It lives in each crate's own Cargo.toml, as +# `[package.metadata.feature-combinations] check`, next to the +# `[features]` table it mirrors, and is read out below with `cargo +# metadata`. Adding a crate to this half means adding that table to its +# manifest — nothing here changes. # # Measured cold on a 14-core machine: the full 209-combination powerset takes # 359s; this split takes 242s, a 33% saving. That is what brings the job in @@ -213,80 +218,75 @@ jobs: check echo "LEAF_SECS=$(( $(date +%s) - start ))" >> "$GITHUB_ENV" - # Curated combinations for the user-facing crates. Each line is - # `crate|cargo feature flags`; an empty flags field means default - # features. Every entry is a configuration someone can actually select - # and that the manifests or ffi-builds.yml document. + # Curated combinations for the user-facing crates. The list itself is + # not here: it lives in each crate's own Cargo.toml, under + # `[package.metadata.feature-combinations] check`, right next to the + # `[features]` table it mirrors — so whoever adds a feature is already + # editing the file that decides whether CI covers it. Every workspace + # crate that declares that table is picked up, so adding a crate to this + # half of the job means adding the table and nothing else. + # + # livekit-ffi and livekit-uniffi deliberately do NOT declare one. + # Measured cold, their 8 combinations cost 103s of a 345s total — 30% of + # the budget for two crates, because each TLS change rebuilds livekit + # underneath livekit-ffi. - name: Top-level crates — curated feature combinations shell: bash run: | set -uo pipefail - COMBOS=$(cat <<'EOF' - # --- livekit ------------------------------------------------------- - livekit| - livekit|--no-default-features - livekit|--no-default-features --features native - livekit|--no-default-features --features native,native-tls - livekit|--no-default-features --features native,native-tls-vendored - livekit|--no-default-features --features native,rustls-tls-native-roots - livekit|--no-default-features --features native,rustls-tls-webpki-roots - livekit|--no-default-features --features native,rustls-tls-webpki-roots,glib-main-loop - livekit|--no-default-features --features native,__lk-internal - # --- livekit-api --------------------------------------------------- - livekit-api| - livekit-api|--no-default-features - livekit-api|--no-default-features --features access-token - livekit-api|--no-default-features --features webhooks - livekit-api|--no-default-features --features services,native-tls - livekit-api|--no-default-features --features services,native-tls-vendored - livekit-api|--no-default-features --features services,rustls-tls-webpki-roots - livekit-api|--no-default-features --features services,rustls-tls-native-roots - livekit-api|--no-default-features --features signal-client-native,rustls-tls-webpki-roots - livekit-api|--features rustls-tls-webpki-roots - # livekit-ffi and livekit-uniffi are deliberately NOT here. Measured - # cold, their 8 combinations cost 103s of a 345s total — 30% of the - # budget for two crates, because each TLS change rebuilds livekit - # underneath livekit-ffi. Dropping them takes the job from 345s to - # 242s, which is what makes per-commit affordable. Their shipped - # configurations are still covered: builds.yml builds the whole - # workspace (livekit-ffi at default features) on every PR, and - # ffi-builds.yml builds every shipped per-platform TLS variant on - # push to main. The residual gap — livekit-ffi's non-default TLS - # variants on PRs — is knowingly accepted: covering it cost 103s of a - # 345s budget, for two crates whose shipped configurations are - # already built on every PR and on every push to main. - EOF - ) + + # One `crate|feat,feat` line per combination. `.metadata` is null for + # a crate without the table, and jq indexes null to null, so the + # `// []` covers every other workspace member. + COMBOS=$(cargo metadata --no-deps --format-version 1 | jq -r ' + .packages[] + | .name as $crate + | (.metadata["feature-combinations"].check // [])[] + | "\($crate)|\(join(","))" + ') + + # A renamed table or a bad jq path would otherwise turn this step into + # a silent no-op that still reports success. + if [ -z "$COMBOS" ]; then + echo "::error::no [package.metadata.feature-combinations] table found in the workspace" + exit 1 + fi + + # Features that cannot build on this target (see the matrix comment). + EXCLUDED="${{ matrix.exclude_features || '' }}" start=$(date +%s) - total=0; failed=0 + total=0; failed=0; skipped=0 FAILURES="" - while IFS='|' read -r crate flags; do - # Skip comments and blanks. - [[ -z "${crate// /}" || "$crate" =~ ^[[:space:]]*# ]] && continue - crate="${crate// /}" + while IFS='|' read -r crate features; do + [ -n "$crate" ] || continue - # Android has no OpenSSL to link: drop bare `native-tls` (but keep - # `native-tls-vendored`, which builds it from source). - if [ -n "${{ matrix.exclude_features || '' }}" ] \ - && printf '%s' "$flags" | grep -qE '(^|[ ,])native-tls($|[ ,])'; then - echo "::notice::skipping (no Android OpenSSL): $crate $flags" + skip="" + for ex in ${EXCLUDED//,/ }; do + case ",$features," in + *",$ex,"*) skip="$ex"; break ;; + esac + done + if [ -n "$skip" ]; then + echo "::notice::skipping ($skip unsupported on ${{ matrix.target }}): $crate [$features]" + skipped=$((skipped + 1)) continue fi total=$((total + 1)) - echo "::group::$crate $flags" - if ! $CHECK_CMD -p "$crate" $flags; then + echo "::group::$crate [$features]" + if ! $CHECK_CMD -p "$crate" --no-default-features ${features:+--features "$features"}; then failed=$((failed + 1)) - FAILURES="${FAILURES}\n $crate $flags" + FAILURES="${FAILURES}\n $crate [$features]" fi echo "::endgroup::" done <<< "$COMBOS" echo "CURATED_SECS=$(( $(date +%s) - start ))" >> "$GITHUB_ENV" echo "CURATED_TOTAL=$total" >> "$GITHUB_ENV" + echo "CURATED_SKIPPED=$skipped" >> "$GITHUB_ENV" echo "CURATED_FAILED=$failed" >> "$GITHUB_ENV" - echo "Ran $total curated combinations, $failed failed." + echo "Ran $total curated combinations, $failed failed, $skipped skipped." if [ "$failed" -gt 0 ]; then echo -e "::error::failed curated combinations:$FAILURES" exit 1 @@ -304,11 +304,12 @@ jobs: echo "| --- | --- |" echo "| Baseline \`cargo check\` (default features) | $(secs "${BASELINE_SECS:-}") |" echo "| Leaf crates, full powerset (83 combos) | $(secs "${LEAF_SECS:-}") |" - echo "| Top-level crates, curated (${CURATED_TOTAL:-?} combos) | $(secs "${CURATED_SECS:-}") |" + echo "| Top-level crates, curated from Cargo.toml (${CURATED_TOTAL:-?} combos) | $(secs "${CURATED_SECS:-}") |" if [ -n "${LEAF_SECS:-}" ] && [ -n "${CURATED_SECS:-}" ]; then awk -v l="$LEAF_SECS" -v c="$CURATED_SECS" -v b="${BASELINE_SECS:-0}" \ 'BEGIN { printf "| **Feature checking total** | **%.1f min** |\n", (l+c)/60 if (b > 0) printf "| Total as a multiple of baseline | %.1fx |\n", (l+c)/b }' fi + echo "| Curated skipped (unsupported on target) | ${CURATED_SKIPPED:-n/a} |" echo "| Curated failures | ${CURATED_FAILED:-n/a} |" } >> "$GITHUB_STEP_SUMMARY" diff --git a/AGENTS.md b/AGENTS.md index d5ce57ea5..7e93359c8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -79,18 +79,24 @@ Several crates export items to Swift/Kotlin/Node/Python through UniFFI — `live `.github/workflows/feature-combinations-curated.yml` is the only job in CI that exercises features in *combination*. A `dep:` that one feature pulls in but the code uses unconditionally, or a `?/` forward that silently no-ops, compiles fine under default features and breaks only for the caller who picks a particular set — nothing else catches that. It has two halves, maintained differently, and both need updating by hand. -- **When adding a feature to `livekit` or `livekit-api`**, add the configurations a user would plausibly select to the `COMBOS` list in the "Top-level crates" step - - These two crates get a hand-picked list rather than a powerset, so a new feature is invisible to this job until it is listed there - - One line per configuration, `crate|cargo feature flags`; an empty flags field means default features +- **When adding a feature to `livekit` or `livekit-api`**, add the configurations a user would plausibly select to that crate's own `[package.metadata.feature-combinations] check` table, directly below its `[features]` table + - These two crates get a hand-picked list rather than a powerset, so a new feature is invisible to this job until it is listed there. The workflow reads the tables with `cargo metadata`, so it needs no edit + - Each entry is a complete feature set run as `--no-default-features --features `. List `"default"` to mean a plain `cargo check`, and `[]` for no features at all — `["default", "rustls-tls-webpki-roots"]` is defaults *plus* that feature - Pair the feature with what it realistically ships alongside (a TLS backend together with `native`, say) rather than listing it on its own - Do not add combinations nobody can select — internal `__lk-*` flags and two-TLS-backends-at-once are deliberately absent, and were most of what made the full powerset expensive + - Check the table parses and says what you meant without building anything: + ```bash + cargo metadata --no-deps --format-version 1 \ + | jq -r '.packages[] | .name as $c | (.metadata["feature-combinations"].check // [])[] | "\($c) \(join(","))"' + ``` +- **To give another crate a curated list**, add the same table to its manifest. Any workspace crate that declares one is picked up; `livekit-ffi` and `livekit-uniffi` deliberately do not (see below) - **When adding a workspace crate below `livekit`/`livekit-api`**, add `-p ` to `LEAF_PACKAGES` - Leaf crates get the full `cargo hack --feature-powerset --depth 2`, so listing the crate is all that is needed; its features are then picked up automatically as they are added - cargo-hack only varies the features of packages named with `-p`. An unlisted crate still gets built as a dependency, which makes it easy to assume it is covered when it is not - Keep `--depth 2`. `livekit` alone goes from 67 combinations to 232 at depth 3, and pairwise interactions are where these bugs actually live - A feature that pulls in `openssl-sys` cannot build for the three Android targets — there is no Android OpenSSL to link against. Add it to their `exclude_features` rather than trying to make it work; Android ships rustls (see `ffi-builds.yml`) - `livekit-ffi` and `livekit-uniffi` are excluded on purpose: their combinations cost more than everything else in the job combined, because each TLS change rebuilds `livekit` underneath them. Their shipped configurations are covered by `builds.yml` and `ffi-builds.yml` instead -- Check a change to either list without building anything by appending `--print-command-list`, which enumerates the combinations cargo-hack would run +- Check a change to `LEAF_PACKAGES` without building anything by appending `--print-command-list`, which enumerates the combinations cargo-hack would run (the curated tables have their own `cargo metadata` check, above) ## Documenting changes diff --git a/livekit-api/Cargo.toml b/livekit-api/Cargo.toml index 94ffb722b..d4e7d1255 100644 --- a/livekit-api/Cargo.toml +++ b/livekit-api/Cargo.toml @@ -55,6 +55,23 @@ rustls-tls-native-roots = ["livekit-signaling?/rustls-tls-native-roots", "reqwes rustls-tls-webpki-roots = ["livekit-signaling?/rustls-tls-webpki-roots", "reqwest?/rustls-tls-webpki-roots"] __rustls-tls = ["livekit-signaling?/__rustls-tls", "reqwest?/__rustls"] +# Feature combinations checked in CI. See the equivalent table in +# `livekit/Cargo.toml` for the encoding and why this crate is hand-picked +# rather than run through cargo-hack's powerset. +[package.metadata.feature-combinations] +check = [ + ["default"], + [], + ["access-token"], + ["webhooks"], + ["services", "native-tls"], + ["services", "native-tls-vendored"], + ["services", "rustls-tls-webpki-roots"], + ["services", "rustls-tls-native-roots"], + ["signal-client-native", "rustls-tls-webpki-roots"], + ["default", "rustls-tls-webpki-roots"], +] + [dependencies] livekit-signaling = { workspace = true, optional = true } livekit-protocol = { workspace = true } diff --git a/livekit/Cargo.toml b/livekit/Cargo.toml index 80d058fba..eed51a617 100644 --- a/livekit/Cargo.toml +++ b/livekit/Cargo.toml @@ -32,6 +32,29 @@ __rustls-tls = ["livekit-signaling/__rustls-tls"] __lk-internal = [] # internal features (used by livekit-ffi) __lk-e2e-test = ["livekit-data-stream/test-utils"] # end-to-end testing with a LiveKit server +# Feature combinations checked in CI, read by +# `.github/workflows/feature-combinations-curated.yml` via `cargo metadata`. +# Each entry is a complete feature set, run as +# `cargo check --no-default-features --features `: +# list `"default"` to mean a plain `cargo check`, and `[]` for no features at +# all. This crate gets a hand-picked list rather than cargo-hack's powerset +# (most of the powerset is unselectable — two TLS backends at once, or an +# internal `__lk-*` flag paired with something unrelated), so a new feature is +# invisible to CI until it is listed here. Pair a feature with what it +# realistically ships alongside rather than listing it alone. +[package.metadata.feature-combinations] +check = [ + ["default"], + [], + ["native"], + ["native", "native-tls"], + ["native", "native-tls-vendored"], + ["native", "rustls-tls-native-roots"], + ["native", "rustls-tls-webpki-roots"], + ["native", "rustls-tls-webpki-roots", "glib-main-loop"], + ["native", "__lk-internal"], +] + [dependencies] livekit-signaling = { workspace = true } libwebrtc = { workspace = true } From 5401938badb152ff9e486c5a44a1bd0d7836d38c Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Tue, 8 Sep 2026 17:06:02 -0400 Subject: [PATCH 10/12] feat: add mode powerset / curated to Cargo.toml files --- .../feature-combinations-curated.yml | 130 ++++++++++++------ AGENTS.md | 58 +++++--- libwebrtc/Cargo.toml | 5 + livekit-api/Cargo.toml | 11 +- livekit-common/Cargo.toml | 6 + livekit-data-stream/Cargo.toml | 5 + livekit-datatrack/Cargo.toml | 5 + livekit-net/Cargo.toml | 5 + livekit-protocol/Cargo.toml | 5 + livekit-region/Cargo.toml | 5 + livekit-signaling/Cargo.toml | 5 + livekit-token-source/Cargo.toml | 5 + livekit-token/Cargo.toml | 5 + livekit/Cargo.toml | 16 +-- webrtc-sys/Cargo.toml | 5 + 15 files changed, 195 insertions(+), 76 deletions(-) diff --git a/.github/workflows/feature-combinations-curated.yml b/.github/workflows/feature-combinations-curated.yml index f4fef3e9d..683994557 100644 --- a/.github/workflows/feature-combinations-curated.yml +++ b/.github/workflows/feature-combinations-curated.yml @@ -13,26 +13,25 @@ # limitations under the License. # Feature checking, split by where combinatorial coverage is actually worth -# paying for. Two halves: +# paying for. Which half a crate lands in is declared by the crate itself, in +# `[package.metadata.feature-combinations] mode` in its Cargo.toml, and read +# out here with `cargo metadata`. Nothing in this file lists crate names. # -# 1. Leaf crates get the full `--feature-powerset --depth 2`. Their feature +# mode = "powerset" — the full `cargo hack --feature-powerset --depth 2`. +# 104 combinations across the eleven crates that declare it. Their feature # sets are small, so exhaustive coverage is cheap, and they are where the -# `dep:`/`?/` plumbing bugs actually live. The set is every publishable -# workspace crate that has features and is not one of the two below — note -# that is *not* the same as "everything livekit depends on": -# livekit-token-source is a sibling crate, not a dependency, and belongs -# here all the same. +# `dep:`/`?/` plumbing bugs actually live. Adding a feature to one of them +# needs no change anywhere: cargo-hack picks it up. # -# 2. livekit and livekit-api get a hand-picked list of 19 combinations -# instead of their 125-combination powerset. Most of those 125 are -# configurations nobody can actually select — two TLS backends at once, or -# an internal `__lk-*` feature paired with something unrelated. +# mode = "curated" — only the combinations listed in that crate's `check` +# list. livekit and livekit-api use this: 19 combinations instead of their +# 125-combination powerset, because most of those 125 are configurations +# nobody can actually select — two TLS backends at once, or an internal +# `__lk-*` feature paired with something unrelated. The tradeoff is that a +# new feature is invisible to this job until it is added to that list. # -# That list is not kept here. It lives in each crate's own Cargo.toml, as -# `[package.metadata.feature-combinations] check`, next to the -# `[features]` table it mirrors, and is read out below with `cargo -# metadata`. Adding a crate to this half means adding that table to its -# manifest — nothing here changes. +# A crate with no such table is not feature-checked here at all. livekit-ffi +# and livekit-uniffi are the deliberate omissions; see the curated step below. # # Measured cold on a 14-core machine: the full 209-combination powerset takes # 359s; this split takes 242s, a 33% saving. That is what brings the job in @@ -43,9 +42,10 @@ # dropped combinations is not caught anywhere. There is no exhaustive backstop # — the nightly full-powerset workflow was removed once this proved to be the # better cost/benefit. To get the full 209 back for a one-off investigation, -# run it by hand: +# run it by hand over every crate that declares a mode: # -# cargo hack -p livekit -p livekit-api $LEAF_PACKAGES \ +# cargo hack $(cargo metadata --no-deps --format-version 1 \ +# | jq -r '.packages[] | select(.metadata["feature-combinations"]) | "-p \(.name)"') \ # --feature-powerset --depth 2 --keep-going check name: Feature Combinations (per-commit) @@ -73,19 +73,6 @@ concurrency: env: CARGO_TERM_COLOR: always - # Leaf/internal crates: full powerset. 83 combinations across the ten. - LEAF_PACKAGES: >- - -p livekit-signaling - -p livekit-net - -p livekit-region - -p livekit-protocol - -p livekit-datatrack - -p livekit-data-stream - -p livekit-common - -p libwebrtc - -p webrtc-sys - -p livekit-token - -p livekit-token-source jobs: curated: @@ -197,26 +184,81 @@ jobs: restore-keys: | ${{ runner.os }}-${{ matrix.target }}-cargo-registry- + # The two package sets, and the curated combinations, all come out of + # `[package.metadata.feature-combinations]` in each crate's own Cargo.toml + # — so a crate opts into this job by editing its own manifest, next to the + # `[features]` table that decides what needs covering. + - name: Resolve feature-combination sets from Cargo.toml + shell: bash + run: | + set -uo pipefail + META=$(cargo metadata --no-deps --format-version 1) + + # `.metadata` is null for a crate that declares nothing, and jq + # indexes null to null, so these select() calls simply skip it. + by_mode() { + printf '%s' "$META" | jq -r --arg mode "$1" ' + .packages[] + | select(.metadata["feature-combinations"].mode == $mode) + | "-p \(.name)" + ' | sort | tr '\n' ' ' + } + + # A typo'd mode, or a curated crate with nothing to run, would + # otherwise drop that crate from CI without failing anything. + BAD=$(printf '%s' "$META" | jq -r ' + .packages[] + | .name as $crate + | .metadata["feature-combinations"] as $fc + | select($fc != null) + | if ($fc.mode != "powerset" and $fc.mode != "curated") then + "\($crate): unknown mode \($fc.mode // "") (want \"powerset\" or \"curated\")" + elif ($fc.mode == "curated" and (($fc.check // []) | length) == 0) then + "\($crate): mode is \"curated\" but check list is missing or empty" + elif ($fc.mode == "powerset" and $fc.check != null) then + "\($crate): mode is \"powerset\", so its check list is silently ignored" + else empty end + ') + if [ -n "$BAD" ]; then + echo "::error::bad [package.metadata.feature-combinations] tables:" + echo "$BAD" + exit 1 + fi + + POWERSET_PACKAGES=$(by_mode powerset) + CURATED_PACKAGES=$(by_mode curated) + if [ -z "${POWERSET_PACKAGES// /}" ] || [ -z "${CURATED_PACKAGES// /}" ]; then + echo "::error::resolved an empty package set (powerset='$POWERSET_PACKAGES' curated='$CURATED_PACKAGES')" + exit 1 + fi + + echo "POWERSET_PACKAGES=$POWERSET_PACKAGES" >> "$GITHUB_ENV" + echo "CURATED_PACKAGES=$CURATED_PACKAGES" >> "$GITHUB_ENV" + echo "powerset: $POWERSET_PACKAGES" + echo "curated: $CURATED_PACKAGES" + # A plain default-feature check of the same crates, so the feature steps # below have something to be measured against in the run summary. - name: Baseline check (default features) shell: bash run: | start=$(date +%s) - $CHECK_CMD $LEAF_PACKAGES -p livekit -p livekit-api + $CHECK_CMD $POWERSET_PACKAGES $CURATED_PACKAGES echo "BASELINE_SECS=$(( $(date +%s) - start ))" >> "$GITHUB_ENV" - - name: Leaf crates — full feature powerset (depth 2) + - name: Powerset crates — full feature powerset (depth 2) shell: bash run: | + HACK_ARGS="--feature-powerset --depth 2 --keep-going ${{ matrix.exclude_features && format('--exclude-features {0}', matrix.exclude_features) || '' }}" + + # Counted rather than hardcoded, so the run summary cannot drift from + # what actually ran. --print-command-list compiles nothing. + POWERSET_TOTAL=$($HACK_CMD $POWERSET_PACKAGES $HACK_ARGS --print-command-list check 2>/dev/null | grep -c '^cargo' || true) + echo "POWERSET_TOTAL=$POWERSET_TOTAL" >> "$GITHUB_ENV" + start=$(date +%s) - $HACK_CMD $LEAF_PACKAGES \ - --feature-powerset \ - --depth 2 \ - --keep-going \ - ${{ matrix.exclude_features && format('--exclude-features {0}', matrix.exclude_features) || '' }} \ - check - echo "LEAF_SECS=$(( $(date +%s) - start ))" >> "$GITHUB_ENV" + $HACK_CMD $POWERSET_PACKAGES $HACK_ARGS check + echo "POWERSET_SECS=$(( $(date +%s) - start ))" >> "$GITHUB_ENV" # Curated combinations for the user-facing crates. The list itself is # not here: it lives in each crate's own Cargo.toml, under @@ -303,10 +345,10 @@ jobs: echo "| Measurement | Value |" echo "| --- | --- |" echo "| Baseline \`cargo check\` (default features) | $(secs "${BASELINE_SECS:-}") |" - echo "| Leaf crates, full powerset (83 combos) | $(secs "${LEAF_SECS:-}") |" - echo "| Top-level crates, curated from Cargo.toml (${CURATED_TOTAL:-?} combos) | $(secs "${CURATED_SECS:-}") |" - if [ -n "${LEAF_SECS:-}" ] && [ -n "${CURATED_SECS:-}" ]; then - awk -v l="$LEAF_SECS" -v c="$CURATED_SECS" -v b="${BASELINE_SECS:-0}" \ + echo "| Powerset crates (${POWERSET_TOTAL:-?} combos) | $(secs "${POWERSET_SECS:-}") |" + echo "| Curated crates (${CURATED_TOTAL:-?} combos) | $(secs "${CURATED_SECS:-}") |" + if [ -n "${POWERSET_SECS:-}" ] && [ -n "${CURATED_SECS:-}" ]; then + awk -v l="$POWERSET_SECS" -v c="$CURATED_SECS" -v b="${BASELINE_SECS:-0}" \ 'BEGIN { printf "| **Feature checking total** | **%.1f min** |\n", (l+c)/60 if (b > 0) printf "| Total as a multiple of baseline | %.1fx |\n", (l+c)/b }' fi diff --git a/AGENTS.md b/AGENTS.md index 7e93359c8..9cfcfc105 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -77,26 +77,50 @@ Several crates export items to Swift/Kotlin/Node/Python through UniFFI — `live ## Feature combinations -`.github/workflows/feature-combinations-curated.yml` is the only job in CI that exercises features in *combination*. A `dep:` that one feature pulls in but the code uses unconditionally, or a `?/` forward that silently no-ops, compiles fine under default features and breaks only for the caller who picks a particular set — nothing else catches that. It has two halves, maintained differently, and both need updating by hand. - -- **When adding a feature to `livekit` or `livekit-api`**, add the configurations a user would plausibly select to that crate's own `[package.metadata.feature-combinations] check` table, directly below its `[features]` table - - These two crates get a hand-picked list rather than a powerset, so a new feature is invisible to this job until it is listed there. The workflow reads the tables with `cargo metadata`, so it needs no edit - - Each entry is a complete feature set run as `--no-default-features --features `. List `"default"` to mean a plain `cargo check`, and `[]` for no features at all — `["default", "rustls-tls-webpki-roots"]` is defaults *plus* that feature +`.github/workflows/feature-combinations-curated.yml` is the only job in CI that exercises features in *combination*. A `dep:` that one feature pulls in but the code uses unconditionally, or a `?/` forward that silently no-ops, compiles fine under default features and breaks only for the caller who picks a particular set — nothing else catches that. + +**The workflow lists no crate names.** Each crate declares how it wants to be checked, in its own `Cargo.toml` directly below its `[features]` table, and the workflow reads those declarations with `cargo metadata`: + +```toml +[package.metadata.feature-combinations] +mode = "powerset" # cargo hack --feature-powerset --depth 2 +``` + +```toml +[package.metadata.feature-combinations] +mode = "curated" # only the combinations listed below +check = [ + ["default"], # a plain `cargo check` + [], # --no-default-features + ["native", "rustls-tls-webpki-roots"], + ["default", "rustls-tls-webpki-roots"], # defaults *plus* one +] +``` + +A crate with no such table is not feature-checked at all. Today eleven crates are `"powerset"`, `livekit` and `livekit-api` are `"curated"`, and `livekit-ffi`/`livekit-uniffi` deliberately declare nothing. + +- **When adding a workspace crate**, give it `mode = "powerset"` and stop there + - Its features are then picked up automatically as they are added, and no CI file changes + - Reach for `"curated"` only when the powerset is genuinely too expensive — that is a real loss of coverage, so it needs a reason recorded next to the table + - cargo-hack only varies the features of packages it is given. A crate with no table still gets built as a dependency, which makes it easy to assume it is covered when it is not +- **When adding a feature to a `"curated"` crate** (`livekit`, `livekit-api`), add the configurations a user would plausibly select to that crate's `check` list + - A curated crate's new feature is invisible to this job until it is listed there — this is the standing cost of `"curated"` + - Each entry is a complete feature set, run as `--no-default-features --features `. Spell defaults as `"default"`, since a bare `[]` already means "no features at all" - Pair the feature with what it realistically ships alongside (a TLS backend together with `native`, say) rather than listing it on its own - Do not add combinations nobody can select — internal `__lk-*` flags and two-TLS-backends-at-once are deliberately absent, and were most of what made the full powerset expensive - - Check the table parses and says what you meant without building anything: - ```bash - cargo metadata --no-deps --format-version 1 \ - | jq -r '.packages[] | .name as $c | (.metadata["feature-combinations"].check // [])[] | "\($c) \(join(","))"' - ``` -- **To give another crate a curated list**, add the same table to its manifest. Any workspace crate that declares one is picked up; `livekit-ffi` and `livekit-uniffi` deliberately do not (see below) -- **When adding a workspace crate below `livekit`/`livekit-api`**, add `-p ` to `LEAF_PACKAGES` - - Leaf crates get the full `cargo hack --feature-powerset --depth 2`, so listing the crate is all that is needed; its features are then picked up automatically as they are added - - cargo-hack only varies the features of packages named with `-p`. An unlisted crate still gets built as a dependency, which makes it easy to assume it is covered when it is not +- **Check any change to these tables without building anything.** This prints exactly what CI will do: + ```bash + cargo metadata --no-deps --format-version 1 | jq -r ' + .packages[] | .name as $c | .metadata["feature-combinations"] as $fc + | select($fc != null) + | if $fc.mode == "curated" then ($fc.check[] | "curated \($c) \(join(","))") + else "powerset \($c)" end' + ``` + For a `"powerset"` crate, `cargo hack -p --feature-powerset --depth 2 --print-command-list check` enumerates the combinations cargo-hack would run +- The workflow validates the tables before running anything and fails on a typo'd `mode`, a `"curated"` crate with an empty `check`, a `"powerset"` crate carrying a `check` list that would be silently ignored, or either package set resolving to empty. A malformed table fails the job rather than quietly dropping a crate from CI - Keep `--depth 2`. `livekit` alone goes from 67 combinations to 232 at depth 3, and pairwise interactions are where these bugs actually live -- A feature that pulls in `openssl-sys` cannot build for the three Android targets — there is no Android OpenSSL to link against. Add it to their `exclude_features` rather than trying to make it work; Android ships rustls (see `ffi-builds.yml`) -- `livekit-ffi` and `livekit-uniffi` are excluded on purpose: their combinations cost more than everything else in the job combined, because each TLS change rebuilds `livekit` underneath them. Their shipped configurations are covered by `builds.yml` and `ffi-builds.yml` instead -- Check a change to `LEAF_PACKAGES` without building anything by appending `--print-command-list`, which enumerates the combinations cargo-hack would run (the curated tables have their own `cargo metadata` check, above) +- A feature that pulls in `openssl-sys` cannot build for the three Android targets — there is no Android OpenSSL to link against. Add it to their `exclude_features` in the workflow matrix rather than trying to make it work; Android ships rustls (see `ffi-builds.yml`). It is honoured by both halves of the job +- `livekit-ffi` and `livekit-uniffi` declare no table on purpose: their combinations cost more than everything else in the job combined, because each TLS change rebuilds `livekit` underneath them. Their shipped configurations are covered by `builds.yml` and `ffi-builds.yml` instead ## Documenting changes diff --git a/libwebrtc/Cargo.toml b/libwebrtc/Cargo.toml index e288933c2..c01fce568 100644 --- a/libwebrtc/Cargo.toml +++ b/libwebrtc/Cargo.toml @@ -16,6 +16,11 @@ default = [] # running in your application. glib-main-loop = [ "dep:glib" ] +# How CI checks this crate's features, read by +# `.github/workflows/feature-combinations-curated.yml` via `cargo metadata`. +[package.metadata.feature-combinations] +mode = "powerset" + [dependencies] log = { workspace = true } serde = { workspace = true, features = ["derive"] } diff --git a/livekit-api/Cargo.toml b/livekit-api/Cargo.toml index d4e7d1255..7aa4eb60d 100644 --- a/livekit-api/Cargo.toml +++ b/livekit-api/Cargo.toml @@ -56,12 +56,13 @@ rustls-tls-webpki-roots = ["livekit-signaling?/rustls-tls-webpki-roots", "reqwes __rustls-tls = ["livekit-signaling?/__rustls-tls", "reqwest?/__rustls"] # Feature combinations checked in CI. See the equivalent table in -# `livekit/Cargo.toml` for the encoding and why this crate is hand-picked -# rather than run through cargo-hack's powerset. +# `livekit/Cargo.toml` for the encoding, and what `mode = "curated"` costs +# relative to cargo-hack's powerset. [package.metadata.feature-combinations] +mode = "curated" check = [ - ["default"], - [], + ["default"], # a plain `cargo check` + [], # --no-default-features ["access-token"], ["webhooks"], ["services", "native-tls"], @@ -69,7 +70,7 @@ check = [ ["services", "rustls-tls-webpki-roots"], ["services", "rustls-tls-native-roots"], ["signal-client-native", "rustls-tls-webpki-roots"], - ["default", "rustls-tls-webpki-roots"], + ["default", "rustls-tls-webpki-roots"], # defaults *plus* one ] [dependencies] diff --git a/livekit-common/Cargo.toml b/livekit-common/Cargo.toml index a4075b76a..610a466f2 100644 --- a/livekit-common/Cargo.toml +++ b/livekit-common/Cargo.toml @@ -16,3 +16,9 @@ uniffi = { workspace = true, features = ["scaffolding-ffi-buffer-fns"], optional # Exposes this crate's shared FFI type registrations. Enabled transitively by # livekit-uniffi so every component borrows one converter per type. uniffi = ["dep:uniffi", "dep:bytes"] + +# How CI checks this crate's features, read by +# `.github/workflows/feature-combinations-curated.yml` via `cargo metadata`. +# `mode = "powerset"` is the full `cargo hack --feature-powerset --depth 2`: +[package.metadata.feature-combinations] +mode = "powerset" diff --git a/livekit-data-stream/Cargo.toml b/livekit-data-stream/Cargo.toml index fb906af54..0eefc1bed 100644 --- a/livekit-data-stream/Cargo.toml +++ b/livekit-data-stream/Cargo.toml @@ -10,6 +10,11 @@ repository.workspace = true [features] test-utils = ["dep:rand"] +# How CI checks this crate's features, read by +# `.github/workflows/feature-combinations-curated.yml` via `cargo metadata`. +[package.metadata.feature-combinations] +mode = "powerset" + [dependencies] livekit-common = { workspace = true } livekit-protocol = { workspace = true } diff --git a/livekit-datatrack/Cargo.toml b/livekit-datatrack/Cargo.toml index a212f0fa7..6c30b644b 100644 --- a/livekit-datatrack/Cargo.toml +++ b/livekit-datatrack/Cargo.toml @@ -28,6 +28,11 @@ indexmap = "2" uniffi = ["dep:uniffi", "dep:livekit-common", "livekit-common/uniffi"] __fuzz = ["dep:fake"] +# How CI checks this crate's features, read by +# `.github/workflows/feature-combinations-curated.yml` via `cargo metadata`. +[package.metadata.feature-combinations] +mode = "powerset" + [dev-dependencies] test-case = "3.3" fake = { version = "4.4", features = ["derive"] } diff --git a/livekit-net/Cargo.toml b/livekit-net/Cargo.toml index bc643d26e..d42d72692 100644 --- a/livekit-net/Cargo.toml +++ b/livekit-net/Cargo.toml @@ -42,6 +42,11 @@ rustls-tls-webpki-roots = [ ] __rustls-tls = ["tokio-tungstenite?/__rustls-tls", "reqwest?/__rustls"] +# How CI checks this crate's features, read by +# `.github/workflows/feature-combinations-curated.yml` via `cargo metadata`. +[package.metadata.feature-combinations] +mode = "powerset" + [dependencies] async-trait = "0.1" diff --git a/livekit-protocol/Cargo.toml b/livekit-protocol/Cargo.toml index 3bbbe36a0..2f8daa902 100644 --- a/livekit-protocol/Cargo.toml +++ b/livekit-protocol/Cargo.toml @@ -11,3 +11,8 @@ prost = "0.12" serde = { workspace = true } pbjson = "0.6" pbjson-types = "0.6" + +# How CI checks this crate's features, read by +# `.github/workflows/feature-combinations-curated.yml` via `cargo metadata`. +[package.metadata.feature-combinations] +mode = "powerset" diff --git a/livekit-region/Cargo.toml b/livekit-region/Cargo.toml index 67cbbfb06..1deb20e5b 100644 --- a/livekit-region/Cargo.toml +++ b/livekit-region/Cargo.toml @@ -9,3 +9,8 @@ readme = "README.md" [dependencies] serde = { workspace = true, features = ["derive"] } + +# How CI checks this crate's features, read by +# `.github/workflows/feature-combinations-curated.yml` via `cargo metadata`. +[package.metadata.feature-combinations] +mode = "powerset" diff --git a/livekit-signaling/Cargo.toml b/livekit-signaling/Cargo.toml index 3c0464cc9..12b09f311 100644 --- a/livekit-signaling/Cargo.toml +++ b/livekit-signaling/Cargo.toml @@ -23,6 +23,11 @@ rustls-tls-native-roots = ["livekit-net/rustls-tls-native-roots"] rustls-tls-webpki-roots = ["livekit-net/rustls-tls-webpki-roots"] __rustls-tls = ["livekit-net/__rustls-tls"] +# How CI checks this crate's features, read by +# `.github/workflows/feature-combinations-curated.yml` via `cargo metadata`. +[package.metadata.feature-combinations] +mode = "powerset" + [dependencies] livekit-net = { workspace = true } livekit-protocol = { workspace = true } diff --git a/livekit-token-source/Cargo.toml b/livekit-token-source/Cargo.toml index 458a4ec35..89012c51c 100644 --- a/livekit-token-source/Cargo.toml +++ b/livekit-token-source/Cargo.toml @@ -22,6 +22,11 @@ native-tls-vendored = ["livekit-net/native-tls-vendored"] rustls-tls-native-roots = ["livekit-net/rustls-tls-native-roots"] rustls-tls-webpki-roots = ["livekit-net/rustls-tls-webpki-roots"] +# How CI checks this crate's features, read by +# `.github/workflows/feature-combinations-curated.yml` via `cargo metadata`. +[package.metadata.feature-combinations] +mode = "powerset" + [dependencies] async-trait = "0.1" base64 = { version = "0.21", features = ["std"] } diff --git a/livekit-token/Cargo.toml b/livekit-token/Cargo.toml index faf786fff..6034f6698 100644 --- a/livekit-token/Cargo.toml +++ b/livekit-token/Cargo.toml @@ -18,3 +18,8 @@ sha2 = "0.10" jsonwebtoken = { version = "10", default-features = false } hmac = "0.12" signature = "2" + +# How CI checks this crate's features, read by +# `.github/workflows/feature-combinations-curated.yml` via `cargo metadata`. +[package.metadata.feature-combinations] +mode = "powerset" diff --git a/livekit/Cargo.toml b/livekit/Cargo.toml index eed51a617..c1bfc6955 100644 --- a/livekit/Cargo.toml +++ b/livekit/Cargo.toml @@ -32,20 +32,16 @@ __rustls-tls = ["livekit-signaling/__rustls-tls"] __lk-internal = [] # internal features (used by livekit-ffi) __lk-e2e-test = ["livekit-data-stream/test-utils"] # end-to-end testing with a LiveKit server -# Feature combinations checked in CI, read by +# How CI checks this crate's features, read by # `.github/workflows/feature-combinations-curated.yml` via `cargo metadata`. +# # Each entry is a complete feature set, run as -# `cargo check --no-default-features --features `: -# list `"default"` to mean a plain `cargo check`, and `[]` for no features at -# all. This crate gets a hand-picked list rather than cargo-hack's powerset -# (most of the powerset is unselectable — two TLS backends at once, or an -# internal `__lk-*` flag paired with something unrelated), so a new feature is -# invisible to CI until it is listed here. Pair a feature with what it -# realistically ships alongside rather than listing it alone. +# `cargo check --no-default-features --features `. [package.metadata.feature-combinations] +mode = "curated" check = [ - ["default"], - [], + ["default"], # a plain `cargo check` + [], # --no-default-features ["native"], ["native", "native-tls"], ["native", "native-tls-vendored"], diff --git a/webrtc-sys/Cargo.toml b/webrtc-sys/Cargo.toml index 0dc40648e..22af9e0a9 100644 --- a/webrtc-sys/Cargo.toml +++ b/webrtc-sys/Cargo.toml @@ -10,6 +10,11 @@ repository.workspace = true [features] default = [] +# How CI checks this crate's features, read by +# `.github/workflows/feature-combinations-curated.yml` via `cargo metadata`. +[package.metadata.feature-combinations] +mode = "powerset" + [dependencies] cxx = "1.0" log = { workspace = true } From 7486053ec8f2481c4df7c7816cf3dfc26b72382b Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Wed, 9 Sep 2026 11:39:27 -0400 Subject: [PATCH 11/12] fix: address line ending inconsistency jq.exe uses native windows line endings which the script wasn't taking into account properly. --- .../workflows/feature-combinations-curated.yml | 18 ++++++++++++------ AGENTS.md | 2 +- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/.github/workflows/feature-combinations-curated.yml b/.github/workflows/feature-combinations-curated.yml index 683994557..770ca2db0 100644 --- a/.github/workflows/feature-combinations-curated.yml +++ b/.github/workflows/feature-combinations-curated.yml @@ -45,8 +45,8 @@ # run it by hand over every crate that declares a mode: # # cargo hack $(cargo metadata --no-deps --format-version 1 \ -# | jq -r '.packages[] | select(.metadata["feature-combinations"]) | "-p \(.name)"') \ -# --feature-powerset --depth 2 --keep-going check +# | jq -r '.packages[] | select(.metadata["feature-combinations"]) | "-p \(.name)"' \ +# | tr -d '\r') --feature-powerset --depth 2 --keep-going check name: Feature Combinations (per-commit) on: @@ -196,12 +196,16 @@ jobs: # `.metadata` is null for a crate that declares nothing, and jq # indexes null to null, so these select() calls simply skip it. + # + # `tr -d '\r'` is not optional: jq.exe on the Windows runners writes + # CRLF, and a CR that survives rides into the package name, which + # cargo rejects with `invalid character in package name`. by_mode() { printf '%s' "$META" | jq -r --arg mode "$1" ' .packages[] | select(.metadata["feature-combinations"].mode == $mode) | "-p \(.name)" - ' | sort | tr '\n' ' ' + ' | tr -d '\r' | sort | tr '\n' ' ' } # A typo'd mode, or a curated crate with nothing to run, would @@ -218,7 +222,7 @@ jobs: elif ($fc.mode == "powerset" and $fc.check != null) then "\($crate): mode is \"powerset\", so its check list is silently ignored" else empty end - ') + ' | tr -d '\r') if [ -n "$BAD" ]; then echo "::error::bad [package.metadata.feature-combinations] tables:" echo "$BAD" @@ -279,13 +283,15 @@ jobs: # One `crate|feat,feat` line per combination. `.metadata` is null for # a crate without the table, and jq indexes null to null, so the - # `// []` covers every other workspace member. + # `// []` covers every other workspace member. `tr -d '\r'` because + # jq.exe on the Windows runners writes CRLF, and a surviving CR ends + # up inside the `--features` value. COMBOS=$(cargo metadata --no-deps --format-version 1 | jq -r ' .packages[] | .name as $crate | (.metadata["feature-combinations"].check // [])[] | "\($crate)|\(join(","))" - ') + ' | tr -d '\r') # A renamed table or a bad jq path would otherwise turn this step into # a silent no-op that still reports success. diff --git a/AGENTS.md b/AGENTS.md index 9cfcfc105..ec4245124 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -114,7 +114,7 @@ A crate with no such table is not feature-checked at all. Today eleven crates ar .packages[] | .name as $c | .metadata["feature-combinations"] as $fc | select($fc != null) | if $fc.mode == "curated" then ($fc.check[] | "curated \($c) \(join(","))") - else "powerset \($c)" end' + else "powerset \($c)" end' | tr -d '\r' ``` For a `"powerset"` crate, `cargo hack -p --feature-powerset --depth 2 --print-command-list check` enumerates the combinations cargo-hack would run - The workflow validates the tables before running anything and fails on a typo'd `mode`, a `"curated"` crate with an empty `check`, a `"powerset"` crate carrying a `check` list that would be silently ignored, or either package set resolving to empty. A malformed table fails the job rather than quietly dropping a crate from CI From 5ae3178b6542048fecb593a5e4bc6cac4a0862b7 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Thu, 10 Sep 2026 09:24:32 -0400 Subject: [PATCH 12/12] fix: drop add-all-features-complication push trigger --- .github/workflows/feature-combinations-curated.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/feature-combinations-curated.yml b/.github/workflows/feature-combinations-curated.yml index 770ca2db0..37daf0bf8 100644 --- a/.github/workflows/feature-combinations-curated.yml +++ b/.github/workflows/feature-combinations-curated.yml @@ -63,9 +63,6 @@ on: push: branches: - "main" - # TEMPORARY: so this runs on the branch that introduces it. Remove before - # merging. - - "add-all-features-complication" concurrency: group: ${{ github.workflow }}-${{ github.ref }}