From 91dcf95e6bfd79cb3ea5265a347e67135904cc93 Mon Sep 17 00:00:00 2001 From: andrejs Date: Thu, 5 Nov 2026 08:56:37 +0000 Subject: [PATCH 1/9] Add muted test case retrieval from testplan.json --- .../lib/allure_ruby_commons/testplan.rb | 23 ++++++++++++++++++- .../spec/unit/testplan_spec.rb | 4 ++-- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/allure-ruby-commons/lib/allure_ruby_commons/testplan.rb b/allure-ruby-commons/lib/allure_ruby_commons/testplan.rb index 8f6afb9..54ef02c 100644 --- a/allure-ruby-commons/lib/allure_ruby_commons/testplan.rb +++ b/allure-ruby-commons/lib/allure_ruby_commons/testplan.rb @@ -22,13 +22,34 @@ def test_names @test_names ||= tests&.map { |test| test[:selector] } end + # Test case ids of muted tests + # + # @return [Array] + def muted_test_ids + @muted_test_ids ||= muted_tests&.map { |test| test[:testCaseId] } + end + private + # Parsed test plan object + # + # @return [Hash] + def test_plan + @test_plan ||= load_json(test_plan_path) + end + # Tests to execute from allure testplan.json # # @return [Array] def tests - @tests ||= load_json(test_plan_path)&.fetch(:tests) if test_plan_path + @tests ||= test_plan&.fetch(:tests, nil) if test_plan_path + end + + # Tests to skip from allure testplan.json + # + # @return [Array] + def muted_tests + @muted_tests ||= test_plan&.fetch(:mutedTests, nil) if test_plan_path end # Fetch test plan path diff --git a/allure-ruby-commons/spec/unit/testplan_spec.rb b/allure-ruby-commons/spec/unit/testplan_spec.rb index 02de04b..258e89f 100644 --- a/allure-ruby-commons/spec/unit/testplan_spec.rb +++ b/allure-ruby-commons/spec/unit/testplan_spec.rb @@ -9,7 +9,7 @@ described_class.instance_variable_set(:@test_ids, nil) described_class.instance_variable_set(:@test_names, nil) - :@test_plan_path.tap do |var| + %i[@test_plan_path @test_plan].each do |var| described_class.send(:remove_instance_variable, var) if described_class.instance_variable_defined?(var) end end @@ -30,7 +30,7 @@ context "handles" do let(:path) { "#{Dir.pwd}/spec/fixtures/test_plan/malformed/testplan.json" } - it "malformed testplan.json" do + it "malformed testplan.json", :focus do expect(described_class.test_ids).to eq(nil) expect(described_class.test_names).to eq(nil) end From 33bf2353d788a69b99337b1c449155c0c21deb3f Mon Sep 17 00:00:00 2001 From: andrejs Date: Thu, 5 Nov 2026 10:58:05 +0000 Subject: [PATCH 2/9] Add functionality to skip tests present in test plan --- .../lib/allure_cucumber/formatter.rb | 22 ++++++++++-- .../allure_cucumber/models/cucumber_model.rb | 7 ++++ allure-cucumber/spec/cucumber_helper.rb | 4 ++- allure-cucumber/spec/spec_helper.rb | 2 +- .../unit/formatter_test_run_finished_spec.rb | 34 +++++++++++++++++-- allure-rspec/lib/allure_rspec/formatter.rb | 9 ++++- allure-rspec/spec/spec_helper.rb | 2 +- .../unit/formatter_example_started_spec.rb | 19 +++++++++++ .../lib/allure_ruby_commons/testplan.rb | 6 ++-- .../fixtures/test_plan/partial/testplan.json | 5 +++ .../spec/unit/testplan_spec.rb | 21 ++++++++++++ 11 files changed, 119 insertions(+), 12 deletions(-) create mode 100644 allure-ruby-commons/spec/fixtures/test_plan/partial/testplan.json diff --git a/allure-cucumber/lib/allure_cucumber/formatter.rb b/allure-cucumber/lib/allure_cucumber/formatter.rb index c9ce002..3acb00e 100644 --- a/allure-cucumber/lib/allure_cucumber/formatter.rb +++ b/allure-cucumber/lib/allure_cucumber/formatter.rb @@ -1,10 +1,13 @@ # frozen_string_literal: true require "cucumber/core" +require "cucumber/glue/dsl" module AllureCucumber # Main formatter class. Translates cucumber event to allure lifecycle class CucumberFormatter + include Cucumber::Glue::Dsl + # @return [Hash] hook handler methods HOOK_HANDLERS = { "Before hook" => :start_prepare_fixture, @@ -37,6 +40,7 @@ def initialize(config) config.on_event(:test_case_finished) { |event| on_test_case_finished(event) } register_test_run_hook_finished(config) + register_muting_hook end # Clean test result directory before starting run @@ -69,8 +73,12 @@ def on_test_run_hook_finished(event) # @param [Cucumber::Events::TestCaseStarted] event # @return [void] def on_test_case_started(event) - lifecycle.start_test_container(Allure::TestResultContainer.new(name: event.test_case.name)) - lifecycle.start_test_case(cucumber_model.test_result(event.test_case)) + test_case = event.test_case + result = cucumber_model.test_result(test_case) + result.status_details.muted = true if cucumber_model.muted?(test_case) + + lifecycle.start_test_container(Allure::TestResultContainer.new(name: test_case.name)) + lifecycle.start_test_case(result) end # Handle test step started event @@ -114,6 +122,16 @@ def on_test_case_finished(event) attr_reader :lifecycle, :cucumber_model + # Skip matched scenarios through Cucumber's public hook DSL. + # @return [void] + def register_muting_hook + model = cucumber_model + Around do |scenario, run| + skip_this_scenario("Skipped by Allure test plan") if model.muted?(scenario) + run.call + end + end + # Register global hook finished event # @param [Cucumber::Configuration] config # @return [void] diff --git a/allure-cucumber/lib/allure_cucumber/models/cucumber_model.rb b/allure-cucumber/lib/allure_cucumber/models/cucumber_model.rb index f08d4d9..ae5928f 100644 --- a/allure-cucumber/lib/allure_cucumber/models/cucumber_model.rb +++ b/allure-cucumber/lib/allure_cucumber/models/cucumber_model.rb @@ -33,6 +33,13 @@ def test_result(test_case) ) end + # Match a scenario or outline against the test plan's muted identities. + # @param [Cucumber::Core::Test::Case, Cucumber::RunningTestCase::TestCase] test_case + # @return [Boolean, nil] + def muted?(test_case) + Allure::TestPlan.muted_test_ids&.include?(Scenario.new(test_case, ast_lookup).test_case_id) + end + # Convert to allure step result # @param [Cucumber::Core::Test::Step] test_step # @return [Hash] diff --git a/allure-cucumber/spec/cucumber_helper.rb b/allure-cucumber/spec/cucumber_helper.rb index 076c89f..7f904b1 100644 --- a/allure-cucumber/spec/cucumber_helper.rb +++ b/allure-cucumber/spec/cucumber_helper.rb @@ -3,7 +3,9 @@ require "cucumber/cli/main" class MockKernel - def exit(status); end + def exit(status) + status + end end class CucumberHelper diff --git a/allure-cucumber/spec/spec_helper.rb b/allure-cucumber/spec/spec_helper.rb index 2ea2afa..eb314eb 100644 --- a/allure-cucumber/spec/spec_helper.rb +++ b/allure-cucumber/spec/spec_helper.rb @@ -50,6 +50,6 @@ def run_cucumber_cli(feature, support_code: CucumberHelper::ENV) Thread.new do CucumberHelper.new(test_tmp_dir).execute(feature, support_code: support_code) - end.join + end.value end end diff --git a/allure-cucumber/spec/unit/formatter_test_run_finished_spec.rb b/allure-cucumber/spec/unit/formatter_test_run_finished_spec.rb index 90e50c0..d99ee13 100644 --- a/allure-cucumber/spec/unit/formatter_test_run_finished_spec.rb +++ b/allure-cucumber/spec/unit/formatter_test_run_finished_spec.rb @@ -4,7 +4,7 @@ include_context "allure mock" include_context "cucumber runner" - before do + it "writes run-level files" do run_cucumber_cli(<<~FEATURE) Feature: Simple feature @@ -12,10 +12,38 @@ Simple scenario description Given a is 5 FEATURE - end - it "writes run-level files" do expect(lifecycle).to have_received(:write_environment).once expect(lifecycle).to have_received(:write_globals).once end + + context "with muted tests" do + let(:muted_id) { Digest::MD5.hexdigest("#{test_tmp_dir}/features/test.feature#Muted scenario") } + + before do + allow(Allure::TestPlan).to receive(:muted_test_ids).and_return([muted_id]) + allow(Allure::AllureLifecycle).to receive(:new).and_call_original + end + + it "skips a scenario whose test case id is muted" do + status = run_cucumber_cli(<<~FEATURE) + Feature: Simple feature + + Scenario: Muted scenario + Then step fails with simple exception + + Scenario: Add a to b + Simple scenario description + Given a is 5 + FEATURE + + results = Dir["#{test_tmp_dir}/reports/allure-results/*-result.json"].flat_map { |f| JSON.parse(File.read(f)) } + + aggregate_failures do + expect(status).to eq(0) + expect(results.find { |t| t["name"] == "Add a to b" }["status"]).to eq("passed") + expect(results.find { |t| t["name"] == "Muted scenario" }["status"]).to eq("skipped") + end + end + end end diff --git a/allure-rspec/lib/allure_rspec/formatter.rb b/allure-rspec/lib/allure_rspec/formatter.rb index 80cdda7..2f9fbb0 100644 --- a/allure-rspec/lib/allure_rspec/formatter.rb +++ b/allure-rspec/lib/allure_rspec/formatter.rb @@ -78,7 +78,14 @@ def example_group_started(example_group_notification) # @param [RSpec::Core::Notifications::ExampleNotification] example_notification # @return [void] def example_started(example_notification) - lifecycle.start_test_case(test_result(example_notification.example)) + example = example_notification.example + result = test_result(example) + if Allure::TestPlan.muted_test_ids&.include?(result.test_case_id) + result.status_details.muted = true + example.metadata[:skip] = "Skipped by Allure test plan" + end + + lifecycle.start_test_case(result) end # Finishes example diff --git a/allure-rspec/spec/spec_helper.rb b/allure-rspec/spec/spec_helper.rb index f938dd2..000bcbc 100644 --- a/allure-rspec/spec/spec_helper.rb +++ b/allure-rspec/spec/spec_helper.rb @@ -60,6 +60,6 @@ def run_rspec(spec) allow(RSpec).to receive(:configuration).and_return(configuration) allow(RSpec).to receive(:world).and_return(world) - Thread.new { RspecRunner.new(test_tmp_dir).run(spec) }.join + Thread.new { RspecRunner.new(test_tmp_dir).run(spec) }.value end end diff --git a/allure-rspec/spec/unit/formatter_example_started_spec.rb b/allure-rspec/spec/unit/formatter_example_started_spec.rb index 9582738..86463a3 100644 --- a/allure-rspec/spec/unit/formatter_example_started_spec.rb +++ b/allure-rspec/spec/unit/formatter_example_started_spec.rb @@ -21,6 +21,25 @@ ] end + context "with muted tests" do + let(:muted_id) { Digest::MD5.hexdigest("#{test_tmp_dir}/spec/test_spec.rb##{suite} #{spec}") } + + before do + allow(Allure::TestPlan).to receive(:muted_test_ids).and_return([muted_id]) + end + it "adds skip metadata to the matching example" do + run_rspec(<<~SPEC) + describe "#{suite}" do + it "#{spec}" do + end + end + SPEC + + example = RSpec.world.example_groups.first.examples.first + expect(example.metadata[:skip]).to eq("Skipped by Allure test plan") + end + end + context "default parameters" do it "are added to allure test case" do run_rspec(<<~SPEC) diff --git a/allure-ruby-commons/lib/allure_ruby_commons/testplan.rb b/allure-ruby-commons/lib/allure_ruby_commons/testplan.rb index 54ef02c..19426b0 100644 --- a/allure-ruby-commons/lib/allure_ruby_commons/testplan.rb +++ b/allure-ruby-commons/lib/allure_ruby_commons/testplan.rb @@ -12,21 +12,21 @@ class << self # # @return [Array] def test_ids - @test_ids ||= tests&.map { |test| test[:id] } + @test_ids ||= tests&.filter_map { |test| test[:id] } end # Test names of executable tests # # @return [Array] def test_names - @test_names ||= tests&.map { |test| test[:selector] } + @test_names ||= tests&.filter_map { |test| test[:selector] } end # Test case ids of muted tests # # @return [Array] def muted_test_ids - @muted_test_ids ||= muted_tests&.map { |test| test[:testCaseId] } + @muted_test_ids ||= muted_tests&.filter_map { |test| test[:testCaseId] } end private diff --git a/allure-ruby-commons/spec/fixtures/test_plan/partial/testplan.json b/allure-ruby-commons/spec/fixtures/test_plan/partial/testplan.json new file mode 100644 index 0000000..26ccfdf --- /dev/null +++ b/allure-ruby-commons/spec/fixtures/test_plan/partial/testplan.json @@ -0,0 +1,5 @@ +{ + "version": "1.0", + "tests": [{}, {"id": "id only"}, {"selector": "selector only"}], + "mutedTests": [{}, {"testCaseId": "muted-id"}] +} diff --git a/allure-ruby-commons/spec/unit/testplan_spec.rb b/allure-ruby-commons/spec/unit/testplan_spec.rb index 258e89f..60e5207 100644 --- a/allure-ruby-commons/spec/unit/testplan_spec.rb +++ b/allure-ruby-commons/spec/unit/testplan_spec.rb @@ -8,6 +8,8 @@ described_class.instance_variable_set(:@tests, nil) described_class.instance_variable_set(:@test_ids, nil) described_class.instance_variable_set(:@test_names, nil) + described_class.instance_variable_set(:@muted_tests, nil) + described_class.instance_variable_set(:@muted_test_ids, nil) %i[@test_plan_path @test_plan].each do |var| described_class.send(:remove_instance_variable, var) if described_class.instance_variable_defined?(var) @@ -24,6 +26,7 @@ it "correct testplan.json" do expect(described_class.test_ids).to eq(test_ids) expect(described_class.test_names).to eq(test_names) + expect(described_class.muted_test_ids).to be_nil end end @@ -45,6 +48,24 @@ end end + context "with partial entries" do + let(:path) { "#{Dir.pwd}/spec/fixtures/test_plan/partial/testplan.json" } + + it "reads muted IDs and ignores absent IDs and selectors" do + expect(described_class.test_ids).to eq(["id only"]) + expect(described_class.test_names).to eq(["selector only"]) + expect(described_class.muted_test_ids).to eq(["muted-id"]) + end + end + + context "without a plan" do + let(:path) { nil } + + it "has no muted IDs" do + expect(described_class.muted_test_ids).to be_nil + end + end + context "handles" do let(:path) { "#{Dir.pwd}/spec/fixtures/test_plan/correct" } From 4a9c683d56d6f5e16ff0075425fc53f8e5a3806f Mon Sep 17 00:00:00 2001 From: andrejs Date: Thu, 5 Nov 2026 13:12:56 +0000 Subject: [PATCH 3/9] Use set for test case id storage --- .../allure_cucumber/models/cucumber_model.rb | 4 ++-- .../unit/formatter_test_run_finished_spec.rb | 3 ++- allure-rspec/lib/allure_rspec/formatter.rb | 2 +- .../unit/formatter_example_started_spec.rb | 2 +- .../lib/allure_ruby_commons/testplan.rb | 18 ++++++++++++++---- allure-ruby-commons/spec/unit/testplan_spec.rb | 18 +++++++++++++----- 6 files changed, 33 insertions(+), 14 deletions(-) diff --git a/allure-cucumber/lib/allure_cucumber/models/cucumber_model.rb b/allure-cucumber/lib/allure_cucumber/models/cucumber_model.rb index ae5928f..67efab8 100644 --- a/allure-cucumber/lib/allure_cucumber/models/cucumber_model.rb +++ b/allure-cucumber/lib/allure_cucumber/models/cucumber_model.rb @@ -35,9 +35,9 @@ def test_result(test_case) # Match a scenario or outline against the test plan's muted identities. # @param [Cucumber::Core::Test::Case, Cucumber::RunningTestCase::TestCase] test_case - # @return [Boolean, nil] + # @return [Boolean] def muted?(test_case) - Allure::TestPlan.muted_test_ids&.include?(Scenario.new(test_case, ast_lookup).test_case_id) + Allure::TestPlan.muted?(Scenario.new(test_case, ast_lookup).test_case_id) end # Convert to allure step result diff --git a/allure-cucumber/spec/unit/formatter_test_run_finished_spec.rb b/allure-cucumber/spec/unit/formatter_test_run_finished_spec.rb index d99ee13..6683f72 100644 --- a/allure-cucumber/spec/unit/formatter_test_run_finished_spec.rb +++ b/allure-cucumber/spec/unit/formatter_test_run_finished_spec.rb @@ -21,7 +21,8 @@ let(:muted_id) { Digest::MD5.hexdigest("#{test_tmp_dir}/features/test.feature#Muted scenario") } before do - allow(Allure::TestPlan).to receive(:muted_test_ids).and_return([muted_id]) + allow(Allure::TestPlan).to receive(:muted?).and_call_original + allow(Allure::TestPlan).to receive(:muted?).with(muted_id).and_return(true) allow(Allure::AllureLifecycle).to receive(:new).and_call_original end diff --git a/allure-rspec/lib/allure_rspec/formatter.rb b/allure-rspec/lib/allure_rspec/formatter.rb index 2f9fbb0..81072ba 100644 --- a/allure-rspec/lib/allure_rspec/formatter.rb +++ b/allure-rspec/lib/allure_rspec/formatter.rb @@ -80,7 +80,7 @@ def example_group_started(example_group_notification) def example_started(example_notification) example = example_notification.example result = test_result(example) - if Allure::TestPlan.muted_test_ids&.include?(result.test_case_id) + if Allure::TestPlan.muted?(result.test_case_id) result.status_details.muted = true example.metadata[:skip] = "Skipped by Allure test plan" end diff --git a/allure-rspec/spec/unit/formatter_example_started_spec.rb b/allure-rspec/spec/unit/formatter_example_started_spec.rb index 86463a3..caf08da 100644 --- a/allure-rspec/spec/unit/formatter_example_started_spec.rb +++ b/allure-rspec/spec/unit/formatter_example_started_spec.rb @@ -25,7 +25,7 @@ let(:muted_id) { Digest::MD5.hexdigest("#{test_tmp_dir}/spec/test_spec.rb##{suite} #{spec}") } before do - allow(Allure::TestPlan).to receive(:muted_test_ids).and_return([muted_id]) + allow(Allure::TestPlan).to receive(:muted?).with(muted_id).and_return(true) end it "adds skip metadata to the matching example" do run_rspec(<<~SPEC) diff --git a/allure-ruby-commons/lib/allure_ruby_commons/testplan.rb b/allure-ruby-commons/lib/allure_ruby_commons/testplan.rb index 19426b0..6ee3c3a 100644 --- a/allure-ruby-commons/lib/allure_ruby_commons/testplan.rb +++ b/allure-ruby-commons/lib/allure_ruby_commons/testplan.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require "set" + module Allure class TestPlan extend JsonHelper @@ -22,15 +24,23 @@ def test_names @test_names ||= tests&.filter_map { |test| test[:selector] } end - # Test case ids of muted tests + # Whether a test case identity is muted # - # @return [Array] - def muted_test_ids - @muted_test_ids ||= muted_tests&.filter_map { |test| test[:testCaseId] } + # @param [String] test_case_id + # @return [Boolean] + def muted?(test_case_id) + muted_test_ids.include?(test_case_id) end private + # Muted test case ids + # + # @return [Set] + def muted_test_ids + @muted_test_ids ||= Set.new(muted_tests&.filter_map { |test| test[:testCaseId] }).freeze + end + # Parsed test plan object # # @return [Hash] diff --git a/allure-ruby-commons/spec/unit/testplan_spec.rb b/allure-ruby-commons/spec/unit/testplan_spec.rb index 60e5207..fc95e8a 100644 --- a/allure-ruby-commons/spec/unit/testplan_spec.rb +++ b/allure-ruby-commons/spec/unit/testplan_spec.rb @@ -26,7 +26,7 @@ it "correct testplan.json" do expect(described_class.test_ids).to eq(test_ids) expect(described_class.test_names).to eq(test_names) - expect(described_class.muted_test_ids).to be_nil + expect(described_class.muted?("muted-id")).to be(false) end end @@ -51,18 +51,26 @@ context "with partial entries" do let(:path) { "#{Dir.pwd}/spec/fixtures/test_plan/partial/testplan.json" } - it "reads muted IDs and ignores absent IDs and selectors" do + it "matches muted identities and ignores absent IDs and selectors" do + matched = described_class.muted?("muted-id") + unmatched = described_class.muted?("other-id") + missing = described_class.muted?(nil) + expect(described_class.test_ids).to eq(["id only"]) expect(described_class.test_names).to eq(["selector only"]) - expect(described_class.muted_test_ids).to eq(["muted-id"]) + expect(matched).to be(true) + expect(unmatched).to be(false) + expect(missing).to be(false) end end context "without a plan" do let(:path) { nil } - it "has no muted IDs" do - expect(described_class.muted_test_ids).to be_nil + it "does not mute tests" do + muted = described_class.muted?("muted-id") + + expect(muted).to be(false) end end From 562938a79dc12ecd642180b4a48adcc69e4f6e39 Mon Sep 17 00:00:00 2001 From: andrejs Date: Thu, 5 Nov 2026 14:48:09 +0000 Subject: [PATCH 4/9] Remove duplicate test case id construction --- allure-cucumber/lib/allure_cucumber/formatter.rb | 4 +++- .../spec/unit/formatter_test_case_started_spec.rb | 5 +++++ .../spec/unit/formatter_test_run_finished_spec.rb | 8 +++++++- allure-ruby-commons/lib/allure_ruby_commons/testplan.rb | 7 +++++++ allure-ruby-commons/spec/unit/testplan_spec.rb | 3 +++ 5 files changed, 25 insertions(+), 2 deletions(-) diff --git a/allure-cucumber/lib/allure_cucumber/formatter.rb b/allure-cucumber/lib/allure_cucumber/formatter.rb index 3acb00e..51e1666 100644 --- a/allure-cucumber/lib/allure_cucumber/formatter.rb +++ b/allure-cucumber/lib/allure_cucumber/formatter.rb @@ -75,7 +75,7 @@ def on_test_run_hook_finished(event) def on_test_case_started(event) test_case = event.test_case result = cucumber_model.test_result(test_case) - result.status_details.muted = true if cucumber_model.muted?(test_case) + result.status_details.muted = true if Allure::TestPlan.muted?(result.test_case_id) lifecycle.start_test_container(Allure::TestResultContainer.new(name: test_case.name)) lifecycle.start_test_case(result) @@ -125,6 +125,8 @@ def on_test_case_finished(event) # Skip matched scenarios through Cucumber's public hook DSL. # @return [void] def register_muting_hook + return unless Allure::TestPlan.muted_tests? + model = cucumber_model Around do |scenario, run| skip_this_scenario("Skipped by Allure test plan") if model.muted?(scenario) diff --git a/allure-cucumber/spec/unit/formatter_test_case_started_spec.rb b/allure-cucumber/spec/unit/formatter_test_case_started_spec.rb index ffbfbf8..6602ad4 100644 --- a/allure-cucumber/spec/unit/formatter_test_case_started_spec.rb +++ b/allure-cucumber/spec/unit/formatter_test_case_started_spec.rb @@ -43,6 +43,10 @@ end it "are added to allure test case" do + allow(Allure::TestPlan).to receive(:muted_tests?).and_return(false) + allow(AllureCucumber::Scenario).to receive(:new).and_call_original + expect_any_instance_of(AllureCucumber::CucumberFormatter).not_to receive(:Around) + run_cucumber_cli(<<~FEATURE) Feature: #{feature} @@ -63,6 +67,7 @@ expect(arg.labels).to match_array(labels) end end + expect(AllureCucumber::Scenario).to have_received(:new).once end end diff --git a/allure-cucumber/spec/unit/formatter_test_run_finished_spec.rb b/allure-cucumber/spec/unit/formatter_test_run_finished_spec.rb index 6683f72..7ca1bab 100644 --- a/allure-cucumber/spec/unit/formatter_test_run_finished_spec.rb +++ b/allure-cucumber/spec/unit/formatter_test_run_finished_spec.rb @@ -21,12 +21,15 @@ let(:muted_id) { Digest::MD5.hexdigest("#{test_tmp_dir}/features/test.feature#Muted scenario") } before do + allow(Allure::TestPlan).to receive(:muted_tests?).and_return(true) allow(Allure::TestPlan).to receive(:muted?).and_call_original allow(Allure::TestPlan).to receive(:muted?).with(muted_id).and_return(true) allow(Allure::AllureLifecycle).to receive(:new).and_call_original end it "skips a scenario whose test case id is muted" do + allow(AllureCucumber::Scenario).to receive(:new).and_call_original + status = run_cucumber_cli(<<~FEATURE) Feature: Simple feature @@ -43,7 +46,10 @@ aggregate_failures do expect(status).to eq(0) expect(results.find { |t| t["name"] == "Add a to b" }["status"]).to eq("passed") - expect(results.find { |t| t["name"] == "Muted scenario" }["status"]).to eq("skipped") + muted = results.find { |t| t["name"] == "Muted scenario" } + expect(muted["status"]).to eq("skipped") + expect(muted["statusDetails"]["muted"]).to be(true) + expect(AllureCucumber::Scenario).to have_received(:new).exactly(4).times end end end diff --git a/allure-ruby-commons/lib/allure_ruby_commons/testplan.rb b/allure-ruby-commons/lib/allure_ruby_commons/testplan.rb index 6ee3c3a..9102797 100644 --- a/allure-ruby-commons/lib/allure_ruby_commons/testplan.rb +++ b/allure-ruby-commons/lib/allure_ruby_commons/testplan.rb @@ -32,6 +32,13 @@ def muted?(test_case_id) muted_test_ids.include?(test_case_id) end + # Whether the test plan contains any muted identities + # + # @return [Boolean] + def muted_tests? + !muted_test_ids.empty? + end + private # Muted test case ids diff --git a/allure-ruby-commons/spec/unit/testplan_spec.rb b/allure-ruby-commons/spec/unit/testplan_spec.rb index fc95e8a..299d61f 100644 --- a/allure-ruby-commons/spec/unit/testplan_spec.rb +++ b/allure-ruby-commons/spec/unit/testplan_spec.rb @@ -27,6 +27,7 @@ expect(described_class.test_ids).to eq(test_ids) expect(described_class.test_names).to eq(test_names) expect(described_class.muted?("muted-id")).to be(false) + expect(described_class.muted_tests?).to be(false) end end @@ -61,6 +62,7 @@ expect(matched).to be(true) expect(unmatched).to be(false) expect(missing).to be(false) + expect(described_class.muted_tests?).to be(true) end end @@ -71,6 +73,7 @@ muted = described_class.muted?("muted-id") expect(muted).to be(false) + expect(described_class.muted_tests?).to be(false) end end From 5857ed40c29a9905fe275a00998cd4bfb328d5b7 Mon Sep 17 00:00:00 2001 From: andrejs Date: Thu, 5 Nov 2026 15:23:14 +0000 Subject: [PATCH 5/9] Use proper file name for test plan --- .../lib/allure_ruby_commons/{testplan.rb => test_plan.rb} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename allure-ruby-commons/lib/allure_ruby_commons/{testplan.rb => test_plan.rb} (100%) diff --git a/allure-ruby-commons/lib/allure_ruby_commons/testplan.rb b/allure-ruby-commons/lib/allure_ruby_commons/test_plan.rb similarity index 100% rename from allure-ruby-commons/lib/allure_ruby_commons/testplan.rb rename to allure-ruby-commons/lib/allure_ruby_commons/test_plan.rb From 59c199483428fa704a70fa6aabad5da4635b4ee2 Mon Sep 17 00:00:00 2001 From: andrejs Date: Fri, 6 Nov 2026 10:47:53 +0200 Subject: [PATCH 6/9] Support native allure ids in muted tests --- .../lib/allure_cucumber/formatter.rb | 1 - .../allure_cucumber/models/cucumber_model.rb | 12 +++++-- .../lib/allure_cucumber/models/scenario.rb | 6 ++++ .../unit/formatter_test_run_finished_spec.rb | 32 ++++++++++++------- allure-rspec/lib/allure_rspec/formatter.rb | 2 +- .../unit/formatter_example_started_spec.rb | 23 ++++++++++--- .../lib/allure_ruby_commons/test_plan.rb | 16 ++++++++-- .../fixtures/test_plan/ids_only/testplan.json | 4 +++ .../fixtures/test_plan/partial/testplan.json | 2 +- .../spec/unit/testplan_spec.rb | 19 +++++++++++ 10 files changed, 93 insertions(+), 24 deletions(-) create mode 100644 allure-ruby-commons/spec/fixtures/test_plan/ids_only/testplan.json diff --git a/allure-cucumber/lib/allure_cucumber/formatter.rb b/allure-cucumber/lib/allure_cucumber/formatter.rb index 51e1666..1248f76 100644 --- a/allure-cucumber/lib/allure_cucumber/formatter.rb +++ b/allure-cucumber/lib/allure_cucumber/formatter.rb @@ -75,7 +75,6 @@ def on_test_run_hook_finished(event) def on_test_case_started(event) test_case = event.test_case result = cucumber_model.test_result(test_case) - result.status_details.muted = true if Allure::TestPlan.muted?(result.test_case_id) lifecycle.start_test_container(Allure::TestResultContainer.new(name: test_case.name)) lifecycle.start_test_case(result) diff --git a/allure-cucumber/lib/allure_cucumber/models/cucumber_model.rb b/allure-cucumber/lib/allure_cucumber/models/cucumber_model.rb index 67efab8..f11171f 100644 --- a/allure-cucumber/lib/allure_cucumber/models/cucumber_model.rb +++ b/allure-cucumber/lib/allure_cucumber/models/cucumber_model.rb @@ -30,14 +30,16 @@ def test_result(test_case) title_path: scenario.title_path, environment: config.environment, **test_result_metadata(parser) - ) + ).tap do |result| + result.status_details.muted = true if muted_scenario?(scenario) + end end # Match a scenario or outline against the test plan's muted identities. # @param [Cucumber::Core::Test::Case, Cucumber::RunningTestCase::TestCase] test_case # @return [Boolean] def muted?(test_case) - Allure::TestPlan.muted?(Scenario.new(test_case, ast_lookup).test_case_id) + muted_scenario?(Scenario.new(test_case, ast_lookup)) end # Convert to allure step result @@ -75,6 +77,12 @@ def failure_details(result) attr_reader :ast_lookup, :config + # @param [Scenario] scenario + # @return [Boolean] + def muted_scenario?(scenario) + Allure::TestPlan.muted?(scenario.test_case_id, allure_id: scenario.allure_id) + end + def test_result_metadata(parser) { labels: parser.labels, diff --git a/allure-cucumber/lib/allure_cucumber/models/scenario.rb b/allure-cucumber/lib/allure_cucumber/models/scenario.rb index 9e3c6b2..6ec88fb 100644 --- a/allure-cucumber/lib/allure_cucumber/models/scenario.rb +++ b/allure-cucumber/lib/allure_cucumber/models/scenario.rb @@ -25,6 +25,12 @@ def test_case_id @test_case_id ||= Allure::ResultUtils.test_case_id(location_file, scenario.name) end + # Explicit Allure ID from scenario or inherited tags + # @return [String, nil] + def allure_id + @allure_id ||= tags.find { |tag| tag.start_with?("@allure_id:") }&.delete_prefix("@allure_id:") + end + # Feature name scenario belongs to # @return [String] def feature_name diff --git a/allure-cucumber/spec/unit/formatter_test_run_finished_spec.rb b/allure-cucumber/spec/unit/formatter_test_run_finished_spec.rb index 7ca1bab..2abcfbe 100644 --- a/allure-cucumber/spec/unit/formatter_test_run_finished_spec.rb +++ b/allure-cucumber/spec/unit/formatter_test_run_finished_spec.rb @@ -23,33 +23,43 @@ before do allow(Allure::TestPlan).to receive(:muted_tests?).and_return(true) allow(Allure::TestPlan).to receive(:muted?).and_call_original - allow(Allure::TestPlan).to receive(:muted?).with(muted_id).and_return(true) + allow(Allure::TestPlan).to receive(:muted?).with(anything, allure_id: "123").and_return(true) + allow(Allure::TestPlan).to receive(:muted?).with(muted_id, allure_id: "456").and_return(true) allow(Allure::AllureLifecycle).to receive(:new).and_call_original end - it "skips a scenario whose test case id is muted" do + it "skips scenarios matching either an inherited Allure ID or a generated identity" do allow(AllureCucumber::Scenario).to receive(:new).and_call_original + muted_names = ["Allure ID scenario", "Muted scenario"] status = run_cucumber_cli(<<~FEATURE) Feature: Simple feature - Scenario: Muted scenario - Then step fails with simple exception + @allure_id:123 + Rule: Explicit IDs + Scenario: Allure ID scenario + Then step fails with simple exception - Scenario: Add a to b - Simple scenario description - Given a is 5 + Rule: Generated IDs + @allure_id:456 + Scenario: Muted scenario + Then step fails with simple exception + + Scenario: Add a to b + Simple scenario description + Given a is 5 FEATURE results = Dir["#{test_tmp_dir}/reports/allure-results/*-result.json"].flat_map { |f| JSON.parse(File.read(f)) } + muted = results.select { |t| muted_names.include?(t["name"]) } aggregate_failures do expect(status).to eq(0) expect(results.find { |t| t["name"] == "Add a to b" }["status"]).to eq("passed") - muted = results.find { |t| t["name"] == "Muted scenario" } - expect(muted["status"]).to eq("skipped") - expect(muted["statusDetails"]["muted"]).to be(true) - expect(AllureCucumber::Scenario).to have_received(:new).exactly(4).times + expect(muted.size).to eq(2) + expect(muted.map { |t| t["status"] }).to all(eq("skipped")) + expect(muted.map { |t| t["statusDetails"]["muted"] }).to all(be(true)) + expect(AllureCucumber::Scenario).to have_received(:new).exactly(6).times end end end diff --git a/allure-rspec/lib/allure_rspec/formatter.rb b/allure-rspec/lib/allure_rspec/formatter.rb index 81072ba..72c053c 100644 --- a/allure-rspec/lib/allure_rspec/formatter.rb +++ b/allure-rspec/lib/allure_rspec/formatter.rb @@ -80,7 +80,7 @@ def example_group_started(example_group_notification) def example_started(example_notification) example = example_notification.example result = test_result(example) - if Allure::TestPlan.muted?(result.test_case_id) + if Allure::TestPlan.muted?(result.test_case_id, allure_id: example.metadata[:allure_id]) result.status_details.muted = true example.metadata[:skip] = "Skipped by Allure test plan" end diff --git a/allure-rspec/spec/unit/formatter_example_started_spec.rb b/allure-rspec/spec/unit/formatter_example_started_spec.rb index caf08da..3018d2e 100644 --- a/allure-rspec/spec/unit/formatter_example_started_spec.rb +++ b/allure-rspec/spec/unit/formatter_example_started_spec.rb @@ -25,18 +25,31 @@ let(:muted_id) { Digest::MD5.hexdigest("#{test_tmp_dir}/spec/test_spec.rb##{suite} #{spec}") } before do - allow(Allure::TestPlan).to receive(:muted?).with(muted_id).and_return(true) + allow(Allure::TestPlan).to receive(:muted?).and_call_original + allow(Allure::TestPlan).to receive(:muted?).with(anything, allure_id: 123).and_return(true) + allow(Allure::TestPlan).to receive(:muted?).with(muted_id, allure_id: 456).and_return(true) end - it "adds skip metadata to the matching example" do + + it "adds skip metadata to examples matching either identity" do run_rspec(<<~SPEC) describe "#{suite}" do - it "#{spec}" do + it "Explicit ID", allure_id: 123 do + raise "Muted body should not run" + end + + it "#{spec}", allure_id: 456 do + raise "Muted body should not run" + end + + it "Unmatched", allure_id: 456 do end end SPEC - example = RSpec.world.example_groups.first.examples.first - expect(example.metadata[:skip]).to eq("Skipped by Allure test plan") + examples = RSpec.world.example_groups.first.examples + expect(examples.map { |example| example.metadata[:skip] }).to eq( + ["Skipped by Allure test plan", "Skipped by Allure test plan", nil] + ) end end diff --git a/allure-ruby-commons/lib/allure_ruby_commons/test_plan.rb b/allure-ruby-commons/lib/allure_ruby_commons/test_plan.rb index 9102797..b8d9641 100644 --- a/allure-ruby-commons/lib/allure_ruby_commons/test_plan.rb +++ b/allure-ruby-commons/lib/allure_ruby_commons/test_plan.rb @@ -24,11 +24,14 @@ def test_names @test_names ||= tests&.filter_map { |test| test[:selector] } end - # Whether a test case identity is muted + # Match an Allure ID first, falling back to the generated test case identity # # @param [String] test_case_id + # @param [String, Integer, nil] allure_id # @return [Boolean] - def muted?(test_case_id) + def muted?(test_case_id, allure_id: nil) + return true if allure_id && muted_allure_ids.include?(allure_id.to_s) + muted_test_ids.include?(test_case_id) end @@ -36,11 +39,18 @@ def muted?(test_case_id) # # @return [Boolean] def muted_tests? - !muted_test_ids.empty? + muted_allure_ids.any? || muted_test_ids.any? end private + # Muted explicit Allure ids + # + # @return [Set] + def muted_allure_ids + @muted_allure_ids ||= Set.new(muted_tests&.filter_map { |test| test[:id]&.to_s }).freeze + end + # Muted test case ids # # @return [Set] diff --git a/allure-ruby-commons/spec/fixtures/test_plan/ids_only/testplan.json b/allure-ruby-commons/spec/fixtures/test_plan/ids_only/testplan.json new file mode 100644 index 0000000..0f62fa8 --- /dev/null +++ b/allure-ruby-commons/spec/fixtures/test_plan/ids_only/testplan.json @@ -0,0 +1,4 @@ +{ + "version": "1.0", + "mutedTests": [{"id": 123}] +} diff --git a/allure-ruby-commons/spec/fixtures/test_plan/partial/testplan.json b/allure-ruby-commons/spec/fixtures/test_plan/partial/testplan.json index 26ccfdf..404aa21 100644 --- a/allure-ruby-commons/spec/fixtures/test_plan/partial/testplan.json +++ b/allure-ruby-commons/spec/fixtures/test_plan/partial/testplan.json @@ -1,5 +1,5 @@ { "version": "1.0", "tests": [{}, {"id": "id only"}, {"selector": "selector only"}], - "mutedTests": [{}, {"testCaseId": "muted-id"}] + "mutedTests": [{}, {"id": 123}, {"testCaseId": "muted-id"}] } diff --git a/allure-ruby-commons/spec/unit/testplan_spec.rb b/allure-ruby-commons/spec/unit/testplan_spec.rb index 299d61f..86187bb 100644 --- a/allure-ruby-commons/spec/unit/testplan_spec.rb +++ b/allure-ruby-commons/spec/unit/testplan_spec.rb @@ -10,6 +10,7 @@ described_class.instance_variable_set(:@test_names, nil) described_class.instance_variable_set(:@muted_tests, nil) described_class.instance_variable_set(:@muted_test_ids, nil) + described_class.instance_variable_set(:@muted_allure_ids, nil) %i[@test_plan_path @test_plan].each do |var| described_class.send(:remove_instance_variable, var) if described_class.instance_variable_defined?(var) @@ -56,16 +57,34 @@ matched = described_class.muted?("muted-id") unmatched = described_class.muted?("other-id") missing = described_class.muted?(nil) + matched_by_allure_id = described_class.muted?("other-id", allure_id: "123") + fallback = described_class.muted?("muted-id", allure_id: 456) + unmatched_ids = described_class.muted?("other-id", allure_id: 456) expect(described_class.test_ids).to eq(["id only"]) expect(described_class.test_names).to eq(["selector only"]) expect(matched).to be(true) expect(unmatched).to be(false) expect(missing).to be(false) + expect(matched_by_allure_id).to be(true) + expect(fallback).to be(true) + expect(unmatched_ids).to be(false) expect(described_class.muted_tests?).to be(true) end end + context "with only muted Allure IDs" do + let(:path) { "#{Dir.pwd}/spec/fixtures/test_plan/ids_only/testplan.json" } + + it "activates muting without generated identities" do + available = described_class.muted_tests? + matched = described_class.muted?("other-id", allure_id: 123) + + expect(available).to be(true) + expect(matched).to be(true) + end + end + context "without a plan" do let(:path) { nil } From 58a2f137dfbf5ad67f2d6cf6572a8c17eab604ee Mon Sep 17 00:00:00 2001 From: andrejs Date: Fri, 6 Nov 2026 15:08:34 +0200 Subject: [PATCH 7/9] Split full_name and test_case_id in to separate fields --- .../allure_cucumber/models/cucumber_model.rb | 2 +- .../lib/allure_cucumber/models/scenario.rb | 10 +++++++-- .../unit/formatter_test_case_started_spec.rb | 3 ++- allure-rspec/lib/allure_rspec/formatter.rb | 5 +++-- .../unit/formatter_example_started_spec.rb | 3 ++- .../lib/allure_ruby_commons/result_utils.rb | 5 +++-- .../spec/unit/result_utils_spec.rb | 22 ++++++++++++------- 7 files changed, 33 insertions(+), 17 deletions(-) diff --git a/allure-cucumber/lib/allure_cucumber/models/cucumber_model.rb b/allure-cucumber/lib/allure_cucumber/models/cucumber_model.rb index f11171f..1809413 100644 --- a/allure-cucumber/lib/allure_cucumber/models/cucumber_model.rb +++ b/allure-cucumber/lib/allure_cucumber/models/cucumber_model.rb @@ -25,7 +25,7 @@ def test_result(test_case) description: scenario.description, description_html: scenario.description, history_id: scenario.id, - full_name: scenario.name, + full_name: scenario.full_name, test_case_id: scenario.test_case_id, title_path: scenario.title_path, environment: config.environment, diff --git a/allure-cucumber/lib/allure_cucumber/models/scenario.rb b/allure-cucumber/lib/allure_cucumber/models/scenario.rb index 6ec88fb..e08f5bd 100644 --- a/allure-cucumber/lib/allure_cucumber/models/scenario.rb +++ b/allure-cucumber/lib/allure_cucumber/models/scenario.rb @@ -19,10 +19,16 @@ def id @id ||= Digest::MD5.hexdigest(test_case.inspect) end - # Stable scenario identity shared by all rows of an outline + # Stable hashed scenario identity # @return [String] def test_case_id - @test_case_id ||= Allure::ResultUtils.test_case_id(location_file, scenario.name) + @test_case_id ||= Digest::MD5.hexdigest(full_name) + end + + # Stable scenario identity + # @return [String] + def full_name + @full_name ||= Allure::ResultUtils.full_name(location_file, scenario.name) end # Explicit Allure ID from scenario or inherited tags diff --git a/allure-cucumber/spec/unit/formatter_test_case_started_spec.rb b/allure-cucumber/spec/unit/formatter_test_case_started_spec.rb index 6602ad4..de80daf 100644 --- a/allure-cucumber/spec/unit/formatter_test_case_started_spec.rb +++ b/allure-cucumber/spec/unit/formatter_test_case_started_spec.rb @@ -59,7 +59,7 @@ aggregate_failures "Should have correct args" do expect(arg.name).to eq(scenario) expect(arg.description).to eq("Simple scenario description") - expect(arg.full_name).to eq(scenario) + expect(arg.full_name).to eq("#{test_tmp_dir}/features/test.feature##{scenario}") expect(arg.test_case_id).to eq(Digest::MD5.hexdigest("#{test_tmp_dir}/features/test.feature##{scenario}")) expect(arg.title_path).to eq(["#{test_tmp_dir}/features/test.feature", feature]) expect(arg.links).to be_empty @@ -238,6 +238,7 @@ expect(lifecycle).to(have_received(:start_test_case).twice) { |arg| examples.push(arg) } expect(examples[0].name).to include("Add a to b, Examples (#1)") expect(examples[1].name).to include("Add a to b, Examples (#2)") + expect(examples.map(&:full_name)).to all(eq("#{test_tmp_dir}/features/test.feature#Add a to b")) expected_id = Digest::MD5.hexdigest("#{test_tmp_dir}/features/test.feature#Add a to b") expect(examples.map(&:test_case_id)).to all(eq(expected_id)) expect(examples[0].parameters).to contain_exactly( diff --git a/allure-rspec/lib/allure_rspec/formatter.rb b/allure-rspec/lib/allure_rspec/formatter.rb index 72c053c..ca6479d 100644 --- a/allure-rspec/lib/allure_rspec/formatter.rb +++ b/allure-rspec/lib/allure_rspec/formatter.rb @@ -123,14 +123,15 @@ def message(notification) # @return [Allure::TestResult] def test_result(example) parser = RspecMetadataParser.new(example, allure_config) + full_name = Allure::ResultUtils.full_name(example.metadata[:rerun_file_path], example.full_description) Allure::TestResult.new( name: example.description, description: "Location - #{strip_relative(parser.location)}", description_html: "Location - #{strip_relative(parser.location)}", history_id: example.id, - full_name: example.full_description, - test_case_id: Allure::ResultUtils.test_case_id(example.metadata[:rerun_file_path], example.full_description), + full_name: full_name, + test_case_id: Digest::MD5.hexdigest(full_name), title_path: parser.title_path, labels: parser.labels, links: parser.links, diff --git a/allure-rspec/spec/unit/formatter_example_started_spec.rb b/allure-rspec/spec/unit/formatter_example_started_spec.rb index 3018d2e..666b663 100644 --- a/allure-rspec/spec/unit/formatter_example_started_spec.rb +++ b/allure-rspec/spec/unit/formatter_example_started_spec.rb @@ -75,7 +75,7 @@ aggregate_failures "Should have correct args" do expect(arg.name).to eq(spec) expect(arg.description).to eq("Location - #{test_tmp_dir}/spec/test_spec.rb:10") - expect(arg.full_name).to eq("#{suite} #{spec}") + expect(arg.full_name).to eq("#{test_tmp_dir}/spec/test_spec.rb##{suite} #{spec}") expect(arg.test_case_id).to eq(Digest::MD5.hexdigest("#{test_tmp_dir}/spec/test_spec.rb##{suite} #{spec}")) expect(arg.title_path).to eq(["#{test_tmp_dir}/spec/test_spec.rb", suite]) expect(arg.links).to be_empty @@ -103,6 +103,7 @@ expect(lifecycle).to have_received(:start_test_case).once do |arg| aggregate_failures "Should have correct args" do expect(arg.description).to eq("Location - #{test_tmp_dir}/spec/test_spec.rb:8") + expect(arg.full_name).to eq("#{test_tmp_dir}/spec/test_spec.rb##{suite} behaves like shared #{spec}") end end end diff --git a/allure-ruby-commons/lib/allure_ruby_commons/result_utils.rb b/allure-ruby-commons/lib/allure_ruby_commons/result_utils.rb index 31e04ae..b94b4fc 100644 --- a/allure-ruby-commons/lib/allure_ruby_commons/result_utils.rb +++ b/allure-ruby-commons/lib/allure_ruby_commons/result_utils.rb @@ -34,9 +34,10 @@ class << self # @param [String] file_path # @param [String] full_name # @return [String] - def test_case_id(file_path, full_name) + def full_name(file_path, full_name) relative_path = Pathname.new(File.expand_path(file_path)).relative_path_from(Pathname.pwd).to_s - Digest::MD5.hexdigest("#{relative_path}##{full_name}") + + "#{relative_path}##{full_name}" end # @param [Time] time diff --git a/allure-ruby-commons/spec/unit/result_utils_spec.rb b/allure-ruby-commons/spec/unit/result_utils_spec.rb index a08432e..b3178ad 100644 --- a/allure-ruby-commons/spec/unit/result_utils_spec.rb +++ b/allure-ruby-commons/spec/unit/result_utils_spec.rb @@ -64,31 +64,37 @@ def raise_multi_error expect(utils.issue_link("issue", "http://jira.com/123", "{}")).to eq(Allure::Link.new("issue", "issue", "http://jira.com/123")) end - describe ".test_case_id" do - it "hashes the normalized relative file and full name" do + describe ".full_name" do + it "combines the normalized relative file and test name" do paths = [ "spec/test_spec.rb", "./spec/test_spec.rb", "spec/nested/../test_spec.rb", File.expand_path("spec/test_spec.rb") ] - expected = Digest::MD5.hexdigest("spec/test_spec.rb#Suite example") + expected = "spec/test_spec.rb#Suite example" - ids = paths.map { |path| utils.test_case_id(path, "Suite example") } + full_names = paths.map { |path| utils.full_name(path, "Suite example") } - expect(ids).to all(eq(expected)) + expect(full_names).to all(eq(expected)) end - it "distinguishes directories and full names" do + it "distinguishes directories and test names" do inputs = [ ["spec/api/test_spec.rb", "Suite example"], ["spec/ui/test_spec.rb", "Suite example"], ["spec/api/test_spec.rb", "Suite another example"] ] - ids = inputs.map { |file, name| utils.test_case_id(file, name) } + expected = [ + "spec/api/test_spec.rb#Suite example", + "spec/ui/test_spec.rb#Suite example", + "spec/api/test_spec.rb#Suite another example" + ] + + full_names = inputs.map { |file, name| utils.full_name(file, name) } - expect(ids.uniq.size).to eq(inputs.size) + expect(full_names).to eq(expected) end end From fa0854b32327f5bcf33965f2d25b774b59d67f82 Mon Sep 17 00:00:00 2001 From: andrejs Date: Fri, 6 Nov 2026 17:34:46 +0200 Subject: [PATCH 8/9] Add support for test plan filters using updated full_name --- .../lib/allure_cucumber/formatter.rb | 4 +++- .../allure_cucumber/models/cucumber_model.rb | 8 +++++++ .../lib/allure_cucumber/test_plan_filter.rb | 15 ++++++++++++ .../spec/unit/allure_cucumber_spec.rb | 23 +++++++++++++++++++ allure-rspec/lib/allure_rspec/_utils.rb | 9 ++++++++ allure-rspec/lib/allure_rspec/formatter.rb | 2 +- allure-rspec/spec/unit/allure_rspec_spec.rb | 12 ++++++++++ 7 files changed, 71 insertions(+), 2 deletions(-) create mode 100644 allure-cucumber/lib/allure_cucumber/test_plan_filter.rb diff --git a/allure-cucumber/lib/allure_cucumber/formatter.rb b/allure-cucumber/lib/allure_cucumber/formatter.rb index 1248f76..20840da 100644 --- a/allure-cucumber/lib/allure_cucumber/formatter.rb +++ b/allure-cucumber/lib/allure_cucumber/formatter.rb @@ -3,6 +3,8 @@ require "cucumber/core" require "cucumber/glue/dsl" +require_relative "test_plan_filter" + module AllureCucumber # Main formatter class. Translates cucumber event to allure lifecycle class CucumberFormatter @@ -30,7 +32,7 @@ def initialize(config) @cucumber_model ||= AllureCucumberModel.new(config, allure_config) names = Allure::TestPlan.test_names - config.name_regexps.push(*names.map { |name| /#{name}/ }) if names + config.filters << TestPlanFilter.new(cucumber_model, names.to_set) if names config.on_event(:test_run_started) { |event| on_test_run_started(event) } config.on_event(:test_run_finished) { |event| on_test_run_finished(event) } diff --git a/allure-cucumber/lib/allure_cucumber/models/cucumber_model.rb b/allure-cucumber/lib/allure_cucumber/models/cucumber_model.rb index 1809413..dd0f392 100644 --- a/allure-cucumber/lib/allure_cucumber/models/cucumber_model.rb +++ b/allure-cucumber/lib/allure_cucumber/models/cucumber_model.rb @@ -35,6 +35,14 @@ def test_result(test_case) end end + # Match a scenario or outline against exact reported full-name selectors. + # @param [Cucumber::Core::Test::Case] test_case + # @param [Set] selectors + # @return [Boolean] + def selected?(test_case, selectors) + selectors.include?(Scenario.new(test_case, ast_lookup).full_name) + end + # Match a scenario or outline against the test plan's muted identities. # @param [Cucumber::Core::Test::Case, Cucumber::RunningTestCase::TestCase] test_case # @return [Boolean] diff --git a/allure-cucumber/lib/allure_cucumber/test_plan_filter.rb b/allure-cucumber/lib/allure_cucumber/test_plan_filter.rb new file mode 100644 index 0000000..13c8394 --- /dev/null +++ b/allure-cucumber/lib/allure_cucumber/test_plan_filter.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +require "cucumber/core/filter" + +module AllureCucumber + # Select test-plan cases before hooks and execution are applied. + class TestPlanFilter < Cucumber::Core::Filter.new(:model, :selectors) + # @param [Cucumber::Core::Test::Case] test_case + # @return [TestPlanFilter] + def test_case(test_case) + test_case.describe_to(receiver) if model.selected?(test_case, selectors) + self + end + end +end diff --git a/allure-cucumber/spec/unit/allure_cucumber_spec.rb b/allure-cucumber/spec/unit/allure_cucumber_spec.rb index 428265b..a3b39b7 100644 --- a/allure-cucumber/spec/unit/allure_cucumber_spec.rb +++ b/allure-cucumber/spec/unit/allure_cucumber_spec.rb @@ -28,6 +28,29 @@ expect(AllureCucumber.configuration.tms_prefix).to eq("TMS") end + it "forwards only cases whose file-qualified full name matches a selector" do + selector = "features/test.feature#Scenario [case]" + allow(Allure::TestPlan).to receive(:test_names).and_return([selector]) + configuration = Cucumber::Configuration.new + receiver = spy("receiver") + full_names = [selector, "features/other.feature#Scenario [case]", "features/test.feature#Scenario c"] + cases = full_names.map do |full_name| + test_case = instance_double(Cucumber::Core::Test::Case, describe_to: nil) + scenario = instance_double(AllureCucumber::Scenario, full_name: full_name) + allow(AllureCucumber::Scenario).to receive(:new).with(test_case, anything).and_return(scenario) + test_case + end + + Thread.new { AllureCucumber::CucumberFormatter.new(configuration) }.value + expect(configuration.filters.size).to eq(1) + filter = configuration.filters.first.with_receiver(receiver) + cases.each { |test_case| filter.test_case(test_case) } + + expect(cases[0]).to have_received(:describe_to).with(receiver).once + expect(cases[1]).not_to have_received(:describe_to) + expect(cases[2]).not_to have_received(:describe_to) + end + context "with global labels" do include_context "cucumber runner" diff --git a/allure-rspec/lib/allure_rspec/_utils.rb b/allure-rspec/lib/allure_rspec/_utils.rb index 592151d..9fd170d 100644 --- a/allure-rspec/lib/allure_rspec/_utils.rb +++ b/allure-rspec/lib/allure_rspec/_utils.rb @@ -2,6 +2,15 @@ module AllureRspec module Utils + # Match reported full names through RSpec's metadata filter API. + # @param [Set] names + # @return [Proc] + def self.full_name_filter(names) + lambda do |file_path, metadata| + names.include?(Allure::ResultUtils.full_name(file_path, metadata[:full_description])) + end + end + # Strip relative ./ form path # @param [String] path # @return [String] diff --git a/allure-rspec/lib/allure_rspec/formatter.rb b/allure-rspec/lib/allure_rspec/formatter.rb index ca6479d..80502cd 100644 --- a/allure-rspec/lib/allure_rspec/formatter.rb +++ b/allure-rspec/lib/allure_rspec/formatter.rb @@ -34,7 +34,7 @@ class RSpecFormatter < RSpec::Core::Formatters::BaseFormatter names = Allure::TestPlan.test_names config.filter_run_when_matching(*ids.map { |id| { allure_id: id } }) if ids - config.full_description = names if names + config.filter_run_including(rerun_file_path: Utils.full_name_filter(names.to_set)) if names end def initialize(output) diff --git a/allure-rspec/spec/unit/allure_rspec_spec.rb b/allure-rspec/spec/unit/allure_rspec_spec.rb index 51b407b..ae47d15 100644 --- a/allure-rspec/spec/unit/allure_rspec_spec.rb +++ b/allure-rspec/spec/unit/allure_rspec_spec.rb @@ -28,6 +28,18 @@ expect(AllureRspec.configuration.tms_tag).to eq("TMS") end + it "matches selectors against exact file-qualified full names" do + filter = AllureRspec::Utils.full_name_filter(["spec/test_spec.rb#Suite [case]"]) + + matches = [ + filter.call("./spec/test_spec.rb", full_description: "Suite [case]"), + filter.call("spec/other_spec.rb", full_description: "Suite [case]"), + filter.call("spec/test_spec.rb", full_description: "Suite c") + ] + + expect(matches).to eq([true, false, false]) + end + context "with global labels" do include_context "rspec runner" From f9bb98614a5415469a0fdd76959924f1f085131e Mon Sep 17 00:00:00 2001 From: andrejs Date: Fri, 6 Nov 2026 18:04:10 +0200 Subject: [PATCH 9/9] Add selector field support to muted test list --- .../allure_cucumber/models/cucumber_model.rb | 4 +- .../unit/formatter_test_run_finished_spec.rb | 23 ++++++++--- allure-rspec/lib/allure_rspec/formatter.rb | 4 +- .../unit/formatter_example_started_spec.rb | 20 +++++++-- .../lib/allure_ruby_commons/test_plan.rb | 17 ++++++-- .../test_plan/selectors_only/testplan.json | 4 ++ .../spec/unit/testplan_spec.rb | 41 +++++++++++++++---- 7 files changed, 88 insertions(+), 25 deletions(-) create mode 100644 allure-ruby-commons/spec/fixtures/test_plan/selectors_only/testplan.json diff --git a/allure-cucumber/lib/allure_cucumber/models/cucumber_model.rb b/allure-cucumber/lib/allure_cucumber/models/cucumber_model.rb index dd0f392..06fca95 100644 --- a/allure-cucumber/lib/allure_cucumber/models/cucumber_model.rb +++ b/allure-cucumber/lib/allure_cucumber/models/cucumber_model.rb @@ -88,7 +88,9 @@ def failure_details(result) # @param [Scenario] scenario # @return [Boolean] def muted_scenario?(scenario) - Allure::TestPlan.muted?(scenario.test_case_id, allure_id: scenario.allure_id) + Allure::TestPlan.muted?( + allure_id: scenario.allure_id, selector: scenario.full_name, test_case_id: scenario.test_case_id + ) end def test_result_metadata(parser) diff --git a/allure-cucumber/spec/unit/formatter_test_run_finished_spec.rb b/allure-cucumber/spec/unit/formatter_test_run_finished_spec.rb index 2abcfbe..0fd1e32 100644 --- a/allure-cucumber/spec/unit/formatter_test_run_finished_spec.rb +++ b/allure-cucumber/spec/unit/formatter_test_run_finished_spec.rb @@ -19,18 +19,26 @@ context "with muted tests" do let(:muted_id) { Digest::MD5.hexdigest("#{test_tmp_dir}/features/test.feature#Muted scenario") } + let(:muted_selector) { "#{test_tmp_dir}/features/test.feature#Selector scenario" } before do allow(Allure::TestPlan).to receive(:muted_tests?).and_return(true) allow(Allure::TestPlan).to receive(:muted?).and_call_original - allow(Allure::TestPlan).to receive(:muted?).with(anything, allure_id: "123").and_return(true) - allow(Allure::TestPlan).to receive(:muted?).with(muted_id, allure_id: "456").and_return(true) + allow(Allure::TestPlan).to receive(:muted?).with( + allure_id: "123", selector: anything, test_case_id: anything + ).and_return(true) + allow(Allure::TestPlan).to receive(:muted?).with( + allure_id: "456", selector: anything, test_case_id: muted_id + ).and_return(true) + allow(Allure::TestPlan).to receive(:muted?).with( + allure_id: nil, selector: muted_selector, test_case_id: anything + ).and_return(true) allow(Allure::AllureLifecycle).to receive(:new).and_call_original end - it "skips scenarios matching either an inherited Allure ID or a generated identity" do + it "skips scenarios matching an inherited Allure ID, generated identity or selector" do allow(AllureCucumber::Scenario).to receive(:new).and_call_original - muted_names = ["Allure ID scenario", "Muted scenario"] + muted_names = ["Allure ID scenario", "Muted scenario", "Selector scenario"] status = run_cucumber_cli(<<~FEATURE) Feature: Simple feature @@ -45,6 +53,9 @@ Scenario: Muted scenario Then step fails with simple exception + Scenario: Selector scenario + Then step fails with simple exception + Scenario: Add a to b Simple scenario description Given a is 5 @@ -56,10 +67,10 @@ aggregate_failures do expect(status).to eq(0) expect(results.find { |t| t["name"] == "Add a to b" }["status"]).to eq("passed") - expect(muted.size).to eq(2) + expect(muted.size).to eq(3) expect(muted.map { |t| t["status"] }).to all(eq("skipped")) expect(muted.map { |t| t["statusDetails"]["muted"] }).to all(be(true)) - expect(AllureCucumber::Scenario).to have_received(:new).exactly(6).times + expect(AllureCucumber::Scenario).to have_received(:new).exactly(8).times end end end diff --git a/allure-rspec/lib/allure_rspec/formatter.rb b/allure-rspec/lib/allure_rspec/formatter.rb index 80502cd..b1ab917 100644 --- a/allure-rspec/lib/allure_rspec/formatter.rb +++ b/allure-rspec/lib/allure_rspec/formatter.rb @@ -80,7 +80,9 @@ def example_group_started(example_group_notification) def example_started(example_notification) example = example_notification.example result = test_result(example) - if Allure::TestPlan.muted?(result.test_case_id, allure_id: example.metadata[:allure_id]) + if Allure::TestPlan.muted?( + allure_id: example.metadata[:allure_id], selector: result.full_name, test_case_id: result.test_case_id + ) result.status_details.muted = true example.metadata[:skip] = "Skipped by Allure test plan" end diff --git a/allure-rspec/spec/unit/formatter_example_started_spec.rb b/allure-rspec/spec/unit/formatter_example_started_spec.rb index 666b663..a810513 100644 --- a/allure-rspec/spec/unit/formatter_example_started_spec.rb +++ b/allure-rspec/spec/unit/formatter_example_started_spec.rb @@ -23,14 +23,22 @@ context "with muted tests" do let(:muted_id) { Digest::MD5.hexdigest("#{test_tmp_dir}/spec/test_spec.rb##{suite} #{spec}") } + let(:muted_selector) { "#{test_tmp_dir}/spec/test_spec.rb##{suite} Selector" } before do allow(Allure::TestPlan).to receive(:muted?).and_call_original - allow(Allure::TestPlan).to receive(:muted?).with(anything, allure_id: 123).and_return(true) - allow(Allure::TestPlan).to receive(:muted?).with(muted_id, allure_id: 456).and_return(true) + allow(Allure::TestPlan).to receive(:muted?).with( + allure_id: 123, selector: anything, test_case_id: anything + ).and_return(true) + allow(Allure::TestPlan).to receive(:muted?).with( + allure_id: 456, selector: anything, test_case_id: muted_id + ).and_return(true) + allow(Allure::TestPlan).to receive(:muted?).with( + allure_id: 456, selector: muted_selector, test_case_id: anything + ).and_return(true) end - it "adds skip metadata to examples matching either identity" do + it "adds skip metadata to examples matching any muted identity" do run_rspec(<<~SPEC) describe "#{suite}" do it "Explicit ID", allure_id: 123 do @@ -41,6 +49,10 @@ raise "Muted body should not run" end + it "Selector", allure_id: 456 do + raise "Muted body should not run" + end + it "Unmatched", allure_id: 456 do end end @@ -48,7 +60,7 @@ examples = RSpec.world.example_groups.first.examples expect(examples.map { |example| example.metadata[:skip] }).to eq( - ["Skipped by Allure test plan", "Skipped by Allure test plan", nil] + ["Skipped by Allure test plan", "Skipped by Allure test plan", "Skipped by Allure test plan", nil] ) end end diff --git a/allure-ruby-commons/lib/allure_ruby_commons/test_plan.rb b/allure-ruby-commons/lib/allure_ruby_commons/test_plan.rb index b8d9641..19a7880 100644 --- a/allure-ruby-commons/lib/allure_ruby_commons/test_plan.rb +++ b/allure-ruby-commons/lib/allure_ruby_commons/test_plan.rb @@ -24,13 +24,15 @@ def test_names @test_names ||= tests&.filter_map { |test| test[:selector] } end - # Match an Allure ID first, falling back to the generated test case identity + # Match an Allure ID first, falling back to the selector and generated identity # - # @param [String] test_case_id # @param [String, Integer, nil] allure_id + # @param [String, nil] selector file-qualified full name + # @param [String] test_case_id # @return [Boolean] - def muted?(test_case_id, allure_id: nil) + def muted?(allure_id:, selector:, test_case_id:) return true if allure_id && muted_allure_ids.include?(allure_id.to_s) + return true if muted_selectors.include?(selector) muted_test_ids.include?(test_case_id) end @@ -39,7 +41,7 @@ def muted?(test_case_id, allure_id: nil) # # @return [Boolean] def muted_tests? - muted_allure_ids.any? || muted_test_ids.any? + muted_allure_ids.any? || muted_test_ids.any? || muted_selectors.any? end private @@ -58,6 +60,13 @@ def muted_test_ids @muted_test_ids ||= Set.new(muted_tests&.filter_map { |test| test[:testCaseId] }).freeze end + # Muted file-qualified full-name selectors + # + # @return [Set] + def muted_selectors + @muted_selectors ||= Set.new(muted_tests&.filter_map { |test| test[:selector] }).freeze + end + # Parsed test plan object # # @return [Hash] diff --git a/allure-ruby-commons/spec/fixtures/test_plan/selectors_only/testplan.json b/allure-ruby-commons/spec/fixtures/test_plan/selectors_only/testplan.json new file mode 100644 index 0000000..267c462 --- /dev/null +++ b/allure-ruby-commons/spec/fixtures/test_plan/selectors_only/testplan.json @@ -0,0 +1,4 @@ +{ + "version": "1.0", + "mutedTests": [{"selector": "spec/test_spec.rb#Suite [case]"}] +} diff --git a/allure-ruby-commons/spec/unit/testplan_spec.rb b/allure-ruby-commons/spec/unit/testplan_spec.rb index 86187bb..ec01340 100644 --- a/allure-ruby-commons/spec/unit/testplan_spec.rb +++ b/allure-ruby-commons/spec/unit/testplan_spec.rb @@ -11,6 +11,7 @@ described_class.instance_variable_set(:@muted_tests, nil) described_class.instance_variable_set(:@muted_test_ids, nil) described_class.instance_variable_set(:@muted_allure_ids, nil) + described_class.instance_variable_set(:@muted_selectors, nil) %i[@test_plan_path @test_plan].each do |var| described_class.send(:remove_instance_variable, var) if described_class.instance_variable_defined?(var) @@ -27,7 +28,7 @@ it "correct testplan.json" do expect(described_class.test_ids).to eq(test_ids) expect(described_class.test_names).to eq(test_names) - expect(described_class.muted?("muted-id")).to be(false) + expect(described_class.muted?(allure_id: nil, selector: nil, test_case_id: "muted-id")).to be(false) expect(described_class.muted_tests?).to be(false) end end @@ -54,12 +55,12 @@ let(:path) { "#{Dir.pwd}/spec/fixtures/test_plan/partial/testplan.json" } it "matches muted identities and ignores absent IDs and selectors" do - matched = described_class.muted?("muted-id") - unmatched = described_class.muted?("other-id") - missing = described_class.muted?(nil) - matched_by_allure_id = described_class.muted?("other-id", allure_id: "123") - fallback = described_class.muted?("muted-id", allure_id: 456) - unmatched_ids = described_class.muted?("other-id", allure_id: 456) + matched = described_class.muted?(allure_id: nil, selector: nil, test_case_id: "muted-id") + unmatched = described_class.muted?(allure_id: nil, selector: nil, test_case_id: "other-id") + missing = described_class.muted?(allure_id: nil, selector: nil, test_case_id: nil) + matched_by_allure_id = described_class.muted?(allure_id: "123", selector: nil, test_case_id: "other-id") + fallback = described_class.muted?(allure_id: 456, selector: nil, test_case_id: "muted-id") + unmatched_ids = described_class.muted?(allure_id: 456, selector: nil, test_case_id: "other-id") expect(described_class.test_ids).to eq(["id only"]) expect(described_class.test_names).to eq(["selector only"]) @@ -78,18 +79,40 @@ it "activates muting without generated identities" do available = described_class.muted_tests? - matched = described_class.muted?("other-id", allure_id: 123) + matched = described_class.muted?(allure_id: 123, selector: nil, test_case_id: "other-id") expect(available).to be(true) expect(matched).to be(true) end end + context "with only muted selectors" do + let(:path) { "#{Dir.pwd}/spec/fixtures/test_plan/selectors_only/testplan.json" } + + it "activates muting and matches exact full names when IDs do not match" do + available = described_class.muted_tests? + matched = described_class.muted?( + allure_id: 456, selector: "spec/test_spec.rb#Suite [case]", test_case_id: "other-id" + ) + other_file = described_class.muted?( + allure_id: nil, selector: "spec/other_spec.rb#Suite [case]", test_case_id: "other-id" + ) + regex_like = described_class.muted?( + allure_id: nil, selector: "spec/test_spec.rb#Suite c", test_case_id: "other-id" + ) + + expect(available).to be(true) + expect(matched).to be(true) + expect(other_file).to be(false) + expect(regex_like).to be(false) + end + end + context "without a plan" do let(:path) { nil } it "does not mute tests" do - muted = described_class.muted?("muted-id") + muted = described_class.muted?(allure_id: nil, selector: nil, test_case_id: "muted-id") expect(muted).to be(false) expect(described_class.muted_tests?).to be(false)