Support muted tests from Allure test plans

allure-cucumber/lib/allure_cucumber/formatter.rb

@@ -1,10 +1,15 @@
 # frozen_string_literal: true
 
 require "cucumber/core"
+require "cucumber/glue/dsl"
+
+require_relative "test_plan_filter"
 
 module AllureCucumber
   # Main formatter class. Translates cucumber event to allure lifecycle
   class CucumberFormatter
+    include Cucumber::Glue::Dsl
+
     # @return [Hash] hook handler methods
     HOOK_HANDLERS = {
       "Before hook" => :start_prepare_fixture,
@@ -27,7 +32,7 @@ def initialize(config)
       @cucumber_model ||= AllureCucumberModel.new(config, allure_config)
 
       names = Allure::TestPlan.test_names
-      config.name_regexps.push(*names.map { |name| /#{name}/ }) if names
+      config.filters << TestPlanFilter.new(cucumber_model, names.to_set) if names
 
       config.on_event(:test_run_started) { |event| on_test_run_started(event) }
       config.on_event(:test_run_finished) { |event| on_test_run_finished(event) }
@@ -37,6 +42,7 @@ def initialize(config)
       config.on_event(:test_case_finished) { |event| on_test_case_finished(event) }
 
       register_test_run_hook_finished(config)
+      register_muting_hook
     end
 
     # Clean test result directory before starting run
@@ -69,8 +75,11 @@ def on_test_run_hook_finished(event)
     # @param [Cucumber::Events::TestCaseStarted] event
     # @return [void]
     def on_test_case_started(event)
-      lifecycle.start_test_container(Allure::TestResultContainer.new(name: event.test_case.name))
-      lifecycle.start_test_case(cucumber_model.test_result(event.test_case))
+      test_case = event.test_case
+      result = cucumber_model.test_result(test_case)
+
+      lifecycle.start_test_container(Allure::TestResultContainer.new(name: test_case.name))
+      lifecycle.start_test_case(result)
     end
 
     # Handle test step started event
@@ -114,6 +123,18 @@ def on_test_case_finished(event)
 
     attr_reader :lifecycle, :cucumber_model
 
+    # Skip matched scenarios through Cucumber's public hook DSL.
+    # @return [void]
+    def register_muting_hook
+      return unless Allure::TestPlan.muted_tests?
+
+      model = cucumber_model
+      Around do |scenario, run|
+        skip_this_scenario("Skipped by Allure test plan") if model.muted?(scenario)
+        run.call
+      end
+    end
+
     # Register global hook finished event
     # @param [Cucumber::Configuration] config
     # @return [void]

allure-cucumber/lib/allure_cucumber/models/cucumber_model.rb

@@ -25,12 +25,29 @@ def test_result(test_case)
         description: scenario.description,
         description_html: scenario.description,
         history_id: scenario.id,
-        full_name: scenario.name,
+        full_name: scenario.full_name,
         test_case_id: scenario.test_case_id,
         title_path: scenario.title_path,
         environment: config.environment,
         **test_result_metadata(parser)
-      )
+      ).tap do |result|
+        result.status_details.muted = true if muted_scenario?(scenario)
+      end
+    end
+
+    # Match a scenario or outline against exact reported full-name selectors.
+    # @param [Cucumber::Core::Test::Case] test_case
+    # @param [Set<String>] selectors
+    # @return [Boolean]
+    def selected?(test_case, selectors)
+      selectors.include?(Scenario.new(test_case, ast_lookup).full_name)
+    end
+
+    # Match a scenario or outline against the test plan's muted identities.
+    # @param [Cucumber::Core::Test::Case, Cucumber::RunningTestCase::TestCase] test_case
+    # @return [Boolean]
+    def muted?(test_case)
+      muted_scenario?(Scenario.new(test_case, ast_lookup))
     end
 
     # Convert to allure step result
@@ -68,6 +85,14 @@ def failure_details(result)
 
     attr_reader :ast_lookup, :config
 
+    # @param [Scenario] scenario
+    # @return [Boolean]
+    def muted_scenario?(scenario)
+      Allure::TestPlan.muted?(
+        allure_id: scenario.allure_id, selector: scenario.full_name, test_case_id: scenario.test_case_id
+      )
+    end
+
     def test_result_metadata(parser)
       {
         labels: parser.labels,

allure-cucumber/lib/allure_cucumber/models/scenario.rb

@@ -19,10 +19,22 @@ def id
       @id ||= Digest::MD5.hexdigest(test_case.inspect)
     end
 
-    # Stable scenario identity shared by all rows of an outline
+    # Stable hashed scenario identity
     # @return [String]
     def test_case_id
-      @test_case_id ||= Allure::ResultUtils.test_case_id(location_file, scenario.name)
+      @test_case_id ||= Digest::MD5.hexdigest(full_name)
+    end
+
+    # Stable scenario identity
+    # @return [String]
+    def full_name
+      @full_name ||= Allure::ResultUtils.full_name(location_file, scenario.name)
+    end
+
+    # Explicit Allure ID from scenario or inherited tags
+    # @return [String, nil]
+    def allure_id
+      @allure_id ||= tags.find { |tag| tag.start_with?("@allure_id:") }&.delete_prefix("@allure_id:")
     end
 
     # Feature name scenario belongs to

allure-cucumber/lib/allure_cucumber/test_plan_filter.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require "cucumber/core/filter"
+
+module AllureCucumber
+  # Select test-plan cases before hooks and execution are applied.
+  class TestPlanFilter < Cucumber::Core::Filter.new(:model, :selectors)
+    # @param [Cucumber::Core::Test::Case] test_case
+    # @return [TestPlanFilter]
+    def test_case(test_case)
+      test_case.describe_to(receiver) if model.selected?(test_case, selectors)
+      self
+    end
+  end
+end

allure-cucumber/spec/cucumber_helper.rb

@@ -3,7 +3,9 @@
 require "cucumber/cli/main"
 
 class MockKernel
-  def exit(status); end
+  def exit(status)
+    status
+  end
 end
 
 class CucumberHelper

allure-cucumber/spec/spec_helper.rb

@@ -50,6 +50,6 @@
   def run_cucumber_cli(feature, support_code: CucumberHelper::ENV)
     Thread.new do
       CucumberHelper.new(test_tmp_dir).execute(feature, support_code: support_code)
-    end.join
+    end.value
   end
 end

allure-cucumber/spec/unit/allure_cucumber_spec.rb

@@ -28,6 +28,29 @@
     expect(AllureCucumber.configuration.tms_prefix).to eq("TMS")
   end
 
+  it "forwards only cases whose file-qualified full name matches a selector" do
+    selector = "features/test.feature#Scenario [case]"
+    allow(Allure::TestPlan).to receive(:test_names).and_return([selector])
+    configuration = Cucumber::Configuration.new
+    receiver = spy("receiver")
+    full_names = [selector, "features/other.feature#Scenario [case]", "features/test.feature#Scenario c"]
+    cases = full_names.map do |full_name|
+      test_case = instance_double(Cucumber::Core::Test::Case, describe_to: nil)
+      scenario = instance_double(AllureCucumber::Scenario, full_name: full_name)
+      allow(AllureCucumber::Scenario).to receive(:new).with(test_case, anything).and_return(scenario)
+      test_case
+    end
+
+    Thread.new { AllureCucumber::CucumberFormatter.new(configuration) }.value
+    expect(configuration.filters.size).to eq(1)
+    filter = configuration.filters.first.with_receiver(receiver)
+    cases.each { |test_case| filter.test_case(test_case) }
+
+    expect(cases[0]).to have_received(:describe_to).with(receiver).once
+    expect(cases[1]).not_to have_received(:describe_to)
+    expect(cases[2]).not_to have_received(:describe_to)
+  end
+
   context "with global labels" do
     include_context "cucumber runner"
 

allure-cucumber/spec/unit/formatter_test_case_started_spec.rb

@@ -43,6 +43,10 @@
     end
 
     it "are added to allure test case" do
+      allow(Allure::TestPlan).to receive(:muted_tests?).and_return(false)
+      allow(AllureCucumber::Scenario).to receive(:new).and_call_original
+      expect_any_instance_of(AllureCucumber::CucumberFormatter).not_to receive(:Around)
+
       run_cucumber_cli(<<~FEATURE)
         Feature: #{feature}
 
@@ -55,14 +59,15 @@
         aggregate_failures "Should have correct args" do
           expect(arg.name).to eq(scenario)
           expect(arg.description).to eq("Simple scenario description")
-          expect(arg.full_name).to eq(scenario)
+          expect(arg.full_name).to eq("#{test_tmp_dir}/features/test.feature##{scenario}")
           expect(arg.test_case_id).to eq(Digest::MD5.hexdigest("#{test_tmp_dir}/features/test.feature##{scenario}"))
           expect(arg.title_path).to eq(["#{test_tmp_dir}/features/test.feature", feature])
           expect(arg.links).to be_empty
           expect(arg.parameters).to be_empty
           expect(arg.labels).to match_array(labels)
         end
       end
+      expect(AllureCucumber::Scenario).to have_received(:new).once
     end
   end
 
@@ -233,6 +238,7 @@
         expect(lifecycle).to(have_received(:start_test_case).twice) { |arg| examples.push(arg) }
         expect(examples[0].name).to include("Add a to b, Examples (#1)")
         expect(examples[1].name).to include("Add a to b, Examples (#2)")
+        expect(examples.map(&:full_name)).to all(eq("#{test_tmp_dir}/features/test.feature#Add a to b"))
         expected_id = Digest::MD5.hexdigest("#{test_tmp_dir}/features/test.feature#Add a to b")
         expect(examples.map(&:test_case_id)).to all(eq(expected_id))
         expect(examples[0].parameters).to contain_exactly(

allure-cucumber/spec/unit/formatter_test_run_finished_spec.rb

@@ -4,18 +4,74 @@
   include_context "allure mock"
   include_context "cucumber runner"
 
-  before do
+  it "writes run-level files" do
     run_cucumber_cli(<<~FEATURE)
       Feature: Simple feature
 
       Scenario: Add a to b
         Simple scenario description
         Given a is 5
     FEATURE
-  end
 
-  it "writes run-level files" do
     expect(lifecycle).to have_received(:write_environment).once
     expect(lifecycle).to have_received(:write_globals).once
   end
+
+  context "with muted tests" do
+    let(:muted_id) { Digest::MD5.hexdigest("#{test_tmp_dir}/features/test.feature#Muted scenario") }
+    let(:muted_selector) { "#{test_tmp_dir}/features/test.feature#Selector scenario" }
+
+    before do
+      allow(Allure::TestPlan).to receive(:muted_tests?).and_return(true)
+      allow(Allure::TestPlan).to receive(:muted?).and_call_original
+      allow(Allure::TestPlan).to receive(:muted?).with(
+        allure_id: "123", selector: anything, test_case_id: anything
+      ).and_return(true)
+      allow(Allure::TestPlan).to receive(:muted?).with(
+        allure_id: "456", selector: anything, test_case_id: muted_id
+      ).and_return(true)
+      allow(Allure::TestPlan).to receive(:muted?).with(
+        allure_id: nil, selector: muted_selector, test_case_id: anything
+      ).and_return(true)
+      allow(Allure::AllureLifecycle).to receive(:new).and_call_original
+    end
+
+    it "skips scenarios matching an inherited Allure ID, generated identity or selector" do
+      allow(AllureCucumber::Scenario).to receive(:new).and_call_original
+      muted_names = ["Allure ID scenario", "Muted scenario", "Selector scenario"]
+
+      status = run_cucumber_cli(<<~FEATURE)
+        Feature: Simple feature
+
+        @allure_id:123
+        Rule: Explicit IDs
+          Scenario: Allure ID scenario
+            Then step fails with simple exception
+
+        Rule: Generated IDs
+          @allure_id:456
+          Scenario: Muted scenario
+            Then step fails with simple exception
+
+          Scenario: Selector scenario
+            Then step fails with simple exception
+
+          Scenario: Add a to b
+            Simple scenario description
+            Given a is 5
+      FEATURE
+
+      results = Dir["#{test_tmp_dir}/reports/allure-results/*-result.json"].flat_map { |f| JSON.parse(File.read(f)) }
+      muted = results.select { |t| muted_names.include?(t["name"]) }
+
+      aggregate_failures do
+        expect(status).to eq(0)
+        expect(results.find { |t| t["name"] == "Add a to b" }["status"]).to eq("passed")
+        expect(muted.size).to eq(3)
+        expect(muted.map { |t| t["status"] }).to all(eq("skipped"))
+        expect(muted.map { |t| t["statusDetails"]["muted"] }).to all(be(true))
+        expect(AllureCucumber::Scenario).to have_received(:new).exactly(8).times
+      end
+    end
+  end
 end

allure-rspec/lib/allure_rspec/_utils.rb

@@ -2,6 +2,15 @@
 
 module AllureRspec
   module Utils
+    # Match reported full names through RSpec's metadata filter API.
+    # @param [Set<String>] names
+    # @return [Proc]
+    def self.full_name_filter(names)
+      lambda do |file_path, metadata|
+        names.include?(Allure::ResultUtils.full_name(file_path, metadata[:full_description]))
+      end
+    end
+
     # Strip relative ./ form path
     # @param [String] path
     # @return [String]

allure-rspec/lib/allure_rspec/formatter.rb

@@ -34,7 +34,7 @@ class RSpecFormatter < RSpec::Core::Formatters::BaseFormatter
       names = Allure::TestPlan.test_names
 
       config.filter_run_when_matching(*ids.map { |id| { allure_id: id } }) if ids
-      config.full_description = names if names
+      config.filter_run_including(rerun_file_path: Utils.full_name_filter(names.to_set)) if names
     end
 
     def initialize(output)
@@ -78,7 +78,16 @@ def example_group_started(example_group_notification)
     # @param [RSpec::Core::Notifications::ExampleNotification] example_notification
     # @return [void]
     def example_started(example_notification)
-      lifecycle.start_test_case(test_result(example_notification.example))
+      example = example_notification.example
+      result = test_result(example)
+      if Allure::TestPlan.muted?(
+        allure_id: example.metadata[:allure_id], selector: result.full_name, test_case_id: result.test_case_id
+      )
+        result.status_details.muted = true
+        example.metadata[:skip] = "Skipped by Allure test plan"
+      end
+
+      lifecycle.start_test_case(result)
     end
 
     # Finishes example
@@ -116,14 +125,15 @@ def message(notification)
     # @return [Allure::TestResult]
     def test_result(example)
       parser = RspecMetadataParser.new(example, allure_config)
+      full_name = Allure::ResultUtils.full_name(example.metadata[:rerun_file_path], example.full_description)
 
       Allure::TestResult.new(
         name: example.description,
         description: "Location - #{strip_relative(parser.location)}",
         description_html: "Location - #{strip_relative(parser.location)}",
         history_id: example.id,
-        full_name: example.full_description,
-        test_case_id: Allure::ResultUtils.test_case_id(example.metadata[:rerun_file_path], example.full_description),
+        full_name: full_name,
+        test_case_id: Digest::MD5.hexdigest(full_name),
         title_path: parser.title_path,
         labels: parser.labels,
         links: parser.links,

allure-rspec/spec/spec_helper.rb

@@ -60,6 +60,6 @@ def run_rspec(spec)
     allow(RSpec).to receive(:configuration).and_return(configuration)
     allow(RSpec).to receive(:world).and_return(world)
 
-    Thread.new { RspecRunner.new(test_tmp_dir).run(spec) }.join
+    Thread.new { RspecRunner.new(test_tmp_dir).run(spec) }.value
   end
 end