diff --git a/allure-cucumber/lib/allure_cucumber/formatter.rb b/allure-cucumber/lib/allure_cucumber/formatter.rb index c9ce0029..20840da0 100644 --- a/allure-cucumber/lib/allure_cucumber/formatter.rb +++ b/allure-cucumber/lib/allure_cucumber/formatter.rb @@ -1,10 +1,15 @@ # frozen_string_literal: true require "cucumber/core" +require "cucumber/glue/dsl" + +require_relative "test_plan_filter" module AllureCucumber # Main formatter class. Translates cucumber event to allure lifecycle class CucumberFormatter + include Cucumber::Glue::Dsl + # @return [Hash] hook handler methods HOOK_HANDLERS = { "Before hook" => :start_prepare_fixture, @@ -27,7 +32,7 @@ def initialize(config) @cucumber_model ||= AllureCucumberModel.new(config, allure_config) names = Allure::TestPlan.test_names - config.name_regexps.push(*names.map { |name| /#{name}/ }) if names + config.filters << TestPlanFilter.new(cucumber_model, names.to_set) if names config.on_event(:test_run_started) { |event| on_test_run_started(event) } config.on_event(:test_run_finished) { |event| on_test_run_finished(event) } @@ -37,6 +42,7 @@ def initialize(config) config.on_event(:test_case_finished) { |event| on_test_case_finished(event) } register_test_run_hook_finished(config) + register_muting_hook end # Clean test result directory before starting run @@ -69,8 +75,11 @@ def on_test_run_hook_finished(event) # @param [Cucumber::Events::TestCaseStarted] event # @return [void] def on_test_case_started(event) - lifecycle.start_test_container(Allure::TestResultContainer.new(name: event.test_case.name)) - lifecycle.start_test_case(cucumber_model.test_result(event.test_case)) + test_case = event.test_case + result = cucumber_model.test_result(test_case) + + lifecycle.start_test_container(Allure::TestResultContainer.new(name: test_case.name)) + lifecycle.start_test_case(result) end # Handle test step started event @@ -114,6 +123,18 @@ def on_test_case_finished(event) attr_reader :lifecycle, :cucumber_model + # Skip matched scenarios through Cucumber's public hook DSL. + # @return [void] + def register_muting_hook + return unless Allure::TestPlan.muted_tests? + + model = cucumber_model + Around do |scenario, run| + skip_this_scenario("Skipped by Allure test plan") if model.muted?(scenario) + run.call + end + end + # Register global hook finished event # @param [Cucumber::Configuration] config # @return [void] diff --git a/allure-cucumber/lib/allure_cucumber/models/cucumber_model.rb b/allure-cucumber/lib/allure_cucumber/models/cucumber_model.rb index f08d4d94..06fca956 100644 --- a/allure-cucumber/lib/allure_cucumber/models/cucumber_model.rb +++ b/allure-cucumber/lib/allure_cucumber/models/cucumber_model.rb @@ -25,12 +25,29 @@ def test_result(test_case) description: scenario.description, description_html: scenario.description, history_id: scenario.id, - full_name: scenario.name, + full_name: scenario.full_name, test_case_id: scenario.test_case_id, title_path: scenario.title_path, environment: config.environment, **test_result_metadata(parser) - ) + ).tap do |result| + result.status_details.muted = true if muted_scenario?(scenario) + end + end + + # Match a scenario or outline against exact reported full-name selectors. + # @param [Cucumber::Core::Test::Case] test_case + # @param [Set] selectors + # @return [Boolean] + def selected?(test_case, selectors) + selectors.include?(Scenario.new(test_case, ast_lookup).full_name) + end + + # Match a scenario or outline against the test plan's muted identities. + # @param [Cucumber::Core::Test::Case, Cucumber::RunningTestCase::TestCase] test_case + # @return [Boolean] + def muted?(test_case) + muted_scenario?(Scenario.new(test_case, ast_lookup)) end # Convert to allure step result @@ -68,6 +85,14 @@ def failure_details(result) attr_reader :ast_lookup, :config + # @param [Scenario] scenario + # @return [Boolean] + def muted_scenario?(scenario) + Allure::TestPlan.muted?( + allure_id: scenario.allure_id, selector: scenario.full_name, test_case_id: scenario.test_case_id + ) + end + def test_result_metadata(parser) { labels: parser.labels, diff --git a/allure-cucumber/lib/allure_cucumber/models/scenario.rb b/allure-cucumber/lib/allure_cucumber/models/scenario.rb index 9e3c6b29..e08f5bdb 100644 --- a/allure-cucumber/lib/allure_cucumber/models/scenario.rb +++ b/allure-cucumber/lib/allure_cucumber/models/scenario.rb @@ -19,10 +19,22 @@ def id @id ||= Digest::MD5.hexdigest(test_case.inspect) end - # Stable scenario identity shared by all rows of an outline + # Stable hashed scenario identity # @return [String] def test_case_id - @test_case_id ||= Allure::ResultUtils.test_case_id(location_file, scenario.name) + @test_case_id ||= Digest::MD5.hexdigest(full_name) + end + + # Stable scenario identity + # @return [String] + def full_name + @full_name ||= Allure::ResultUtils.full_name(location_file, scenario.name) + end + + # Explicit Allure ID from scenario or inherited tags + # @return [String, nil] + def allure_id + @allure_id ||= tags.find { |tag| tag.start_with?("@allure_id:") }&.delete_prefix("@allure_id:") end # Feature name scenario belongs to diff --git a/allure-cucumber/lib/allure_cucumber/test_plan_filter.rb b/allure-cucumber/lib/allure_cucumber/test_plan_filter.rb new file mode 100644 index 00000000..13c83948 --- /dev/null +++ b/allure-cucumber/lib/allure_cucumber/test_plan_filter.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +require "cucumber/core/filter" + +module AllureCucumber + # Select test-plan cases before hooks and execution are applied. + class TestPlanFilter < Cucumber::Core::Filter.new(:model, :selectors) + # @param [Cucumber::Core::Test::Case] test_case + # @return [TestPlanFilter] + def test_case(test_case) + test_case.describe_to(receiver) if model.selected?(test_case, selectors) + self + end + end +end diff --git a/allure-cucumber/spec/cucumber_helper.rb b/allure-cucumber/spec/cucumber_helper.rb index 076c89f8..7f904b14 100644 --- a/allure-cucumber/spec/cucumber_helper.rb +++ b/allure-cucumber/spec/cucumber_helper.rb @@ -3,7 +3,9 @@ require "cucumber/cli/main" class MockKernel - def exit(status); end + def exit(status) + status + end end class CucumberHelper diff --git a/allure-cucumber/spec/spec_helper.rb b/allure-cucumber/spec/spec_helper.rb index 2ea2afae..eb314eb4 100644 --- a/allure-cucumber/spec/spec_helper.rb +++ b/allure-cucumber/spec/spec_helper.rb @@ -50,6 +50,6 @@ def run_cucumber_cli(feature, support_code: CucumberHelper::ENV) Thread.new do CucumberHelper.new(test_tmp_dir).execute(feature, support_code: support_code) - end.join + end.value end end diff --git a/allure-cucumber/spec/unit/allure_cucumber_spec.rb b/allure-cucumber/spec/unit/allure_cucumber_spec.rb index 428265b6..a3b39b7d 100644 --- a/allure-cucumber/spec/unit/allure_cucumber_spec.rb +++ b/allure-cucumber/spec/unit/allure_cucumber_spec.rb @@ -28,6 +28,29 @@ expect(AllureCucumber.configuration.tms_prefix).to eq("TMS") end + it "forwards only cases whose file-qualified full name matches a selector" do + selector = "features/test.feature#Scenario [case]" + allow(Allure::TestPlan).to receive(:test_names).and_return([selector]) + configuration = Cucumber::Configuration.new + receiver = spy("receiver") + full_names = [selector, "features/other.feature#Scenario [case]", "features/test.feature#Scenario c"] + cases = full_names.map do |full_name| + test_case = instance_double(Cucumber::Core::Test::Case, describe_to: nil) + scenario = instance_double(AllureCucumber::Scenario, full_name: full_name) + allow(AllureCucumber::Scenario).to receive(:new).with(test_case, anything).and_return(scenario) + test_case + end + + Thread.new { AllureCucumber::CucumberFormatter.new(configuration) }.value + expect(configuration.filters.size).to eq(1) + filter = configuration.filters.first.with_receiver(receiver) + cases.each { |test_case| filter.test_case(test_case) } + + expect(cases[0]).to have_received(:describe_to).with(receiver).once + expect(cases[1]).not_to have_received(:describe_to) + expect(cases[2]).not_to have_received(:describe_to) + end + context "with global labels" do include_context "cucumber runner" diff --git a/allure-cucumber/spec/unit/formatter_test_case_started_spec.rb b/allure-cucumber/spec/unit/formatter_test_case_started_spec.rb index ffbfbf8a..de80daf5 100644 --- a/allure-cucumber/spec/unit/formatter_test_case_started_spec.rb +++ b/allure-cucumber/spec/unit/formatter_test_case_started_spec.rb @@ -43,6 +43,10 @@ end it "are added to allure test case" do + allow(Allure::TestPlan).to receive(:muted_tests?).and_return(false) + allow(AllureCucumber::Scenario).to receive(:new).and_call_original + expect_any_instance_of(AllureCucumber::CucumberFormatter).not_to receive(:Around) + run_cucumber_cli(<<~FEATURE) Feature: #{feature} @@ -55,7 +59,7 @@ aggregate_failures "Should have correct args" do expect(arg.name).to eq(scenario) expect(arg.description).to eq("Simple scenario description") - expect(arg.full_name).to eq(scenario) + expect(arg.full_name).to eq("#{test_tmp_dir}/features/test.feature##{scenario}") expect(arg.test_case_id).to eq(Digest::MD5.hexdigest("#{test_tmp_dir}/features/test.feature##{scenario}")) expect(arg.title_path).to eq(["#{test_tmp_dir}/features/test.feature", feature]) expect(arg.links).to be_empty @@ -63,6 +67,7 @@ expect(arg.labels).to match_array(labels) end end + expect(AllureCucumber::Scenario).to have_received(:new).once end end @@ -233,6 +238,7 @@ expect(lifecycle).to(have_received(:start_test_case).twice) { |arg| examples.push(arg) } expect(examples[0].name).to include("Add a to b, Examples (#1)") expect(examples[1].name).to include("Add a to b, Examples (#2)") + expect(examples.map(&:full_name)).to all(eq("#{test_tmp_dir}/features/test.feature#Add a to b")) expected_id = Digest::MD5.hexdigest("#{test_tmp_dir}/features/test.feature#Add a to b") expect(examples.map(&:test_case_id)).to all(eq(expected_id)) expect(examples[0].parameters).to contain_exactly( diff --git a/allure-cucumber/spec/unit/formatter_test_run_finished_spec.rb b/allure-cucumber/spec/unit/formatter_test_run_finished_spec.rb index 90e50c00..0fd1e32f 100644 --- a/allure-cucumber/spec/unit/formatter_test_run_finished_spec.rb +++ b/allure-cucumber/spec/unit/formatter_test_run_finished_spec.rb @@ -4,7 +4,7 @@ include_context "allure mock" include_context "cucumber runner" - before do + it "writes run-level files" do run_cucumber_cli(<<~FEATURE) Feature: Simple feature @@ -12,10 +12,66 @@ Simple scenario description Given a is 5 FEATURE - end - it "writes run-level files" do expect(lifecycle).to have_received(:write_environment).once expect(lifecycle).to have_received(:write_globals).once end + + context "with muted tests" do + let(:muted_id) { Digest::MD5.hexdigest("#{test_tmp_dir}/features/test.feature#Muted scenario") } + let(:muted_selector) { "#{test_tmp_dir}/features/test.feature#Selector scenario" } + + before do + allow(Allure::TestPlan).to receive(:muted_tests?).and_return(true) + allow(Allure::TestPlan).to receive(:muted?).and_call_original + allow(Allure::TestPlan).to receive(:muted?).with( + allure_id: "123", selector: anything, test_case_id: anything + ).and_return(true) + allow(Allure::TestPlan).to receive(:muted?).with( + allure_id: "456", selector: anything, test_case_id: muted_id + ).and_return(true) + allow(Allure::TestPlan).to receive(:muted?).with( + allure_id: nil, selector: muted_selector, test_case_id: anything + ).and_return(true) + allow(Allure::AllureLifecycle).to receive(:new).and_call_original + end + + it "skips scenarios matching an inherited Allure ID, generated identity or selector" do + allow(AllureCucumber::Scenario).to receive(:new).and_call_original + muted_names = ["Allure ID scenario", "Muted scenario", "Selector scenario"] + + status = run_cucumber_cli(<<~FEATURE) + Feature: Simple feature + + @allure_id:123 + Rule: Explicit IDs + Scenario: Allure ID scenario + Then step fails with simple exception + + Rule: Generated IDs + @allure_id:456 + Scenario: Muted scenario + Then step fails with simple exception + + Scenario: Selector scenario + Then step fails with simple exception + + Scenario: Add a to b + Simple scenario description + Given a is 5 + FEATURE + + results = Dir["#{test_tmp_dir}/reports/allure-results/*-result.json"].flat_map { |f| JSON.parse(File.read(f)) } + muted = results.select { |t| muted_names.include?(t["name"]) } + + aggregate_failures do + expect(status).to eq(0) + expect(results.find { |t| t["name"] == "Add a to b" }["status"]).to eq("passed") + expect(muted.size).to eq(3) + expect(muted.map { |t| t["status"] }).to all(eq("skipped")) + expect(muted.map { |t| t["statusDetails"]["muted"] }).to all(be(true)) + expect(AllureCucumber::Scenario).to have_received(:new).exactly(8).times + end + end + end end diff --git a/allure-rspec/lib/allure_rspec/_utils.rb b/allure-rspec/lib/allure_rspec/_utils.rb index 592151dd..9fd170d1 100644 --- a/allure-rspec/lib/allure_rspec/_utils.rb +++ b/allure-rspec/lib/allure_rspec/_utils.rb @@ -2,6 +2,15 @@ module AllureRspec module Utils + # Match reported full names through RSpec's metadata filter API. + # @param [Set] names + # @return [Proc] + def self.full_name_filter(names) + lambda do |file_path, metadata| + names.include?(Allure::ResultUtils.full_name(file_path, metadata[:full_description])) + end + end + # Strip relative ./ form path # @param [String] path # @return [String] diff --git a/allure-rspec/lib/allure_rspec/formatter.rb b/allure-rspec/lib/allure_rspec/formatter.rb index 80cdda74..b1ab9170 100644 --- a/allure-rspec/lib/allure_rspec/formatter.rb +++ b/allure-rspec/lib/allure_rspec/formatter.rb @@ -34,7 +34,7 @@ class RSpecFormatter < RSpec::Core::Formatters::BaseFormatter names = Allure::TestPlan.test_names config.filter_run_when_matching(*ids.map { |id| { allure_id: id } }) if ids - config.full_description = names if names + config.filter_run_including(rerun_file_path: Utils.full_name_filter(names.to_set)) if names end def initialize(output) @@ -78,7 +78,16 @@ def example_group_started(example_group_notification) # @param [RSpec::Core::Notifications::ExampleNotification] example_notification # @return [void] def example_started(example_notification) - lifecycle.start_test_case(test_result(example_notification.example)) + example = example_notification.example + result = test_result(example) + if Allure::TestPlan.muted?( + allure_id: example.metadata[:allure_id], selector: result.full_name, test_case_id: result.test_case_id + ) + result.status_details.muted = true + example.metadata[:skip] = "Skipped by Allure test plan" + end + + lifecycle.start_test_case(result) end # Finishes example @@ -116,14 +125,15 @@ def message(notification) # @return [Allure::TestResult] def test_result(example) parser = RspecMetadataParser.new(example, allure_config) + full_name = Allure::ResultUtils.full_name(example.metadata[:rerun_file_path], example.full_description) Allure::TestResult.new( name: example.description, description: "Location - #{strip_relative(parser.location)}", description_html: "Location - #{strip_relative(parser.location)}", history_id: example.id, - full_name: example.full_description, - test_case_id: Allure::ResultUtils.test_case_id(example.metadata[:rerun_file_path], example.full_description), + full_name: full_name, + test_case_id: Digest::MD5.hexdigest(full_name), title_path: parser.title_path, labels: parser.labels, links: parser.links, diff --git a/allure-rspec/spec/spec_helper.rb b/allure-rspec/spec/spec_helper.rb index f938dd28..000bcbce 100644 --- a/allure-rspec/spec/spec_helper.rb +++ b/allure-rspec/spec/spec_helper.rb @@ -60,6 +60,6 @@ def run_rspec(spec) allow(RSpec).to receive(:configuration).and_return(configuration) allow(RSpec).to receive(:world).and_return(world) - Thread.new { RspecRunner.new(test_tmp_dir).run(spec) }.join + Thread.new { RspecRunner.new(test_tmp_dir).run(spec) }.value end end diff --git a/allure-rspec/spec/unit/allure_rspec_spec.rb b/allure-rspec/spec/unit/allure_rspec_spec.rb index 51b407bc..ae47d159 100644 --- a/allure-rspec/spec/unit/allure_rspec_spec.rb +++ b/allure-rspec/spec/unit/allure_rspec_spec.rb @@ -28,6 +28,18 @@ expect(AllureRspec.configuration.tms_tag).to eq("TMS") end + it "matches selectors against exact file-qualified full names" do + filter = AllureRspec::Utils.full_name_filter(["spec/test_spec.rb#Suite [case]"]) + + matches = [ + filter.call("./spec/test_spec.rb", full_description: "Suite [case]"), + filter.call("spec/other_spec.rb", full_description: "Suite [case]"), + filter.call("spec/test_spec.rb", full_description: "Suite c") + ] + + expect(matches).to eq([true, false, false]) + end + context "with global labels" do include_context "rspec runner" diff --git a/allure-rspec/spec/unit/formatter_example_started_spec.rb b/allure-rspec/spec/unit/formatter_example_started_spec.rb index 9582738e..a8105130 100644 --- a/allure-rspec/spec/unit/formatter_example_started_spec.rb +++ b/allure-rspec/spec/unit/formatter_example_started_spec.rb @@ -21,6 +21,50 @@ ] end + context "with muted tests" do + let(:muted_id) { Digest::MD5.hexdigest("#{test_tmp_dir}/spec/test_spec.rb##{suite} #{spec}") } + let(:muted_selector) { "#{test_tmp_dir}/spec/test_spec.rb##{suite} Selector" } + + before do + allow(Allure::TestPlan).to receive(:muted?).and_call_original + allow(Allure::TestPlan).to receive(:muted?).with( + allure_id: 123, selector: anything, test_case_id: anything + ).and_return(true) + allow(Allure::TestPlan).to receive(:muted?).with( + allure_id: 456, selector: anything, test_case_id: muted_id + ).and_return(true) + allow(Allure::TestPlan).to receive(:muted?).with( + allure_id: 456, selector: muted_selector, test_case_id: anything + ).and_return(true) + end + + it "adds skip metadata to examples matching any muted identity" do + run_rspec(<<~SPEC) + describe "#{suite}" do + it "Explicit ID", allure_id: 123 do + raise "Muted body should not run" + end + + it "#{spec}", allure_id: 456 do + raise "Muted body should not run" + end + + it "Selector", allure_id: 456 do + raise "Muted body should not run" + end + + it "Unmatched", allure_id: 456 do + end + end + SPEC + + examples = RSpec.world.example_groups.first.examples + expect(examples.map { |example| example.metadata[:skip] }).to eq( + ["Skipped by Allure test plan", "Skipped by Allure test plan", "Skipped by Allure test plan", nil] + ) + end + end + context "default parameters" do it "are added to allure test case" do run_rspec(<<~SPEC) @@ -43,7 +87,7 @@ aggregate_failures "Should have correct args" do expect(arg.name).to eq(spec) expect(arg.description).to eq("Location - #{test_tmp_dir}/spec/test_spec.rb:10") - expect(arg.full_name).to eq("#{suite} #{spec}") + expect(arg.full_name).to eq("#{test_tmp_dir}/spec/test_spec.rb##{suite} #{spec}") expect(arg.test_case_id).to eq(Digest::MD5.hexdigest("#{test_tmp_dir}/spec/test_spec.rb##{suite} #{spec}")) expect(arg.title_path).to eq(["#{test_tmp_dir}/spec/test_spec.rb", suite]) expect(arg.links).to be_empty @@ -71,6 +115,7 @@ expect(lifecycle).to have_received(:start_test_case).once do |arg| aggregate_failures "Should have correct args" do expect(arg.description).to eq("Location - #{test_tmp_dir}/spec/test_spec.rb:8") + expect(arg.full_name).to eq("#{test_tmp_dir}/spec/test_spec.rb##{suite} behaves like shared #{spec}") end end end diff --git a/allure-ruby-commons/lib/allure_ruby_commons/result_utils.rb b/allure-ruby-commons/lib/allure_ruby_commons/result_utils.rb index 31e04aea..b94b4fc2 100644 --- a/allure-ruby-commons/lib/allure_ruby_commons/result_utils.rb +++ b/allure-ruby-commons/lib/allure_ruby_commons/result_utils.rb @@ -34,9 +34,10 @@ class << self # @param [String] file_path # @param [String] full_name # @return [String] - def test_case_id(file_path, full_name) + def full_name(file_path, full_name) relative_path = Pathname.new(File.expand_path(file_path)).relative_path_from(Pathname.pwd).to_s - Digest::MD5.hexdigest("#{relative_path}##{full_name}") + + "#{relative_path}##{full_name}" end # @param [Time] time diff --git a/allure-ruby-commons/lib/allure_ruby_commons/test_plan.rb b/allure-ruby-commons/lib/allure_ruby_commons/test_plan.rb new file mode 100644 index 00000000..19a78801 --- /dev/null +++ b/allure-ruby-commons/lib/allure_ruby_commons/test_plan.rb @@ -0,0 +1,110 @@ +# frozen_string_literal: true + +require "set" + +module Allure + class TestPlan + extend JsonHelper + + # @return [String] test plan path env var name + TESTPLAN_PATH = "ALLURE_TESTPLAN_PATH" + + class << self + # Allure id's of executable tests + # + # @return [Array] + def test_ids + @test_ids ||= tests&.filter_map { |test| test[:id] } + end + + # Test names of executable tests + # + # @return [Array] + def test_names + @test_names ||= tests&.filter_map { |test| test[:selector] } + end + + # Match an Allure ID first, falling back to the selector and generated identity + # + # @param [String, Integer, nil] allure_id + # @param [String, nil] selector file-qualified full name + # @param [String] test_case_id + # @return [Boolean] + def muted?(allure_id:, selector:, test_case_id:) + return true if allure_id && muted_allure_ids.include?(allure_id.to_s) + return true if muted_selectors.include?(selector) + + muted_test_ids.include?(test_case_id) + end + + # Whether the test plan contains any muted identities + # + # @return [Boolean] + def muted_tests? + muted_allure_ids.any? || muted_test_ids.any? || muted_selectors.any? + end + + private + + # Muted explicit Allure ids + # + # @return [Set] + def muted_allure_ids + @muted_allure_ids ||= Set.new(muted_tests&.filter_map { |test| test[:id]&.to_s }).freeze + end + + # Muted test case ids + # + # @return [Set] + def muted_test_ids + @muted_test_ids ||= Set.new(muted_tests&.filter_map { |test| test[:testCaseId] }).freeze + end + + # Muted file-qualified full-name selectors + # + # @return [Set] + def muted_selectors + @muted_selectors ||= Set.new(muted_tests&.filter_map { |test| test[:selector] }).freeze + end + + # Parsed test plan object + # + # @return [Hash] + def test_plan + @test_plan ||= load_json(test_plan_path) + end + + # Tests to execute from allure testplan.json + # + # @return [Array] + def tests + @tests ||= test_plan&.fetch(:tests, nil) if test_plan_path + end + + # Tests to skip from allure testplan.json + # + # @return [Array] + def muted_tests + @muted_tests ||= test_plan&.fetch(:mutedTests, nil) if test_plan_path + end + + # Fetch test plan path + # + # @return [] + def test_plan_path + return @test_plan_path if defined?(@test_plan_path) + + @test_plan_path = ENV[TESTPLAN_PATH].then do |path| + next unless path + next path if File.file?(path) + + json = File.join(path, "testplan.json") + next json if File.exist?(json) + + Allure.configuration.logger.warn("'ALLURE_TESTPLAN_PATH' env var is set, but no testplan.json found!") + nil + end + end + end + end +end diff --git a/allure-ruby-commons/lib/allure_ruby_commons/testplan.rb b/allure-ruby-commons/lib/allure_ruby_commons/testplan.rb deleted file mode 100644 index 8f6afb99..00000000 --- a/allure-ruby-commons/lib/allure_ruby_commons/testplan.rb +++ /dev/null @@ -1,53 +0,0 @@ -# frozen_string_literal: true - -module Allure - class TestPlan - extend JsonHelper - - # @return [String] test plan path env var name - TESTPLAN_PATH = "ALLURE_TESTPLAN_PATH" - - class << self - # Allure id's of executable tests - # - # @return [Array] - def test_ids - @test_ids ||= tests&.map { |test| test[:id] } - end - - # Test names of executable tests - # - # @return [Array] - def test_names - @test_names ||= tests&.map { |test| test[:selector] } - end - - private - - # Tests to execute from allure testplan.json - # - # @return [Array] - def tests - @tests ||= load_json(test_plan_path)&.fetch(:tests) if test_plan_path - end - - # Fetch test plan path - # - # @return [] - def test_plan_path - return @test_plan_path if defined?(@test_plan_path) - - @test_plan_path = ENV[TESTPLAN_PATH].then do |path| - next unless path - next path if File.file?(path) - - json = File.join(path, "testplan.json") - next json if File.exist?(json) - - Allure.configuration.logger.warn("'ALLURE_TESTPLAN_PATH' env var is set, but no testplan.json found!") - nil - end - end - end - end -end diff --git a/allure-ruby-commons/spec/fixtures/test_plan/ids_only/testplan.json b/allure-ruby-commons/spec/fixtures/test_plan/ids_only/testplan.json new file mode 100644 index 00000000..0f62fa84 --- /dev/null +++ b/allure-ruby-commons/spec/fixtures/test_plan/ids_only/testplan.json @@ -0,0 +1,4 @@ +{ + "version": "1.0", + "mutedTests": [{"id": 123}] +} diff --git a/allure-ruby-commons/spec/fixtures/test_plan/partial/testplan.json b/allure-ruby-commons/spec/fixtures/test_plan/partial/testplan.json new file mode 100644 index 00000000..404aa21d --- /dev/null +++ b/allure-ruby-commons/spec/fixtures/test_plan/partial/testplan.json @@ -0,0 +1,5 @@ +{ + "version": "1.0", + "tests": [{}, {"id": "id only"}, {"selector": "selector only"}], + "mutedTests": [{}, {"id": 123}, {"testCaseId": "muted-id"}] +} diff --git a/allure-ruby-commons/spec/fixtures/test_plan/selectors_only/testplan.json b/allure-ruby-commons/spec/fixtures/test_plan/selectors_only/testplan.json new file mode 100644 index 00000000..267c462d --- /dev/null +++ b/allure-ruby-commons/spec/fixtures/test_plan/selectors_only/testplan.json @@ -0,0 +1,4 @@ +{ + "version": "1.0", + "mutedTests": [{"selector": "spec/test_spec.rb#Suite [case]"}] +} diff --git a/allure-ruby-commons/spec/unit/result_utils_spec.rb b/allure-ruby-commons/spec/unit/result_utils_spec.rb index a08432e8..b3178ad4 100644 --- a/allure-ruby-commons/spec/unit/result_utils_spec.rb +++ b/allure-ruby-commons/spec/unit/result_utils_spec.rb @@ -64,31 +64,37 @@ def raise_multi_error expect(utils.issue_link("issue", "http://jira.com/123", "{}")).to eq(Allure::Link.new("issue", "issue", "http://jira.com/123")) end - describe ".test_case_id" do - it "hashes the normalized relative file and full name" do + describe ".full_name" do + it "combines the normalized relative file and test name" do paths = [ "spec/test_spec.rb", "./spec/test_spec.rb", "spec/nested/../test_spec.rb", File.expand_path("spec/test_spec.rb") ] - expected = Digest::MD5.hexdigest("spec/test_spec.rb#Suite example") + expected = "spec/test_spec.rb#Suite example" - ids = paths.map { |path| utils.test_case_id(path, "Suite example") } + full_names = paths.map { |path| utils.full_name(path, "Suite example") } - expect(ids).to all(eq(expected)) + expect(full_names).to all(eq(expected)) end - it "distinguishes directories and full names" do + it "distinguishes directories and test names" do inputs = [ ["spec/api/test_spec.rb", "Suite example"], ["spec/ui/test_spec.rb", "Suite example"], ["spec/api/test_spec.rb", "Suite another example"] ] - ids = inputs.map { |file, name| utils.test_case_id(file, name) } + expected = [ + "spec/api/test_spec.rb#Suite example", + "spec/ui/test_spec.rb#Suite example", + "spec/api/test_spec.rb#Suite another example" + ] + + full_names = inputs.map { |file, name| utils.full_name(file, name) } - expect(ids.uniq.size).to eq(inputs.size) + expect(full_names).to eq(expected) end end diff --git a/allure-ruby-commons/spec/unit/testplan_spec.rb b/allure-ruby-commons/spec/unit/testplan_spec.rb index 02de04b9..ec01340a 100644 --- a/allure-ruby-commons/spec/unit/testplan_spec.rb +++ b/allure-ruby-commons/spec/unit/testplan_spec.rb @@ -8,8 +8,12 @@ described_class.instance_variable_set(:@tests, nil) described_class.instance_variable_set(:@test_ids, nil) described_class.instance_variable_set(:@test_names, nil) + described_class.instance_variable_set(:@muted_tests, nil) + described_class.instance_variable_set(:@muted_test_ids, nil) + described_class.instance_variable_set(:@muted_allure_ids, nil) + described_class.instance_variable_set(:@muted_selectors, nil) - :@test_plan_path.tap do |var| + %i[@test_plan_path @test_plan].each do |var| described_class.send(:remove_instance_variable, var) if described_class.instance_variable_defined?(var) end end @@ -24,13 +28,15 @@ it "correct testplan.json" do expect(described_class.test_ids).to eq(test_ids) expect(described_class.test_names).to eq(test_names) + expect(described_class.muted?(allure_id: nil, selector: nil, test_case_id: "muted-id")).to be(false) + expect(described_class.muted_tests?).to be(false) end end context "handles" do let(:path) { "#{Dir.pwd}/spec/fixtures/test_plan/malformed/testplan.json" } - it "malformed testplan.json" do + it "malformed testplan.json", :focus do expect(described_class.test_ids).to eq(nil) expect(described_class.test_names).to eq(nil) end @@ -45,6 +51,74 @@ end end + context "with partial entries" do + let(:path) { "#{Dir.pwd}/spec/fixtures/test_plan/partial/testplan.json" } + + it "matches muted identities and ignores absent IDs and selectors" do + matched = described_class.muted?(allure_id: nil, selector: nil, test_case_id: "muted-id") + unmatched = described_class.muted?(allure_id: nil, selector: nil, test_case_id: "other-id") + missing = described_class.muted?(allure_id: nil, selector: nil, test_case_id: nil) + matched_by_allure_id = described_class.muted?(allure_id: "123", selector: nil, test_case_id: "other-id") + fallback = described_class.muted?(allure_id: 456, selector: nil, test_case_id: "muted-id") + unmatched_ids = described_class.muted?(allure_id: 456, selector: nil, test_case_id: "other-id") + + expect(described_class.test_ids).to eq(["id only"]) + expect(described_class.test_names).to eq(["selector only"]) + expect(matched).to be(true) + expect(unmatched).to be(false) + expect(missing).to be(false) + expect(matched_by_allure_id).to be(true) + expect(fallback).to be(true) + expect(unmatched_ids).to be(false) + expect(described_class.muted_tests?).to be(true) + end + end + + context "with only muted Allure IDs" do + let(:path) { "#{Dir.pwd}/spec/fixtures/test_plan/ids_only/testplan.json" } + + it "activates muting without generated identities" do + available = described_class.muted_tests? + matched = described_class.muted?(allure_id: 123, selector: nil, test_case_id: "other-id") + + expect(available).to be(true) + expect(matched).to be(true) + end + end + + context "with only muted selectors" do + let(:path) { "#{Dir.pwd}/spec/fixtures/test_plan/selectors_only/testplan.json" } + + it "activates muting and matches exact full names when IDs do not match" do + available = described_class.muted_tests? + matched = described_class.muted?( + allure_id: 456, selector: "spec/test_spec.rb#Suite [case]", test_case_id: "other-id" + ) + other_file = described_class.muted?( + allure_id: nil, selector: "spec/other_spec.rb#Suite [case]", test_case_id: "other-id" + ) + regex_like = described_class.muted?( + allure_id: nil, selector: "spec/test_spec.rb#Suite c", test_case_id: "other-id" + ) + + expect(available).to be(true) + expect(matched).to be(true) + expect(other_file).to be(false) + expect(regex_like).to be(false) + end + end + + context "without a plan" do + let(:path) { nil } + + it "does not mute tests" do + muted = described_class.muted?(allure_id: nil, selector: nil, test_case_id: "muted-id") + + expect(muted).to be(false) + expect(described_class.muted_tests?).to be(false) + end + end + context "handles" do let(:path) { "#{Dir.pwd}/spec/fixtures/test_plan/correct" }