From 6a50e26d1dc57f49f568025d5adac32cb9c4fc94 Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Sat, 5 Sep 2026 23:37:44 +0200 Subject: [PATCH 1/3] feat: migrate to html2rss default strategy and httpx webmock adapter - Point Gemfile to sibling html2rss path for httpx experiment - Configure VCR to hook into :webmock with HTTPX adapter in spec_helper - Update strategy display names for default, faraday, and httpx - Explicitly require net/http in error_classifier for Net::OpenTimeout/ReadTimeout --- Gemfile | 8 ++-- Gemfile.lock | 76 ++++++++++++------------------ app/web/api/v1/strategies.rb | 2 +- app/web/errors/error_classifier.rb | 1 + spec/spec_helper.rb | 5 +- 5 files changed, 41 insertions(+), 51 deletions(-) diff --git a/Gemfile b/Gemfile index a85b669c..18a82dd1 100644 --- a/Gemfile +++ b/Gemfile @@ -4,13 +4,13 @@ source 'https://rubygems.org' git_source(:github) { |repo_name| "https://github.com/#{repo_name}" } -gem 'html2rss', '~> 0.29' +# gem 'html2rss', '~> 0.29' # gem 'html2rss', github: 'html2rss/html2rss', branch: 'master' -gem 'html2rss-configs', github: 'html2rss/html2rss-configs' +# gem 'html2rss-configs', github: 'html2rss/html2rss-configs' # Use these instead of the two above (uncomment them) when developing locally: -# gem 'html2rss', path: '../html2rss' -# gem 'html2rss-configs', path: '../html2rss-configs' +gem 'html2rss', path: '../html2rss' +gem 'html2rss-configs', path: '../html2rss-configs' gem 'base64' gem 'rack-cache' diff --git a/Gemfile.lock b/Gemfile.lock index 71007563..2ff5196a 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,10 +1,31 @@ -GIT - remote: https://github.com/html2rss/html2rss-configs - revision: 13cdb5f27bc7c9e943f27ee222eaf13f3dd0c93c +PATH + remote: ../html2rss-configs specs: html2rss-configs (0.2.0) html2rss +PATH + remote: ../html2rss + specs: + html2rss (0.29.1) + addressable (~> 2.7) + brotli + dry-validation + httpx (~> 1.8) + kramdown + mcp (~> 1.2) + mime-types (> 3.0) + nokogiri (>= 1.10, < 2.0) + rack (~> 3.0) + rackup (~> 2.0) + regexp_parser + rss + sanitize + thor + tzinfo + webrick (~> 1.9) + zeitwerk + GEM remote: https://rubygems.org/ specs: @@ -90,39 +111,11 @@ GEM zeitwerk (~> 2.6) erb (6.0.7) erubi (1.13.1) - faraday (2.14.3) - faraday-net_http (>= 2.0, < 3.5) - json - logger - faraday-follow_redirects (0.5.0) - faraday (>= 1, < 3) - faraday-gzip (3.1.0) - faraday (>= 2.0, < 3) - zlib (~> 3.0) - faraday-net_http (3.4.4) - net-http (~> 0.5) hana (1.3.7) hashdiff (1.2.1) - html2rss (0.29.1) - addressable (~> 2.7) - brotli - dry-validation - faraday (> 2.0.1, < 3.0) - faraday-follow_redirects - faraday-gzip (~> 3) - kramdown - mcp (~> 1.2) - mime-types (> 3.0) - nokogiri (>= 1.10, < 2.0) - rack (~> 3.0) - rackup (~> 2.0) - regexp_parser - rss - sanitize - thor - tzinfo - webrick (~> 1.9) - zeitwerk + http-2 (1.2.2) + httpx (1.8.3) + http-2 (>= 1.2.0) i18n (1.15.2) concurrent-ruby (~> 1.0) io-console (0.9.2) @@ -154,8 +147,6 @@ GEM minitest (6.0.6) drb (~> 2.0) prism (~> 1.5) - net-http (0.9.1) - uri (>= 0.11.1) nio4r (2.7.5) nokogiri (1.19.4-aarch64-linux-gnu) racc (~> 1.4) @@ -301,7 +292,6 @@ GEM webrick (1.9.2) yard (0.9.45) zeitwerk (2.8.3) - zlib (3.2.3) PLATFORMS aarch64-linux-gnu @@ -316,7 +306,7 @@ PLATFORMS DEPENDENCIES base64 climate_control - html2rss (~> 0.29) + html2rss! html2rss-configs! irb puma @@ -368,14 +358,12 @@ CHECKSUMS dry-validation (1.11.1) sha256=70900bb5a2d911c8aab566d3e360c6bff389b8bf92ea8e04885ce51c41ff8085 erb (6.0.7) sha256=c5ca6dc25b0ef974a44dc8f59fe847577122483b1968a38dec305c60bf91ee92 erubi (1.13.1) sha256=a082103b0885dbc5ecf1172fede897f9ebdb745a4b97a5e8dc63953db1ee4ad9 - faraday (2.14.3) sha256=1882247e6766615c8220b4392bf1d27f6ebb63d8e28267587cef1fb0bf37f278 - faraday-follow_redirects (0.5.0) sha256=5cde93c894b30943a5d2b93c2fe9284216a6b756f7af406a1e55f211d97d10ad - faraday-gzip (3.1.0) sha256=320783690be169f9b7ddde11598b77156951343753f66a9ab98b1f6694433ff8 - faraday-net_http (3.4.4) sha256=0e78af151747ed1b00f33e25973b4bc220d7f16c00c39676817c8b12331eb588 hana (1.3.7) sha256=5425db42d651fea08859811c29d20446f16af196308162894db208cac5ce9b0d hashdiff (1.2.1) sha256=9c079dbc513dfc8833ab59c0c2d8f230fa28499cc5efb4b8dd276cf931457cd1 - html2rss (0.29.1) sha256=60e4514b3d066480f7172646b2df10e030d2f1a24f0f7e756b130889605f6326 + html2rss (0.29.1) html2rss-configs (0.2.0) + http-2 (1.2.2) sha256=81b5d45f50fd4cd5f8c5d09651184bec9401e3ef169c3eb3e5b003d5614a92b9 + httpx (1.8.3) sha256=cb88f2285c4ef17164d803a640dc393d28722cba7f282ad0e7f9f926cd02dc47 i18n (1.15.2) sha256=00f9eb62412fe593b2a65a97daa75300d37abb8f7202ec748e94b6d46a9dd1b5 io-console (0.9.2) sha256=efa74f891dd03c0939a931dfc6e74c2813d904763d456ea9762b0525e748db08 irb (1.18.0) sha256=de9454a0703a54704b9811a5ef31a60c86949fbf4013fcf244fabc7c775248e3 @@ -390,7 +378,6 @@ CHECKSUMS mime-types (3.7.0) sha256=dcebf61c246f08e15a4de34e386ebe8233791e868564a470c3fe77c00eed5e56 mime-types-data (3.2026.0701) sha256=cd8811e1fb89d836499ba0582368a10ee74cef929ba956d1d5ddca045e6a730f minitest (6.0.6) sha256=153ea36d1d987a62942382b61075745042a2b3123b1cd48f4c3675af9cc7d6f1 - net-http (0.9.1) sha256=25ba0b67c63e89df626ed8fac771d0ad24ad151a858af2cc8e6a716ca4336996 nio4r (2.7.5) sha256=6c90168e48fb5f8e768419c93abb94ba2b892a1d0602cb06eef16d8b7df1dca1 nokogiri (1.19.4-aarch64-linux-gnu) sha256=1269fb644a6de405057a53dd5c762b1209b43ca7424f839454d3dbc677c31a8f nokogiri (1.19.4-aarch64-linux-musl) sha256=35c65b9ce72b3bb03207bdbe7067915019dc18c1b9b59139684bd6690fdd01af @@ -457,7 +444,6 @@ CHECKSUMS webrick (1.9.2) sha256=beb4a15fc474defed24a3bda4ffd88a490d517c9e4e6118c3edce59e45864131 yard (0.9.45) sha256=52e211493f7cb8a3ebf7e104a25a1e73937a3103092545d34cb88fafebb3dc51 zeitwerk (2.8.3) sha256=2c85125a8467ce069e20123d1e709a08955c9d29c118c25b46b7b7fafdbb92e5 - zlib (3.2.3) sha256=5bd316698b32f31a64ab910a8b6c282442ca1626a81bbd6a1674e8522e319c20 BUNDLED WITH 4.0.9 diff --git a/app/web/api/v1/strategies.rb b/app/web/api/v1/strategies.rb index 84902102..57ddae88 100644 --- a/app/web/api/v1/strategies.rb +++ b/app/web/api/v1/strategies.rb @@ -33,7 +33,7 @@ def index(_request) def display_name_for(name) case name.to_s - when 'faraday' then 'Default' + when 'default', 'faraday', 'httpx' then 'Default' when 'botasaurus' then 'Browser (Botasaurus)' else name.to_s.split('_').map(&:capitalize).join(' ') end diff --git a/app/web/errors/error_classifier.rb b/app/web/errors/error_classifier.rb index 46f80a69..8e831b96 100644 --- a/app/web/errors/error_classifier.rb +++ b/app/web/errors/error_classifier.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require 'timeout' +require 'net/http' module Html2rss module Web diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 4a7811d7..c5d27b58 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -19,11 +19,14 @@ end require 'rack/test' +require 'httpx' require 'vcr' +require 'webmock/rspec' +require 'httpx/adapters/webmock' VCR.configure do |config| config.cassette_library_dir = 'spec/fixtures/vcr_cassettes' - config.hook_into :faraday, :webmock + config.hook_into :webmock config.ignore_localhost = true end From aa117a505d7fe5d061089e2c543cce1487e46bfd Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Sat, 5 Sep 2026 23:59:48 +0200 Subject: [PATCH 2/3] fix: align error classifier with HTTPX and constrain strategy listing --- app/web/api/v1/strategies.rb | 7 ++++-- app/web/errors/error_classifier.rb | 26 ++++++++++++++++++++-- spec/html2rss/web/api/v1_spec.rb | 4 +++- spec/html2rss/web/error_classifier_spec.rb | 25 +++++++++++++++++++++ 4 files changed, 57 insertions(+), 5 deletions(-) diff --git a/app/web/api/v1/strategies.rb b/app/web/api/v1/strategies.rb index 57ddae88..b9049ae1 100644 --- a/app/web/api/v1/strategies.rb +++ b/app/web/api/v1/strategies.rb @@ -10,6 +10,8 @@ module V1 # Exposes only lightweight strategy metadata so clients can render # choices without coupling to backend strategy internals. module Strategies + PUBLIC_STRATEGIES = %i[default botasaurus].freeze + class << self # @param _request [Rack::Request] # @return [Hash{Symbol=>Object}] response with strategy list. @@ -18,7 +20,8 @@ class << self # @option return [Hash] :meta list metadata. # @option return [Integer] :total number of strategies. def index(_request) - strategies = Html2rss::RequestService.strategy_names.map do |name| + available = PUBLIC_STRATEGIES.select { |name| Html2rss::RequestService.strategy_registered?(name) } + strategies = available.map do |name| { id: name.to_s, name: name.to_s, @@ -33,7 +36,7 @@ def index(_request) def display_name_for(name) case name.to_s - when 'default', 'faraday', 'httpx' then 'Default' + when 'default' then 'Default' when 'botasaurus' then 'Browser (Botasaurus)' else name.to_s.split('_').map(&:capitalize).join(' ') end diff --git a/app/web/errors/error_classifier.rb b/app/web/errors/error_classifier.rb index 8e831b96..cfadb9e5 100644 --- a/app/web/errors/error_classifier.rb +++ b/app/web/errors/error_classifier.rb @@ -213,6 +213,17 @@ def initialize(decision) retry_action: 'primary' ).freeze + PRIVATE_NETWORK_DENIED = Decision.new( + status: 403, + code: 'FORBIDDEN', + message: 'Requests to private or loopback networks are forbidden.', + kind: 'input', + cacheable: true, + retryable: false, + next_action: 'correct_input', + retry_action: 'none' + ).freeze + INTERNAL_NETWORK_ERROR = Decision.new( status: 500, code: 'INTERNAL_SERVER_ERROR', @@ -236,6 +247,10 @@ def initialize(decision) defined?(::Html2rss::RequestService::BotasaurusConnectionFailed) && c.any?(::Html2rss::RequestService::BotasaurusConnectionFailed) }, SCRAPER_UNAVAILABLE], + [lambda { |c, _| + defined?(::Html2rss::RequestService::PrivateNetworkDenied) && + c.any?(::Html2rss::RequestService::PrivateNetworkDenied) + }, PRIVATE_NETWORK_DENIED], [lambda { |c, _| # queue/boot = our capacity or Chromium — not the target site. return false unless defined?(::Html2rss::RequestService::RequestTimedOut) @@ -255,7 +270,10 @@ def initialize(decision) [lambda { |_, err| defined?(::Rack::Timeout::RequestTimeoutException) && err.is_a?(::Rack::Timeout::RequestTimeoutException) }, SERVICE_UNAVAILABLE], - [->(_, err) { err.is_a?(Timeout::Error) || err.is_a?(Errno::ETIMEDOUT) }, GATEWAY_TIMEOUT] + [lambda { |c, err| + err.is_a?(Timeout::Error) || err.is_a?(Errno::ETIMEDOUT) || + (defined?(::HTTPX::TimeoutError) && (err.is_a?(::HTTPX::TimeoutError) || c.any?(::HTTPX::TimeoutError))) + }, GATEWAY_TIMEOUT] ].freeze class << self @@ -327,7 +345,11 @@ def decision_for_http_error(error, meta, default_message: nil) end def network_error?(error) - error_chain(error).any? { NETWORK_ERRORS.include?(it.class) } + error_chain(error).any? do |err| + NETWORK_ERRORS.include?(err.class) || + (defined?(::HTTPX::Error) && + (err.is_a?(::HTTPX::ConnectionError) || err.is_a?(::HTTPX::TLSError) || err.is_a?(::HTTPX::TimeoutError))) + end end end end diff --git a/spec/html2rss/web/api/v1_spec.rb b/spec/html2rss/web/api/v1_spec.rb index e327afa5..7d2425dc 100644 --- a/spec/html2rss/web/api/v1_spec.rb +++ b/spec/html2rss/web/api/v1_spec.rb @@ -451,7 +451,9 @@ def relative_feed_link_header(token) expect(last_response.status).to eq(200) expect(last_response.content_type).to include('application/json') json = expect_success_response(last_response) - expect(json.dig('data', 'strategies')).to be_an(Array) + strategies = json.dig('data', 'strategies') + expect(strategies.map { |s| s['id'] }).to eq(%w[default botasaurus]) + expect(strategies.map { |s| s['display_name'] }).to eq(['Default', 'Browser (Botasaurus)']) end end diff --git a/spec/html2rss/web/error_classifier_spec.rb b/spec/html2rss/web/error_classifier_spec.rb index 7c0a0937..697de449 100644 --- a/spec/html2rss/web/error_classifier_spec.rb +++ b/spec/html2rss/web/error_classifier_spec.rb @@ -198,6 +198,16 @@ def initialize(message = nil, timeout_phase: nil) described_class::SERVICE_UNAVAILABLE ) expect(described_class.classify(Net::OpenTimeout.new('timeout'))).to eq(described_class::GATEWAY_TIMEOUT) + expect(described_class.classify(HTTPX::TimeoutError.new(5, 'timeout'))).to eq( + described_class::GATEWAY_TIMEOUT + ) + end + + it 'returns the forbidden decision for PrivateNetworkDenied' do + stub_const('Html2rss::RequestService::PrivateNetworkDenied', Class.new(Html2rss::Error)) + error = Html2rss::RequestService::PrivateNetworkDenied.new('private network blocked') + + expect(described_class.classify(error)).to eq(described_class::PRIVATE_NETWORK_DENIED) end it 'classifies low-level network errors as internal server error with network kind' do @@ -208,6 +218,21 @@ def initialize(message = nil, timeout_phase: nil) retryable: true ) end + + it 'classifies HTTPX network errors as internal server error with network kind', :aggregate_failures do + expect(described_class.classify(HTTPX::ConnectionError.new('conn reset'))).to have_attributes( + status: 500, + code: 'INTERNAL_SERVER_ERROR', + kind: 'network', + retryable: true + ) + expect(described_class.classify(HTTPX::TLSError.new('tls error'))).to have_attributes( + status: 500, + code: 'INTERNAL_SERVER_ERROR', + kind: 'network', + retryable: true + ) + end end describe 'Diagnostics' do From b3ff14d650a4ec86b9471a346c37d5cd4ef3fcc1 Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Sun, 6 Sep 2026 00:14:18 +0200 Subject: [PATCH 3/3] fix(build): point html2rss to GitHub branch instead of local path --- Gemfile | 8 ++++---- Gemfile.lock | 19 +++++++++++-------- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/Gemfile b/Gemfile index 18a82dd1..b99277b6 100644 --- a/Gemfile +++ b/Gemfile @@ -5,12 +5,12 @@ source 'https://rubygems.org' git_source(:github) { |repo_name| "https://github.com/#{repo_name}" } # gem 'html2rss', '~> 0.29' -# gem 'html2rss', github: 'html2rss/html2rss', branch: 'master' -# gem 'html2rss-configs', github: 'html2rss/html2rss-configs' +gem 'html2rss', github: 'html2rss/html2rss', branch: 'experiment/httpx-default' +gem 'html2rss-configs', github: 'html2rss/html2rss-configs' # Use these instead of the two above (uncomment them) when developing locally: -gem 'html2rss', path: '../html2rss' -gem 'html2rss-configs', path: '../html2rss-configs' +# gem 'html2rss', path: '../html2rss' +# gem 'html2rss-configs', path: '../html2rss-configs' gem 'base64' gem 'rack-cache' diff --git a/Gemfile.lock b/Gemfile.lock index 2ff5196a..a393308e 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,11 +1,7 @@ -PATH - remote: ../html2rss-configs - specs: - html2rss-configs (0.2.0) - html2rss - -PATH - remote: ../html2rss +GIT + remote: https://github.com/html2rss/html2rss + revision: f0b73c2ae11dcc4aabc3386ecc0d7b6ee1b54013 + branch: experiment/httpx-default specs: html2rss (0.29.1) addressable (~> 2.7) @@ -26,6 +22,13 @@ PATH webrick (~> 1.9) zeitwerk +GIT + remote: https://github.com/html2rss/html2rss-configs + revision: 13cdb5f27bc7c9e943f27ee222eaf13f3dd0c93c + specs: + html2rss-configs (0.2.0) + html2rss + GEM remote: https://rubygems.org/ specs: