feat: adopt html2rss default HTTPX strategy and WebMock adapter

Gemfile

@@ -4,8 +4,8 @@ source 'https://rubygems.org'
 
 git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
 
-gem 'html2rss', '~> 0.29'
-# gem 'html2rss', github: 'html2rss/html2rss', branch: 'master'
+# gem 'html2rss', '~> 0.29'
+gem 'html2rss', github: 'html2rss/html2rss', branch: 'experiment/httpx-default'
 gem 'html2rss-configs', github: 'html2rss/html2rss-configs'
 
 # Use these instead of the two above (uncomment them) when developing locally:

Gemfile.lock

@@ -1,3 +1,27 @@
+GIT
+  remote: https://github.com/html2rss/html2rss
+  revision: f0b73c2ae11dcc4aabc3386ecc0d7b6ee1b54013
+  branch: experiment/httpx-default
+  specs:
+    html2rss (0.29.1)
+      addressable (~> 2.7)
+      brotli
+      dry-validation
+      httpx (~> 1.8)
+      kramdown
+      mcp (~> 1.2)
+      mime-types (> 3.0)
+      nokogiri (>= 1.10, < 2.0)
+      rack (~> 3.0)
+      rackup (~> 2.0)
+      regexp_parser
+      rss
+      sanitize
+      thor
+      tzinfo
+      webrick (~> 1.9)
+      zeitwerk
+
 GIT
   remote: https://github.com/html2rss/html2rss-configs
   revision: 13cdb5f27bc7c9e943f27ee222eaf13f3dd0c93c
@@ -90,39 +114,11 @@ GEM
       zeitwerk (~> 2.6)
     erb (6.0.7)
     erubi (1.13.1)
-    faraday (2.14.3)
-      faraday-net_http (>= 2.0, < 3.5)
-      json
-      logger
-    faraday-follow_redirects (0.5.0)
-      faraday (>= 1, < 3)
-    faraday-gzip (3.1.0)
-      faraday (>= 2.0, < 3)
-      zlib (~> 3.0)
-    faraday-net_http (3.4.4)
-      net-http (~> 0.5)
     hana (1.3.7)
     hashdiff (1.2.1)
-    html2rss (0.29.1)
-      addressable (~> 2.7)
-      brotli
-      dry-validation
-      faraday (> 2.0.1, < 3.0)
-      faraday-follow_redirects
-      faraday-gzip (~> 3)
-      kramdown
-      mcp (~> 1.2)
-      mime-types (> 3.0)
-      nokogiri (>= 1.10, < 2.0)
-      rack (~> 3.0)
-      rackup (~> 2.0)
-      regexp_parser
-      rss
-      sanitize
-      thor
-      tzinfo
-      webrick (~> 1.9)
-      zeitwerk
+    http-2 (1.2.2)
+    httpx (1.8.3)
+      http-2 (>= 1.2.0)
     i18n (1.15.2)
       concurrent-ruby (~> 1.0)
     io-console (0.9.2)
@@ -154,8 +150,6 @@ GEM
     minitest (6.0.6)
       drb (~> 2.0)
       prism (~> 1.5)
-    net-http (0.9.1)
-      uri (>= 0.11.1)
     nio4r (2.7.5)
     nokogiri (1.19.4-aarch64-linux-gnu)
       racc (~> 1.4)
@@ -301,7 +295,6 @@ GEM
     webrick (1.9.2)
     yard (0.9.45)
     zeitwerk (2.8.3)
-    zlib (3.2.3)
 
 PLATFORMS
   aarch64-linux-gnu
@@ -316,7 +309,7 @@ PLATFORMS
 DEPENDENCIES
   base64
   climate_control
-  html2rss (~> 0.29)
+  html2rss!
   html2rss-configs!
   irb
   puma
@@ -368,14 +361,12 @@ CHECKSUMS
   dry-validation (1.11.1) sha256=70900bb5a2d911c8aab566d3e360c6bff389b8bf92ea8e04885ce51c41ff8085
   erb (6.0.7) sha256=c5ca6dc25b0ef974a44dc8f59fe847577122483b1968a38dec305c60bf91ee92
   erubi (1.13.1) sha256=a082103b0885dbc5ecf1172fede897f9ebdb745a4b97a5e8dc63953db1ee4ad9
-  faraday (2.14.3) sha256=1882247e6766615c8220b4392bf1d27f6ebb63d8e28267587cef1fb0bf37f278
-  faraday-follow_redirects (0.5.0) sha256=5cde93c894b30943a5d2b93c2fe9284216a6b756f7af406a1e55f211d97d10ad
-  faraday-gzip (3.1.0) sha256=320783690be169f9b7ddde11598b77156951343753f66a9ab98b1f6694433ff8
-  faraday-net_http (3.4.4) sha256=0e78af151747ed1b00f33e25973b4bc220d7f16c00c39676817c8b12331eb588
   hana (1.3.7) sha256=5425db42d651fea08859811c29d20446f16af196308162894db208cac5ce9b0d
   hashdiff (1.2.1) sha256=9c079dbc513dfc8833ab59c0c2d8f230fa28499cc5efb4b8dd276cf931457cd1
-  html2rss (0.29.1) sha256=60e4514b3d066480f7172646b2df10e030d2f1a24f0f7e756b130889605f6326
+  html2rss (0.29.1)
   html2rss-configs (0.2.0)
+  http-2 (1.2.2) sha256=81b5d45f50fd4cd5f8c5d09651184bec9401e3ef169c3eb3e5b003d5614a92b9
+  httpx (1.8.3) sha256=cb88f2285c4ef17164d803a640dc393d28722cba7f282ad0e7f9f926cd02dc47
   i18n (1.15.2) sha256=00f9eb62412fe593b2a65a97daa75300d37abb8f7202ec748e94b6d46a9dd1b5
   io-console (0.9.2) sha256=efa74f891dd03c0939a931dfc6e74c2813d904763d456ea9762b0525e748db08
   irb (1.18.0) sha256=de9454a0703a54704b9811a5ef31a60c86949fbf4013fcf244fabc7c775248e3
@@ -390,7 +381,6 @@ CHECKSUMS
   mime-types (3.7.0) sha256=dcebf61c246f08e15a4de34e386ebe8233791e868564a470c3fe77c00eed5e56
   mime-types-data (3.2026.0701) sha256=cd8811e1fb89d836499ba0582368a10ee74cef929ba956d1d5ddca045e6a730f
   minitest (6.0.6) sha256=153ea36d1d987a62942382b61075745042a2b3123b1cd48f4c3675af9cc7d6f1
-  net-http (0.9.1) sha256=25ba0b67c63e89df626ed8fac771d0ad24ad151a858af2cc8e6a716ca4336996
   nio4r (2.7.5) sha256=6c90168e48fb5f8e768419c93abb94ba2b892a1d0602cb06eef16d8b7df1dca1
   nokogiri (1.19.4-aarch64-linux-gnu) sha256=1269fb644a6de405057a53dd5c762b1209b43ca7424f839454d3dbc677c31a8f
   nokogiri (1.19.4-aarch64-linux-musl) sha256=35c65b9ce72b3bb03207bdbe7067915019dc18c1b9b59139684bd6690fdd01af
@@ -457,7 +447,6 @@ CHECKSUMS
   webrick (1.9.2) sha256=beb4a15fc474defed24a3bda4ffd88a490d517c9e4e6118c3edce59e45864131
   yard (0.9.45) sha256=52e211493f7cb8a3ebf7e104a25a1e73937a3103092545d34cb88fafebb3dc51
   zeitwerk (2.8.3) sha256=2c85125a8467ce069e20123d1e709a08955c9d29c118c25b46b7b7fafdbb92e5
-  zlib (3.2.3) sha256=5bd316698b32f31a64ab910a8b6c282442ca1626a81bbd6a1674e8522e319c20
 
 BUNDLED WITH
   4.0.9

app/web/api/v1/strategies.rb

@@ -10,6 +10,8 @@ module V1
         # Exposes only lightweight strategy metadata so clients can render
         # choices without coupling to backend strategy internals.
         module Strategies
+          PUBLIC_STRATEGIES = %i[default botasaurus].freeze
+
           class << self
             # @param _request [Rack::Request]
             # @return [Hash{Symbol=>Object}] response with strategy list.
@@ -18,7 +20,8 @@ class << self
             # @option return [Hash] :meta list metadata.
             # @option return [Integer] :total number of strategies.
             def index(_request)
-              strategies = Html2rss::RequestService.strategy_names.map do |name|
+              available = PUBLIC_STRATEGIES.select { |name| Html2rss::RequestService.strategy_registered?(name) }
+              strategies = available.map do |name|
                 {
                   id: name.to_s,
                   name: name.to_s,
@@ -33,7 +36,7 @@ def index(_request)
 
             def display_name_for(name)
               case name.to_s
-              when 'faraday' then 'Default'
+              when 'default' then 'Default'
               when 'botasaurus' then 'Browser (Botasaurus)'
               else name.to_s.split('_').map(&:capitalize).join(' ')
               end

app/web/errors/error_classifier.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'timeout'
+require 'net/http'
 
 module Html2rss
   module Web
@@ -212,6 +213,17 @@ def initialize(decision)
         retry_action: 'primary'
       ).freeze
 
+      PRIVATE_NETWORK_DENIED = Decision.new(
+        status: 403,
+        code: 'FORBIDDEN',
+        message: 'Requests to private or loopback networks are forbidden.',
+        kind: 'input',
+        cacheable: true,
+        retryable: false,
+        next_action: 'correct_input',
+        retry_action: 'none'
+      ).freeze
+
       INTERNAL_NETWORK_ERROR = Decision.new(
         status: 500,
         code: 'INTERNAL_SERVER_ERROR',
@@ -235,6 +247,10 @@ def initialize(decision)
            defined?(::Html2rss::RequestService::BotasaurusConnectionFailed) &&
              c.any?(::Html2rss::RequestService::BotasaurusConnectionFailed)
          }, SCRAPER_UNAVAILABLE],
+        [lambda { |c, _|
+           defined?(::Html2rss::RequestService::PrivateNetworkDenied) &&
+             c.any?(::Html2rss::RequestService::PrivateNetworkDenied)
+         }, PRIVATE_NETWORK_DENIED],
         [lambda { |c, _|
            # queue/boot = our capacity or Chromium — not the target site.
            return false unless defined?(::Html2rss::RequestService::RequestTimedOut)
@@ -254,7 +270,10 @@ def initialize(decision)
         [lambda { |_, err|
            defined?(::Rack::Timeout::RequestTimeoutException) && err.is_a?(::Rack::Timeout::RequestTimeoutException)
          }, SERVICE_UNAVAILABLE],
-        [->(_, err) { err.is_a?(Timeout::Error) || err.is_a?(Errno::ETIMEDOUT) }, GATEWAY_TIMEOUT]
+        [lambda { |c, err|
+           err.is_a?(Timeout::Error) || err.is_a?(Errno::ETIMEDOUT) ||
+             (defined?(::HTTPX::TimeoutError) && (err.is_a?(::HTTPX::TimeoutError) || c.any?(::HTTPX::TimeoutError)))
+         }, GATEWAY_TIMEOUT]
       ].freeze
 
       class << self
@@ -326,7 +345,11 @@ def decision_for_http_error(error, meta, default_message: nil)
         end
 
         def network_error?(error)
-          error_chain(error).any? { NETWORK_ERRORS.include?(it.class) }
+          error_chain(error).any? do |err|
+            NETWORK_ERRORS.include?(err.class) ||
+              (defined?(::HTTPX::Error) &&
+                (err.is_a?(::HTTPX::ConnectionError) || err.is_a?(::HTTPX::TLSError) || err.is_a?(::HTTPX::TimeoutError)))
+          end
         end
       end
     end

spec/html2rss/web/api/v1_spec.rb

@@ -451,7 +451,9 @@ def relative_feed_link_header(token)
       expect(last_response.status).to eq(200)
       expect(last_response.content_type).to include('application/json')
       json = expect_success_response(last_response)
-      expect(json.dig('data', 'strategies')).to be_an(Array)
+      strategies = json.dig('data', 'strategies')
+      expect(strategies.map { |s| s['id'] }).to eq(%w[default botasaurus])
+      expect(strategies.map { |s| s['display_name'] }).to eq(['Default', 'Browser (Botasaurus)'])
     end
   end
 

spec/html2rss/web/error_classifier_spec.rb

@@ -198,6 +198,16 @@ def initialize(message = nil, timeout_phase: nil)
         described_class::SERVICE_UNAVAILABLE
       )
       expect(described_class.classify(Net::OpenTimeout.new('timeout'))).to eq(described_class::GATEWAY_TIMEOUT)
+      expect(described_class.classify(HTTPX::TimeoutError.new(5, 'timeout'))).to eq(
+        described_class::GATEWAY_TIMEOUT
+      )
+    end
+
+    it 'returns the forbidden decision for PrivateNetworkDenied' do
+      stub_const('Html2rss::RequestService::PrivateNetworkDenied', Class.new(Html2rss::Error))
+      error = Html2rss::RequestService::PrivateNetworkDenied.new('private network blocked')
+
+      expect(described_class.classify(error)).to eq(described_class::PRIVATE_NETWORK_DENIED)
     end
 
     it 'classifies low-level network errors as internal server error with network kind' do
@@ -208,6 +218,21 @@ def initialize(message = nil, timeout_phase: nil)
         retryable: true
       )
     end
+
+    it 'classifies HTTPX network errors as internal server error with network kind', :aggregate_failures do
+      expect(described_class.classify(HTTPX::ConnectionError.new('conn reset'))).to have_attributes(
+        status: 500,
+        code: 'INTERNAL_SERVER_ERROR',
+        kind: 'network',
+        retryable: true
+      )
+      expect(described_class.classify(HTTPX::TLSError.new('tls error'))).to have_attributes(
+        status: 500,
+        code: 'INTERNAL_SERVER_ERROR',
+        kind: 'network',
+        retryable: true
+      )
+    end
   end
 
   describe 'Diagnostics' do

spec/spec_helper.rb

@@ -19,11 +19,14 @@
 end
 
 require 'rack/test'
+require 'httpx'
 require 'vcr'
+require 'webmock/rspec'
+require 'httpx/adapters/webmock'
 
 VCR.configure do |config|
   config.cassette_library_dir = 'spec/fixtures/vcr_cassettes'
-  config.hook_into :faraday, :webmock
+  config.hook_into :webmock
   config.ignore_localhost = true
 end