From 72d6d9c7e4c2a8e03afcb7f3b2828755d3f8b66d Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Sun, 6 Sep 2026 08:06:53 +0000 Subject: [PATCH 1/5] refactor(server): embrace Falcon fiber reactor, strip thread mutexes, and modernize caching --- .devcontainer/docker-compose.yml | 1 + Gemfile | 2 +- Gemfile.lock | 4 +- app/web/boot/setup.rb | 1 + app/web/config/flags.rb | 24 ----- app/web/config/runtime_env.rb | 28 +++++- app/web/errors/error_classifier.rb | 18 ++-- app/web/feeds/cache.rb | 93 +++++++++++++----- app/web/feeds/http_cache.rb | 49 --------- app/web/feeds/renderer.rb | 42 ++++++-- app/web/request/rate_limiter.rb | 109 +++++++-------------- app/web/request/request_context.rb | 6 +- app/web/security/account_manager.rb | 18 ++-- app/web/telemetry/app_logger.rb | 2 + app/web/telemetry/log_event.rb | 2 +- docker-compose.yml | 10 +- frontend/src/api/generated/types.gen.ts | 2 +- public/openapi.yaml | 2 +- spec/html2rss/web/api/v1_spec.rb | 9 +- spec/html2rss/web/error_classifier_spec.rb | 5 +- spec/html2rss/web/error_responder_spec.rb | 8 +- spec/html2rss/web/feeds/cache_spec.rb | 53 +++++++--- spec/html2rss/web/feeds/renderer_spec.rb | 30 +++--- spec/html2rss/web/flags_spec.rb | 6 -- spec/html2rss/web/rate_limiter_spec.rb | 35 ------- 25 files changed, 250 insertions(+), 309 deletions(-) delete mode 100644 app/web/feeds/http_cache.rb diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index 2e23f37f4..bfd2411f1 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -5,6 +5,7 @@ services: dockerfile: .devcontainer/Dockerfile volumes: - ../:/workspace:cached + - ../../:/Users/gil/versioned/html2rss:cached - bundle-cache:/usr/local/bundle ports: - "4000:4000" diff --git a/Gemfile b/Gemfile index c253e7525..2eb46a713 100644 --- a/Gemfile +++ b/Gemfile @@ -5,7 +5,7 @@ source 'https://rubygems.org' git_source(:github) { |repo_name| "https://github.com/#{repo_name}" } # gem 'html2rss', '~> 0.29' -gem 'html2rss', github: 'html2rss/html2rss', branch: 'master' +gem 'html2rss', github: 'html2rss/html2rss', branch: 'refactor/httpx-transport-modernization' gem 'html2rss-configs', github: 'html2rss/html2rss-configs' # Use these instead of the two above (uncomment them) when developing locally: diff --git a/Gemfile.lock b/Gemfile.lock index 42cf66317..3598d3141 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ GIT remote: https://github.com/html2rss/html2rss - revision: 4260c842a5632fbef712cf3bee3a4fddcfee3654 - branch: master + revision: da5cf9400f758c956b49177962b26f76299898bf + branch: refactor/httpx-transport-modernization specs: html2rss (0.29.1) addressable (~> 2.7) diff --git a/app/web/boot/setup.rb b/app/web/boot/setup.rb index 17b855c0d..33b952180 100644 --- a/app/web/boot/setup.rb +++ b/app/web/boot/setup.rb @@ -16,6 +16,7 @@ def sentry_enabled? # # @return [void] def call! + Boot.eager_load! unless EnvironmentValidator.development? validate_environment! capture_runtime_env! configure_sentry! diff --git a/app/web/config/flags.rb b/app/web/config/flags.rb index dc80ef642..412f106e3 100644 --- a/app/web/config/flags.rb +++ b/app/web/config/flags.rb @@ -31,20 +31,6 @@ module Flags # rubocop:disable Metrics/ModuleLength default: true, validator: nil ), - async_feed_refresh_enabled: Definition.new( - name: :async_feed_refresh_enabled, - env_key: 'ASYNC_FEED_REFRESH_ENABLED', - type: :boolean, - default: false, - validator: nil - ), - async_feed_refresh_stale_factor: Definition.new( - name: :async_feed_refresh_stale_factor, - env_key: 'ASYNC_FEED_REFRESH_STALE_FACTOR', - type: :integer, - default: 3, - validator: ->(value) { value >= 1 } - ), feeds_cache_max_size: Definition.new( name: :feeds_cache_max_size, env_key: 'FEEDS_CACHE_MAX_SIZE', @@ -121,16 +107,6 @@ def auto_source_enabled? fetch(:auto_source_enabled) end - # @return [Boolean] - def async_feed_refresh_enabled? - fetch(:async_feed_refresh_enabled) - end - - # @return [Integer] - def async_feed_refresh_stale_factor - fetch(:async_feed_refresh_stale_factor) - end - # Validates all known flags and managed env key prefixes. # # @return [void] diff --git a/app/web/config/runtime_env.rb b/app/web/config/runtime_env.rb index 8718e645a..9eec7119c 100644 --- a/app/web/config/runtime_env.rb +++ b/app/web/config/runtime_env.rb @@ -9,21 +9,23 @@ module RuntimeEnv ADMIN_ACCESS_TOKEN_PLACEHOLDER = 'CHANGE_ME_ADMIN_TOKEN' HEALTH_CHECK_TOKEN_PLACEHOLDER = 'CHANGE_ME_HEALTH_CHECK_TOKEN' SENSITIVE_KEYS = %w[HTML2RSS_SECRET_KEY HTML2RSS_ACCESS_TOKEN HEALTH_CHECK_TOKEN SENTRY_DSN].freeze - BOOT_METADATA_KEYS = %w[BUILD_TAG GIT_SHA RACK_ENV SENTRY_ENABLE_LOGS].freeze - @mutex = Mutex.new + BOOT_METADATA_KEYS = %w[ + BUILD_TAG GIT_SHA RACK_ENV SENTRY_ENABLE_LOGS PORT WEB_CONCURRENCY REQUEST_TIMEOUT_SECONDS + ].freeze + # rubocop:disable ThreadSafety/ClassInstanceVariable @values = nil class << self # @return [void] def capture! - @mutex.synchronize { @values = tracked_env_values.freeze } + @values = tracked_env_values.freeze scrub_sensitive_env! nil end # @return [void] def reset! - @mutex.synchronize { @values = nil } + @values = nil end # @return [String] @@ -78,6 +80,21 @@ def rack_env fetch('RACK_ENV', ENV.fetch('RACK_ENV', 'development')) end + # @return [Integer] + def port + fetch('PORT', 4000).to_i + end + + # @return [Integer] + def web_concurrency + fetch('WEB_CONCURRENCY', 2).to_i + end + + # @return [Float] + def request_timeout_seconds + fetch('REQUEST_TIMEOUT_SECONDS', 55.0).to_f + end + private # @param key [String] @@ -86,7 +103,8 @@ def rack_env def fetch(key, default = :__missing__) return ENV.fetch(key) if ENV.key?(key) - current_values = @mutex.synchronize { @values || {} } + current_values = @values || {} + # rubocop:enable ThreadSafety/ClassInstanceVariable return current_values.fetch(key) if current_values.key?(key) return default unless default == :__missing__ diff --git a/app/web/errors/error_classifier.rb b/app/web/errors/error_classifier.rb index cfadb9e5a..8af50767b 100644 --- a/app/web/errors/error_classifier.rb +++ b/app/web/errors/error_classifier.rb @@ -267,11 +267,9 @@ def initialize(decision) defined?(::Html2rss::RequestService::RequestTimedOut) && c.any?(::Html2rss::RequestService::RequestTimedOut) }, GATEWAY_TIMEOUT], - [lambda { |_, err| - defined?(::Rack::Timeout::RequestTimeoutException) && err.is_a?(::Rack::Timeout::RequestTimeoutException) - }, SERVICE_UNAVAILABLE], [lambda { |c, err| err.is_a?(Timeout::Error) || err.is_a?(Errno::ETIMEDOUT) || + (defined?(::Async::TimeoutError) && (err.is_a?(::Async::TimeoutError) || c.any?(::Async::TimeoutError))) || (defined?(::HTTPX::TimeoutError) && (err.is_a?(::HTTPX::TimeoutError) || c.any?(::HTTPX::TimeoutError))) }, GATEWAY_TIMEOUT] ].freeze @@ -345,11 +343,15 @@ def decision_for_http_error(error, meta, default_message: nil) end def network_error?(error) - error_chain(error).any? do |err| - NETWORK_ERRORS.include?(err.class) || - (defined?(::HTTPX::Error) && - (err.is_a?(::HTTPX::ConnectionError) || err.is_a?(::HTTPX::TLSError) || err.is_a?(::HTTPX::TimeoutError))) - end + error_chain(error).any? { |err| network_error_class?(err) } + end + + def network_error_class?(err) + return true if NETWORK_ERRORS.include?(err.class) + return true if defined?(::Async::TimeoutError) && err.is_a?(::Async::TimeoutError) + return false unless defined?(::HTTPX::Error) + + err.is_a?(::HTTPX::ConnectionError) || err.is_a?(::HTTPX::TLSError) || err.is_a?(::HTTPX::TimeoutError) end end end diff --git a/app/web/feeds/cache.rb b/app/web/feeds/cache.rb index efffab346..af61c983a 100644 --- a/app/web/feeds/cache.rb +++ b/app/web/feeds/cache.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true -require 'concurrent/ivar' -require 'concurrent/map' +require 'async' +require 'async/notification' require 'digest' require 'time' @@ -9,16 +9,16 @@ module Html2rss module Web module Feeds ## - # Small synchronous cache for canonical feed results. + # Fiber-native cache for canonical feed results. module Cache # rubocop:disable ThreadSafety/ClassInstanceVariable def self.entries - @entries ||= Concurrent::Map.new + @entries ||= {} end private_class_method :entries def self.in_flight - @in_flight ||= Concurrent::Map.new + @in_flight ||= {} end private_class_method :in_flight # rubocop:enable ThreadSafety/ClassInstanceVariable @@ -26,6 +26,45 @@ def self.in_flight Entry = Data.define(:result, :expires_at) DEFAULT_TTL_SECONDS = 3600 + ## + # Coordinates in-flight fiber coalescing for identical cache keys. + class InFlight + def initialize + @notification = Async::Notification.new + @completed = false + @result = nil + @error = nil + end + + # @return [Object] + def wait + return @result if @completed && !@error + raise @error if @completed && @error + + @notification.wait + raise @error if @error + + @result + end + + # @param result [Object] + # @return [void] + def success!(result) + @result = result + @completed = true + @notification.signal(result) + end + + # @param error [Exception] + # @return [void] + def failure!(error) + @error = error + @completed = true + @notification.signal(error) + end + end + private_constant :InFlight + class << self # Converts feed-provided minutes to seconds with a safe fallback. # @@ -44,26 +83,12 @@ def seconds_from_minutes(value, default: DEFAULT_TTL_SECONDS) # @param cacheable [Boolean, Proc] # @yieldreturn [Html2rss::Web::Feeds::Contracts::RenderResult] # @return [Html2rss::Web::Feeds::Contracts::RenderResult] - # rubocop:disable-next Metrics/MethodLength - def fetch(key, ttl_seconds:, cacheable: true) + def fetch(key, ttl_seconds:, cacheable: true, &) entry = read_entry(key) return entry.result if fresh?(entry) + return in_flight[key].wait if in_flight.key?(key) - ivar = Concurrent::IVar.new - actual_ivar = in_flight.put_if_absent(key, ivar) - return actual_ivar.value! if actual_ivar - - begin - result = yield - write_entry(key, ttl_seconds, result) if cacheable_result?(cacheable, result) - ivar.set(result) - result - rescue StandardError => error - ivar.fail(error) - raise - ensure - in_flight.delete_pair(key, ivar) - end + execute_fetch(key, ttl_seconds, cacheable, &) end # @param reason [String] @@ -114,20 +139,34 @@ def prune_if_needed def prune_expired now = Time.now.utc - entries.each_pair { |k, v| entries.delete(k) if v && now >= v.expires_at } + entries.delete_if { |_k, v| v && now >= v.expires_at } + end + + def execute_fetch(key, ttl_seconds, cacheable) # rubocop:disable Metrics/MethodLength + job = in_flight[key] = InFlight.new + begin + result = yield + write_entry(key, ttl_seconds, result) if cacheable_result?(cacheable, result) + job.success!(result) + result + rescue Exception => error # rubocop:disable Lint/RescueException -- Async::Stop inherits from Exception + job.failure!(error) + raise + ensure + in_flight.delete(key) + end end def prune_excess(max) excess = entries.size - (max * 0.9).to_i return if excess <= 0 - entries_by_expiration.first(excess).each { entries.delete(it.first) } + entries_by_expiration.first(excess).each { |pair| entries.delete(pair.first) } end def entries_by_expiration - candidates = [] - entries.each_pair { |k, v| candidates << [k, v.expires_at] if v&.expires_at } - candidates.sort_by!(&:last) + entries.select { |_k, v| v&.expires_at } + .sort_by { |_k, v| v.expires_at } end # @param cacheable [Boolean, Proc] diff --git a/app/web/feeds/http_cache.rb b/app/web/feeds/http_cache.rb deleted file mode 100644 index 511000cd9..000000000 --- a/app/web/feeds/http_cache.rb +++ /dev/null @@ -1,49 +0,0 @@ -# frozen_string_literal: true - -require 'time' - -module Html2rss - module Web - module Feeds - ## - # Collection of methods which set HTTP Caching related headers in the response. - module HttpCache - class << self - ## - # Sets Expires and Cache-Control headers to cache for `seconds`. - # @param response [Hash] - # @param seconds [Integer] - # @param cache_control [String, nil] - # @return [void] - def expires(response, seconds, cache_control: nil) - expires_now(response) and return if seconds <= 0 - - response['Expires'] = (Time.now + seconds).httpdate - - cache_value = "max-age=#{seconds}" - cache_value += ",#{cache_control}" if cache_control - response['Cache-Control'] = cache_value - end - - ## - # Sets Expires and Cache-Control headers to invalidate existing cache and - # prevent caching. - # @param response [Hash] - # @return [void] - def expires_now(response) - response['Expires'] = '0' - response['Cache-Control'] = 'private,max-age=0,no-cache,no-store,must-revalidate' - end - - # @param response [Hash] - # @param fields [Array] - # @return [void] - def vary(response, *fields) - existing = response['Vary'].to_s.split(',').map(&:strip).reject(&:empty?) - response['Vary'] = (existing + fields).uniq.join(', ') - end - end - end - end - end -end diff --git a/app/web/feeds/renderer.rb b/app/web/feeds/renderer.rb index edecb064b..60cf47e54 100644 --- a/app/web/feeds/renderer.rb +++ b/app/web/feeds/renderer.rb @@ -8,8 +8,8 @@ module Web module Feeds ## # Builds feed HTTP envelopes: status, headers, and serialized bodies. - module Renderer - class << self + module Renderer # rubocop:disable Metrics/ModuleLength + class << self # rubocop:disable Metrics/ClassLength # Renders a RenderResult and configures the HTTP response headers and status. # # @param result [Html2rss::Web::Feeds::Contracts::RenderResult] @@ -30,7 +30,7 @@ def render(result, response:, request:) # @return [String] plain-text error body def render_error(message, response:) response['Content-Type'] = FormatNegotiation::TEXT_PLAIN_CONTENT_TYPE - HttpCache.expires_now(response) + expires_now(response) call_error(message: message) end @@ -91,9 +91,9 @@ def set_header_if_present(response, header_name, value) def apply_vary_and_links(response, result, request) if result.status == :ok apply_alternate_links(response, request) - HttpCache.vary(response, 'Accept', 'Host') + vary(response, 'Accept', 'Host') else - HttpCache.vary(response, 'Accept') + vary(response, 'Accept') end end @@ -141,9 +141,37 @@ def plain_response?(result) # @param result [Html2rss::Web::Feeds::Contracts::RenderResult] # @return [void] def apply_cache_headers(response, result) - return HttpCache.expires_now(response) if result.status == :error + return expires_now(response) if result.status == :error - HttpCache.expires(response, result.ttl_seconds, cache_control: 'public') + expires(response, result.ttl_seconds, cache_control: 'public') + end + + # @param response [Rack::Response] + # @param seconds [Integer] + # @param cache_control [String, nil] + # @return [void] + def expires(response, seconds, cache_control: nil) + expires_now(response) and return if seconds <= 0 + + response['Expires'] = (Time.now + seconds).httpdate + cache_value = "max-age=#{seconds}" + cache_value += ",#{cache_control}" if cache_control + response['Cache-Control'] = cache_value + end + + # @param response [Rack::Response] + # @return [void] + def expires_now(response) + response['Expires'] = '0' + response['Cache-Control'] = 'private,max-age=0,no-cache,no-store,must-revalidate' + end + + # @param response [Rack::Response] + # @param fields [Array] + # @return [void] + def vary(response, *fields) + existing = response['Vary'].to_s.split(',').map(&:strip).reject(&:empty?) + response['Vary'] = (existing + fields).uniq.join(', ') end # @param response [Rack::Response] diff --git a/app/web/request/rate_limiter.rb b/app/web/request/rate_limiter.rb index 8983a68e6..28fb06a80 100644 --- a/app/web/request/rate_limiter.rb +++ b/app/web/request/rate_limiter.rb @@ -1,6 +1,5 @@ # frozen_string_literal: true -require 'concurrent/map' require 'rack/request' require 'rack/response' require 'rack/utils' @@ -8,16 +7,18 @@ module Html2rss module Web ## - # Rack middleware providing IP-based rate limiting with thread-safe tracking, + # Rack middleware providing IP-based rate limiting with fiber-native tracking, # automated pruning, and standardized 429 error formatting. class RateLimiter ## # Encapsulates timestamp tracking and rate limit logic for a single client IP. class RequestTrack + # @return [Array] + attr_reader :timestamps + private :timestamps + def initialize - @mutex = Mutex.new @timestamps = [] - @deleted = false end # Records request time, prunes old timestamps, and checks if limit is exceeded. @@ -25,61 +26,39 @@ def initialize # @param now [Integer] # @param window_seconds [Integer] # @param max_requests [Integer] - # @return [Array<(Boolean, Integer, Boolean)>] limit exceeded flag, retry_after seconds, and deleted flag. - # rubocop:disable-next Metrics/MethodLength + # @return [Array<(Boolean, Integer)>] limit exceeded flag and retry_after seconds. def record_and_check_limit(now, window_seconds, max_requests) - @mutex.synchronize do - return [false, 0, true] if @deleted - - window_start = now - window_seconds - @timestamps.reject! { |t| t < window_start } - - if @timestamps.size >= max_requests - oldest = @timestamps.first - retry_after = [1, oldest + window_seconds - now].max - [true, retry_after, false] - else - @timestamps << now - [false, 0, false] - end + window_start = now - window_seconds + @timestamps.reject! { |t| t < window_start } + + if @timestamps.size >= max_requests + oldest = @timestamps.first + retry_after = [1, oldest + window_seconds - now].max + [true, retry_after] + else + @timestamps << now + [false, 0] end end # Prunes expired timestamps and deletes the key from history if empty. - # Uses non-blocking try_lock to avoid blocking the pruning thread. # # @param window_start [Integer] - # @param history [Concurrent::Map] + # @param history [Hash] # @param key [String] - # @return [Boolean] true if key was pruned and deleted. - # rubocop:disable-next Metrics/MethodLength + # @return [void] def prune(window_start, history, key) - return false unless @mutex.try_lock - - begin - @timestamps.reject! { |t| t < window_start } - if @timestamps.empty? - if history.delete_pair(key, self) - @deleted = true - true - else - false - end - else - false - end - ensure - @mutex.unlock - end + @timestamps.reject! { |t| t < window_start } + history.delete(key) if @timestamps.empty? + nil end end # @param app [#call] def initialize(app) @app = app - @history = Concurrent::Map.new + @history = {} @last_pruned = 0 - @prune_mutex = Mutex.new end # @param env [Hash] @@ -97,27 +76,12 @@ def call(env) client_key = request.ip now = Time.now.to_i - limit_exceeded = false - retry_after = nil - - loop do - track = @history.compute_if_absent(client_key) { RequestTrack.new } - - exceeded, after, deleted = track.record_and_check_limit( - now, - Flags.rate_limit_window_seconds, - Flags.rate_limit_max_requests - ) - - if deleted - @history.delete_pair(client_key, track) - next - end - - limit_exceeded = exceeded - retry_after = after - break - end + track = (@history[client_key] ||= RequestTrack.new) + limit_exceeded, retry_after = track.record_and_check_limit( + now, + Flags.rate_limit_window_seconds, + Flags.rate_limit_max_requests + ) if limit_exceeded SecurityLogger.log_rate_limit_exceeded(client_key, path, Flags.rate_limit_max_requests) @@ -154,26 +118,19 @@ def bypass?(path) end # Prunes inactive IP tracks when history grows too large. - # Utilizes a prune mutex and a time-based throttle to minimize CPU overhead. + # Utilizes a time-based throttle to minimize CPU overhead. # Hard-caps the history size to prevent OOM. # # @return [void] - # rubocop:disable-next Metrics/MethodLength def prune_history_if_needed now = Time.now.to_i size = @history.size if size > 20_000 - @prune_mutex.synchronize do - handle_overflow(now) if @history.size > 20_000 - end - elsif size > 1000 && (now - @last_pruned) > 10 && @prune_mutex.try_lock - begin - @last_pruned = now - prune_all_expired(now) - ensure - @prune_mutex.unlock - end + handle_overflow(now) + elsif size > 1000 && (now - @last_pruned) > 10 + @last_pruned = now + prune_all_expired(now) end nil end diff --git a/app/web/request/request_context.rb b/app/web/request/request_context.rb index 59e884c5c..7195ded14 100644 --- a/app/web/request/request_context.rb +++ b/app/web/request/request_context.rb @@ -13,12 +13,12 @@ class << self # @param context [Context] # @return [Context] def set!(context) - Thread.current[:request_context] = context + Fiber[:request_context] = context end # @return [Context, nil] def current - Thread.current[:request_context] + Fiber[:request_context] end # @return [Hash{Symbol=>Object}] @@ -31,7 +31,7 @@ def current_h # @return [nil] def clear! - Thread.current[:request_context] = nil + Fiber[:request_context] = nil nil end diff --git a/app/web/security/account_manager.rb b/app/web/security/account_manager.rb index e4ab9dad2..79650662d 100644 --- a/app/web/security/account_manager.rb +++ b/app/web/security/account_manager.rb @@ -3,12 +3,12 @@ module Html2rss module Web ## - # Thread-safe account snapshot cache. + # Fiber-native account snapshot cache. # # Keeps config reads cheap by materializing one immutable snapshot and # exposing narrow lookup helpers for auth and authorization flows. module AccountManager - @mutex = Mutex.new + Snapshot = Data.define(:accounts, :token_index, :username_index) @snapshot = nil class << self @@ -17,7 +17,8 @@ class << self # @param reason [String] # @return [nil] def reload!(reason: 'manual') - @mutex.synchronize { @snapshot = nil } + # rubocop:disable-next ThreadSafety/ClassInstanceVariable + @snapshot = nil Observability.emit( event_name: 'cache.lifecycle', outcome: 'success', @@ -31,12 +32,12 @@ def reload!(reason: 'manual') def get_account(token) return nil unless token - snapshot[:token_index][token] + snapshot.token_index[token] end # @return [ArrayObject}>] def accounts - snapshot[:accounts] + snapshot.accounts end # @param username [String, nil] @@ -44,13 +45,14 @@ def accounts def get_account_by_username(username) return nil unless username - snapshot[:username_index][username] + snapshot.username_index[username] end private def snapshot - @mutex.synchronize { @snapshot ||= build_snapshot } + # rubocop:disable-next ThreadSafety/ClassInstanceVariable + @snapshot ||= build_snapshot end def build_snapshot @@ -64,7 +66,7 @@ def build_snapshot outcome: 'success', details: { component: 'account_manager', event: 'build', accounts_count: accounts.length } ) - { accounts: accounts, token_index: token_index, username_index: username_index }.freeze + Snapshot.new(accounts:, token_index:, username_index:) end # @param raw_accounts [Array, nil] diff --git a/app/web/telemetry/app_logger.rb b/app/web/telemetry/app_logger.rb index 11f10da3d..46191dc39 100644 --- a/app/web/telemetry/app_logger.rb +++ b/app/web/telemetry/app_logger.rb @@ -53,6 +53,8 @@ def base_payload(severity, datetime) # @param message [Object] # @return [Hash{Symbol=>Object}] def normalize_message(message) + return message if message.is_a?(Hash) + message_string = message.to_s return parsed_json(message_string) if json_like?(message_string) diff --git a/app/web/telemetry/log_event.rb b/app/web/telemetry/log_event.rb index 17f280ad7..a67636063 100644 --- a/app/web/telemetry/log_event.rb +++ b/app/web/telemetry/log_event.rb @@ -10,7 +10,7 @@ class << self # @param level [Symbol] # @return [void] def emit(payload:, level: :info) - logger.public_send(level, build_payload(payload).to_json) + logger.public_send(level, build_payload(payload)) rescue StandardError => error warn_fallback(error, payload) end diff --git a/docker-compose.yml b/docker-compose.yml index 89ac942b4..2590d9376 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -28,7 +28,7 @@ services: SENTRY_DSN: ${SENTRY_DSN:-} SENTRY_ENABLE_LOGS: ${SENTRY_ENABLE_LOGS:-false} HTML2RSS_TOTAL_TIMEOUT_SECONDS: 50 - RACK_TIMEOUT_SERVICE_TIMEOUT: 55 + REQUEST_TIMEOUT_SECONDS: 55 BOTASAURUS_SCRAPE_TIMEOUT_SECONDS: 45 BOTASAURUS_SCRAPE_WORK_TIMEOUT_SECONDS: 30 BOTASAURUS_SCRAPER_URL: http://botasaurus:4010 @@ -40,14 +40,6 @@ services: # target: /app/config/feeds.yml # read_only: true - watchtower: - image: containrrr/watchtower - restart: unless-stopped - volumes: - - /var/run/docker.sock:/var/run/docker.sock - - "${HOME}/.docker/config.json:/config.json" - command: --cleanup --interval 7200 - botasaurus: image: html2rss/botasaurus-scrape-api:latest restart: unless-stopped diff --git a/frontend/src/api/generated/types.gen.ts b/frontend/src/api/generated/types.gen.ts index 39f0afc3b..f24843d17 100644 --- a/frontend/src/api/generated/types.gen.ts +++ b/frontend/src/api/generated/types.gen.ts @@ -278,7 +278,7 @@ export type RenderFeedByTokenErrors = { */ 500: string; /** - * returns 503 when the server times out + * returns 503 when the scraper queue times out */ 503: string; /** diff --git a/public/openapi.yaml b/public/openapi.yaml index e5ad1eee3..e3248428a 100644 --- a/public/openapi.yaml +++ b/public/openapi.yaml @@ -596,7 +596,7 @@ paths: example: 'Failed to generate feed: Invalid token' schema: type: string - description: returns 503 when the server times out + description: returns 503 when the scraper queue times out headers: Retry-After: description: The number of seconds to wait before retrying the request. diff --git a/spec/html2rss/web/api/v1_spec.rb b/spec/html2rss/web/api/v1_spec.rb index 295e38902..d55f8edd7 100644 --- a/spec/html2rss/web/api/v1_spec.rb +++ b/spec/html2rss/web/api/v1_spec.rb @@ -669,12 +669,11 @@ def relative_feed_link_header(token) expect(last_response.headers['Retry-After']).not_to be_nil end - it 'returns 503 when the server times out', :aggregate_failures do - token = Html2rss::Web::Auth.generate_feed_token('admin', "#{feed_url}/timeout-503", strategy: 'faraday') - stub_const('Rack::Timeout::RequestTimeoutException', Class.new(StandardError)) + it 'returns 503 when the scraper queue times out', :aggregate_failures do + token = Html2rss::Web::Auth.generate_feed_token('admin', "#{feed_url}/timeout-503", strategy: 'default') - allow(Html2rss::Web::Feeds::Service).to receive(:call) - .and_raise(Rack::Timeout::RequestTimeoutException.new('service timeout')) + error = Html2rss::RequestService::RequestTimedOut.new('queue timed out', timeout_phase: 'queue') + allow(Html2rss::Web::Feeds::Service).to receive(:call).and_raise(error) get "/api/v1/feeds/#{token}.xml" diff --git a/spec/html2rss/web/error_classifier_spec.rb b/spec/html2rss/web/error_classifier_spec.rb index 697de449b..b39d405c8 100644 --- a/spec/html2rss/web/error_classifier_spec.rb +++ b/spec/html2rss/web/error_classifier_spec.rb @@ -193,10 +193,7 @@ def initialize(message = nil, timeout_phase: nil) end it 'classifies timeout errors correctly', :aggregate_failures do - stub_const('Rack::Timeout::RequestTimeoutException', Class.new(StandardError)) - expect(described_class.classify(Rack::Timeout::RequestTimeoutException.new)).to eq( - described_class::SERVICE_UNAVAILABLE - ) + expect(described_class.classify(Async::TimeoutError.new)).to eq(described_class::GATEWAY_TIMEOUT) expect(described_class.classify(Net::OpenTimeout.new('timeout'))).to eq(described_class::GATEWAY_TIMEOUT) expect(described_class.classify(HTTPX::TimeoutError.new(5, 'timeout'))).to eq( described_class::GATEWAY_TIMEOUT diff --git a/spec/html2rss/web/error_responder_spec.rb b/spec/html2rss/web/error_responder_spec.rb index 72b736066..d32379285 100644 --- a/spec/html2rss/web/error_responder_spec.rb +++ b/spec/html2rss/web/error_responder_spec.rb @@ -112,14 +112,14 @@ def expected_api_error_response # rubocop:disable Metrics/MethodLength expect(response['Retry-After']).to eq('60') end - it 'maps Rack::Timeout::RequestTimeoutException to 503 and injects Retry-After header', :aggregate_failures do - stub_const('Rack::Timeout::RequestTimeoutException', Class.new(StandardError)) + it 'maps Async::TimeoutError to 504 and injects Retry-After header', :aggregate_failures do + require 'async' response, _body = respond_with( - error: Rack::Timeout::RequestTimeoutException.new('timeout'), + error: Async::TimeoutError.new, path: '/api/v1/feeds', target: Html2rss::Web::RequestTarget::API ) - expect(response.status).to eq(503) + expect(response.status).to eq(504) expect(response['Retry-After']).to eq('300') end diff --git a/spec/html2rss/web/feeds/cache_spec.rb b/spec/html2rss/web/feeds/cache_spec.rb index 25733f078..80e5ea732 100644 --- a/spec/html2rss/web/feeds/cache_spec.rb +++ b/spec/html2rss/web/feeds/cache_spec.rb @@ -43,6 +43,31 @@ expect(calls).to eq(1) end + it 'propagates cancellation and cleans in-flight entry when the leader is stopped', + :aggregate_failures do # rubocop:disable RSpec/ExampleLength + waiter_error = nil + + Async do |task| + task.async do + described_class.fetch('feed_result:stopped_test', ttl_seconds: 60) do + sleep 5 + result + end + end + task.yield + waiter = task.async do + described_class.fetch('feed_result:stopped_test', ttl_seconds: 60) { result } + rescue Exception => error # rubocop:disable Lint/RescueException + waiter_error = error + end + task.children.first.stop + waiter.wait + end + + expect(waiter_error).to be_a(Exception) + expect(described_class.send(:in_flight)).to be_empty + end + describe '.seconds_from_minutes' do it 'converts positive minute values to seconds', :aggregate_failures do expect(described_class.seconds_from_minutes(5)).to eq(300) @@ -77,22 +102,20 @@ def fetch_calls @fetch_calls ||= 0 end - # rubocop:disable-next Metrics/MethodLength, ThreadSafety/NewThread - def run_concurrent_fetches(key, concurrency) - computation_calls = Concurrent::AtomicFixnum.new(0) - barrier = Concurrent::CyclicBarrier.new(concurrency) - - threads = Array.new(concurrency) do - Thread.new do - barrier.wait - described_class.fetch(key, ttl_seconds: 60) do - computation_calls.increment - sleep 0.05 - result + def run_concurrent_fetches(key, concurrency) # rubocop:disable Metrics/MethodLength + calls = 0 + results = [] + Async do |task| + Array.new(concurrency) do + task.async do + results << described_class.fetch(key, ttl_seconds: 60) do + calls += 1 + sleep 0.05 + result + end end - end + end.each(&:wait) end - - [threads.map(&:value), computation_calls.value] + [results, calls] end end diff --git a/spec/html2rss/web/feeds/renderer_spec.rb b/spec/html2rss/web/feeds/renderer_spec.rb index d7fa83e35..688c7cfe3 100644 --- a/spec/html2rss/web/feeds/renderer_spec.rb +++ b/spec/html2rss/web/feeds/renderer_spec.rb @@ -99,22 +99,18 @@ def render_body(result, path:, accept: nil) end it 'sets status, content-type, link alternates, vary, and cache control on the response', :aggregate_failures do - allow(response).to receive(:status=) - allow(response).to receive(:[]=) - allow(Html2rss::Web::Feeds::HttpCache).to receive(:vary) - allow(Html2rss::Web::Feeds::HttpCache).to receive(:expires) + real_response = Rack::Response.new + described_class.render(ok_result, response: real_response, request: request) - described_class.render(ok_result, response: response, request: request) - - expect(response).to have_received(:status=).with(200) - expect(response).to have_received(:[]=).with('Content-Type', 'application/xml') - expect(response).to have_received(:[]=).with( - 'Link', + expect(real_response.status).to eq(200) + expect(real_response['Content-Type']).to eq('application/xml') + expect(real_response['Link']).to eq( '; rel="alternate"; type="application/rss+xml", ' \ '; rel="alternate"; type="application/feed+json"' ) - expect(Html2rss::Web::Feeds::HttpCache).to have_received(:vary).with(response, 'Accept', 'Host') - expect(Html2rss::Web::Feeds::HttpCache).to have_received(:expires).with(response, 300, cache_control: 'public') + expect(real_response['Vary']).to eq('Accept, Host') + expect(real_response['Cache-Control']).to eq('max-age=300,public') + expect(real_response['Expires']).to be_a(String) end it 'omits reverse-proxy Docker DNS Hosts from Link targets' do @@ -171,17 +167,15 @@ def render_body(result, path:, accept: nil) end describe '.render_error' do - let(:response) { instance_double(Rack::Response, :[]= => nil) } + let(:response) { Rack::Response.new } it 'sets plain text content-type and disables cache on the response', :aggregate_failures do - allow(response).to receive(:[]=) - allow(Html2rss::Web::Feeds::HttpCache).to receive(:expires_now) - body = described_class.render_error('Test Error', response: response) expect(body).to eq('Failed to generate feed: Test Error') - expect(response).to have_received(:[]=).with('Content-Type', 'text/plain; charset=utf-8') - expect(Html2rss::Web::Feeds::HttpCache).to have_received(:expires_now).with(response) + expect(response['Content-Type']).to eq('text/plain; charset=utf-8') + expect(response['Cache-Control']).to eq('private,max-age=0,no-cache,no-store,must-revalidate') + expect(response['Expires']).to eq('0') end end # rubocop:enable RSpec/ExampleLength diff --git a/spec/html2rss/web/flags_spec.rb b/spec/html2rss/web/flags_spec.rb index 9c5c4c815..4795b3aed 100644 --- a/spec/html2rss/web/flags_spec.rb +++ b/spec/html2rss/web/flags_spec.rb @@ -48,12 +48,6 @@ end end - it 'raises for malformed stale factor' do - ClimateControl.modify('ASYNC_FEED_REFRESH_STALE_FACTOR' => '0') do - expect { described_class.validate! }.to raise_error(ArgumentError, /failed constraints/) - end - end - it 'raises for invalid feeds cache max size' do ClimateControl.modify('FEEDS_CACHE_MAX_SIZE' => '0') do expect { described_class.validate! }.to raise_error(ArgumentError, /failed constraints/) diff --git a/spec/html2rss/web/rate_limiter_spec.rb b/spec/html2rss/web/rate_limiter_spec.rb index 7532049a6..783d83959 100644 --- a/spec/html2rss/web/rate_limiter_spec.rb +++ b/spec/html2rss/web/rate_limiter_spec.rb @@ -141,40 +141,5 @@ anything, '/api/v1/feeds', 3 ) end - - it 'skips pruning locked tracks using non-blocking try_lock' do - history_map = middleware.instance_variable_get(:@history) - track = described_class::RequestTrack.new - history_map['999.999.999.999'] = track - - # Stub try_lock to return false, mimicking lock contention - allow(track.instance_variable_get(:@mutex)).to receive(:try_lock).and_return(false) - - # Populate history past 1000 so pruning is triggered - 1005.times do |i| - history_map["192.168.1.#{i}"] = described_class::RequestTrack.new - end - - request_builder.get('/api/v1/feeds') - - # Check that the locked track was NOT pruned/deleted, even though it was empty - expect(history_map.key?('999.999.999.999')).to be(true) - end - - it 'recovers and retries when a track is deleted during check' do - track = described_class::RequestTrack.new - track.instance_variable_set(:@deleted, true) - - call_count = 0 - original_new = described_class::RequestTrack.method(:new) - allow(described_class::RequestTrack).to receive(:new) do - call_count += 1 - call_count == 1 ? track : original_new.call - end - - response = request_builder.get('/api/v1/feeds') - expect(response.status).to eq(200) - expect(call_count).to eq(2) - end end end From 9188ac9a822c546b58c14cb7fefdafd9aa2b237c Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Sun, 6 Sep 2026 08:08:18 +0000 Subject: [PATCH 2/5] chore(git): ignore local agent artifacts --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 380efe19e..44b0b160b 100644 --- a/.gitignore +++ b/.gitignore @@ -60,3 +60,6 @@ frontend/.astro # Bundler/vendored dependencies /vendor/ + +# Local agent artifacts +/.agents/ From ae25e4e0480d2c15d5bb2e213c756aa46e381607 Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Sun, 6 Sep 2026 08:34:36 +0000 Subject: [PATCH 3/5] refactor(feeds): standardize feed token specs and docs on default strategy --- Gemfile.lock | 2 +- app/web/errors/error_classifier.rb | 2 +- docs/README.md | 2 +- spec/html2rss/web/api/v1_spec.rb | 26 +++++++++---------- spec/html2rss/web/app_integration_spec.rb | 8 +++--- spec/html2rss/web/error_classifier_spec.rb | 12 ++++----- .../web/feeds/feed_result_integration_spec.rb | 4 +-- spec/html2rss/web/feeds/renderer_spec.rb | 6 ++--- spec/html2rss/web/feeds/responder_spec.rb | 18 ++++++------- spec/html2rss/web/feeds/service_spec.rb | 8 +++--- .../web/feeds/source_resolver_spec.rb | 16 +++++++++--- spec/html2rss/web/log_sanitizer_spec.rb | 4 +-- spec/html2rss/web/sentry_logs_spec.rb | 4 +-- spec/smoke/docker_spec.rb | 6 ++--- 14 files changed, 63 insertions(+), 55 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 3598d3141..b0d3f1ce4 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,6 +1,6 @@ GIT remote: https://github.com/html2rss/html2rss - revision: da5cf9400f758c956b49177962b26f76299898bf + revision: 5f2ab31855f092c1ed18a7a5150448bd66e46ade branch: refactor/httpx-transport-modernization specs: html2rss (0.29.1) diff --git a/app/web/errors/error_classifier.rb b/app/web/errors/error_classifier.rb index 8af50767b..60b52fd0b 100644 --- a/app/web/errors/error_classifier.rb +++ b/app/web/errors/error_classifier.rb @@ -262,7 +262,7 @@ def initialize(decision) %w[queue boot].include?(phase) }, SERVICE_UNAVAILABLE], [lambda { |c, _| - # Gem wall-clock timeout (Botasaurus 504 / Faraday timeout) — not Timeout::Error. + # Gem wall-clock timeout (Botasaurus 504 / HTTPX timeout) — not Timeout::Error. # work / nil (transport hop) → site-shaped 504. defined?(::Html2rss::RequestService::RequestTimedOut) && c.any?(::Html2rss::RequestService::RequestTimedOut) diff --git a/docs/README.md b/docs/README.md index a8035bacd..e10adec69 100644 --- a/docs/README.md +++ b/docs/README.md @@ -235,7 +235,7 @@ Default `docker-compose.yml` aligns botasaurus-scrape-api, the html2rss gem clie | --- | --- | --- | --- | | `SCRAPE_TIMEOUT_SECONDS` | botasaurus-scrape-api | `45` | Handler wall (queue, boot, navigate, wait) | | `SCRAPE_WORK_TIMEOUT_SECONDS` | botasaurus-scrape-api | `30` | Post-boot navigate, selector wait, scroll | -| `BOTASAURUS_SCRAPE_TIMEOUT_SECONDS` | html2rss (web) | `45` | Faraday POST `/scrape` cap (mirrors scrape total) | +| `BOTASAURUS_SCRAPE_TIMEOUT_SECONDS` | html2rss (web) | `45` | HTTPX POST `/scrape` cap (mirrors scrape total) | | `BOTASAURUS_SCRAPE_WORK_TIMEOUT_SECONDS` | html2rss | `30` | Max `wait_timeout_seconds` in feed YAML | | `HTML2RSS_TOTAL_TIMEOUT_SECONDS` | html2rss-web | `50` | Feed build budget (scrape + extraction) | | `REQUEST_TIMEOUT_SECONDS` | html2rss-web | `55` | Falcon server request timeout | diff --git a/spec/html2rss/web/api/v1_spec.rb b/spec/html2rss/web/api/v1_spec.rb index d55f8edd7..b6e5cc56b 100644 --- a/spec/html2rss/web/api/v1_spec.rb +++ b/spec/html2rss/web/api/v1_spec.rb @@ -478,7 +478,7 @@ def relative_feed_link_header(token) end it 'renders feed for a valid token', :aggregate_failures do - token = Html2rss::Web::Auth.generate_feed_token('admin', feed_url, strategy: 'faraday') + token = Html2rss::Web::Auth.generate_feed_token('admin', feed_url, strategy: 'default') allow(Html2rss::Web::Feeds::Service).to receive(:call).and_return(feed_result) stub_feed_renderer @@ -490,7 +490,7 @@ def relative_feed_link_header(token) end it 'returns alternate Link headers for successful feeds', :aggregate_failures do - token = Html2rss::Web::Auth.generate_feed_token('admin', "#{feed_url}/link-headers", strategy: 'faraday') + token = Html2rss::Web::Auth.generate_feed_token('admin', "#{feed_url}/link-headers", strategy: 'default') allow(Html2rss::Web::Feeds::Service).to receive(:call).and_return(ok_feed_result_with_payload) get "/api/v1/feeds/#{token}.xml", {}, { 'HTTP_HOST' => 'example.test' } @@ -501,7 +501,7 @@ def relative_feed_link_header(token) end it 'uses relative Link targets and varies JSON feed_url by Host', :aggregate_failures do - token = Html2rss::Web::Auth.generate_feed_token('admin', "#{feed_url}/host-vary", strategy: 'faraday') + token = Html2rss::Web::Auth.generate_feed_token('admin', "#{feed_url}/host-vary", strategy: 'default') feed = link_header_feed_double allow(Html2rss::Web::Feeds::Service).to receive(:call) .and_return(ok_render_result(feed: feed, cache_key: 'feed_result:host-vary')) @@ -525,7 +525,7 @@ def relative_feed_link_header(token) end it 'prefers xml when Accept quality outranks json', :aggregate_failures do - token = Html2rss::Web::Auth.generate_feed_token('admin', feed_url, strategy: 'faraday') + token = Html2rss::Web::Auth.generate_feed_token('admin', feed_url, strategy: 'default') allow(Html2rss::Web::Feeds::Service).to receive(:call).and_return(feed_result) stub_feed_renderer @@ -537,7 +537,7 @@ def relative_feed_link_header(token) end it 'ignores query param strategy overrides', :aggregate_failures, openapi: false do - token = Html2rss::Web::Auth.generate_feed_token('admin', feed_url, strategy: 'faraday') + token = Html2rss::Web::Auth.generate_feed_token('admin', feed_url, strategy: 'default') allow(Html2rss::Web::Feeds::Service).to receive(:call).and_return(feed_result) stub_feed_renderer @@ -574,7 +574,7 @@ def relative_feed_link_header(token) it 'returns forbidden when auto source is disabled', :aggregate_failures do unique_url = "#{feed_url}/disabled" - token = Html2rss::Web::Auth.generate_feed_token('admin', unique_url, strategy: 'faraday') + token = Html2rss::Web::Auth.generate_feed_token('admin', unique_url, strategy: 'default') ClimateControl.modify(AUTO_SOURCE_ENABLED: 'false') do get "/api/v1/feeds/#{token}", {}, { 'HTTP_ACCEPT' => 'application/xml' } @@ -587,7 +587,7 @@ def relative_feed_link_header(token) it 'returns plain text forbidden errors when requested through Accept', :aggregate_failures do unique_url = "#{feed_url}/disabled-json" - token = Html2rss::Web::Auth.generate_feed_token('admin', unique_url, strategy: 'faraday') + token = Html2rss::Web::Auth.generate_feed_token('admin', unique_url, strategy: 'default') ClimateControl.modify(AUTO_SOURCE_ENABLED: 'false') do get "/api/v1/feeds/#{token}", {}, { 'HTTP_ACCEPT' => 'application/feed+json' } @@ -600,7 +600,7 @@ def relative_feed_link_header(token) it 'returns non-cacheable feed errors when service generation fails', :aggregate_failures do unique_url = "#{feed_url}/service-error-xml" - token = Html2rss::Web::Auth.generate_feed_token('admin', unique_url, strategy: 'faraday') + token = Html2rss::Web::Auth.generate_feed_token('admin', unique_url, strategy: 'default') allow(Html2rss::Web::Feeds::Service).to receive(:call).and_return(service_error_result) @@ -615,7 +615,7 @@ def relative_feed_link_header(token) it 'returns non-cacheable plain text errors when service generation fails for json', :aggregate_failures, openapi: false do unique_url = "#{feed_url}/service-error-json" - token = Html2rss::Web::Auth.generate_feed_token('admin', unique_url, strategy: 'faraday') + token = Html2rss::Web::Auth.generate_feed_token('admin', unique_url, strategy: 'default') status, content_type, cache_control, body = json_feed_service_error_tuple(token) @@ -626,7 +626,7 @@ def relative_feed_link_header(token) end it 'returns 422 for empty extraction feeds in xml representation', :aggregate_failures do - token = Html2rss::Web::Auth.generate_feed_token('admin', "#{feed_url}/empty-xml", strategy: 'faraday') + token = Html2rss::Web::Auth.generate_feed_token('admin', "#{feed_url}/empty-xml", strategy: 'default') allow(Html2rss::Web::Feeds::Service).to receive(:call).and_return(extraction_empty_result) get "/api/v1/feeds/#{token}.xml" @@ -638,7 +638,7 @@ def relative_feed_link_header(token) end it 'returns 422 for empty extraction feeds in json feed representation', :aggregate_failures do - token = Html2rss::Web::Auth.generate_feed_token('admin', "#{feed_url}/empty-json", strategy: 'faraday') + token = Html2rss::Web::Auth.generate_feed_token('admin', "#{feed_url}/empty-json", strategy: 'default') allow(Html2rss::Web::Feeds::Service).to receive(:call).and_return(extraction_empty_result) get "/api/v1/feeds/#{token}.json" @@ -657,7 +657,7 @@ def relative_feed_link_header(token) rate_limit_window_seconds: 60 ) - token = Html2rss::Web::Auth.generate_feed_token('admin', "#{feed_url}/rate-limited-429", strategy: 'faraday') + token = Html2rss::Web::Auth.generate_feed_token('admin', "#{feed_url}/rate-limited-429", strategy: 'default') allow(Html2rss::Web::Feeds::Service).to receive(:call).and_return(feed_result) stub_feed_renderer @@ -682,7 +682,7 @@ def relative_feed_link_header(token) end it 'returns 504 when the gateway times out', :aggregate_failures do - token = Html2rss::Web::Auth.generate_feed_token('admin', "#{feed_url}/timeout-504", strategy: 'faraday') + token = Html2rss::Web::Auth.generate_feed_token('admin', "#{feed_url}/timeout-504", strategy: 'default') allow(Html2rss::Web::Feeds::Service).to receive(:call).and_raise(Timeout::Error.new('gateway timeout')) diff --git a/spec/html2rss/web/app_integration_spec.rb b/spec/html2rss/web/app_integration_spec.rb index 181ab59b8..ef2cc03c4 100644 --- a/spec/html2rss/web/app_integration_spec.rb +++ b/spec/html2rss/web/app_integration_spec.rb @@ -15,7 +15,7 @@ let(:feed_url) { 'https://example.com/articles' } let(:feed_token) do - Html2rss::Web::Auth.generate_feed_token(account[:username], feed_url, strategy: 'faraday') + Html2rss::Web::Auth.generate_feed_token(account[:username], feed_url, strategy: 'default') end let(:encoded_feed_token) { CGI.escape(feed_token) } @@ -148,7 +148,7 @@ 'RACK_ENV' => 'production', 'HTML2RSS_SECRET_KEY' => 'scrubbed-secret-key-for-request-specs' ) do - generated_token = Html2rss::Web::Auth.generate_feed_token(account[:username], feed_url, strategy: 'faraday') + generated_token = Html2rss::Web::Auth.generate_feed_token(account[:username], feed_url, strategy: 'default') get "/api/v1/feeds/#{generated_token}", {}, { 'HTTP_ACCEPT' => 'application/xml' } expect(ENV.fetch('HTML2RSS_SECRET_KEY', nil)).to be_nil @@ -216,7 +216,7 @@ it 'returns 422 when extraction yields an empty feed warning', :aggregate_failures do unique_empty_url = "#{feed_url}/empty-warning" - empty_token = Html2rss::Web::Auth.generate_feed_token(account[:username], unique_empty_url, strategy: 'faraday') + empty_token = Html2rss::Web::Auth.generate_feed_token(account[:username], unique_empty_url, strategy: 'default') stub_empty_feed_warning_result get "/api/v1/feeds/#{empty_token}.json" @@ -232,7 +232,7 @@ def stub_escaped_feed_token(raw_token:, encoded_token:) Html2rss::Web::FeedToken, url: feed_url, username: account[:username], - strategy: 'faraday' + strategy: 'default' ) allow(Html2rss::Web::FeedToken::Codec).to receive(:decode).with(raw_token).and_return(escaped_token_payload) diff --git a/spec/html2rss/web/error_classifier_spec.rb b/spec/html2rss/web/error_classifier_spec.rb index b39d405c8..00e02d854 100644 --- a/spec/html2rss/web/error_classifier_spec.rb +++ b/spec/html2rss/web/error_classifier_spec.rb @@ -56,7 +56,7 @@ def initialize(message = nil, timeout_phase: nil) it 'ignores attempts payload when mapping HTTP semantics' do klass = stub_no_feed_items_extracted_with_attempts - error = klass.new(attempts: [{ strategy: :faraday, items_count: 0 }]) + error = klass.new(attempts: [{ strategy: :default, items_count: 0 }]) expect(described_class.classify(error)).to have_attributes( status: 422, @@ -95,7 +95,7 @@ def initialize(message = nil, timeout_phase: nil) expect(described_class.classify(error)).to eq(described_class::SCRAPER_UNAVAILABLE) end - it 'returns gateway timeout for RequestTimedOut (Botasaurus/Faraday wall-clock)' do + it 'returns gateway timeout for RequestTimedOut (Botasaurus/HTTPX wall-clock)' do stub_request_timed_out error = Html2rss::RequestService::RequestTimedOut.new('Botasaurus scrape timed out') @@ -238,12 +238,12 @@ def initialize(message = nil, timeout_phase: nil) klass.new( attempts: [ { - strategy: :faraday, + strategy: :default, items_count: 0, transport_meta: { 'request_id' => 'req-123', 'render_ms' => 45, - 'strategy_used' => 'faraday', + 'strategy_used' => 'default', 'timeout_phase' => 'work' } } @@ -254,12 +254,12 @@ def initialize(message = nil, timeout_phase: nil) it 'extracts strategy attempts and transport meta when present', :aggregate_failures do diagnostics = described_class::Diagnostics.from_error(diagnostic_error) expect(diagnostics).to have_attributes( - request_id: 'req-123', render_ms: 45, strategy_used: 'faraday', timeout_phase: 'work' + request_id: 'req-123', render_ms: 45, strategy_used: 'default', timeout_phase: 'work' ) expect(diagnostics.strategy_attempts.size).to eq(1) expect(diagnostics.to_h).to include( strategy_attempts: diagnostics.strategy_attempts, - request_id: 'req-123', render_ms: 45, strategy_used: 'faraday', timeout_phase: 'work' + request_id: 'req-123', render_ms: 45, strategy_used: 'default', timeout_phase: 'work' ) end diff --git a/spec/html2rss/web/feeds/feed_result_integration_spec.rb b/spec/html2rss/web/feeds/feed_result_integration_spec.rb index 6ff968525..0275664a9 100644 --- a/spec/html2rss/web/feeds/feed_result_integration_spec.rb +++ b/spec/html2rss/web/feeds/feed_result_integration_spec.rb @@ -12,7 +12,7 @@ generator_input: generator_input, ttl_seconds: 600, url: page_url, - strategy: :faraday, + strategy: :default, feed_name: nil, directory_defaults: {}, request_params: {} @@ -21,7 +21,7 @@ let(:generator_input) do { channel: { url: page_url, title: 'Integration Feed' }, - strategy: :faraday, + strategy: :default, selectors: { items: { selector: 'article' }, title: { selector: 'h1 a' }, diff --git a/spec/html2rss/web/feeds/renderer_spec.rb b/spec/html2rss/web/feeds/renderer_spec.rb index 688c7cfe3..2fb58b20e 100644 --- a/spec/html2rss/web/feeds/renderer_spec.rb +++ b/spec/html2rss/web/feeds/renderer_spec.rb @@ -141,16 +141,16 @@ def render_body(result, path:, accept: nil) it 'sets diagnostic headers when telemetry is available in feed status', :aggregate_failures do status_double = instance_double( Html2rss::Status, - selected_strategy: :faraday, + selected_strategy: :default, strategy_attempts: [ - { strategy: :faraday, items_count: 5, transport_meta: { 'render_ms' => 120, 'request_id' => 'req-abc' } } + { strategy: :default, items_count: 5, transport_meta: { 'render_ms' => 120, 'request_id' => 'req-abc' } } ] ) allow(mock_feed_result).to receive(:status).and_return(status_double) resp = Rack::Response.new described_class.render(ok_result, response: resp, request: request) - expect(resp['X-Html2rss-Strategy']).to eq('faraday') + expect(resp['X-Html2rss-Strategy']).to eq('default') expect(resp['X-Html2rss-Render-Ms']).to eq('120') expect(resp['X-Html2rss-Request-Id']).to eq('req-abc') end diff --git a/spec/html2rss/web/feeds/responder_spec.rb b/spec/html2rss/web/feeds/responder_spec.rb index 1d8e1a9d5..199bff9aa 100644 --- a/spec/html2rss/web/feeds/responder_spec.rb +++ b/spec/html2rss/web/feeds/responder_spec.rb @@ -20,7 +20,7 @@ let(:static_config) do { channel: { url: 'https://example.com', ttl: 10 }, - strategy: :faraday + strategy: :default } end @@ -67,7 +67,7 @@ expect(Html2rss::Web::Observability).to have_received(:emit).with( event_name: 'feed.render', outcome: 'success', - details: include(strategy: :faraday, url: 'https://example.com', feed_name: 'example'), + details: include(strategy: :default, url: 'https://example.com', feed_name: 'example'), level: :info ) end @@ -134,7 +134,7 @@ end it 'emits hard error from RenderResult fields including diagnostics', :aggregate_failures do # rubocop:disable RSpec/ExampleLength - attempts = [{ strategy: :faraday, items_count: 0, error_class: 'Timeout::Error' }] + attempts = [{ strategy: :default, items_count: 0, error_class: 'Timeout::Error' }] allow(Html2rss::Web::Feeds::Service).to receive(:call).and_return( Html2rss::Web::Feeds::Contracts::RenderResult.new( status: :error, @@ -153,7 +153,7 @@ diagnostics: have_attributes(strategy_attempts: attempts), event_name: 'feed.render', details: include( - strategy: :faraday, + strategy: :default, url: 'https://example.com', feed_name: 'example', error_code: 'INTERNAL_SERVER_ERROR', @@ -161,7 +161,7 @@ strategy_attempts: attempts ), level: :warn, - context: { url: 'https://example.com', strategy: :faraday } + context: { url: 'https://example.com', strategy: :default } ) end end @@ -189,7 +189,7 @@ empty_reason: 'content_extraction_empty', diagnostics: Html2rss::Web::ErrorClassifier::Diagnostics.from_attempts( [ - { strategy: :faraday, items_count: 0, error_class: nil }, + { strategy: :default, items_count: 0, error_class: nil }, { strategy: :botasaurus, items_count: 0, error_class: nil } ] ) @@ -214,7 +214,7 @@ event_name: 'feed.render', outcome: 'failure', details: include( - strategy: :faraday, + strategy: :default, url: 'https://example.com', reason: 'content_extraction_empty', strategy_attempts: result.diagnostics.strategy_attempts @@ -259,7 +259,7 @@ expect(Html2rss::Web::Observability).to have_received(:emit).with( event_name: 'feed.render', outcome: 'failure', - details: include(strategy: :faraday, url: 'https://example.com', reason: 'feed_empty'), + details: include(strategy: :default, url: 'https://example.com', reason: 'feed_empty'), level: :warn ) end @@ -324,7 +324,7 @@ def expect_resolved_static_source have_attributes( source_kind: :static, cache_identity: a_string_starting_with('static:example:'), - generator_input: include(strategy: :faraday, channel: { url: 'https://example.com', ttl: 10 }), + generator_input: include(strategy: :default, channel: { url: 'https://example.com', ttl: 10 }), ttl_seconds: 600 ) ) diff --git a/spec/html2rss/web/feeds/service_spec.rb b/spec/html2rss/web/feeds/service_spec.rb index d1fabacc5..0adea554e 100644 --- a/spec/html2rss/web/feeds/service_spec.rb +++ b/spec/html2rss/web/feeds/service_spec.rb @@ -205,7 +205,7 @@ def initialize(attempts:) allow(Html2rss).to receive(:feed_result).with(resolved_source.generator_input).and_raise( no_feed_items_extracted_class.new( attempts: [ - { strategy: :faraday, items_count: 0, error_class: nil }, + { strategy: :default, items_count: 0, error_class: nil }, { strategy: :botasaurus, items_count: 0, error_class: nil } ] ) @@ -219,7 +219,7 @@ def initialize(attempts:) expect(result.error_message).to include('No feed items extracted after auto fallback') expect(result.diagnostics.strategy_attempts).to eq( [ - { strategy: :faraday, items_count: 0, error_class: nil }, + { strategy: :default, items_count: 0, error_class: nil }, { strategy: :botasaurus, items_count: 0, error_class: nil } ] ) @@ -238,7 +238,7 @@ def initialize(attempts:) it 'maps NoFeedItemsExtracted nested in Exception#cause to empty extraction', :aggregate_failures do root = no_feed_items_extracted_class.new( - attempts: [{ strategy: :faraday, items_count: 0, error_class: nil }] + attempts: [{ strategy: :default, items_count: 0, error_class: nil }] ) wrapper = StandardError.new('strategy failed') allow(wrapper).to receive(:cause).and_return(root) @@ -246,7 +246,7 @@ def initialize(attempts:) expect(result.status).to eq(:empty) expect(result.empty_reason).to eq('content_extraction_empty') - expect(result.diagnostics.strategy_attempts).to eq([{ strategy: :faraday, items_count: 0, error_class: nil }]) + expect(result.diagnostics.strategy_attempts).to eq([{ strategy: :default, items_count: 0, error_class: nil }]) end end diff --git a/spec/html2rss/web/feeds/source_resolver_spec.rb b/spec/html2rss/web/feeds/source_resolver_spec.rb index b8c6fa1fd..618d2140f 100644 --- a/spec/html2rss/web/feeds/source_resolver_spec.rb +++ b/spec/html2rss/web/feeds/source_resolver_spec.rb @@ -90,7 +90,7 @@ def resolved_tuple(resolved) Html2rss::Web::FeedToken, username: 'admin', url: 'https://example.com/private', - strategy: 'faraday' + strategy: 'default' ) end @@ -102,7 +102,7 @@ def resolved_tuple(resolved) allow(Html2rss::Web::UrlValidator).to receive(:url_allowed?) .with({ username: 'admin' }, 'https://example.com/private').and_return(true) allow(Html2rss::Web::Flags).to receive(:auto_source_enabled?).and_return(true) - allow(Html2rss::RequestService).to receive(:strategy_names).and_return([:faraday]) + allow(Html2rss::RequestService).to receive(:strategy_names).and_return(%i[auto default faraday botasaurus]) allow(Html2rss::Web::LocalConfig).to receive(:global) .and_return({ headers: { 'User-Agent' => 'html2rss-web' } }) end @@ -112,14 +112,22 @@ def resolved_tuple(resolved) expect(resolved_tuple(resolved)).to match( [:token, start_with('token:'), 300, - include(strategy: :faraday, channel: { url: 'https://example.com/private' }, auto_source: {})] + include(strategy: :default, channel: { url: 'https://example.com/private' }, auto_source: {})] ) expect(resolved).to have_attributes( url: 'https://example.com/private', - strategy: :faraday + strategy: :default ) end + it 'accepts legacy faraday token strategy for backwards compatibility' do + allow(feed_token).to receive(:strategy).and_return('faraday') + + resolved = described_class.call(feed_request) + + expect(resolved.strategy).to eq(:faraday) + end + it 'defaults blank token strategy to auto', :aggregate_failures do allow(feed_token).to receive(:strategy).and_return(nil) diff --git a/spec/html2rss/web/log_sanitizer_spec.rb b/spec/html2rss/web/log_sanitizer_spec.rb index 5de5c5d25..9afcd7c36 100644 --- a/spec/html2rss/web/log_sanitizer_spec.rb +++ b/spec/html2rss/web/log_sanitizer_spec.rb @@ -27,7 +27,7 @@ http_method: 'GET', route_group: 'api_v1', actor: nil, - strategy: 'faraday', + strategy: 'default', started_at: '2026-03-21T00:00:00Z' ) end @@ -104,7 +104,7 @@ Html2rss::Web::Observability.emit( event_name: 'feed.render', outcome: 'success', - details: { url: 'https://news.ycombinator.com', strategy: 'faraday' } + details: { url: 'https://news.ycombinator.com', strategy: 'default' } ) lines = io.string.lines.map { |line| JSON.parse(line, symbolize_names: true) } diff --git a/spec/html2rss/web/sentry_logs_spec.rb b/spec/html2rss/web/sentry_logs_spec.rb index 495bd437a..8ef4d5f3d 100644 --- a/spec/html2rss/web/sentry_logs_spec.rb +++ b/spec/html2rss/web/sentry_logs_spec.rb @@ -114,7 +114,7 @@ def breadcrumb_payload outcome: 'failure', request_id: 'req-123', route_group: 'api_v1', - strategy: 'faraday', + strategy: 'default', details: { url: 'https://example.com/articles', fallback: 'botasaurus' } } end @@ -134,7 +134,7 @@ def breadcrumb_data_matcher outcome: 'failure', request_id: 'req-123', route_group: 'api_v1', - strategy: 'faraday', + strategy: 'default', details: breadcrumb_details_matcher ) end diff --git a/spec/smoke/docker_spec.rb b/spec/smoke/docker_spec.rb index 115d2b76b..c4a4c6bec 100644 --- a/spec/smoke/docker_spec.rb +++ b/spec/smoke/docker_spec.rb @@ -71,7 +71,7 @@ def expect_json_feed_response(path) it 'creates a feed when provided with valid credentials', :aggregate_failures do payload = { url: feed_url, - strategy: 'faraday' + strategy: 'default' } response, body = post_json('/api/v1/feeds', body: payload) @@ -84,7 +84,7 @@ def expect_json_feed_response(path) payload = { url: feed_url, - strategy: 'faraday' + strategy: 'default' } response, body = post_json('/api/v1/feeds', @@ -101,7 +101,7 @@ def expect_json_feed_response(path) payload = { url: feed_url, - strategy: 'faraday' + strategy: 'default' } response, body = post_json('/api/v1/feeds', From 7b32fbc3b3480c8f876dbe288dc7dd766871512d Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Sun, 6 Sep 2026 10:40:15 +0200 Subject: [PATCH 4/5] chore(deps): point html2rss to master and clean up devcontainer mounts --- .devcontainer/docker-compose.yml | 1 - Gemfile | 2 +- Gemfile.lock | 4 ++-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index bfd2411f1..2e23f37f4 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -5,7 +5,6 @@ services: dockerfile: .devcontainer/Dockerfile volumes: - ../:/workspace:cached - - ../../:/Users/gil/versioned/html2rss:cached - bundle-cache:/usr/local/bundle ports: - "4000:4000" diff --git a/Gemfile b/Gemfile index 2eb46a713..c253e7525 100644 --- a/Gemfile +++ b/Gemfile @@ -5,7 +5,7 @@ source 'https://rubygems.org' git_source(:github) { |repo_name| "https://github.com/#{repo_name}" } # gem 'html2rss', '~> 0.29' -gem 'html2rss', github: 'html2rss/html2rss', branch: 'refactor/httpx-transport-modernization' +gem 'html2rss', github: 'html2rss/html2rss', branch: 'master' gem 'html2rss-configs', github: 'html2rss/html2rss-configs' # Use these instead of the two above (uncomment them) when developing locally: diff --git a/Gemfile.lock b/Gemfile.lock index b0d3f1ce4..54591b80a 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ GIT remote: https://github.com/html2rss/html2rss - revision: 5f2ab31855f092c1ed18a7a5150448bd66e46ade - branch: refactor/httpx-transport-modernization + revision: e1d2550ea9df109f58c8feb0ebe82ebb0e21d8ac + branch: master specs: html2rss (0.29.1) addressable (~> 2.7) From e816a091782629d07a5c67897d8922f91d5b308c Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Sun, 6 Sep 2026 10:48:05 +0200 Subject: [PATCH 5/5] chore(deps): bump html2rss to 0.30.0 --- Gemfile | 4 ++-- Gemfile.lock | 46 ++++++++++++++++++++-------------------------- 2 files changed, 22 insertions(+), 28 deletions(-) diff --git a/Gemfile b/Gemfile index c253e7525..6c1c1a75d 100644 --- a/Gemfile +++ b/Gemfile @@ -4,8 +4,8 @@ source 'https://rubygems.org' git_source(:github) { |repo_name| "https://github.com/#{repo_name}" } -# gem 'html2rss', '~> 0.29' -gem 'html2rss', github: 'html2rss/html2rss', branch: 'master' +gem 'html2rss', '~> 0.30' +# gem 'html2rss', github: 'html2rss/html2rss', branch: 'master' gem 'html2rss-configs', github: 'html2rss/html2rss-configs' # Use these instead of the two above (uncomment them) when developing locally: diff --git a/Gemfile.lock b/Gemfile.lock index 54591b80a..5461d1a01 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,27 +1,3 @@ -GIT - remote: https://github.com/html2rss/html2rss - revision: e1d2550ea9df109f58c8feb0ebe82ebb0e21d8ac - branch: master - specs: - html2rss (0.29.1) - addressable (~> 2.7) - brotli - dry-validation - httpx (~> 1.8) - kramdown - mcp (~> 1.2) - mime-types (> 3.0) - nokogiri (>= 1.10, < 2.0) - rack (~> 3.0) - rackup (~> 2.0) - regexp_parser - rss - sanitize - thor - tzinfo - webrick (~> 1.9) - zeitwerk - GIT remote: https://github.com/html2rss/html2rss-configs revision: 13cdb5f27bc7c9e943f27ee222eaf13f3dd0c93c @@ -165,6 +141,24 @@ GEM fiber-storage (1.0.1) hana (1.3.7) hashdiff (1.2.1) + html2rss (0.30.0) + addressable (~> 2.7) + brotli + dry-validation + httpx (~> 1.8) + kramdown + mcp (~> 1.2) + mime-types (> 3.0) + nokogiri (>= 1.10, < 2.0) + rack (~> 3.0) + rackup (~> 2.0) + regexp_parser + rss + sanitize + thor + tzinfo + webrick (~> 1.9) + zeitwerk http-2 (1.2.2) httpx (1.8.3) http-2 (>= 1.2.0) @@ -376,7 +370,7 @@ DEPENDENCIES base64 climate_control falcon - html2rss! + html2rss (~> 0.30) html2rss-configs! irb rack-test @@ -440,7 +434,7 @@ CHECKSUMS fiber-storage (1.0.1) sha256=f48e5b6d8b0be96dac486332b55cee82240057065dc761c1ea692b2e719240e1 hana (1.3.7) sha256=5425db42d651fea08859811c29d20446f16af196308162894db208cac5ce9b0d hashdiff (1.2.1) sha256=9c079dbc513dfc8833ab59c0c2d8f230fa28499cc5efb4b8dd276cf931457cd1 - html2rss (0.29.1) + html2rss (0.30.0) sha256=a0ec169cf4a41954b2ec327625b15e81d718061a59a0bd27670ca01f238a2852 html2rss-configs (0.2.0) http-2 (1.2.2) sha256=81b5d45f50fd4cd5f8c5d09651184bec9401e3ef169c3eb3e5b003d5614a92b9 httpx (1.8.3) sha256=cb88f2285c4ef17164d803a640dc393d28722cba7f282ad0e7f9f926cd02dc47