diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index a994bdbb..2e23f37f 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -13,6 +13,7 @@ services: - RACK_ENV=development - BUNDLE_PATH=/usr/local/bundle - PORT=4000 + - BOTASAURUS_SCRAPER_URL=${BOTASAURUS_SCRAPER_URL:-http://host.docker.internal:4010} command: sleep infinity user: vscode working_dir: /workspace diff --git a/Dockerfile b/Dockerfile index fe75a3bb..544784c3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,12 +27,13 @@ RUN apk add --no-cache \ 'libc-dev>=0.7' \ 'make>=4' \ 'openssl=~3' \ + 'openssl-dev=~3' \ 'libxml2-dev>=2' \ 'libxslt-dev>=1' \ && gem install bundler:$(tail -1 Gemfile.lock | tr -d ' ') \ && bundle config set --local without 'development test' \ && bundle install --retry=5 --jobs=$(nproc) \ - && bundle binstubs bundler html2rss \ + && bundle binstubs bundler html2rss falcon \ && bundle clean --force \ && rm -rf /usr/local/bundle/cache \ /usr/local/bundle/bundler/gems/*/.git \ @@ -76,8 +77,6 @@ RUN apk add --no-cache \ --no-create-home \ --uid "$UID" "$USER" \ && mkdir -p /app \ - && mkdir -p /app/tmp/rack-cache-body \ - && mkdir -p /app/tmp/rack-cache-meta \ && chown "$USER":"$USER" -R /app WORKDIR /app @@ -92,4 +91,4 @@ COPY --chown=$USER:$USER config ./config COPY --chown=$USER:$USER public ./public COPY --from=frontend-builder --chown=$USER:$USER /app/frontend/dist ./frontend/dist -CMD ["bundle", "exec", "puma", "-C", "./config/puma.rb"] +CMD ["bundle", "exec", "falcon", "host", "./config/falcon.rb"] diff --git a/Gemfile b/Gemfile index b99277b6..c253e752 100644 --- a/Gemfile +++ b/Gemfile @@ -5,7 +5,7 @@ source 'https://rubygems.org' git_source(:github) { |repo_name| "https://github.com/#{repo_name}" } # gem 'html2rss', '~> 0.29' -gem 'html2rss', github: 'html2rss/html2rss', branch: 'experiment/httpx-default' +gem 'html2rss', github: 'html2rss/html2rss', branch: 'master' gem 'html2rss-configs', github: 'html2rss/html2rss-configs' # Use these instead of the two above (uncomment them) when developing locally: @@ -13,13 +13,10 @@ gem 'html2rss-configs', github: 'html2rss/html2rss-configs' # gem 'html2rss-configs', path: '../html2rss-configs' gem 'base64' -gem 'rack-cache' -gem 'rack-timeout' +gem 'falcon' gem 'roda' gem 'zeitwerk' -gem 'puma', require: false - group :development do gem 'irb', require: false gem 'rake', require: false diff --git a/Gemfile.lock b/Gemfile.lock index a393308e..42cf6631 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ GIT remote: https://github.com/html2rss/html2rss - revision: f0b73c2ae11dcc4aabc3386ecc0d7b6ee1b54013 - branch: experiment/httpx-default + revision: 4260c842a5632fbef712cf3bee3a4fddcfee3654 + branch: master specs: html2rss (0.29.1) addressable (~> 2.7) @@ -64,6 +64,34 @@ GEM addressable (2.9.0) public_suffix (>= 2.0.2, < 8.0) ast (2.4.3) + async (2.45.1) + console (~> 1.29) + fiber-annotation + io-event (~> 1.21) + async-container (0.38.0) + async (~> 2.44) + async-http (0.103.0) + async (>= 2.35.1) + async-pool (~> 0.12) + io-endpoint (~> 0.18) + io-stream (~> 0.14) + protocol-http (~> 0.66) + protocol-http1 (~> 0.41) + protocol-http2 (~> 0.28) + protocol-url (~> 0.2) + async-http-cache (0.4.6) + async-http (~> 0.56) + async-pool (0.12.0) + async (>= 2.0) + async-service (0.25.0) + async + async-container (~> 0.34) + string-format (~> 0.2) + async-utilization (0.5.0) + console (~> 1.0) + bake (0.25.0) + bigdecimal + samovar (~> 2.1) base64 (0.3.0) bigdecimal (4.1.2) brotli (0.8.0) @@ -71,6 +99,10 @@ GEM climate_control (1.2.0) concurrent-ruby (1.3.8) connection_pool (3.0.2) + console (1.37.0) + fiber-annotation + fiber-local (~> 1.1) + json crack (1.0.1) bigdecimal rexml @@ -114,6 +146,23 @@ GEM zeitwerk (~> 2.6) erb (6.0.7) erubi (1.13.1) + falcon (0.57.0) + async + async-container (~> 0.20) + async-http (~> 0.97) + async-http-cache (~> 0.4) + async-service (~> 0.19) + async-utilization (~> 0.3) + bundler + localhost (~> 1.1) + openssl (>= 3.0) + protocol-http (~> 0.31) + protocol-rack (~> 0.7) + samovar (~> 2.3) + fiber-annotation (0.2.0) + fiber-local (1.1.0) + fiber-storage + fiber-storage (1.0.1) hana (1.3.7) hashdiff (1.2.1) http-2 (1.2.2) @@ -122,6 +171,11 @@ GEM i18n (1.15.2) concurrent-ruby (~> 1.0) io-console (0.9.2) + io-endpoint (0.18.0) + openssl (>= 3.3.0) + io-event (1.21.1) + io-stream (0.14.0) + openssl (>= 3.3) irb (1.18.0) pp (>= 0.6.0) prism (>= 1.3.0) @@ -137,6 +191,8 @@ GEM rexml (>= 3.4.4) language_server-protocol (3.17.0.6) lint_roller (1.1.0) + localhost (1.8.0) + bake logger (1.7.0) loofah (2.25.2) crass (~> 1.0.2) @@ -150,7 +206,6 @@ GEM minitest (6.0.6) drb (~> 2.0) prism (~> 1.5) - nio4r (2.7.5) nokogiri (1.19.4-aarch64-linux-gnu) racc (~> 1.4) nokogiri (1.19.4-aarch64-linux-musl) @@ -167,6 +222,7 @@ GEM racc (~> 1.4) nokogiri (1.19.4-x86_64-linux-musl) racc (~> 1.4) + openssl (4.0.2) parallel (2.1.0) parser (3.3.12.0) ast (~> 2.4.1) @@ -175,19 +231,26 @@ GEM prettyprint prettyprint (0.2.0) prism (1.9.0) + protocol-hpack (1.5.1) + protocol-http (0.71.0) + protocol-http1 (0.41.0) + protocol-http (~> 0.68) + protocol-http2 (0.28.0) + protocol-hpack (~> 1.4) + protocol-http (~> 0.62) + protocol-rack (0.22.1) + io-stream (>= 0.10) + protocol-http (~> 0.58) + rack (>= 1.0) + protocol-url (0.18.0) public_suffix (7.0.5) - puma (8.0.2) - nio4r (~> 2.0) racc (1.8.1) rack (3.2.7) - rack-cache (1.17.0) - rack (>= 0.4) rack-session (2.1.2) base64 (>= 0.1.0) rack (>= 3.0.0) rack-test (2.2.0) rack (>= 1.3) - rack-timeout (0.7.0) rackup (2.3.1) rack (>= 3) rails-dom-testing (2.3.0) @@ -267,6 +330,8 @@ GEM prism (>= 1.2, < 2.0) rbs (>= 3, < 5) ruby-progressbar (1.13.0) + samovar (2.5.1) + console (~> 1.0) sanitize (7.0.0) crass (~> 1.0.2) nokogiri (>= 1.16.8) @@ -278,6 +343,7 @@ GEM simplecov (1.2.0) simpleidn (0.3.0) stackprof (0.2.28) + string-format (0.2.0) thor (1.5.0) tsort (0.2.0) tzinfo (2.0.6) @@ -309,13 +375,11 @@ PLATFORMS DEPENDENCIES base64 climate_control + falcon html2rss! html2rss-configs! irb - puma - rack-cache rack-test - rack-timeout rake roda rspec @@ -340,6 +404,14 @@ CHECKSUMS activesupport (8.1.3.1) sha256=85458765f25ea48b9019c46b6bb3fa5683197bf4280d9f06710a6e8d7a831376 addressable (2.9.0) sha256=7fdf6ac3660f7f4e867a0838be3f6cf722ace541dd97767fa42bc6cfa980c7af ast (2.4.3) sha256=954615157c1d6a382bc27d690d973195e79db7f55e9765ac7c481c60bdb4d383 + async (2.45.1) sha256=95849f7432eca535146c69f3078dab3c7228d0075ed71f48733f3e585dd7c267 + async-container (0.38.0) sha256=71479d8551870d684e602f463d7b8368909c835a64e38b555e1821512ffffc88 + async-http (0.103.0) sha256=137e3d5d419f7d72d5fa33cf04f30232e79c57b690025a650b8f9e48ece8bc41 + async-http-cache (0.4.6) sha256=2038d1f093182f16b50b4db271c25085e3938da10bfcfc2904cadb0530fddfd6 + async-pool (0.12.0) sha256=5580493dc024bd704dab2ed0bf1307100722e49cbb825c7965f8501194f58506 + async-service (0.25.0) sha256=71d9d75c2898dddf6549150cecd57156453e5b61da69fa87f10a0c97becfb6d1 + async-utilization (0.5.0) sha256=96c347634e09edcbcb7f70cf8c5a1ae058c64d88186341e3f924a44440f95981 + bake (0.25.0) sha256=a47bdc6a26addc048827debc36fe27bb4d5d71ac2958ad910b5386e9baf49869 base64 (0.3.0) sha256=27337aeabad6ffae05c265c450490628ef3ebd4b67be58257393227588f5a97b bigdecimal (4.1.2) sha256=53d217666027eab4280346fba98e7d5b66baaae1b9c3c1c0ffe89d48188a3fbd brotli (0.8.0) sha256=0c5a42046b3b603fb109656881147fd76064c034b7d19c1b4fcc32a093a4d55d @@ -347,6 +419,7 @@ CHECKSUMS climate_control (1.2.0) sha256=36b21896193fa8c8536fa1cd843a07cf8ddbd03aaba43665e26c53ec1bd70aa5 concurrent-ruby (1.3.8) sha256=b2f1be836e968ccc78ccfce277ea79c72a88633f22306782c16ff23fb415d1e1 connection_pool (3.0.2) sha256=33fff5ba71a12d2aa26cb72b1db8bba2a1a01823559fb01d29eb74c286e62e0a + console (1.37.0) sha256=8093b286c2595a063849c098594fee5155ecc7967d36f3aa8cbe7569c8f3efd7 crack (1.0.1) sha256=ff4a10390cd31d66440b7524eb1841874db86201d5b70032028553130b6d4c7e crass (1.0.7) sha256=94868719948664c89ddcaf0a37c65048413dfcb1c869470a5f7a7ceb5390b295 diff-lcs (1.6.2) sha256=9ae0d2cba7d4df3075fe8cd8602a8604993efc0dfa934cff568969efb1909962 @@ -361,6 +434,10 @@ CHECKSUMS dry-validation (1.11.1) sha256=70900bb5a2d911c8aab566d3e360c6bff389b8bf92ea8e04885ce51c41ff8085 erb (6.0.7) sha256=c5ca6dc25b0ef974a44dc8f59fe847577122483b1968a38dec305c60bf91ee92 erubi (1.13.1) sha256=a082103b0885dbc5ecf1172fede897f9ebdb745a4b97a5e8dc63953db1ee4ad9 + falcon (0.57.0) sha256=c9b060bc23a3b90a80376078aae037d8f48e0b2f12f5f30f360d4b29d7baf0f7 + fiber-annotation (0.2.0) sha256=7abfadf1d119f508867d4103bf231c0354d019cc39a5738945dec2edadaf6c03 + fiber-local (1.1.0) sha256=c885f94f210fb9b05737de65d511136ea602e00c5105953748aa0f8793489f06 + fiber-storage (1.0.1) sha256=f48e5b6d8b0be96dac486332b55cee82240057065dc761c1ea692b2e719240e1 hana (1.3.7) sha256=5425db42d651fea08859811c29d20446f16af196308162894db208cac5ce9b0d hashdiff (1.2.1) sha256=9c079dbc513dfc8833ab59c0c2d8f230fa28499cc5efb4b8dd276cf931457cd1 html2rss (0.29.1) @@ -369,19 +446,22 @@ CHECKSUMS httpx (1.8.3) sha256=cb88f2285c4ef17164d803a640dc393d28722cba7f282ad0e7f9f926cd02dc47 i18n (1.15.2) sha256=00f9eb62412fe593b2a65a97daa75300d37abb8f7202ec748e94b6d46a9dd1b5 io-console (0.9.2) sha256=efa74f891dd03c0939a931dfc6e74c2813d904763d456ea9762b0525e748db08 + io-endpoint (0.18.0) sha256=720d186bcd584e8ba490bfc91dab332cc3c5ffee848b5b03894555cb275151aa + io-event (1.21.1) sha256=b3088f89ad67edf9be9fc2f48a5fae0790e92862487551eeb337b49098eb99e6 + io-stream (0.14.0) sha256=07d9b13ea2066ccbbc761fff8d69ad5a31e47561d31e02cc271f3ed9c1a8f1cd irb (1.18.0) sha256=de9454a0703a54704b9811a5ef31a60c86949fbf4013fcf244fabc7c775248e3 json (2.21.2) sha256=1f1d3b7cf2b3ba1a69beca0bb6db13d5438b80bff3cd54cdaaa620b9b07c1c6a json_schemer (2.5.0) sha256=2f01fb4cce721a4e08dd068fc2030cffd0702a7f333f1ea2be6e8991f00ae396 kramdown (2.5.2) sha256=1ba542204c66b6f9111ff00dcc26075b95b220b07f2905d8261740c82f7f02fa language_server-protocol (3.17.0.6) sha256=5ef2c0c138f8267e1bc631d3328347d354f96724b0af22f2c79516120443b7f0 lint_roller (1.1.0) sha256=2c0c845b632a7d172cb849cc90c1bce937a28c5c8ccccb50dfd46a485003cc87 + localhost (1.8.0) sha256=df7ea825b4f64949c588c17efac86bc47ddc4460d723778abe933b71759b2701 logger (1.7.0) sha256=196edec7cc44b66cfb40f9755ce11b392f21f7967696af15d274dde7edff0203 loofah (2.25.2) sha256=2007f746959ac65552456e04b433e83deb22759ab38c838b4445c70e43425918 mcp (1.5.0) sha256=6f97785fa6e069eb667aa3d50299851ea9aca904b4abe8306f156bb8fd9eaff0 mime-types (3.7.0) sha256=dcebf61c246f08e15a4de34e386ebe8233791e868564a470c3fe77c00eed5e56 mime-types-data (3.2026.0701) sha256=cd8811e1fb89d836499ba0582368a10ee74cef929ba956d1d5ddca045e6a730f minitest (6.0.6) sha256=153ea36d1d987a62942382b61075745042a2b3123b1cd48f4c3675af9cc7d6f1 - nio4r (2.7.5) sha256=6c90168e48fb5f8e768419c93abb94ba2b892a1d0602cb06eef16d8b7df1dca1 nokogiri (1.19.4-aarch64-linux-gnu) sha256=1269fb644a6de405057a53dd5c762b1209b43ca7424f839454d3dbc677c31a8f nokogiri (1.19.4-aarch64-linux-musl) sha256=35c65b9ce72b3bb03207bdbe7067915019dc18c1b9b59139684bd6690fdd01af nokogiri (1.19.4-arm-linux-gnu) sha256=a301313e38bb065d68239e79734bcd6f56fb6efaacebde29e9abf2a4735340ca @@ -390,19 +470,23 @@ CHECKSUMS nokogiri (1.19.4-x86_64-darwin) sha256=7fd17057d3e1f00e9954a74b3cd76595d3d4a5ef233b7ed9599047c204f70551 nokogiri (1.19.4-x86_64-linux-gnu) sha256=379fae440b28915e3f19d752ce2dcf8465ed2b2fbefd2a7ca0dd497bc981a06a nokogiri (1.19.4-x86_64-linux-musl) sha256=17dfb7c1fa194ae02fbf7c51a7afc8d278045ab3fdacfd86f91d02d7b274470b + openssl (4.0.2) sha256=1037ad2868ae58df9ad917891c0c0f9815a1172f6846d4bcdd508e4c2ee747c2 parallel (2.1.0) sha256=b35258865c2e31134c5ecb708beaaf6772adf9d5efae28e93e99260877b09356 parser (3.3.12.0) sha256=21a6d7f755d5a24dfbdc6e6b772e4e879a52e7631a88bc5a3a134606052c9828 pp (0.6.4) sha256=dfcb0fce700c41456265922884f9fe195d7fbb0674a3578e6c0f69588e82b570 prettyprint (0.2.0) sha256=2bc9e15581a94742064a3cc8b0fb9d45aae3d03a1baa6ef80922627a0766f193 prism (1.9.0) sha256=7b530c6a9f92c24300014919c9dcbc055bf4cdf51ec30aed099b06cd6674ef85 + protocol-hpack (1.5.1) sha256=6feca238b8078da1cd295677d6f306c6001af92d75fe0643d33e6956cbc3ad91 + protocol-http (0.71.0) sha256=38577bf3df0bf0c3f7ee7e21b43fcb4bd7624e36f697dbe9c3eea344f14bd9ba + protocol-http1 (0.41.0) sha256=9863874a1228948a896c3e7ea16d13a680c8f3ed50706007c78b7033018b0ea3 + protocol-http2 (0.28.0) sha256=b7c6195dcfa80ec2484670e9fd8fc21024cc654a391aa0792433aff1188c1691 + protocol-rack (0.22.1) sha256=1185d245927ef9849a603700d6991ca353bc89724fbf98efa4a4333ed62a9fc3 + protocol-url (0.18.0) sha256=b16f55983094b46d18e04b709333b260e4ab5962a32696728e33a6c396150ed6 public_suffix (7.0.5) sha256=1a8bb08f1bbea19228d3bed6e5ed908d1cb4f7c2726d18bd9cadf60bc676f623 - puma (8.0.2) sha256=c8ed871dfbbe66448ea9ffd46692342d9804d4071522b52b5331b7b6e7b686fb racc (1.8.1) sha256=4a7f6929691dbec8b5209a0b373bc2614882b55fc5d2e447a21aaa691303d62f rack (3.2.7) sha256=93e13e1c24f93556671d85d2d79fa228c3485815c50d7e2f265b5330c6528fb7 - rack-cache (1.17.0) sha256=49592f3ef2173b0f5524df98bb801fb411e839869e7ce84ac428dc492bf0eb90 rack-session (2.1.2) sha256=595434f8c0c3473ae7d7ac56ecda6cc6dfd9d37c0b2b5255330aa1576967ffe8 rack-test (2.2.0) sha256=005a36692c306ac0b4a9350355ee080fd09ddef1148a5f8b2ac636c720f5c463 - rack-timeout (0.7.0) sha256=757337e9793cca999bb73a61fe2a7d4280aa9eefbaf787ce3b98d860749c87d9 rackup (2.3.1) sha256=6c79c26753778e90983761d677a48937ee3192b3ffef6bc963c0950f94688868 rails-dom-testing (2.3.0) sha256=8acc7953a7b911ca44588bf08737bc16719f431a1cc3091a292bca7317925c1d rails-html-sanitizer (1.7.1) sha256=e797a7c9b01e567307e317c576b49ab4168017e63eea4dba9ce3cb587e2f22c2 @@ -429,12 +513,14 @@ CHECKSUMS rubocop-thread_safety (0.7.3) sha256=067cdd52fbf5deffc18995437e45b5194236eaff4f71de3375a1f6052e48f431 ruby-lsp (0.26.11) sha256=4cc8f1587237ff8c9031680b5491df901a82559eeaa1ae96863ffe700b32583d ruby-progressbar (1.13.0) sha256=80fc9c47a9b640d6834e0dc7b3c94c9df37f08cb072b7761e4a71e22cff29b33 + samovar (2.5.1) sha256=8a9fc41eb8868084f0321eb41678c485cfbc7d282fb306c0be67c3284b1d2394 sanitize (7.0.0) sha256=269d1b9d7326e69307723af5643ec032ff86ad616e72a3b36d301ac75a273984 securerandom (0.4.1) sha256=cc5193d414a4341b6e225f0cb4446aceca8e50d5e1888743fac16987638ea0b1 sentry-ruby (7.0.0) sha256=e9616ff521355a983fad404ca575d8bb1f1949a8cac42a7a893dd02156bcdb1a simplecov (1.2.0) sha256=ea6acd05eece5a41990e2a5171c57d15700d329326c7666c85ee8c6a0dd0977e simpleidn (0.3.0) sha256=12ca730bed2f3db04d11e9bfd1bca3e11fb37f55b21eb2e9793fb5814bf54d03 stackprof (0.2.28) sha256=4ec2ace02f386012b40ca20ef80c030ad711831f59511da12e83b34efb0f9a04 + string-format (0.2.0) sha256=bc981c14116b061f12134549f32fa2d61a17b5a35dd6fd36596c21722a789af6 thor (1.5.0) sha256=e3a9e55fe857e44859ce104a84675ab6e8cd59c650a49106a05f55f136425e73 tsort (0.2.0) sha256=9650a793f6859a43b6641671278f79cfead60ac714148aabe4e3f0060480089f tzinfo (2.0.6) sha256=8daf828cc77bcf7d63b0e3bdb6caa47e2272dcfaf4fbfe46f8c3a9df087a829b diff --git a/Makefile b/Makefile index b7f4aad2..c4444eb7 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,6 @@ setup: ## Full development setup cp .env.example .env 2>/dev/null || echo "RACK_ENV=development" > .env; \ echo "Created .env file"; \ fi - @mkdir -p tmp/rack-cache-body tmp/rack-cache-meta @echo "Setting up frontend..." @cd frontend && CI=1 pnpm install --frozen-lockfile @echo "Setup complete!" @@ -146,7 +145,7 @@ openapi-lint-spectral: ## Lint OpenAPI using Spectral OAS rules openai-lint-spectral: openapi-lint-spectral ## Alias for openapi-lint-spectral clean: ## Clean temporary files - @rm -rf tmp/rack-cache-* coverage/ + @rm -rf coverage/ @cd frontend && rm -rf dist/ node_modules/ @echo "Clean complete!" diff --git a/README.md b/README.md index c02b68a2..8feabe25 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,13 @@ feed.example.com { } ``` +Alternatively, Falcon supports direct in-process TLS termination and HTTP/2 negotiation by providing certificate and private key paths: + +```bash +TLS_CERTIFICATE_PATH=/path/to/fullchain.pem +TLS_KEY_PATH=/path/to/privkey.pem +``` + ## Custom feeds (escape hatch) When auto-source is not enough, mount a YAML feeds file: diff --git a/Rakefile b/Rakefile index c0163b50..c62c9361 100644 --- a/Rakefile +++ b/Rakefile @@ -66,7 +66,6 @@ task :test do sh ['docker run', '-d', '-p 4000:4000', - '--env PUMA_LOG_CONFIG=1', "--env BUILD_TAG=#{smoke_build_tag}", "--env GIT_SHA=#{smoke_git_sha}", "--env HEALTH_CHECK_TOKEN=#{smoke_health_token}", diff --git a/app.rb b/app.rb index c0be45bf..818c36b7 100644 --- a/app.rb +++ b/app.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true require 'roda' -require 'rack/cache' require 'json' require 'base64' @@ -54,8 +53,6 @@ def development? = self.class.development? opts.merge!(check_dynamic_arity: false, check_arity: :warn) use RequestContextMiddleware use RateLimiter - use Rack::Cache, metastore: 'file:./tmp/rack-cache-meta', entitystore: 'file:./tmp/rack-cache-body', - verbose: development? plugin :content_security_policy do |csp| csp.default_src :none diff --git a/app/web/api/v1/create_feed.rb b/app/web/api/v1/create_feed.rb index 05bdeb09..b3638f57 100644 --- a/app/web/api/v1/create_feed.rb +++ b/app/web/api/v1/create_feed.rb @@ -9,8 +9,9 @@ module Api module V1 ## # Creates stable feed records from authenticated API requests. - module CreateFeed + module CreateFeed # rubocop:disable Metrics/ModuleLength FEED_ATTRIBUTE_KEYS = %i[id name url feed_token public_url json_public_url created_at updated_at].freeze + MAX_BODY_BYTES = 64 * 1024 ABSOLUTE_URL_REGEXP = %r{\A[a-z][a-z0-9+\-.]*://}i HOSTNAME_INPUT_REGEXP = %r{ \A(localhost(?::\d+)?|(?:\d{1,3}\.){3}\d{1,3}(?::\d+)?|(?:[a-z0-9-]+\.)+[a-z]{2,}(?::\d+)?) @@ -45,6 +46,7 @@ def require_account(request) end def build_create_params(request, account) + enforce_body_limit!(request) params = request_params(request) url = validated_url(params['url'], account) name = params['name'].to_s.strip @@ -52,6 +54,12 @@ def build_create_params(request, account) FeedMetadata::CreateParams.new(url:, name:) end + def enforce_body_limit!(request) + return unless request.content_length.to_i > MAX_BODY_BYTES + + raise Html2rss::Web::BadRequestError, 'Payload too large' + end + def request_params(request) return request.params unless json_request?(request) @@ -59,8 +67,7 @@ def request_params(request) end def parsed_json_body(request) - raw_body = request.body.read - request.body.rewind + raw_body = read_limited_body(request) return {} if raw_body.strip.empty? parsed = JSON.parse(raw_body) @@ -71,6 +78,15 @@ def parsed_json_body(request) raise Html2rss::Web::BadRequestError, 'Invalid JSON payload' end + def read_limited_body(request) + enforce_body_limit!(request) + + request.body.read(MAX_BODY_BYTES + 1).to_s.tap do |raw_body| + request.body.rewind + raise Html2rss::Web::BadRequestError, 'Payload too large' if raw_body.bytesize > MAX_BODY_BYTES + end + end + def json_request?(request) request.env['CONTENT_TYPE'].to_s.include?('application/json') end diff --git a/app/web/boot/development_reloader.rb b/app/web/boot/development_reloader.rb index 82e28a43..25689686 100644 --- a/app/web/boot/development_reloader.rb +++ b/app/web/boot/development_reloader.rb @@ -27,10 +27,12 @@ def initialize(loader:, app_provider:) # @param env [Hash] # @return [Array<(Integer, Hash, #each)>] def call(env) - @reload_mutex.synchronize do + app = @reload_mutex.synchronize do reload_if_needed - @app_provider.call.call(env) + @app_provider.call end + + app.call(env) end private diff --git a/app/web/boot/setup.rb b/app/web/boot/setup.rb index 55ac05b3..17b855c0 100644 --- a/app/web/boot/setup.rb +++ b/app/web/boot/setup.rb @@ -6,8 +6,6 @@ module Boot ## # Applies boot-time runtime configuration outside the Roda class body. module Setup - RACK_TIMEOUT_BUFFER_SECONDS = 5 - class << self # @return [Boolean] def sentry_enabled? @@ -21,8 +19,6 @@ def call! validate_environment! capture_runtime_env! configure_sentry! - configure_request_service! - configure_runtime_logging! configure_gem_defaults! log_startup! end @@ -55,27 +51,6 @@ def configure_sentry! Sentry.configure! end - # @return [void] - def configure_request_service! - return unless defined?(Rack::Timeout) - return unless Rack::Timeout.respond_to?(:service_timeout=) - - Rack::Timeout.service_timeout = - if ENV.key?('RACK_TIMEOUT_SERVICE_TIMEOUT') - Integer(ENV['RACK_TIMEOUT_SERVICE_TIMEOUT']) - else - Html2rss::RequestService::Policy::DEFAULTS[:total_timeout_seconds] + - RACK_TIMEOUT_BUFFER_SECONDS - end - end - - # @return [void] - def configure_runtime_logging! - return unless defined?(Rack::Timeout::Logger) - - Rack::Timeout::Logger.logger = AppLogger.logger - end - # @return [void] def log_startup! AppLogger.logger.info( diff --git a/app/web/feeds/cache.rb b/app/web/feeds/cache.rb index b6ac9eb9..efffab34 100644 --- a/app/web/feeds/cache.rb +++ b/app/web/feeds/cache.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true +require 'concurrent/ivar' require 'concurrent/map' require 'digest' require 'time' @@ -10,12 +11,18 @@ module Feeds ## # Small synchronous cache for canonical feed results. module Cache - # rubocop:disable-next ThreadSafety/ClassInstanceVariable + # rubocop:disable ThreadSafety/ClassInstanceVariable def self.entries @entries ||= Concurrent::Map.new end private_class_method :entries + def self.in_flight + @in_flight ||= Concurrent::Map.new + end + private_class_method :in_flight + # rubocop:enable ThreadSafety/ClassInstanceVariable + Entry = Data.define(:result, :expires_at) DEFAULT_TTL_SECONDS = 3600 @@ -37,22 +44,33 @@ def seconds_from_minutes(value, default: DEFAULT_TTL_SECONDS) # @param cacheable [Boolean, Proc] # @yieldreturn [Html2rss::Web::Feeds::Contracts::RenderResult] # @return [Html2rss::Web::Feeds::Contracts::RenderResult] + # rubocop:disable-next Metrics/MethodLength def fetch(key, ttl_seconds:, cacheable: true) entry = read_entry(key) return entry.result if fresh?(entry) - result = yield - - return result unless cacheable_result?(cacheable, result) - - write_entry(key, ttl_seconds, result) - result + ivar = Concurrent::IVar.new + actual_ivar = in_flight.put_if_absent(key, ivar) + return actual_ivar.value! if actual_ivar + + begin + result = yield + write_entry(key, ttl_seconds, result) if cacheable_result?(cacheable, result) + ivar.set(result) + result + rescue StandardError => error + ivar.fail(error) + raise + ensure + in_flight.delete_pair(key, ivar) + end end # @param reason [String] # @return [nil] def clear!(reason: 'manual') entries.clear + in_flight.clear Observability.emit( event_name: 'cache.lifecycle', outcome: 'success', diff --git a/app/web/telemetry/app_logger.rb b/app/web/telemetry/app_logger.rb index 88d686ad..11f10da3 100644 --- a/app/web/telemetry/app_logger.rb +++ b/app/web/telemetry/app_logger.rb @@ -7,15 +7,17 @@ module Html2rss module Web module AppLogger class << self + # rubocop:disable ThreadSafety/ClassInstanceVariable # @return [Logger] def logger - Thread.current[:app_logger] ||= build_logger + @logger ||= build_logger end # @return [void] def reset_logger! - Thread.current[:app_logger] = nil + @logger = nil end + # rubocop:enable ThreadSafety/ClassInstanceVariable private diff --git a/bin/dev b/bin/dev index 2762d505..a65b8bf1 100755 --- a/bin/dev +++ b/bin/dev @@ -35,7 +35,7 @@ cleanup() { fi # Clean up any remaining processes on our ports - pkill -f "puma.*html2rss-web" 2>/dev/null || true + pkill -f "falcon.*html2rss-web" 2>/dev/null || true pkill -f "vite.*4001" 2>/dev/null || true echo "✅ Development servers stopped" @@ -46,12 +46,12 @@ trap cleanup SIGINT SIGTERM # Prevent silent failures from port conflicts if port_in_use 4000; then - echo "❌ Port 4000 is already in use. Run: \`pkill -f 'puma.*html2rss-web'\`" + echo "❌ Port 4000 is already in use. Run: \`pkill -f 'falcon.*html2rss-web'\`" exit 1 fi # Start Ruby server -bundle exec puma -p 4000 -C config/puma.rb & +bundle exec falcon host config/falcon.rb & RUBY_PID=$! # Verify Ruby server started successfully (allow slower boots in containers) diff --git a/bin/dev-ruby b/bin/dev-ruby index f1b0cb78..0fe5bd90 100755 --- a/bin/dev-ruby +++ b/bin/dev-ruby @@ -12,9 +12,9 @@ export RACK_ENV=${RACK_ENV:-development} # Prevent silent failures from port conflicts if lsof -Pi :${PORT:-4000} -sTCP:LISTEN -t >/dev/null 2>&1; then - echo "❌ Port ${PORT:-4000} is already in use. Run: pkill -f 'puma.*html2rss-web'" + echo "❌ Port ${PORT:-4000} is already in use. Run: pkill -f 'falcon.*html2rss-web'" exit 1 fi echo "Starting Ruby server (code reloading enabled)" -bundle exec puma -p ${PORT:-4000} -C config/puma.rb +bundle exec falcon host config/falcon.rb diff --git a/bin/dev-with-frontend b/bin/dev-with-frontend index ea75ea64..8e8349ce 100755 --- a/bin/dev-with-frontend +++ b/bin/dev-with-frontend @@ -38,7 +38,7 @@ trap cleanup SIGINT SIGTERM # Start Ruby server in background echo "Starting Ruby server..." -bundle exec puma -p ${PORT:-4000} -C config/puma.rb & +bundle exec falcon host config/falcon.rb & RUBY_PID=$! # Wait a moment for Ruby server to start diff --git a/bin/docker-healthcheck b/bin/docker-healthcheck index 617c99ce..6d924aa7 100755 --- a/bin/docker-healthcheck +++ b/bin/docker-healthcheck @@ -3,17 +3,35 @@ require 'uri' require 'net/http' +require 'openssl' port = ENV.fetch('PORT', '4000') token = ENV.fetch('HEALTH_CHECK_TOKEN', nil) token = 'CHANGE_ME_HEALTH_CHECK_TOKEN' if token.nil? || token.empty? -uri = URI("http://localhost:#{port}/api/v1/health") +tls_cert_path = ENV.fetch('TLS_CERTIFICATE_PATH', nil) +tls_enabled = tls_cert_path && !tls_cert_path.empty? + +if tls_enabled && File.exist?(tls_cert_path) + cert = OpenSSL::X509::Certificate.new(File.read(tls_cert_path)) + now = Time.now + exit(1) if now < cert.not_before || now > cert.not_after +end + +scheme = tls_enabled ? 'https' : 'http' +uri = URI("#{scheme}://localhost:#{port}/api/v1/health") request = Net::HTTP::Get.new(uri) request['Authorization'] = "Bearer #{token}" +verify_mode = ENV.fetch('TLS_VERIFY_PEER', 'false') == 'true' ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE + begin - response = Net::HTTP.start(uri.hostname, uri.port) { |http| http.request(request) } + response = Net::HTTP.start( + uri.hostname, + uri.port, + use_ssl: tls_enabled, + verify_mode: verify_mode + ) { |http| http.request(request) } exit(response.is_a?(Net::HTTPSuccess) ? 0 : 1) rescue StandardError exit(1) diff --git a/bin/setup b/bin/setup index 37290355..fbd00a13 100755 --- a/bin/setup +++ b/bin/setup @@ -21,10 +21,7 @@ if [ ! -f .env ]; then echo "Please edit .env file with your configuration" fi -# Create necessary directories -echo "Creating necessary directories..." -mkdir -p tmp/rack-cache-body -mkdir -p tmp/rack-cache-meta + # Run tests to verify setup echo "Running tests to verify setup..." diff --git a/config.ru b/config.ru index afc1875f..4d7a3463 100644 --- a/config.ru +++ b/config.ru @@ -2,7 +2,6 @@ require 'rubygems' require 'bundler/setup' -require 'rack-timeout' require_relative 'app/web/boot/development_reloader' require_relative 'app' @@ -16,8 +15,6 @@ if dev app_provider: -> { Html2rss::Web::App.app } ) else - use Rack::Timeout - Html2rss::Web::Boot.eager_load! run(Html2rss::Web::App.freeze.app) diff --git a/config/falcon.rb b/config/falcon.rb new file mode 100644 index 00000000..3a92228e --- /dev/null +++ b/config/falcon.rb @@ -0,0 +1,91 @@ +# frozen_string_literal: true + +require 'falcon/environment/rack' +require 'openssl' + +## +# Helper for building endpoints and options for Falcon. +module FalconConfig + CIPHERS = %w[ + ECDHE-ECDSA-AES128-GCM-SHA256 + ECDHE-RSA-AES128-GCM-SHA256 + ECDHE-ECDSA-AES256-GCM-SHA384 + ECDHE-RSA-AES256-GCM-SHA384 + ECDHE-ECDSA-CHACHA20-POLY1305 + ECDHE-RSA-CHACHA20-POLY1305 + ].join(':') + + class << self + def project_root + File.expand_path('..', __dir__) + end + + def rackup_file + File.join(project_root, 'config.ru') + end + + def app_file + File.join(project_root, 'app.rb') + end + + def preload_files + ENV['RACK_ENV'] == 'development' ? [] : [app_file] + end + + def worker_count + ENV['RACK_ENV'] == 'development' ? 1 : Integer(ENV.fetch('WEB_CONCURRENCY', 2)) + end + + def timeout_seconds + Float(ENV.fetch('REQUEST_TIMEOUT_SECONDS', 55)) + end + + def endpoint + port = ENV.fetch('PORT', 4000) + cert_path = ENV.fetch('TLS_CERTIFICATE_PATH', nil) + key_path = ENV.fetch('TLS_KEY_PATH', nil) + + if cert_path && !cert_path.empty? && key_path && !key_path.empty? && + File.exist?(cert_path) && File.exist?(key_path) + tls_endpoint(port, cert_path, key_path) + else + Async::HTTP::Endpoint.parse("http://0.0.0.0:#{port}") + end + end + + private + + def tls_endpoint(port, cert_path, key_path) + Async::HTTP::Endpoint.parse( + "https://0.0.0.0:#{port}", + ssl_context: build_ssl_context(cert_path, key_path) + ) + end + + def build_ssl_context(cert_path, key_path) + OpenSSL::SSL::SSLContext.new.tap do |context| + context.cert = OpenSSL::X509::Certificate.new(File.read(cert_path)) + context.key = OpenSSL::PKey.read(File.read(key_path)) + context.alpn_protocols = %w[h2 http/1.1] + context.min_version = OpenSSL::SSL::TLS1_2_VERSION + context.ciphers = CIPHERS + context.options |= OpenSSL::SSL::OP_NO_COMPRESSION if defined?(OpenSSL::SSL::OP_NO_COMPRESSION) + end + end + end +end + +if respond_to?(:service) + service 'html2rss-web' do + include Falcon::Environment::Rack + + preload { FalconConfig.preload_files } + + root { FalconConfig.project_root } + rackup_path { FalconConfig.rackup_file } + + count { FalconConfig.worker_count } + timeout { FalconConfig.timeout_seconds } + endpoint { FalconConfig.endpoint } + end +end diff --git a/config/puma.rb b/config/puma.rb deleted file mode 100644 index 43ffdf75..00000000 --- a/config/puma.rb +++ /dev/null @@ -1,19 +0,0 @@ -# frozen_string_literal: true - -# Single worker in dev enables code reloading (cluster mode prevents reloading) -if ENV['RACK_ENV'] == 'development' - workers 0 - threads_count = Integer(ENV.fetch('WEB_MAX_THREADS', 5)) - threads threads_count, threads_count - plugin :tmp_restart - log_requests true -else - workers Integer(ENV.fetch('WEB_CONCURRENCY', 2)) - threads_count = Integer(ENV.fetch('WEB_MAX_THREADS', 5)) - threads threads_count, threads_count - preload_app! - log_requests false -end - -port ENV.fetch('PORT', 4000) -environment ENV.fetch('RACK_ENV', 'development') diff --git a/docs/README.md b/docs/README.md index e09fc0a1..a8035bac 100644 --- a/docs/README.md +++ b/docs/README.md @@ -17,9 +17,9 @@ Welcome! This is the canonical source of truth for contributing to `html2rss-web `html2rss-web` converts arbitrary websites into RSS 2.0 feeds. -- **Backend**: Ruby + Roda under the `Html2rss::Web` namespace. +- **Backend**: Ruby + Falcon + Roda under the `Html2rss::Web` namespace. - **Frontend**: Preact + Vite, built into `frontend/dist` and served at `/` in production. -- **Feed extraction**: Delegated to the `html2rss` gem. +- **Feed extraction**: Delegated to the `html2rss` gem (HTTPX non-blocking transport layer). - **Distribution**: Docker Compose / Dev Container first. ### Source Of Truth @@ -229,7 +229,7 @@ Compose maps `BOTASAURUS_SENTRY_DSN` into the botasaurus service as optional (`$ ### Compose timeout ladder -Default `docker-compose.yml` aligns botasaurus-scrape-api, the html2rss gem client, and html2rss-web so the scraper exhausts its budget before the web tier aborts the request. Keep **scrape total (45) ≤ feed build (50) ≤ Rack (55)**; the **work** budget (30) applies only after the browser is ready on the scraper. +Default `docker-compose.yml` aligns botasaurus-scrape-api, the html2rss gem client, and html2rss-web so the scraper exhausts its budget before the web tier aborts the request. Keep **scrape total (45) ≤ feed build (50) ≤ Falcon (55)**; the **work** budget (30) applies only after the browser is ready on the scraper. | Variable | Service | Default | Role | | --- | --- | --- | --- | @@ -238,7 +238,7 @@ Default `docker-compose.yml` aligns botasaurus-scrape-api, the html2rss gem clie | `BOTASAURUS_SCRAPE_TIMEOUT_SECONDS` | html2rss (web) | `45` | Faraday POST `/scrape` cap (mirrors scrape total) | | `BOTASAURUS_SCRAPE_WORK_TIMEOUT_SECONDS` | html2rss | `30` | Max `wait_timeout_seconds` in feed YAML | | `HTML2RSS_TOTAL_TIMEOUT_SECONDS` | html2rss-web | `50` | Feed build budget (scrape + extraction) | -| `RACK_TIMEOUT_SERVICE_TIMEOUT` | html2rss-web | `55` | Rack outer wall | +| `REQUEST_TIMEOUT_SECONDS` | html2rss-web | `55` | Falcon server request timeout | When triaging `GATEWAY_TIMEOUT`, capacity-shaped `SERVICE_UNAVAILABLE` (queue/boot), or `error_category:timeout`, confirm both projects use this ladder. Scraper terminal timeouts near **45s** with web failures near **50–55s** indicate aligned budgets; scraper failures near **20–25s** while web waits longer usually mean stale `SCRAPE_*` / `BOTASAURUS_*` env on one side. Split queue/boot timeouts from work timeouts before blaming the ladder (see **Alert baselines**). diff --git a/docs/architecture.md b/docs/architecture.md index 19530d7c..dbcfdcb1 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -2,16 +2,32 @@ This document provides a mental model of how `html2rss-web` processes requests. +## Server & Adapter Stack + +``` +Client (HTTP/1.1 or HTTP/2) + │ + ▼ +[Falcon Server] (Terminates TLS / HTTP/2 multiplexing, fiber-based async reactor) + │ + ▼ +[Roda Routing Tree] (Stateless routing & parameter validation) + │ + ├──▶ [HTTPX] (Non-blocking cooperative fiber I/O to upstream HTML) + └──▶ [Botasaurus Scrape API] (Browser/dynamic scraping via HTTPX) +``` + ## High-Level Data Flow ```mermaid flowchart TD - user["User / RSS Reader"] --> routes["Roda App (app/web/routes)"] + user["User / RSS Reader"] --> falcon["Falcon Server (HTTP/1.1 & HTTP/2)"] + falcon --> routes["Roda App (app/web/routes)"] security["Auth / Security (app/web/security)"] --> routes routes --> feeds["Feeds Service (app/web/feeds)"] cache["Cache (app/web/feeds/cache.rb)"] --> feeds feeds --> gem["html2rss Gem"] - strategies["Request Strategies (Faraday / Botasaurus)"] --> gem + strategies["Request Strategies (HTTPX / Botasaurus)"] --> gem gem --> target["Target Website"] ``` @@ -19,7 +35,7 @@ flowchart TD ### 1. Routing & Auth -Requests enter via `app.rb` and are dispatched to `app/web/routes/`. +Requests enter via `Falcon`, dispatch through `config.ru` and `app.rb`, and are routed to `app/web/routes/`. - **Static feed pages (`/`)**: Routed by `app/web/routes/feed_pages.rb` and resolved as `target_kind: :static`. - Source: static config in `config/feeds.yml` (via `LocalConfig.find`). @@ -54,7 +70,7 @@ The `Html2rss::Web::Feeds::Service` orchestrates extraction behind a gem `FeedRe Strategies are defined by the `html2rss` gem but can be configured here. -- **Faraday**: Default HTTP client for static HTML. +- **HTTPX**: Default non-blocking HTTP transport layer for static HTML. It natively cooperates with the Ruby Fiber scheduler for connection pooling, keep-alive, and HTTP/2 multiplexing across upstream requests. - **Botasaurus**: Used for JavaScript-heavy websites or anti-bot protected pages (`BOTASAURUS_SCRAPER_URL`). To add or configure strategies, see `app/web/feeds/source_resolver.rb` and the `html2rss` gem documentation. diff --git a/frontend/src/api/generated/types.gen.ts b/frontend/src/api/generated/types.gen.ts index 232c1a7d..39f0afc3 100644 --- a/frontend/src/api/generated/types.gen.ts +++ b/frontend/src/api/generated/types.gen.ts @@ -161,6 +161,20 @@ export type CreateFeedData = { }; export type CreateFeedErrors = { + /** + * returns 400 when request body exceeds maximum allowed bytes + */ + 400: { + error: { + code: string; + kind: string; + message: string; + next_action: string; + retry_action: string; + retryable: boolean; + }; + success: boolean; + }; /** * returns 401 with UNAUTHORIZED error payload */ diff --git a/public/openapi.yaml b/public/openapi.yaml index d70b2e2c..e5ad1eee 100644 --- a/public/openapi.yaml +++ b/public/openapi.yaml @@ -382,6 +382,40 @@ paths: - meta type: object description: normalizes hostname-only input to https before feed creation + '400': + content: + application/json: + schema: + properties: + error: + properties: + code: + type: string + kind: + type: string + message: + type: string + next_action: + type: string + retry_action: + type: string + retryable: + type: boolean + required: + - code + - message + - kind + - retryable + - next_action + - retry_action + type: object + success: + type: boolean + required: + - success + - error + type: object + description: returns 400 when request body exceeds maximum allowed bytes '401': content: application/json: diff --git a/spec/config/falcon_config_spec.rb b/spec/config/falcon_config_spec.rb new file mode 100644 index 00000000..d3937cc8 --- /dev/null +++ b/spec/config/falcon_config_spec.rb @@ -0,0 +1,129 @@ +# frozen_string_literal: true + +require 'climate_control' +require 'tempfile' +require_relative '../../config/falcon' + +RSpec.describe FalconConfig do + describe '.project_root' do + it 'returns the repository root directory' do + expect(described_class.project_root).to eq(File.expand_path('../..', __dir__)) + end + end + + describe '.rackup_file' do + it 'returns the path to config.ru' do + expect(described_class.rackup_file).to eq(File.join(described_class.project_root, 'config.ru')) + end + end + + describe '.app_file' do + it 'returns the path to app.rb' do + expect(described_class.app_file).to eq(File.join(described_class.project_root, 'app.rb')) + end + end + + describe '.preload_files' do + it 'returns empty array in development environment' do + ClimateControl.modify('RACK_ENV' => 'development') do + expect(described_class.preload_files).to eq([]) + end + end + + it 'returns app_file in production environment' do + ClimateControl.modify('RACK_ENV' => 'production') do + expect(described_class.preload_files).to eq([described_class.app_file]) + end + end + end + + describe '.worker_count' do + it 'returns 1 in development environment' do + ClimateControl.modify('RACK_ENV' => 'development', 'WEB_CONCURRENCY' => '4') do + expect(described_class.worker_count).to eq(1) + end + end + + it 'returns WEB_CONCURRENCY when in production environment' do + ClimateControl.modify('RACK_ENV' => 'production', 'WEB_CONCURRENCY' => '4') do + expect(described_class.worker_count).to eq(4) + end + end + + it 'defaults to 2 in production when WEB_CONCURRENCY is not set' do + ClimateControl.modify('RACK_ENV' => 'production', 'WEB_CONCURRENCY' => nil) do + expect(described_class.worker_count).to eq(2) + end + end + end + + describe '.timeout_seconds' do + it 'defaults to 55 seconds' do + ClimateControl.modify('REQUEST_TIMEOUT_SECONDS' => nil) do + expect(described_class.timeout_seconds).to eq(55.0) + end + end + + it 'respects REQUEST_TIMEOUT_SECONDS override' do + ClimateControl.modify('REQUEST_TIMEOUT_SECONDS' => '30') do + expect(described_class.timeout_seconds).to eq(30.0) + end + end + end + + describe '.endpoint' do + context 'when TLS certificate or key path is not provided' do + it 'returns an HTTP endpoint' do + ClimateControl.modify('PORT' => '4000', 'TLS_CERTIFICATE_PATH' => nil, 'TLS_KEY_PATH' => nil) do + endpoint = described_class.endpoint + expect(endpoint.scheme).to eq('http') + expect(endpoint.port).to eq(4000) + end + end + end + + context 'when TLS certificate and key files exist' do + let(:cert_file) { Tempfile.new(['cert', '.pem']) } + let(:key_file) { Tempfile.new(['key', '.pem']) } + let(:rsa_key) { OpenSSL::PKey::RSA.new(2048) } + let(:cert) do + OpenSSL::X509::Certificate.new.tap do |c| + c.version = 2 + c.serial = 1 + c.subject = OpenSSL::X509::Name.parse('/CN=localhost') + c.issuer = c.subject + c.public_key = rsa_key.public_key + c.not_before = Time.now - 3600 + c.not_after = Time.now + 3600 + c.sign(rsa_key, OpenSSL::Digest.new('SHA256')) + end + end + + before do + key_file.write(rsa_key.to_pem) + key_file.flush + cert_file.write(cert.to_pem) + cert_file.flush + end + + after do + cert_file.close! + key_file.close! + end + + it 'returns an HTTPS endpoint with hardened TLS options and ciphers', :aggregate_failures do + ClimateControl.modify('PORT' => '4443', 'TLS_CERTIFICATE_PATH' => cert_file.path, + 'TLS_KEY_PATH' => key_file.path) do + endpoint = described_class.endpoint + expect(endpoint.scheme).to eq('https') + expect(endpoint.port).to eq(4443) + ssl_context = endpoint.ssl_context + expect(ssl_context).to be_a(OpenSSL::SSL::SSLContext) + cipher_names = ssl_context.ciphers.map(&:first) + expect(cipher_names).to include('ECDHE-RSA-AES128-GCM-SHA256') + expect(ssl_context.options & OpenSSL::SSL::OP_NO_COMPRESSION).not_to be_zero + end + end + end + end +end diff --git a/spec/html2rss/web/api/v1_spec.rb b/spec/html2rss/web/api/v1_spec.rb index 7d2425dc..295e3890 100644 --- a/spec/html2rss/web/api/v1_spec.rb +++ b/spec/html2rss/web/api/v1_spec.rb @@ -808,5 +808,30 @@ def relative_feed_link_header(token) expect(last_response.status).to eq(429) expect(last_response.headers['Retry-After']).not_to be_nil end + + it 'returns 400 when request body exceeds maximum allowed bytes', :aggregate_failures do + header 'Authorization', "Bearer #{admin_token}" + header 'Content-Type', 'application/json' + + oversized_payload = { url: 'https://example.com/articles', padding: 'x' * 100_000 }.to_json + post '/api/v1/feeds', oversized_payload + + expect(last_response.status).to eq(400) + json = JSON.parse(last_response.body) + expect(json.dig('error', 'message')).to eq('Payload too large') + end + + it 'returns 400 when form-urlencoded request body exceeds maximum allowed bytes', :aggregate_failures, + openapi: false do + header 'Authorization', "Bearer #{admin_token}" + header 'Content-Type', 'application/x-www-form-urlencoded' + + oversized_payload = "url=https://example.com/articles&#{'x' * 100_000}" + post '/api/v1/feeds', oversized_payload + + expect(last_response.status).to eq(400) + json = JSON.parse(last_response.body) + expect(json.dig('error', 'message')).to eq('Payload too large') + end end end diff --git a/spec/html2rss/web/app_logger_spec.rb b/spec/html2rss/web/app_logger_spec.rb index a2557126..dae8f845 100644 --- a/spec/html2rss/web/app_logger_spec.rb +++ b/spec/html2rss/web/app_logger_spec.rb @@ -46,5 +46,13 @@ expect(Html2rss::Web::SentryLogs).not_to have_received(:emit) expect(io.string).to include('"message":"plain-text log line with request details"') end + + it 'shares the logger instance across different fibers' do + described_class.reset_logger! + logger_a = Fiber.new { described_class.logger }.resume + logger_b = Fiber.new { described_class.logger }.resume + + expect(logger_a).to be(logger_b) + end end end diff --git a/spec/html2rss/web/boot/setup_spec.rb b/spec/html2rss/web/boot/setup_spec.rb index ab96f465..47c46a2b 100644 --- a/spec/html2rss/web/boot/setup_spec.rb +++ b/spec/html2rss/web/boot/setup_spec.rb @@ -35,46 +35,6 @@ expect(Html2rss::Web::Flags).to have_received(:validate!).once end - it 'routes rack-timeout logs through the shared app logger' do - stub_const('Rack::Timeout::Logger', Class.new) - logger_holder = { value: nil } - Rack::Timeout::Logger.define_singleton_method(:logger=) { |value| logger_holder[:value] = value } - Rack::Timeout::Logger.define_singleton_method(:logger) { logger_holder[:value] } - - described_class.call! - - expect(Rack::Timeout::Logger.logger).to be(Html2rss::Web::AppLogger.logger) - end - - describe 'Rack::Timeout service timeout' do - let(:timeout_holder) { { value: nil } } - - before do - stub_const('Rack::Timeout', Module.new) - Rack::Timeout.define_singleton_method(:service_timeout=) { |v| v } - allow(Rack::Timeout).to receive(:service_timeout=) { |v| timeout_holder[:value] = v } - stub_environment_validation - end - - it 'sets timeout from RACK_TIMEOUT_SERVICE_TIMEOUT if present' do - ClimateControl.modify(boot_env.merge('RACK_TIMEOUT_SERVICE_TIMEOUT' => '42')) do - described_class.call! - end - - expect(timeout_holder[:value]).to eq(42) - end - - it 'sets timeout from gem default + buffer if RACK_TIMEOUT_SERVICE_TIMEOUT is absent' do - ClimateControl.modify(boot_env.merge('RACK_TIMEOUT_SERVICE_TIMEOUT' => nil)) do - described_class.call! - end - - expected = Html2rss::RequestService::Policy::DEFAULTS[:total_timeout_seconds] + - described_class::RACK_TIMEOUT_BUFFER_SECONDS - expect(timeout_holder[:value]).to eq(expected) - end - end - it 'captures and scrubs sensitive env vars after validation', :aggregate_failures do expect_sensitive_env_during_validation diff --git a/spec/html2rss/web/feeds/cache_spec.rb b/spec/html2rss/web/feeds/cache_spec.rb index 1124e5b1..25733f07 100644 --- a/spec/html2rss/web/feeds/cache_spec.rb +++ b/spec/html2rss/web/feeds/cache_spec.rb @@ -36,6 +36,13 @@ expect { fetch_with_counter }.to change { fetch_calls }.from(1).to(2) end + it 'coalesces concurrent in-flight reads for the same key to a single computation' do + results, calls = run_concurrent_fetches('feed_result:concurrent_test', 5) + + expect(results).to all(eq(result)) + expect(calls).to eq(1) + end + describe '.seconds_from_minutes' do it 'converts positive minute values to seconds', :aggregate_failures do expect(described_class.seconds_from_minutes(5)).to eq(300) @@ -69,4 +76,23 @@ def fetch_with_counter def fetch_calls @fetch_calls ||= 0 end + + # rubocop:disable-next Metrics/MethodLength, ThreadSafety/NewThread + def run_concurrent_fetches(key, concurrency) + computation_calls = Concurrent::AtomicFixnum.new(0) + barrier = Concurrent::CyclicBarrier.new(concurrency) + + threads = Array.new(concurrency) do + Thread.new do + barrier.wait + described_class.fetch(key, ttl_seconds: 60) do + computation_calls.increment + sleep 0.05 + result + end + end + end + + [threads.map(&:value), computation_calls.value] + end end diff --git a/spec/html2rss/web/log_sanitizer_spec.rb b/spec/html2rss/web/log_sanitizer_spec.rb index c4ea809f..5de5c5d2 100644 --- a/spec/html2rss/web/log_sanitizer_spec.rb +++ b/spec/html2rss/web/log_sanitizer_spec.rb @@ -113,12 +113,12 @@ expect(observability_payload.dig(:details, :url)).to eq(expected_news_url) end - it 'formats rack-timeout logfmt as json' do - Html2rss::Web::AppLogger.logger.info('source=rack-timeout id=req-123 timeout=15000ms state=completed') + it 'formats logfmt messages as json' do + Html2rss::Web::AppLogger.logger.info('source=upstream id=req-123 timeout=15000ms state=completed') payload = JSON.parse(io.string.lines.last, symbolize_names: true) expect(payload).to include( - source: 'rack-timeout', + source: 'upstream', id: 'req-123', timeout: '15000ms', state: 'completed'