Revenue grew by eleven percent across the period, driven mainly by renewals in +the enterprise segment and a modest recovery in new logo acquisition.
+Management reaffirmed guidance for the full year and pointed to margin expansion +in the second half as the primary driver of operating leverage.
+I have been bouncing between editors for a year now and I would like to hear what +other people have settled on for long term maintenance work on large codebases.
+Tiny but honest page.
" + assert proxy_module.looks_like_js_shell(html, "Tiny but honest page.") is False + + +class FakeResponse: + def __init__(self, status_code=200, body=b"", headers=None): + self.status_code = status_code + self.content = body + self.headers = headers or {"Content-Type": "text/html; charset=utf-8"} + self.encoding = "utf-8" + + +@pytest.fixture +def allow_public_dns(monkeypatch): + """Resolve every host to a public address unless the test says otherwise.""" + monkeypatch.setattr( + proxy_module.socket, + "getaddrinfo", + lambda host, *a, **k: [(2, 1, 6, "", ("93.184.216.34", 0))], + ) + + +class TestFetchHtml: + def test_returns_body_and_status_on_success(self, monkeypatch, allow_public_dns): + monkeypatch.setattr( + proxy_module.requests, + "get", + lambda *a, **k: FakeResponse(body=b"hello
"), + ) + html, final_url, status, reason = proxy_module.fetch_html( + "https://example.com/a" + ) + assert "hello" in html + assert final_url == "https://example.com/a" + assert status == 200 + assert reason == "ok" + + def test_does_not_request_an_unsafe_url(self, monkeypatch): + def boom(*args, **kwargs): + raise AssertionError("network must not be touched for a blocked URL") + + monkeypatch.setattr(proxy_module.requests, "get", boom) + html, _, status, reason = proxy_module.fetch_html("http://127.0.0.1/admin") + assert html == "" + assert status is None + assert "blocked" in reason.lower() + + def test_follows_a_public_redirect_and_reports_the_final_url( + self, monkeypatch, allow_public_dns + ): + responses = [ + FakeResponse(302, b"", {"Location": "https://example.com/final"}), + FakeResponse(200, b"arrived
"), + ] + monkeypatch.setattr( + proxy_module.requests, "get", lambda *a, **k: responses.pop(0) + ) + html, final_url, status, reason = proxy_module.fetch_html( + "https://example.com/start" + ) + assert "arrived" in html + assert final_url == "https://example.com/final" + assert status == 200 + assert reason == "ok" + + def test_rejects_a_redirect_into_link_local_metadata(self, monkeypatch): + """The pre-flight check alone would miss this; every hop is revalidated.""" + + def resolve(host, *args, **kwargs): + if host == "169.254.169.254": + return [(2, 1, 6, "", ("169.254.169.254", 0))] + return [(2, 1, 6, "", ("93.184.216.34", 0))] + + monkeypatch.setattr(proxy_module.socket, "getaddrinfo", resolve) + monkeypatch.setattr( + proxy_module.requests, + "get", + lambda *a, **k: FakeResponse( + 302, b"", {"Location": "http://169.254.169.254/latest/meta-data/"} + ), + ) + html, _, _, reason = proxy_module.fetch_html("https://example.com/redirector") + assert html == "" + assert "blocked" in reason.lower() + + def test_stops_after_the_redirect_limit(self, monkeypatch, allow_public_dns): + monkeypatch.setattr( + proxy_module.requests, + "get", + lambda *a, **k: FakeResponse( + 302, b"", {"Location": "https://example.com/loop"} + ), + ) + html, _, _, reason = proxy_module.fetch_html("https://example.com/loop") + assert html == "" + assert "redirect" in reason.lower() + + def test_rejects_non_html_content_type(self, monkeypatch, allow_public_dns): + monkeypatch.setattr( + proxy_module.requests, + "get", + lambda *a, **k: FakeResponse(headers={"Content-Type": "application/zip"}), + ) + html, _, _, reason = proxy_module.fetch_html("https://example.com/a.zip") + assert html == "" + assert "content type" in reason.lower() + + def test_truncates_oversized_bodies(self, monkeypatch, allow_public_dns): + oversized = b"" + b"x" * (proxy_module.MAX_HTML_BYTES + 5000) + monkeypatch.setattr( + proxy_module.requests, "get", lambda *a, **k: FakeResponse(body=oversized) + ) + html, _, _, reason = proxy_module.fetch_html("https://example.com/big") + assert len(html) <= proxy_module.MAX_HTML_BYTES + assert reason == "ok" + + def test_reports_http_errors_without_raising(self, monkeypatch, allow_public_dns): + monkeypatch.setattr( + proxy_module.requests, "get", lambda *a, **k: FakeResponse(404, b"nope") + ) + html, _, status, reason = proxy_module.fetch_html("https://example.com/missing") + assert html == "" + assert status == 404 + assert "404" in reason + + def test_reports_transport_failures_without_raising( + self, monkeypatch, allow_public_dns + ): + def boom(*args, **kwargs): + raise proxy_module.requests.RequestException("connection reset") + + monkeypatch.setattr(proxy_module.requests, "get", boom) + html, _, _, reason = proxy_module.fetch_html("https://example.com/flaky") + assert html == "" + assert "connection reset" in reason + + +class TestManifestContract: + def test_manifest_identity_matches_registry_layout(self, skill, manifest): + assert manifest["name"] == "data_engineering/semantic_web_proxy" + assert skill.manifest["name"] == manifest["name"] + assert skill.manifest["version"] == manifest["version"] + + def test_execute_returns_every_declared_output_key(self, skill, manifest): + result = skill.execute({"html_content": ARTICLE_HTML}) + for key in manifest["outputs"]: + assert key in result, f"missing declared output: {key}" + + def test_declared_parameters_validate(self, skill): + assert skill.validate_params( + {"url": "https://example.com/a", "output_format": "markdown"} + ) + + +class TestExecuteOfflinePath: + def test_extracts_from_supplied_html(self, skill): + result = skill.execute({"html_content": ARTICLE_HTML}) + assert result["status"] == "success" + assert result["error"] is None + assert "Revenue grew by eleven percent" in result["semantic_payload"] + assert result["source"]["fetched"] is False + assert result["metadata"]["title"] == "Quarterly Results" + + def test_html_content_wins_over_url_and_skips_the_network(self, skill, monkeypatch): + def boom(*args, **kwargs): + raise AssertionError("fetch must not run when html_content is supplied") + + monkeypatch.setattr(proxy_module, "fetch_html", boom) + result = skill.execute( + {"html_content": ARTICLE_HTML, "url": "https://example.com/a"} + ) + assert result["status"] == "success" + assert result["source"]["fetched"] is False + + def test_missing_both_inputs_is_a_structured_error(self, skill): + result = skill.execute({}) + assert result["status"] == "error" + assert result["semantic_payload"] == "" + assert "url" in result["error"] and "html_content" in result["error"] + + def test_unextractable_html_is_a_structured_error(self, skill): + result = skill.execute({"html_content": ""}) + assert result["status"] == "error" + assert result["error"] + + def test_rejects_unknown_output_format(self, skill): + result = skill.execute({"html_content": ARTICLE_HTML, "output_format": "yaml"}) + assert result["status"] == "error" + assert "output_format" in result["error"] + + def test_javascript_shell_warns_instead_of_reporting_success(self, skill): + result = skill.execute({"html_content": JS_SHELL_HTML}) + assert result["status"] == "warning" + assert "page_likely_requires_javascript" in result["warnings"] + + +class TestExecuteFetchPath: + def test_uses_the_guarded_fetcher_and_records_provenance(self, skill, monkeypatch): + monkeypatch.setattr( + proxy_module, + "fetch_html", + lambda url: (ARTICLE_HTML, "https://example.com/final", 200, "ok"), + ) + result = skill.execute({"url": "https://example.com/start"}) + assert result["status"] == "success" + assert result["source"] == { + "url": "https://example.com/start", + "final_url": "https://example.com/final", + "http_status": 200, + "fetched": True, + } + + def test_blocked_url_surfaces_as_an_error(self, skill, monkeypatch): + def boom(*args, **kwargs): + raise AssertionError("network must not be touched for a blocked URL") + + monkeypatch.setattr(proxy_module.requests, "get", boom) + result = skill.execute({"url": "http://169.254.169.254/latest/meta-data/"}) + assert result["status"] == "error" + assert "blocked" in result["error"].lower() + + def test_fetch_failure_surfaces_as_an_error(self, skill, monkeypatch): + monkeypatch.setattr( + proxy_module, + "fetch_html", + lambda url: ("", url, 404, "Fetch returned HTTP 404."), + ) + result = skill.execute({"url": "https://example.com/missing"}) + assert result["status"] == "error" + assert "404" in result["error"] + assert result["source"]["http_status"] == 404 + + +class TestTokenSavings: + def test_reports_reduction_against_the_raw_document(self, skill): + savings = skill.execute({"html_content": ARTICLE_HTML})["token_savings"] + assert savings["original_tokens"] > savings["semantic_tokens"] + assert ( + savings["tokens_saved"] + == savings["original_tokens"] - savings["semantic_tokens"] + ) + assert 0 < savings["reduction_pct"] <= 100 + assert savings["estimate"] is True + assert savings["tokenizer"] == "heuristic" + + def test_context_share_is_absent_without_a_context_window(self, skill): + savings = skill.execute({"html_content": ARTICLE_HTML})["token_savings"] + assert savings["context_window"] is None + assert savings["context_saved_pct"] is None + + def test_context_share_is_reported_when_the_host_supplies_a_window(self, skill): + savings = skill.execute({"html_content": ARTICLE_HTML, "context_window": 1000})[ + "token_savings" + ] + assert savings["context_window"] == 1000 + expected = round(savings["tokens_saved"] / 1000 * 100, 2) + assert savings["context_saved_pct"] == expected + + def test_tokenizer_fallback_is_reported_as_a_warning(self, skill, monkeypatch): + monkeypatch.setattr(proxy_module, "_find_spec", lambda name: None) + result = skill.execute( + {"html_content": ARTICLE_HTML, "tokenizer": "cl100k_base"} + ) + assert result["token_savings"]["tokenizer"] == "heuristic" + assert "tokenizer_unavailable" in result["warnings"] + + +class TestEmptyJavascriptShell: + """A shell with no noscript fallback extracts nothing at all.""" + + EMPTY_SHELL = ( + '' + '' % ("y" * 3000) + ) + + def test_empty_shell_errors_but_still_names_the_cause(self, skill): + result = skill.execute({"html_content": self.EMPTY_SHELL}) + assert result["status"] == "error" + assert "JavaScript" in result["error"] + assert "page_likely_requires_javascript" in result["warnings"] + + +class TestMalformedClosingTags: + """HTML parsers ignore attributes on end tags, so closes a script. + + A regex that only accepts under-measures script bulk on such pages + and the client-rendered shell goes unreported (CodeQL py/bad-tag-filter). + """ + + def test_script_bulk_is_measured_when_end_tag_carries_attributes(self): + html = ( + '' + "hi
" % ("z" * 4000) + ) + assert proxy_module.looks_like_js_shell(html, "hi") is True + + def test_app_root_is_detected_when_closing_tag_carries_attributes(self): + html = 'The central bank left its benchmark rate unchanged on Tuesday, ending a run of three consecutive increases and signalling that policymakers believe the worst of the inflation episode has passed.
+Officials pointed to a broad deceleration in services prices, which had been the most stubborn component of the index through the second half of last year. Goods prices have been falling outright since October.
+Markets had largely priced the pause in, and the immediate reaction in short-dated government bonds was muted. Equity indices closed modestly higher.
+Attention now turns to the labour market report due at the end of the month, which economists expect to show a further easing in wage growth without a material rise in unemployment.
+A monstera needs repotting when roots start circling the bottom of the container or pushing out of the drainage holes. For most plants in a bright indoor spot that works out to once every eighteen months or so.
+Go up one size only. A pot that is much too large holds water the roots cannot reach, and that is the fastest route to root rot in a plant that otherwise tolerates neglect quite well.
+Use a chunky aroid mix rather than standard potting compost. Bark, perlite and a little coir gives the aeration these roots evolved for while still holding enough moisture between waterings.
+We have a release branch that lives for about six weeks at a time and merging it back is consistently the worst day of the cycle. I am curious what has actually worked for teams that ship on a similar cadence rather than continuously.
+marisol: We stopped merging back entirely and started cherry-picking forward instead. It sounds worse on paper but the conflicts are small and constant rather than enormous and quarterly.
devon: Rebase the release branch onto main every single morning, automatically, and let it fail loudly. The pain is the signal and spreading it across thirty days makes it manageable.
tomas: Honestly the fix was organisational rather than technical for us. We shortened the release cycle to two weeks and most of the branch divergence problems simply evaporated on their own.
alice: I moved back to vim after a decade away and the muscle +memory came back within about a week, which surprised me quite a lot.
bob: The editor matters far less than the language server you +put behind it, in my honest experience across several very different teams.