-- i.e. because the fixture had
+ # stopped reproducing the defect, not because the guard works.
+ import bs4
+
+ vendor = self._vendor()
+ soup = vendor.request(REGISTRY_HTML)
+ table = soup.find_all('table', class_='wikitable')[2]
+ rows = [row for row in table.tbody if isinstance(row, bs4.element.Tag)]
+
+ self.assertEqual(rows[-1].find_all('td'), [],
+ 'the last row must carry no | at all, or this suite proves nothing')
+ self.assertEqual(rows[-1].find_all('th'), [])
+ self.assertEqual(rows[-1].get('class'), ['mw-empty-elt'])
+
+ def test_process_skips_the_cell_less_row_instead_of_raising(self) -> None:
+ # The regression itself. Before the guard this raised
+ # `IndexError: list index out of range` at `line[0]`.
+ vendor = self._vendor()
+ enum = vendor.process(vendor.request(REGISTRY_HTML))
+
+ self.assertEqual(len(enum), 2, f'expected only the two 3-digit codes, got {enum!r}')
+ self.assertIn("CODE_110: 'ReturnCode' = 110, 'Restart marker replay.'", enum[0])
+ self.assertIn("CODE_200: 'ReturnCode' = 200, 'Command okay.'", enum[1])
+
+ def test_process_skips_a_row_with_only_one_cell(self) -> None:
+ # ``len(line) < 2`` rather than ``not line``: a one-cell row would pass
+ # the latter and then raise on ``line[1]``.
+ vendor = self._vendor()
+ enum = vendor.process(vendor.request(SINGLE_CELL_HTML))
+
+ self.assertEqual(len(enum), 2, f'expected only the two 3-digit codes, got {enum!r}')
+ self.assertNotIn('CODE_250', '\n'.join(enum))
+
+ def test_guard_does_not_swallow_legitimate_rows(self) -> None:
+ # The other half of the guard's contract: skipping cell-less rows must not
+ # cost any row that does carry a code and an explanation.
+ vendor = self._vendor()
+ enum = vendor.process(vendor.request(REGISTRY_HTML))
+ rendered = '\n'.join(enum)
+
+ for code in ('110', '200'):
+ with self.subTest(code=code):
+ self.assertIn(f'CODE_{code}', rendered)
+ # ``100 Series`` is dropped by ``len(code) != 3``, not by the guard.
+ self.assertNotIn('100 Series', rendered)
+
+ def test_context_survives_the_cell_less_row(self) -> None:
+ # ``process`` is called from ``context``, which is what ``__init__`` writes
+ # to disk, so the whole generation path has to survive the row too.
+ vendor = self._vendor()
+ context = vendor.context(vendor.request(REGISTRY_HTML))
+
+ self.assertIn("CODE_110: 'ReturnCode' = 110", context)
+ self.assertIn("CODE_200: 'ReturnCode' = 200", context)
+ self.assertIn('class ReturnCode(IntEnum):', context)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/tests/vendor/test_ipx_packet_unit.py b/tests/vendor/test_ipx_packet_unit.py
new file mode 100644
index 0000000000..05ac0104f4
--- /dev/null
+++ b/tests/vendor/test_ipx_packet_unit.py
@@ -0,0 +1,230 @@
+# -*- coding: utf-8 -*-
+"""Regression tests for the retired :mod:`pcapkit.vendor.ipx.packet` scrape.
+
+GitHub issue #518, and the same failure on the same article that #507 found for
+the sibling socket crawler. The packet-type crawler used to scrape
+``find_all('table', class_='wikitable')[1]`` out of the Wikipedia *Internetwork
+Packet Exchange* article, and both halves of that stopped working. Wikipedia
+answers |requests|_' default User-Agent with HTTP 403, so
+:meth:`pcapkit.vendor.default.Vendor._request` could not fetch the page; and the
+table was deleted from the article on 2026-08-25 in revision 1371327031, leaving
+the live page with a single ``wikitable`` -- the IPX header format one -- so that
+index raises :exc:`IndexError` even with the fetch fixed. The scrape is retired
+in favour of the hand-maintained :data:`pcapkit.vendor.ipx.packet.DATA`.
+
+.. |requests| replace:: ``requests``
+.. _requests: https://requests.readthedocs.io
+
+What that leaves worth pinning is the failure mode the issue is actually about: a
+regeneration that quietly drops packet types. :data:`EXPECTED_MEMBERS` spells the
+enumeration out in full, so losing one fails here rather than shipping.
+``Broadcast_4`` is in it verbatim, footnote artefact and all -- ``[4]`` was
+Wikipedia's own citation marker for :rfc:`1132` rather than part of the name, and
+it leaked into the member. Keeping it is what makes byte identity achievable, and
+renaming it would break a public member, so the name is pinned rather than
+tidied.
+
+The suite is unit-tier (see :mod:`tests._tiers`): it reads no capture and, by the
+whole point of the change, makes no network call.
+
+"""
+from __future__ import annotations
+
+import importlib.util
+import pathlib
+import unittest
+from typing import TYPE_CHECKING
+
+from tests._support import purge_modules
+
+if TYPE_CHECKING:
+ from typing import Any
+
+#: Repository root, i.e. the grandparent of the directory holding this file.
+ROOT = pathlib.Path(__file__).resolve().parents[2]
+
+#: Every distribution importing :mod:`pcapkit.vendor` needs. ``requests`` is the
+#: obvious one -- ``pcapkit.vendor.default`` imports it at module scope -- but it
+#: is not sufficient: importing *any* crawler imports the ``pcapkit.vendor``
+#: package, whose :file:`__init__.py` pulls in all seventeen subpackages, seven of
+#: which ``import bs4`` at module scope. So a guard on ``requests`` alone lets the
+#: suite error instead of skipping on a machine that happens to have ``requests``
+#: and not ``beautifulsoup4``.
+VENDOR_DEPS = ('requests', 'bs4', 'html5lib')
+
+#: Whether the crawlers are importable at all. They ship in the ``vendor`` extra
+#: (:file:`pyproject.toml`), not ``test``, and CI installs ``.[test]`` -- so these
+#: tests skip in CI as things stand. Guarded the same way
+#: :file:`tests/protocols/test_dispatch_registry_unit.py` guards its own optional
+#: runtime dependencies, rather than making the whole unit tier depend on the
+#: crawlers' requirements. See #518.
+HAS_VENDOR_DEPS = all(importlib.util.find_spec(name) is not None for name in VENDOR_DEPS)
+
+#: Every member the generated :class:`pcapkit.const.ipx.packet.Packet` is expected
+#: to carry, as ``(name, value)`` in definition order. Spelled out rather than
+#: derived so that a regeneration which loses a packet type -- the exact failure
+#: #518 describes -- fails this test instead of passing a comparison against its
+#: own output.
+EXPECTED_MEMBERS = (
+ ('Unknown', 0),
+ ('RIP', 1),
+ ('Echo_Packet', 2),
+ ('Error_Packet', 3),
+ ('PEP', 4),
+ ('SPX', 5),
+ ('NCP', 17),
+ ('Broadcast_4', 20),
+)
+
+
+def _normalize(context: 'str') -> 'str':
+ """Apply the whitespace normalisation the generator writes files through.
+
+ :meth:`pcapkit.vendor.default.Vendor.__init__` does not write what
+ :meth:`~pcapkit.vendor.default.Vendor.context` returns verbatim: it strips
+ trailing whitespace from every non-blank line, drops whitespace-only lines
+ outright, and ends the file with a newline courtesy of :func:`print`. Byte
+ comparison against the committed file has to do the same, or it reports a
+ difference that regeneration would not actually produce.
+
+ Args:
+ context: Return value of :meth:`~pcapkit.vendor.default.Vendor.context`.
+
+ Returns:
+ The text as the generator would have written it to disk.
+
+ """
+ lines = [] # type: list[str]
+ for line in context.splitlines():
+ if line:
+ if line.strip():
+ lines.append(line.rstrip())
+ else:
+ lines.append(line)
+ return '\n'.join(lines) + '\n'
+
+
+@unittest.skipUnless(HAS_VENDOR_DEPS, f'vendor extra not installed ({", ".join(VENDOR_DEPS)})')
+class IPXPacketVendorTests(unittest.TestCase):
+ """The hand-maintained registry, and the crawler that no longer crawls."""
+
+ if TYPE_CHECKING:
+ vendor_module: 'Any'
+ const_module: 'Any'
+
+ def setUp(self) -> None:
+ purge_modules(['pcapkit'])
+
+ import pcapkit.const.ipx.packet as const_module
+ import pcapkit.vendor.ipx.packet as vendor_module
+
+ # Both modules have to come from this checkout for any of the comparisons
+ # below to mean anything: the generated text is compared against this
+ # repository's constant file, so a vendor module imported from an
+ # installed copy elsewhere would be comparing two trees. That is an
+ # environment mismatch rather than a defect, hence a skip.
+ for module in (vendor_module, const_module):
+ resolved = pathlib.Path(module.__file__).resolve()
+ if ROOT not in resolved.parents:
+ self.skipTest(f'{module.__name__} was imported from {resolved}, which is outside '
+ f'{ROOT}; install this checkout with `pip install -e .` to run this '
+ f'suite against it')
+
+ self.vendor_module = vendor_module
+ self.const_module = const_module
+
+ def _vendor(self) -> 'Any':
+ """A crawler instance with the attributes ``__init__`` would have set.
+
+ ``Vendor.__init__`` regenerates and *writes* the constant file as a side
+ effect of construction, which a test has no business doing to the working
+ tree, so the four attributes it sets are set here instead.
+
+ """
+ vendor = self.vendor_module.Packet.__new__(self.vendor_module.Packet)
+ vendor.NAME = self.vendor_module.Packet.__name__
+ vendor.DOCS = self.vendor_module.Packet.__doc__
+ data = vendor.request()
+ vendor.record = vendor.count(data)
+ return vendor
+
+ def test_link_is_none_so_nothing_is_fetched(self) -> None:
+ # The retirement, stated as an assertion: with no LINK,
+ # Vendor._request() short-circuits to Packet.request() and never reaches
+ # requests.get() -- which is what used to take the 403.
+ self.assertIsNone(self.vendor_module.Packet.LINK)
+
+ def test_request_makes_no_network_call(self) -> None:
+ import requests
+
+ def explode(*args: 'Any', **kwargs: 'Any') -> 'Any':
+ raise AssertionError(f'the crawler made a network call: {args!r}')
+
+ original_get, original_request = requests.get, requests.Session.request
+ requests.get = explode # type: ignore[assignment]
+ requests.Session.request = explode # type: ignore[assignment,method-assign]
+ try:
+ vendor = self.vendor_module.Packet.__new__(self.vendor_module.Packet)
+ vendor.NAME = self.vendor_module.Packet.__name__
+ vendor.DOCS = self.vendor_module.Packet.__doc__
+ self.assertIs(vendor._request(), self.vendor_module.DATA) # pylint: disable=protected-access
+ finally:
+ requests.get = original_get # type: ignore[assignment]
+ requests.Session.request = original_request # type: ignore[method-assign]
+
+ def test_regeneration_reproduces_the_committed_constant_file(self) -> None:
+ # The guard that makes the hand-maintained table trustworthy: running the
+ # crawler must be a no-op against what is checked in, so an edit to DATA
+ # that was never regenerated shows up here.
+ vendor = self._vendor()
+ generated = _normalize(vendor.context(vendor.request()))
+ committed = (ROOT / 'pcapkit' / 'const' / 'ipx' / 'packet.py').read_text(encoding='utf-8')
+ self.assertEqual(generated, committed,
+ 'regenerating pcapkit/const/ipx/packet.py would change it; run '
+ '`python -m pcapkit.vendor.ipx.packet` and commit the result')
+
+ def test_no_packet_type_is_lost(self) -> None:
+ members = tuple((member.name, int(member.value)) for member in self.const_module.Packet)
+ self.assertEqual(members, EXPECTED_MEMBERS)
+
+ def test_registry_and_enumeration_agree(self) -> None:
+ # Every hand-maintained row reaches the enumeration, and nothing in the
+ # enumeration came from anywhere else.
+ vendor = self._vendor()
+ from_data = tuple(
+ (vendor.rename(name, str(code)), code)
+ for code, (name, _) in self.vendor_module.DATA.items()
+ )
+ self.assertEqual(from_data, EXPECTED_MEMBERS)
+
+ def test_broadcast_footnote_artefact_is_preserved(self) -> None:
+ # ``Broadcast_4`` is a public member whose name came from Wikipedia's
+ # footnote marker for its RFC 1132 citation. Renaming it would be a
+ # breaking change, and it is what makes byte identity reachable, so both
+ # the member and the DATA row it comes from are pinned.
+ self.assertEqual(self.const_module.Packet(20), self.const_module.Packet.Broadcast_4)
+ self.assertEqual(self.vendor_module.DATA[20], ('Broadcast[4]', 'Broadcast[4]'))
+
+ def test_unknown_packet_type_survives_the_retirement(self) -> None:
+ # 0 is both the scraped table's "Unknown" row and IPX's own default for
+ # the type field, so it must be present whatever the registry says.
+ self.assertEqual(self.const_module.Packet(0), self.const_module.Packet.Unknown)
+ self.assertEqual(self.const_module.Packet(0).value, 0)
+ self.assertIn(0, self.vendor_module.DATA)
+
+ def test_unlisted_packet_types_still_resolve(self) -> None:
+ # ``_missing_`` has to cover the whole octet, so no legal wire value
+ # raises. Sampled at the bounds and either side of every listed value.
+ for value in (0, 1, 5, 6, 16, 17, 18, 19, 20, 21, 127, 128, 254, 255):
+ with self.subTest(packet=value):
+ self.assertEqual(int(self.const_module.Packet(value)), value)
+
+ def test_out_of_range_packet_types_are_rejected(self) -> None:
+ for value in (-1, 256):
+ with self.subTest(packet=value):
+ with self.assertRaises(ValueError):
+ self.const_module.Packet(value)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/tests/vendor/test_user_agent_unit.py b/tests/vendor/test_user_agent_unit.py
new file mode 100644
index 0000000000..d3fa0f0d72
--- /dev/null
+++ b/tests/vendor/test_user_agent_unit.py
@@ -0,0 +1,384 @@
+# -*- coding: utf-8 -*-
+"""Regression tests for the ``User-Agent`` the vendor crawlers send.
+
+GitHub issue #518: :meth:`pcapkit.vendor.default.Vendor._request` used to call
+``requests.get(self.LINK)`` with no headers at all, in both its direct and its
+proxy branch. Wikimedia rejects |requests|_' default ``python-requests/``
+agent outright -- measured 2026-09-19 as HTTP 403 and 126 bytes of robot-policy
+text, against 200 for a descriptive agent, reproduced on four separate articles
+with nothing but the header changing -- so four crawlers could not fetch their
+registry at all, and burned ``MAX_RETRY`` attempts against a refusal that no
+amount of retrying lifts.
+
+.. |requests| replace:: ``requests``
+.. _requests: https://requests.readthedocs.io
+
+What is worth pinning is narrower than "a header is sent", and there are three
+parts to it:
+
+* **Both** call sites send it. The proxy branch is the one that gets forgotten,
+ because it only runs when the direct fetch has already raised, so a test that
+ exercises only the happy path would have passed on the unfixed code's sibling
+ bug. :meth:`test_the_proxy_branch_sends_it_too` drives that branch deliberately.
+* The agent is **descriptive, not a browser spoof**. Wikimedia's policy asks for
+ an agent that identifies the tool and offers a contact address; sending
+ ``Mozilla/5.0 …`` would satisfy the 403 and misrepresent the client, so
+ :meth:`test_the_agent_is_not_a_browser_spoof` fails on browser tokens.
+* It is **composed from package metadata**, so it tracks
+ :data:`pcapkit.__version__` instead of going stale as a literal that nobody
+ remembers to bump.
+
+No test here makes a network call: ``requests.get`` is replaced by a recorder for
+the whole of each case, and :meth:`test_no_request_goes_out_without_the_header`
+asserts that every call it saw carried one.
+
+The suite is unit-tier (see :mod:`tests._tiers`): it reads no capture.
+
+"""
+from __future__ import annotations
+
+import contextlib
+import importlib.util
+import os
+import pathlib
+import unittest
+from typing import TYPE_CHECKING
+from unittest import mock
+
+from tests._support import purge_modules
+
+if TYPE_CHECKING:
+ from typing import Any, Iterator
+
+#: Repository root, i.e. the grandparent of the directory holding this file.
+ROOT = pathlib.Path(__file__).resolve().parents[2]
+
+#: Every distribution importing :mod:`pcapkit.vendor` needs. ``requests`` is the
+#: obvious one -- ``pcapkit.vendor.default`` imports it at module scope -- but it
+#: is not sufficient: importing ``pcapkit.vendor.default`` imports the
+#: ``pcapkit.vendor`` package first, and its :file:`__init__.py` pulls in all
+#: seventeen subpackages, seven of which ``import bs4`` at module scope. So a
+#: guard on ``requests`` alone lets the suite error instead of skipping on a
+#: machine that happens to have ``requests`` and not ``beautifulsoup4``.
+VENDOR_DEPS = ('requests', 'bs4', 'html5lib')
+
+#: Whether the crawlers are importable at all. They ship in the ``vendor`` extra
+#: (:file:`pyproject.toml`), not ``test``, and CI installs ``.[test]`` -- so these
+#: tests skip in CI as things stand. Guarded the same way
+#: :file:`tests/protocols/test_dispatch_registry_unit.py` guards its own optional
+#: runtime dependencies, rather than making the whole unit tier depend on the
+#: crawlers' requirements. See #518.
+HAS_VENDOR_DEPS = all(importlib.util.find_spec(name) is not None for name in VENDOR_DEPS)
+
+#: Tokens that only appear in a browser's ``User-Agent``. The point of the fix is
+#: an agent that says what the client actually is, so any of these appearing in
+#: it means the fix has been replaced by a spoof. ``Gecko`` covers both the real
+#: token and the ``like Gecko`` every Chromium agent carries.
+BROWSER_TOKENS = ('Mozilla', 'AppleWebKit', 'Chrome', 'Chromium', 'Safari',
+ 'Gecko', 'Edg/', 'OPR/', 'Opera', 'Trident', 'Firefox')
+
+
+#: Repository URL the faked metadata advertises. Deliberately unlike
+#: :data:`pcapkit.vendor.default.PROJECT_URL`, so that a ``get_user_agent`` which
+#: silently fell back to the constant is distinguishable from one that read the
+#: metadata it was given.
+_MOVED_URL = 'https://example.invalid/moved-repo'
+
+#: Homepage URL the faked metadata also advertises, listed *before* the repository
+#: one. Picking this up would mean the label match had degenerated into "first
+#: Project-URL wins".
+_HOMEPAGE_URL = 'https://example.invalid/home'
+
+
+class _Response:
+ """The parts of :class:`requests.Response` that ``_request`` looks at."""
+
+ def __init__(self, text: 'str' = 'ok',
+ ok: 'bool' = True) -> 'None':
+ self.ok = ok
+ self.text = text
+
+
+class _FakeMetadata:
+ """The parts of :class:`importlib.metadata.PackageMetadata` the agent reads.
+
+ Args:
+ name: Value to return for the ``Name`` field.
+ repository_label: Label to file :data:`_MOVED_URL` under. The whole point
+ of the parameter is that the caller chooses its *case*, which no real
+ installed distribution lets a test vary.
+
+ """
+
+ def __init__(self, name: 'str', repository_label: 'str') -> 'None':
+ self.name = name
+ self.repository_label = repository_label
+
+ def get(self, key: 'str', default: 'Any' = None) -> 'Any':
+ return {'Name': self.name}.get(key, default)
+
+ def get_all(self, key: 'str') -> 'Any':
+ if key != 'Project-URL':
+ return []
+ # ``homepage`` first, so that order alone cannot produce a pass.
+ return [f'homepage, {_HOMEPAGE_URL}',
+ f'{self.repository_label}, {_MOVED_URL}',
+ 'changelog, https://example.invalid/changes']
+
+
+@unittest.skipUnless(HAS_VENDOR_DEPS, f'vendor extra not installed ({", ".join(VENDOR_DEPS)})')
+class VendorUserAgentTests(unittest.TestCase):
+ """The descriptive agent, and that both fetch paths actually send it."""
+
+ if TYPE_CHECKING:
+ default: 'Any'
+ requests: 'Any'
+
+ def setUp(self) -> None:
+ purge_modules(['pcapkit'])
+
+ import requests
+
+ import pcapkit
+ import pcapkit.vendor.default as default
+
+ # The module has to come from this checkout for the assertions to mean
+ # anything: a copy imported from an installed distribution elsewhere
+ # would be tested instead of the one being changed. That is an
+ # environment mismatch rather than a defect, hence a skip.
+ resolved = pathlib.Path(default.__file__).resolve()
+ if ROOT not in resolved.parents:
+ self.skipTest(f'{default.__name__} was imported from {resolved}, which is outside '
+ f'{ROOT}; install this checkout with `pip install -e .` to run this '
+ f'suite against it')
+
+ self.default = default
+ self.requests = requests
+ self.pcapkit = pcapkit
+
+ def _crawler(self, link: 'str' = 'https://example.invalid/registry') -> 'Any':
+ """A throwaway crawler with a ``LINK``, built without touching the disk.
+
+ ``Vendor.__init__`` fetches *and writes a constant file* as a side effect
+ of construction, which a test has no business doing to the working tree,
+ so the two attributes it sets that ``_request`` needs are set by hand.
+ ``request`` is reduced to the identity so that the fetched text comes
+ straight back and can be asserted on.
+
+ """
+ class _Crawler(self.default.Vendor): # type: ignore[name-defined,misc]
+ FLAG = 'isinstance(value, int)'
+ LINK = link
+
+ def count(self, data: 'Any') -> 'Any':
+ import collections
+ return collections.Counter()
+
+ def request(self, text: 'str') -> 'str': # type: ignore[override]
+ return text
+
+ def process(self, data: 'Any') -> 'Any':
+ return [], []
+
+ crawler = _Crawler.__new__(_Crawler)
+ crawler.NAME = _Crawler.__name__
+ crawler.DOCS = 'throwaway'
+ return crawler
+
+ def _agent_from_metadata(self, metadata: 'Any') -> 'str':
+ """Build the agent as if the distribution metadata were ``metadata``.
+
+ ``get_user_agent`` is :func:`~functools.lru_cache`\\ d, so the cache is
+ cleared on both sides of the call: once so the faked metadata is actually
+ consulted rather than a cached real answer returned, and once afterwards so
+ the fake does not leak into any later case.
+
+ """
+ import importlib.metadata as md
+
+ self.default.get_user_agent.cache_clear()
+ try:
+ with mock.patch.object(md, 'metadata', return_value=metadata):
+ return self.default.get_user_agent()
+ finally:
+ self.default.get_user_agent.cache_clear()
+
+ @contextlib.contextmanager
+ def _recording_get(self, *responses: 'Any') -> 'Iterator[list[tuple[Any, Any]]]':
+ """Replace ``requests.get`` with a recorder, and hand back its log.
+
+ Each element of ``responses`` is returned by the corresponding call, or
+ raised if it is an exception. Running out of them is an error rather than
+ a silent repeat, so a test that provokes more fetches than it accounted
+ for fails here instead of hanging in ``_request``'s retry loop.
+
+ """
+ calls = [] # type: list[tuple[Any, Any]]
+ queue = list(responses)
+
+ def fake_get(url: 'Any' = None, **kwargs: 'Any') -> 'Any':
+ calls.append((url, kwargs))
+ if not queue:
+ raise AssertionError(f'unexpected fetch #{len(calls)} of {url!r}')
+ reply = queue.pop(0)
+ if isinstance(reply, BaseException):
+ raise reply
+ return reply
+
+ original = self.requests.get
+ self.requests.get = fake_get
+ try:
+ yield calls
+ finally:
+ self.requests.get = original
+
+ # -- the agent itself ---------------------------------------------------
+
+ def test_agent_names_the_package_its_version_and_a_contact_url(self) -> None:
+ agent = self.default.get_user_agent()
+
+ self.assertIsInstance(agent, str)
+ self.assertIn(self.pcapkit.__version__, agent,
+ 'the agent must carry the package version, so it tracks releases')
+ self.assertIn('https://', agent,
+ "Wikimedia's policy asks for a contact address in the agent")
+ self.assertIn(self.default.PROJECT_URL.rstrip('/').rsplit('/', 1)[-1], agent,
+ 'the agent must name the project, so an operator can be identified')
+
+ def test_the_agent_is_not_a_browser_spoof(self) -> None:
+ # Wikimedia asks to be told what the client is. Passing the 403 by
+ # pretending to be a browser would work and would be a lie, so it is
+ # pinned against rather than left to judgement.
+ agent = self.default.get_user_agent()
+ for token in BROWSER_TOKENS:
+ with self.subTest(token=token):
+ self.assertNotIn(token, agent)
+
+ def test_agent_is_composed_from_metadata_not_hardcoded(self) -> None:
+ # The version has to come from pcapkit.__version__ rather than a literal:
+ # pointing that name at something else must move the agent with it.
+ with mock.patch.object(self.default, '__version__', '99.98.97'):
+ self.default.get_user_agent.cache_clear()
+ try:
+ self.assertIn('99.98.97', self.default.get_user_agent())
+ finally:
+ self.default.get_user_agent.cache_clear()
+
+ def test_agent_survives_missing_distribution_metadata(self) -> None:
+ # Running from a source checkout that was never installed must not raise;
+ # the fallback constants stand in for the metadata.
+ import importlib.metadata as md
+
+ self.default.get_user_agent.cache_clear()
+ with mock.patch.object(md, 'metadata', side_effect=md.PackageNotFoundError):
+ try:
+ agent = self.default.get_user_agent()
+ finally:
+ self.default.get_user_agent.cache_clear()
+ self.assertIn(self.default.DISTRIBUTION, agent)
+ self.assertIn(self.default.PROJECT_URL, agent)
+ self.assertIn(self.pcapkit.__version__, agent)
+
+ def test_metadata_is_actually_read_rather_than_hardcoded(self) -> None:
+ # The installed metadata happens to agree with DISTRIBUTION and
+ # PROJECT_URL, so a get_user_agent() that ignored the metadata entirely
+ # would produce the identical string and pass every other case here.
+ # Faking it to disagree is the only way to tell the two apart.
+ agent = self._agent_from_metadata(_FakeMetadata('RenamedDist', 'repository'))
+
+ self.assertIn('RenamedDist/', agent)
+ self.assertIn(_MOVED_URL, agent)
+ self.assertNotIn(_HOMEPAGE_URL, agent,
+ 'homepage was picked up instead of repository')
+
+ def test_project_url_label_is_matched_case_insensitively(self) -> None:
+ # ``get_user_agent`` lowercases the label before comparing it, and the
+ # docstring promises as much -- but every real distribution writes
+ # ``repository`` in lower case already, so dropping the ``.casefold()``
+ # breaks nothing that the installed metadata can reveal. Core metadata
+ # does not constrain the case of a Project-URL label, and a wheel built
+ # from a ``pyproject.toml`` that spells it ``Repository`` is perfectly
+ # legal, so the promise is pinned here with labels that exercise it.
+ for label in ('Repository', 'REPOSITORY', 'RePoSiToRy'):
+ with self.subTest(label=label):
+ agent = self._agent_from_metadata(_FakeMetadata('RenamedDist', label))
+ self.assertIn(_MOVED_URL, agent,
+ f'a Project-URL labelled {label!r} was not recognised; '
+ f'the label comparison is case-sensitive')
+ self.assertNotIn(self.default.PROJECT_URL, agent,
+ 'fell back to PROJECT_URL rather than reading the metadata')
+
+ def test_unrelated_project_url_labels_are_ignored(self) -> None:
+ # The flip side: matching must stay anchored to ``repository`` rather than
+ # becoming "any label that looks close enough".
+ agent = self._agent_from_metadata(_FakeMetadata('RenamedDist', 'repository-mirror'))
+
+ self.assertNotIn(_MOVED_URL, agent)
+ self.assertIn(self.default.PROJECT_URL, agent,
+ 'no label matched, so the fallback URL should have been used')
+
+ # -- that it is actually sent -------------------------------------------
+
+ def test_direct_branch_sends_the_user_agent(self) -> None:
+ crawler = self._crawler()
+ with self._recording_get(_Response()) as calls:
+ crawler._request() # pylint: disable=protected-access
+
+ self.assertEqual(len(calls), 1)
+ _, kwargs = calls[0]
+ self.assertIn('headers', kwargs, 'the direct fetch sent no headers at all')
+ self.assertEqual(kwargs['headers'].get('User-Agent'),
+ self.default.get_user_agent())
+
+ def test_the_proxy_branch_sends_it_too(self) -> None:
+ # The bug this guards is sending the header on one path and not the
+ # other. Make the direct fetch raise so the proxy branch runs, and give
+ # get_proxies() something to find so it is not skipped.
+ crawler = self._crawler()
+ boom = self.requests.exceptions.RequestException('no route')
+
+ with mock.patch.dict(os.environ, {'PCAPKIT_HTTP_PROXY': 'http://127.0.0.1:9',
+ 'PCAPKIT_HTTPS_PROXY': 'http://127.0.0.1:9'}):
+ with self._recording_get(boom, _Response()) as calls:
+ with self.assertWarns(Warning):
+ crawler._request() # pylint: disable=protected-access
+
+ self.assertEqual(len(calls), 2, 'expected a direct attempt then a proxied one')
+ _, proxied = calls[1]
+ self.assertIn('proxies', proxied, 'the second call was not the proxy branch')
+ self.assertIn('headers', proxied, 'the proxy fetch sent no headers at all')
+ self.assertEqual(proxied['headers'].get('User-Agent'),
+ self.default.get_user_agent())
+
+ def test_no_request_goes_out_without_the_header(self) -> None:
+ # Belt and braces over the two cases above: whatever path _request takes,
+ # every fetch it makes carries the agent. Driven through a retry so that
+ # the loop's second and third attempts are covered too, not just the
+ # first -- a header computed inside the loop could regress on one of them.
+ crawler = self._crawler()
+ with self._recording_get(_Response(ok=False), _Response(text=''), _Response()) as calls:
+ with self.assertWarns(Warning):
+ crawler._request() # pylint: disable=protected-access
+
+ self.assertEqual(len(calls), 3)
+ for index, (_, kwargs) in enumerate(calls):
+ with self.subTest(fetch=index):
+ self.assertEqual(kwargs.get('headers', {}).get('User-Agent'),
+ self.default.get_user_agent())
+
+ def test_link_none_still_short_circuits(self) -> None:
+ # The retired crawlers (#507, #518) rely on this: no LINK means
+ # Vendor.request() is called directly and nothing is fetched. Adding the
+ # header must not have moved the short-circuit.
+ crawler = self._crawler()
+ type(crawler).LINK = None
+
+ sentinel = object()
+ with mock.patch.object(type(crawler), 'request', return_value=sentinel):
+ with self._recording_get() as calls:
+ self.assertIs(crawler._request(), sentinel) # pylint: disable=protected-access
+ self.assertEqual(calls, [], 'a crawler with no LINK must not fetch anything')
+
+
+if __name__ == '__main__':
+ unittest.main()
|