From c41a56605e00e08ef153d91cfb35251e84958509 Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Sat, 19 Sep 2026 16:51:21 -0400 Subject: [PATCH 1/2] fix(vendor): send a descriptive User-Agent and retarget the dead pcapng draft Closes #518. - default.py: add get_user_agent(), built from importlib.metadata rather than hard-coded, and pass it at both the direct and the proxy call site. Wikipedia answers requests' default User-Agent with 403, which took out four crawlers. Not a browser spoof: it names the distribution, version and repository. - pcapng/{block_type,option_type,record_type}.py: retarget from the dead ietf.org/staging draft to the -03 archive. The issue's proposed -02 URLs are both wrong -- that revision renders its registries as preformatted ASCII art with one table and no table-N ids, so the crawlers fetch 200 and then raise IndexError. Only -03 reproduces all three files; -04 onward diverge and the renamed draft-ietf-opsawg-pcapng raises on a 0x0A0D0AXX wildcard row. - ipx/packet.py: retire the scrape. The table was deleted from the article on 2026-08-25 and the registry is closed, so DATA is hand-maintained with per-entry citations and the class defines no LINK. - ftp/return_code.py: skip cell-less rows. The article now ends its table with an empty , so line[0] raised IndexError -- which is why this constant file had never been regenerated, and why regenerating it now adds CODE_335 (RFC 2228's ADAT response). 53 -> 54 members, nothing removed. New tests/vendor/test_user_agent_unit.py, test_ipx_packet_unit.py and test_ftp_return_code_unit.py: 25 tests, 35 subtests. 115 of 117 constant files regenerate byte-identically; 87 of 92 response bodies are unchanged by the header, the five that differ being the four 403s and the dead 404. --- pcapkit/const/ftp/return_code.py | 11 +- pcapkit/vendor/default.py | 67 +++- pcapkit/vendor/ftp/return_code.py | 11 + pcapkit/vendor/ipx/packet.py | 191 +++++++---- pcapkit/vendor/pcapng/block_type.py | 47 ++- pcapkit/vendor/pcapng/option_type.py | 5 +- pcapkit/vendor/pcapng/record_type.py | 5 +- tests/vendor/test_ftp_return_code_unit.py | 194 +++++++++++ tests/vendor/test_ipx_packet_unit.py | 230 +++++++++++++ tests/vendor/test_user_agent_unit.py | 384 ++++++++++++++++++++++ 10 files changed, 1079 insertions(+), 66 deletions(-) create mode 100644 tests/vendor/test_ftp_return_code_unit.py create mode 100644 tests/vendor/test_ipx_packet_unit.py create mode 100644 tests/vendor/test_user_agent_unit.py diff --git a/pcapkit/const/ftp/return_code.py b/pcapkit/const/ftp/return_code.py index 05975acdc3..453af8146b 100644 --- a/pcapkit/const/ftp/return_code.py +++ b/pcapkit/const/ftp/return_code.py @@ -159,7 +159,7 @@ def __str__(self) -> 'str': #: Entering Long Passive Mode (long address, port). CODE_228: 'ReturnCode' = 228, 'Entering Long Passive Mode.' - #: Entering Extended Passive Mode (|||port|). + #: Entering Extended Passive Mode ( |||port| ). CODE_229: 'ReturnCode' = 229, 'Entering Extended Passive Mode.' #: User logged in, proceed. @@ -189,6 +189,10 @@ def __str__(self) -> 'str': #: data needs to be exchanged. CODE_334: 'ReturnCode' = 334, 'Server accepts the security mechanism specified by the client; some security data needs to be exchanged.' + #: Server accepts the security data given by the client; more security data + #: needs to be exchanged. + CODE_335: 'ReturnCode' = 335, 'Server accepts the security data given by the client; more security data needs to be exchanged.' + #: Username okay, password okay. Challenge is ". . . . ". CODE_336: 'ReturnCode' = 336, 'Username okay, password okay.' @@ -211,14 +215,13 @@ def __str__(self) -> 'str': #: Requested host unavailable. CODE_434: 'ReturnCode' = 434, 'Requested host unavailable.' - #: Requested file action not taken. + #: Requested file action not taken. File unavailable (e.g., file busy). CODE_450: 'ReturnCode' = 450, 'Requested file action not taken.' #: Requested action aborted. Local error in processing. CODE_451: 'ReturnCode' = 451, 'Requested action aborted.' - #: Requested action not taken. Insufficient storage space in system. File - #: unavailable (e.g., file busy). + #: Requested action not taken. Insufficient storage space in system. CODE_452: 'ReturnCode' = 452, 'Requested action not taken.' #: Syntax error in parameters or arguments. diff --git a/pcapkit/vendor/default.py b/pcapkit/vendor/default.py index ef1947ebb4..c2cee6b309 100644 --- a/pcapkit/vendor/default.py +++ b/pcapkit/vendor/default.py @@ -12,6 +12,8 @@ import collections import contextlib import csv +import functools +import importlib.metadata import inspect import os import re @@ -22,6 +24,7 @@ import requests +from pcapkit import __version__ from pcapkit.utilities.exceptions import VendorNotImplemented from pcapkit.utilities.logging import BOOLEAN_STATES from pcapkit.utilities.warnings import VendorRequestWarning, warn @@ -35,6 +38,16 @@ MAX_RETRY = int(os.environ.get('PCAPKIT_VENDOR_RETRY', 5)) or 1 CI_MODE = BOOLEAN_STATES.get(os.environ.get('PCAPKIT_CI_MODE', 'false').casefold(), False) +#: Distribution name of this package, i.e. the name its metadata is registered +#: under, which is not the import name (:mod:`pcapkit`). +DISTRIBUTION = 'pypcapkit' + +#: Project URL used as the contact address in :func:`get_user_agent` when the +#: distribution metadata cannot be read, i.e. when running straight from a +#: source checkout that was never installed. Kept in step with ``repository`` +#: under ``[project.urls]`` in :file:`pyproject.toml`. +PROJECT_URL = 'https://github.com/JarryShaw/PyPCAPKit' + #: Default constant template of enumerate registry from IANA CSV. LINE = lambda NAME, DOCS, FLAG, ENUM, MISS, MODL: f'''\ # -*- coding: utf-8 -*- @@ -114,6 +127,50 @@ def get_proxies() -> 'dict[str, str]': return PROXIES +@functools.lru_cache(maxsize=1) +def get_user_agent() -> 'str': + """Get the ``User-Agent`` header the crawlers identify themselves with. + + Many :attr:`~Vendor.LINK` registries are Wikipedia articles, and the + Wikimedia Foundation's User-Agent policy refuses |requests|_' default + ``python-requests/`` agent outright -- HTTP 403 with a body reading + *"Please set a user-agent and respect our robot policy"*. What the policy + asks for is an agent that names the tool and gives a contact address, so that + a misbehaving client can be reached instead of simply blocked. It does + **not** ask for a browser agent, and sending one would misrepresent what is + making the request, so this deliberately identifies the crawler as itself. + + The string is composed from the package's own metadata -- distribution name, + :data:`pcapkit.__version__` and the ``repository`` project URL -- rather than + written out as a literal, so that it follows the package instead of going + stale. Where the distribution metadata cannot be read, i.e. when running from + a source checkout that was never installed, :data:`DISTRIBUTION` and + :data:`PROJECT_URL` stand in for it. + + Returns: + Value for the ``User-Agent`` request header. + + See Also: + `Wikimedia Foundation User-Agent policy + `__ + + """ + name = DISTRIBUTION + url = PROJECT_URL + + with contextlib.suppress(importlib.metadata.PackageNotFoundError): + metadata = importlib.metadata.metadata(DISTRIBUTION) + + name = metadata.get('Name') or name + for entry in metadata.get_all('Project-URL') or []: + label, _, value = entry.partition(',') + if label.strip().casefold() == 'repository' and value.strip(): + url = value.strip() + break + + return f'{name}/{__version__} (+{url}) python-requests/{requests.__version__}' + + class VendorMeta(abc.ABCMeta): """Meta class to add dynamic support to :class:`Vendor`. @@ -395,13 +452,19 @@ def _request(self) -> 'list[str]': if self.LINK is None: return self.request() # type: ignore[unreachable] + # NOTE: both branches below send this. Wikimedia rejects ``requests``' + # default agent with HTTP 403, so a crawler without it fetches 126 bytes + # of robot-policy text and retries MAX_RETRY times against a refusal + # that no amount of retrying will lift; see #518. + headers = {'User-Agent': get_user_agent()} + try: counter = 1 while True: if counter > MAX_RETRY: raise requests.exceptions.RequestException - page = requests.get(self.LINK) # nosec: B113 + page = requests.get(self.LINK, headers=headers) # nosec: B113 if not page.ok or not page.text: warn(f'Connection failed; retry for {counter}/{MAX_RETRY}...', VendorRequestWarning, stacklevel=2) @@ -423,7 +486,7 @@ def _request(self) -> 'list[str]': if counter > MAX_RETRY: raise - page = requests.get(self.LINK, proxies=proxies) # nosec: B113 + page = requests.get(self.LINK, headers=headers, proxies=proxies) # nosec: B113 if not page.ok or not page.text: warn(f'Connection failed; retry with proxy for {counter}/{MAX_RETRY}...', VendorRequestWarning, stacklevel=2) diff --git a/pcapkit/vendor/ftp/return_code.py b/pcapkit/vendor/ftp/return_code.py index ecb7f68058..67e831672e 100644 --- a/pcapkit/vendor/ftp/return_code.py +++ b/pcapkit/vendor/ftp/return_code.py @@ -218,6 +218,17 @@ def process(self, soup: 'BeautifulSoup') -> 'list[str]': # type: ignore[overrid for item in content: line = item.find_all('td') + # NOTE: MediaWiki renders a trailing ``|-`` row separator in the + # wikitext as an empty ````, which revision + # 1354125851 of the article carries at the end of this table. It has + # no cells at all, so reading ``line[0]`` raised ``IndexError`` and + # took the whole crawler down. Skipping cell-less rows is the fix + # rather than indexing defensively further down, because a row that + # cannot supply both a code and an explanation has nothing to + # contribute either way; see #518. + if len(line) < 2: + continue + code = ' '.join(line[0].stripped_strings) if len(code) != 3: continue diff --git a/pcapkit/vendor/ipx/packet.py b/pcapkit/vendor/ipx/packet.py index d94811494c..7a1f0c1361 100644 --- a/pcapkit/vendor/ipx/packet.py +++ b/pcapkit/vendor/ipx/packet.py @@ -18,104 +18,185 @@ ############################################################################### import collections -import re from typing import TYPE_CHECKING -import bs4 - from pcapkit.vendor.default import Vendor if TYPE_CHECKING: from collections import Counter - from bs4 import BeautifulSoup - ############################################################################### sys.path.insert(0, path) ############################################################################### __all__ = ['Packet'] +############################################################################### +# NOTE: this crawler no longer crawls, and the registry below is maintained by +# hand; see #518. It is the same failure, on the same article and the same +# removal revision, that retired the sibling socket crawler in #507. +# +# It used to scrape the packet-type table out of +# https://en.wikipedia.org/wiki/Internetwork_Packet_Exchange#IPX_packet_structure, +# and two separate things broke that. Wikipedia now answers ``requests``' +# default ``python-requests/`` User-Agent with HTTP 403 (measured +# 2026-09-19: 403 and 126 bytes of robot-policy text for the default agent, 200 +# and 114117 bytes for a descriptive one). That half is fixed in +# ``Vendor._request``, which now sends the descriptive agent +# :func:`~pcapkit.vendor.default.get_user_agent` builds -- but it only gets the +# crawler as far as a page that no longer holds the data. The table was deleted +# from the article on 2026-08-25, in revision 1371327031 ("deleted some +# irrelevant technical tables"), leaving the live page (revision 1372814585) +# with a single ``wikitable`` -- the IPX header format one, headers +# ``['Octets', 'Field']`` -- so that ``find_all('table', class_='wikitable')[1]`` +# raises ``IndexError`` even once the fetch succeeds. +# +# Pointing ``LINK`` at the last revision that still carries the table +# (``oldid=1368657333``, re-measured 2026-09-19: 200, 118834 bytes, 4 +# ``wikitable``s, ``table[1]`` headers ``['Value', 'Meaning/Protocol']``, 8 body +# rows) would have worked, and running the old ``process()`` against it +# reproduces the committed constant file byte for byte. It is still the wrong +# fix: a network fetch pinned to a frozen snapshot is all of the fragility of a +# crawler with none of the freshness, and it would go on selecting a table by +# index out of a document that nothing stops from changing shape again. +# +# What settles it is that the registry is closed. IPX packet types were Novell's +# to assign -- :rfc:`1362`, :rfc:`1551` and :rfc:`1634` all say "Packet Types +# also need to be assigned by Novell" -- and there is no longer a Novell to +# assign them. IPX will not gain new packet types, so there is nothing for a +# crawler to pick up on its next run. +# +# The table below is therefore transcribed from that last revision that carried +# it, row for row, and each entry names the primary source for the assignment +# where one exists. +############################################################################### + +#: IPX packet type registry, transcribed from the last revision of the Wikipedia +#: article that still carried the table, +#: https://en.wikipedia.org/w/index.php?title=Internetwork_Packet_Exchange&oldid=1368657333 +#: +#: Maps a packet type to the enumeration name it takes and the comment rendered +#: above it. Names go through :meth:`~Vendor.rename`, so the generated member is +#: the :meth:`~Vendor.safe_name` of the first element; comments are stored as the +#: scrape rendered them, reStructuredText markup and all, since the prose-mangling +#: the old ``process()`` did to Wikipedia's cell text has no source left to mangle. +#: +#: Every value below is independently corroborated by Wireshark's IPX dissector, +#: ``epan/dissectors/packet-ipx.h`` (fetched 2026-09-19), which defines +#: ``IPX_PACKET_TYPE_IPX 0``, ``_RIP 1``, ``_ECHO 2``, ``_ERROR 3``, ``_PEP 4``, +#: ``_SPX 5``, ``_NCP 17`` and ``_WANBCAST 20`` -- the same eight values in the +#: same order. Types 1-5 are inherited from Xerox XNS IDP, which IPX was derived +#: from; no RFC assigns them, and the notes below say so where that is the case. +DATA = { + # NOTE: not a protocol assignment. The scraped table listed 0 as "Unknown", + # and it doubles as the IPX protocol's own default for the ``type`` field. + # Wireshark calls the same value plain ``IPX``. The archived revision is the + # only source for the word "Unknown". + 0: ('Unknown', 'Unknown'), + + # :rfc:`1582` ("Extensions to RIP to Support Demand Circuits", 1994) and + # :rfc:`2091` ("Triggered Extensions to RIP to Support Demand Circuits", + # 1997) are the citation the article itself carried, and they do describe + # IPX RIP -- "the Routing Information Protocol (RIP) which runs over the + # Internetwork Packet Exchange (IPX) protocol using socket number 453h". + # Note honestly that they extend IPX RIP rather than assign it packet type + # 1; for the assignment they defer to Novell, "IPX Router Specification", + # Version 1.10, October 1992, which :rfc:`1582` lists as reference [3]. + 1: ('RIP', '``RIP``, Routing Information Protocol ([:rfc:`1582`], [:rfc:`2091`])'), + + # XNS IDP inheritances. The archived revision is the only source found for + # these two names: no RFC and no Novell document reachable today lists + # either against a packet-type number. Wireshark corroborates the values as + # ``ECHO 2`` and ``ERROR 3``. + 2: ('Echo Packet', 'Echo Packet'), + 3: ('Error Packet', 'Error Packet'), + + # NOTE: the best-sourced row in the table, by a distance. :rfc:`1362`, + # :rfc:`1551` and :rfc:`1634` all state "The packets use the IPX defined + # packet type 04 defining a Packet Exchange Packet", and tabulate it as + # ``| Packet Type | 04 | Packet Exchange Packet |``. :rfc:`1791` adds "UDP + # over IPX uses the IPX packet type 4, a normal IPX packet type" and "TCP, + # like UDP, uses IPX packet type 4". :rfc:`1132` gives it from the other + # side -- "IPX packets may be unicast by setting the IPX header Packet Type + # field to 0x04". + 4: ('PEP', '``PEP``, Packet Exchange Protocol, used for SAP (Service Advertising Protocol)'), + + # :rfc:`1553` names the protocol -- "The Sequenced Packet Exchange (SPX) is + # the reliable connection-based transport protocol commonly used by + # applications" -- but not its packet-type number, which rests on Wireshark + # and the archived revision. + 5: ('SPX', '``SPX``, Sequenced Packet Exchange'), + + # :rfc:`1553` names this one too -- "the Netware Core Protocol (NCP), which + # is used for file server access" -- and again without the number. 17 + # (0x11) rests on Wireshark and the archived revision. + 17: ('NCP', '``NCP``, NetWare Core Protocol'), + + # NOTE: :rfc:`1132`, "A Standard for the Transmission of 802.2 Packets over + # IPX Networks", assigns this one outright: "IPX packets may be broadcast by + # setting the IPX header Packet Type field to 0x14" -- 0x14 being 20. + # Wireshark calls it ``WANBCAST`` / "NetBIOS Broadcast". + # + # The archived table rendered the row as ``Broadcast[4]``, where ``[4]`` is + # Wikipedia's own footnote marker for its citation of that RFC rather than + # any part of the name, and it leaked into the generated member name. Both + # the name and the comment are kept verbatim so that regenerating the + # constant file stays a no-op against the last scraped output: renaming the + # member would break :attr:`pcapkit.const.ipx.packet.Packet.Broadcast_4` for + # anyone using it, which is a call for the maintainer rather than for #518. + # This is the same artefact, from the same article, as the ``LLC_4`` member + # #507 flagged in :mod:`pcapkit.vendor.ipx.socket`. + 20: ('Broadcast[4]', 'Broadcast[4]'), +} # type: dict[int, tuple[str, str]] + class Packet(Vendor): """IPX Packet Types""" #: Value limit checker. FLAG = 'isinstance(value, int) and 0 <= value <= 255' - #: Link to registry. - LINK = 'https://en.wikipedia.org/wiki/Internetwork_Packet_Exchange#IPX_packet_structure' - - def count(self, data: 'BeautifulSoup') -> 'Counter[str]': - """Count field records. - Args: - data: Registry data. + def request(self) -> 'dict[int, tuple[str, str]]': # type: ignore[override] # pylint: disable=arguments-differ + """Fetch registry data. Returns: - Field recordings. + Registry data (:data:`~pcapkit.vendor.ipx.packet.DATA`). """ - return collections.Counter() + return DATA - def request(self, text: 'str') -> 'BeautifulSoup': # type: ignore[override] # pylint: disable=signature-differs - """Fetch HTML source. + def count(self, data: 'dict[int, tuple[str, str]]') -> 'Counter[str]': # type: ignore[override] + """Count field records. Args: - text: Context from :attr:`~Vendor.LINK`. + data: Registry data. Returns: - Parsed HTML source. + Field recordings. """ - return bs4.BeautifulSoup(text, 'html5lib') + return collections.Counter(self.safe_name(name) for name, _ in data.values()) - def process(self, soup: 'BeautifulSoup') -> 'tuple[list[str], list[str]]': # pylint: disable=arguments-differ,arguments-renamed - """Process HTML source. + def process(self, data: 'dict[int, tuple[str, str]]') -> 'tuple[list[str], list[str]]': # type: ignore[override] + """Process registry data. Args: - data: Parsed HTML source. + data: Registry data. Returns: Enumeration fields and missing fields. """ - table = soup.find_all('table', class_='wikitable')[1] - content = filter(lambda item: isinstance(item, bs4.element.Tag), table.tbody) - next(content) # header - enum = [] # type: list[str] miss = [ "return extend_enum(cls, 'Unassigned_%d' % value, value)", ] - for item in content: - line = item.find_all('td') - - pval = ''.join(line[0].stripped_strings) - desc = ''.join(line[1].stripped_strings) - - split = desc.split(' (', 1) - if len(split) == 2: - name = split[0] - cmmt = re.sub(r'RFC (\d+)', r'[:rfc:`\1`]', re.sub(r',([^ ])', r', \1', split[1].replace(')', '', 1))) - else: - name, cmmt = desc, '' - renm = self.safe_name(name) - - if cmmt: - name = f'``{name}``' - tmp1 = f', {cmmt}' - else: - tmp1 = '' - desc = self.wrap_comment(f'{name}{tmp1}') - - pres = f"{renm} = {pval}" - sufs = f'#: {desc}' - - # if len(pres) > 74: - # sufs = f"\n{' '*80}{sufs}" - - # enum.append(f'{pres.ljust(76)}{sufs}') - enum.append(f'{sufs}\n {pres}') + + for code, (name, desc) in data.items(): + pval = str(code) + renm = self.rename(name, pval) + + enum.append(f'#: {self.wrap_comment(desc)}\n {renm} = {pval}') return enum, miss diff --git a/pcapkit/vendor/pcapng/block_type.py b/pcapkit/vendor/pcapng/block_type.py index b90d3bf030..8feea75f52 100644 --- a/pcapkit/vendor/pcapng/block_type.py +++ b/pcapkit/vendor/pcapng/block_type.py @@ -24,6 +24,51 @@ from bs4.element import Tag +############################################################################### +# NOTE: on the registry URL, which this module and its two siblings +# (:mod:`~pcapkit.vendor.pcapng.option_type`, +# :mod:`~pcapkit.vendor.pcapng.record_type`) share; see #518. +# +# All three used to point at +# https://www.ietf.org/staging/draft-tuexen-opsawg-pcapng-02.html, which is dead +# -- measured 2026-09-19 as 404 with any User-Agent, serving a 77968-byte HTML +# error page, so ``Vendor._request`` rejects it on ``page.ok`` and retries +# MAX_RETRY times against a page that will never come back. +# +# The revision number in that URL was wrong as well as the path, which is why +# this is now ``-03`` and not ``-02``. The ``-02`` draft renders its registries +# as ASCII art inside ``
``, and carries exactly one HTML ```` -- the
+# running-header one -- so ``soup.select('table#table-9')`` below finds nothing
+# and raises ``IndexError``. The ``table-1`` .. ``table-10`` ids the three
+# crawlers select on first exist in ``-03``, where the registries became real
+# tables. Measured across every published revision, ``-03`` is also the only one
+# that reproduces all three committed constant files byte for byte.
+#
+# Two renderings of ``-03`` were compared, and both reproduce the three constant
+# files byte-identically, so the choice is about exposure rather than data:
+#
+#   * https://www.ietf.org/archive/id/draft-tuexen-opsawg-pcapng-03.html
+#     -- 254640 bytes, 11 ``
`` (the header one plus ``table-1``..``-10``), +# ``Last-Modified: Thu, 24 Jun 2021 01:24:12 GMT``. This is the immutable +# I-D archive: a static file, frozen at publication. +# * https://datatracker.ietf.org/doc/html/draft-tuexen-opsawg-pcapng-03 +# -- 272271 bytes, 13 ``
``, no ``Last-Modified``. Rendered per +# request, and the extra ~17 KB is datatracker chrome: a version selector, a +# "Compare versions" control and a metadata table, plus ten more ```` +# and two more ``
`` elements from a navigation template, which can change +# whenever the service is redeployed, is gratuitous risk for no gain. +# +# NOTE: the draft has since moved to the OPSAWG working group as +# ``draft-ietf-opsawg-pcapng``, currently at ``-05``. Tracking it is a separate +# change, not a URL swap: the newer revisions alter the registries, and from +# ``draft-ietf-opsawg-pcapng-03`` onwards ``process()`` below dies with +# ``ValueError: invalid literal for int() with base 16: '0x0A0D0AXX'`` on a +# wildcard row it has no handling for. +############################################################################### + class BlockType(Vendor): """Block Types""" @@ -31,7 +76,7 @@ class BlockType(Vendor): #: Value limit checker. FLAG = 'isinstance(value, int) and 0 <= value <= 0xFFFFFFFF' #: Link to registry. - LINK = 'https://www.ietf.org/staging/draft-tuexen-opsawg-pcapng-02.html' + LINK = 'https://www.ietf.org/archive/id/draft-tuexen-opsawg-pcapng-03.html' def count(self, data: 'list[str]') -> 'Counter[str]': """Count field records.""" diff --git a/pcapkit/vendor/pcapng/option_type.py b/pcapkit/vendor/pcapng/option_type.py index d512e20ccb..58debb057a 100644 --- a/pcapkit/vendor/pcapng/option_type.py +++ b/pcapkit/vendor/pcapng/option_type.py @@ -150,8 +150,9 @@ class OptionType(Vendor): #: Value limit checker. FLAG = 'isinstance(value, int) and 0 <= value <= 0xFFFF' - #: Link to registry. - LINK = 'https://www.ietf.org/staging/draft-tuexen-opsawg-pcapng-02.html' + #: Link to registry. See :mod:`pcapkit.vendor.pcapng.block_type` for why this + #: is the ``-03`` revision in the immutable I-D archive; see #518. + LINK = 'https://www.ietf.org/archive/id/draft-tuexen-opsawg-pcapng-03.html' def count(self, data: 'list[str]') -> 'Counter[str]': """Count field records.""" diff --git a/pcapkit/vendor/pcapng/record_type.py b/pcapkit/vendor/pcapng/record_type.py index d61a9f0584..dae32c0dfd 100644 --- a/pcapkit/vendor/pcapng/record_type.py +++ b/pcapkit/vendor/pcapng/record_type.py @@ -30,8 +30,9 @@ class RecordType(Vendor): #: Value limit checker. FLAG = 'isinstance(value, int) and 0 <= value <= 0xFFFF' - #: Link to registry. - LINK = 'https://www.ietf.org/staging/draft-tuexen-opsawg-pcapng-02.html' + #: Link to registry. See :mod:`pcapkit.vendor.pcapng.block_type` for why this + #: is the ``-03`` revision in the immutable I-D archive; see #518. + LINK = 'https://www.ietf.org/archive/id/draft-tuexen-opsawg-pcapng-03.html' def count(self, data: 'list[str]') -> 'Counter[str]': """Count field records.""" diff --git a/tests/vendor/test_ftp_return_code_unit.py b/tests/vendor/test_ftp_return_code_unit.py new file mode 100644 index 0000000000..c38f0e6cf5 --- /dev/null +++ b/tests/vendor/test_ftp_return_code_unit.py @@ -0,0 +1,194 @@ +# -*- coding: utf-8 -*- +"""Regression tests for :mod:`pcapkit.vendor.ftp.return_code`'s empty-row guard. + +GitHub issue #518. Fixing the shared ``User-Agent`` got this crawler past the +Wikipedia 403 it had been taking, and it then died one step further on: +:meth:`~pcapkit.vendor.ftp.return_code.ReturnCode.process` read ``line[0]`` on +every row of the registry table, and revision 1354125851 of the article ends that +table with ```` -- which is what MediaWiki renders a +trailing ``|-`` row separator in the wikitext as. It carries no cells at all, so +``line[0]`` raised ``IndexError: list index out of range`` and took the whole +crawler down. The guard is ``if len(line) < 2: continue``. + +That guard is the most fragile line in the #518 change and it had no test: the +suite's other cases all run against the hand-maintained IPX table or a mocked +fetch, so none of them ever sees a cell-less row. This module is that test. + +**The fixture is built inline and parsed with the same ``html5lib`` the crawler +uses**, rather than fetched. Two reasons: the unit tier makes no network call, and +the empty ```` is partly a *parser* artefact -- what matters is the shape +``bs4`` hands ``process()``, so going through +:meth:`~pcapkit.vendor.ftp.return_code.ReturnCode.request` exercises the real path. +:meth:`test_fixture_really_does_carry_a_cell_less_row` guards against the fixture +silently losing its point, which would otherwise leave every assertion here +passing for the wrong reason. + +``process`` is the seam rather than ``count``: this crawler's +:meth:`~pcapkit.vendor.ftp.return_code.ReturnCode.count` ignores its argument and +returns an empty :class:`~collections.Counter`, its real body having been commented +out, so it never touches a row at all. + +The suite is unit-tier (see :mod:`tests._tiers`): it reads no capture and makes no +network call. + +""" +from __future__ import annotations + +import importlib.util +import pathlib +import unittest +from typing import TYPE_CHECKING + +from tests._support import purge_modules + +if TYPE_CHECKING: + from typing import Any + +#: Repository root, i.e. the grandparent of the directory holding this file. +ROOT = pathlib.Path(__file__).resolve().parents[2] + +#: Every distribution importing :mod:`pcapkit.vendor` needs -- see +#: :mod:`tests.vendor.test_user_agent_unit` for why ``requests`` alone is not +#: enough. They ship in the ``vendor`` extra (:file:`pyproject.toml`), not +#: ``test``, and CI installs ``.[test]``. +VENDOR_DEPS = ('requests', 'bs4', 'html5lib') + +#: Whether the crawlers are importable at all. +HAS_VENDOR_DEPS = all(importlib.util.find_spec(name) is not None for name in VENDOR_DEPS) + +#: A cut-down stand-in for the article, carrying the three ``wikitable``s the live +#: page has -- ``process`` indexes the third, ``[2]``, so the two decoys have to be +#: there -- and ending the third with the cell-less row this module is about. +#: +#: The rows mirror the real ones in shape, not just in content: the code sits in a +#: ```` element inside the ``', + '', +) + + +@unittest.skipUnless(HAS_VENDOR_DEPS, f'vendor extra not installed ({", ".join(VENDOR_DEPS)})') +class FTPReturnCodeEmptyRowTests(unittest.TestCase): + """The trailing ```` the live article ends with.""" + + if TYPE_CHECKING: + vendor_module: 'Any' + + def setUp(self) -> None: + purge_modules(['pcapkit']) + + import pcapkit.vendor.ftp.return_code as vendor_module + + # The module has to come from this checkout, or the guard under test is + # not the one being exercised. An environment mismatch, hence a skip. + resolved = pathlib.Path(vendor_module.__file__).resolve() + if ROOT not in resolved.parents: + self.skipTest(f'{vendor_module.__name__} was imported from {resolved}, which is ' + f'outside {ROOT}; install this checkout with `pip install -e .` to run ' + f'this suite against it') + + self.vendor_module = vendor_module + + def _vendor(self) -> 'Any': + """A crawler with the attributes ``__init__`` would have set. + + ``Vendor.__init__`` fetches over the network and *writes* the constant + file as a side effect of construction, neither of which a unit test has + any business doing, so the attributes ``process`` needs are set by hand. + + """ + cls = self.vendor_module.ReturnCode + vendor = cls.__new__(cls) + vendor.NAME = cls.__name__ + vendor.DOCS = cls.__doc__ + vendor.record = vendor.count(None) + return vendor + + def test_fixture_really_does_carry_a_cell_less_row(self) -> None: + # Without this, every other assertion in the module could pass because + # html5lib quietly dropped the empty -- i.e. because the fixture had + # stopped reproducing the defect, not because the guard works. + import bs4 + + vendor = self._vendor() + soup = vendor.request(REGISTRY_HTML) + table = soup.find_all('table', class_='wikitable')[2] + rows = [row for row in table.tbody if isinstance(row, bs4.element.Tag)] + + self.assertEqual(rows[-1].find_all('td'), [], + 'the last row must carry no
``, the "``100 Series``" heading row is a +#: real data row whose code is not three characters (and so is dropped by the +#: separate ``len(code) != 3`` test rather than by the guard), and the final row is +#: written exactly as MediaWiki emits it. +REGISTRY_HTML = """\ + + + + +
RangePurpose
1xxPositive Preliminary reply.
+ + + +
RangePurpose
x0xSyntax.
+ + + + + + +
CodeExplanation
100 SeriesThe requested action is being initiated.
110Restart marker replay.
200Command okay.
+ +""" + +#: A row carrying one cell rather than none. ``len(line) < 2`` covers this too, +#: and a guard written as ``if not line`` would not -- ``line[1]`` would then +#: raise instead of ``line[0]``, which is the same defect one column over. +SINGLE_CELL_HTML = REGISTRY_HTML.replace( + '
250
at all, or this suite proves nothing') + self.assertEqual(rows[-1].find_all('th'), []) + self.assertEqual(rows[-1].get('class'), ['mw-empty-elt']) + + def test_process_skips_the_cell_less_row_instead_of_raising(self) -> None: + # The regression itself. Before the guard this raised + # `IndexError: list index out of range` at `line[0]`. + vendor = self._vendor() + enum = vendor.process(vendor.request(REGISTRY_HTML)) + + self.assertEqual(len(enum), 2, f'expected only the two 3-digit codes, got {enum!r}') + self.assertIn("CODE_110: 'ReturnCode' = 110, 'Restart marker replay.'", enum[0]) + self.assertIn("CODE_200: 'ReturnCode' = 200, 'Command okay.'", enum[1]) + + def test_process_skips_a_row_with_only_one_cell(self) -> None: + # ``len(line) < 2`` rather than ``not line``: a one-cell row would pass + # the latter and then raise on ``line[1]``. + vendor = self._vendor() + enum = vendor.process(vendor.request(SINGLE_CELL_HTML)) + + self.assertEqual(len(enum), 2, f'expected only the two 3-digit codes, got {enum!r}') + self.assertNotIn('CODE_250', '\n'.join(enum)) + + def test_guard_does_not_swallow_legitimate_rows(self) -> None: + # The other half of the guard's contract: skipping cell-less rows must not + # cost any row that does carry a code and an explanation. + vendor = self._vendor() + enum = vendor.process(vendor.request(REGISTRY_HTML)) + rendered = '\n'.join(enum) + + for code in ('110', '200'): + with self.subTest(code=code): + self.assertIn(f'CODE_{code}', rendered) + # ``100 Series`` is dropped by ``len(code) != 3``, not by the guard. + self.assertNotIn('100 Series', rendered) + + def test_context_survives_the_cell_less_row(self) -> None: + # ``process`` is called from ``context``, which is what ``__init__`` writes + # to disk, so the whole generation path has to survive the row too. + vendor = self._vendor() + context = vendor.context(vendor.request(REGISTRY_HTML)) + + self.assertIn("CODE_110: 'ReturnCode' = 110", context) + self.assertIn("CODE_200: 'ReturnCode' = 200", context) + self.assertIn('class ReturnCode(IntEnum):', context) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/vendor/test_ipx_packet_unit.py b/tests/vendor/test_ipx_packet_unit.py new file mode 100644 index 0000000000..05ac0104f4 --- /dev/null +++ b/tests/vendor/test_ipx_packet_unit.py @@ -0,0 +1,230 @@ +# -*- coding: utf-8 -*- +"""Regression tests for the retired :mod:`pcapkit.vendor.ipx.packet` scrape. + +GitHub issue #518, and the same failure on the same article that #507 found for +the sibling socket crawler. The packet-type crawler used to scrape +``find_all('table', class_='wikitable')[1]`` out of the Wikipedia *Internetwork +Packet Exchange* article, and both halves of that stopped working. Wikipedia +answers |requests|_' default User-Agent with HTTP 403, so +:meth:`pcapkit.vendor.default.Vendor._request` could not fetch the page; and the +table was deleted from the article on 2026-08-25 in revision 1371327031, leaving +the live page with a single ``wikitable`` -- the IPX header format one -- so that +index raises :exc:`IndexError` even with the fetch fixed. The scrape is retired +in favour of the hand-maintained :data:`pcapkit.vendor.ipx.packet.DATA`. + +.. |requests| replace:: ``requests`` +.. _requests: https://requests.readthedocs.io + +What that leaves worth pinning is the failure mode the issue is actually about: a +regeneration that quietly drops packet types. :data:`EXPECTED_MEMBERS` spells the +enumeration out in full, so losing one fails here rather than shipping. +``Broadcast_4`` is in it verbatim, footnote artefact and all -- ``[4]`` was +Wikipedia's own citation marker for :rfc:`1132` rather than part of the name, and +it leaked into the member. Keeping it is what makes byte identity achievable, and +renaming it would break a public member, so the name is pinned rather than +tidied. + +The suite is unit-tier (see :mod:`tests._tiers`): it reads no capture and, by the +whole point of the change, makes no network call. + +""" +from __future__ import annotations + +import importlib.util +import pathlib +import unittest +from typing import TYPE_CHECKING + +from tests._support import purge_modules + +if TYPE_CHECKING: + from typing import Any + +#: Repository root, i.e. the grandparent of the directory holding this file. +ROOT = pathlib.Path(__file__).resolve().parents[2] + +#: Every distribution importing :mod:`pcapkit.vendor` needs. ``requests`` is the +#: obvious one -- ``pcapkit.vendor.default`` imports it at module scope -- but it +#: is not sufficient: importing *any* crawler imports the ``pcapkit.vendor`` +#: package, whose :file:`__init__.py` pulls in all seventeen subpackages, seven of +#: which ``import bs4`` at module scope. So a guard on ``requests`` alone lets the +#: suite error instead of skipping on a machine that happens to have ``requests`` +#: and not ``beautifulsoup4``. +VENDOR_DEPS = ('requests', 'bs4', 'html5lib') + +#: Whether the crawlers are importable at all. They ship in the ``vendor`` extra +#: (:file:`pyproject.toml`), not ``test``, and CI installs ``.[test]`` -- so these +#: tests skip in CI as things stand. Guarded the same way +#: :file:`tests/protocols/test_dispatch_registry_unit.py` guards its own optional +#: runtime dependencies, rather than making the whole unit tier depend on the +#: crawlers' requirements. See #518. +HAS_VENDOR_DEPS = all(importlib.util.find_spec(name) is not None for name in VENDOR_DEPS) + +#: Every member the generated :class:`pcapkit.const.ipx.packet.Packet` is expected +#: to carry, as ``(name, value)`` in definition order. Spelled out rather than +#: derived so that a regeneration which loses a packet type -- the exact failure +#: #518 describes -- fails this test instead of passing a comparison against its +#: own output. +EXPECTED_MEMBERS = ( + ('Unknown', 0), + ('RIP', 1), + ('Echo_Packet', 2), + ('Error_Packet', 3), + ('PEP', 4), + ('SPX', 5), + ('NCP', 17), + ('Broadcast_4', 20), +) + + +def _normalize(context: 'str') -> 'str': + """Apply the whitespace normalisation the generator writes files through. + + :meth:`pcapkit.vendor.default.Vendor.__init__` does not write what + :meth:`~pcapkit.vendor.default.Vendor.context` returns verbatim: it strips + trailing whitespace from every non-blank line, drops whitespace-only lines + outright, and ends the file with a newline courtesy of :func:`print`. Byte + comparison against the committed file has to do the same, or it reports a + difference that regeneration would not actually produce. + + Args: + context: Return value of :meth:`~pcapkit.vendor.default.Vendor.context`. + + Returns: + The text as the generator would have written it to disk. + + """ + lines = [] # type: list[str] + for line in context.splitlines(): + if line: + if line.strip(): + lines.append(line.rstrip()) + else: + lines.append(line) + return '\n'.join(lines) + '\n' + + +@unittest.skipUnless(HAS_VENDOR_DEPS, f'vendor extra not installed ({", ".join(VENDOR_DEPS)})') +class IPXPacketVendorTests(unittest.TestCase): + """The hand-maintained registry, and the crawler that no longer crawls.""" + + if TYPE_CHECKING: + vendor_module: 'Any' + const_module: 'Any' + + def setUp(self) -> None: + purge_modules(['pcapkit']) + + import pcapkit.const.ipx.packet as const_module + import pcapkit.vendor.ipx.packet as vendor_module + + # Both modules have to come from this checkout for any of the comparisons + # below to mean anything: the generated text is compared against this + # repository's constant file, so a vendor module imported from an + # installed copy elsewhere would be comparing two trees. That is an + # environment mismatch rather than a defect, hence a skip. + for module in (vendor_module, const_module): + resolved = pathlib.Path(module.__file__).resolve() + if ROOT not in resolved.parents: + self.skipTest(f'{module.__name__} was imported from {resolved}, which is outside ' + f'{ROOT}; install this checkout with `pip install -e .` to run this ' + f'suite against it') + + self.vendor_module = vendor_module + self.const_module = const_module + + def _vendor(self) -> 'Any': + """A crawler instance with the attributes ``__init__`` would have set. + + ``Vendor.__init__`` regenerates and *writes* the constant file as a side + effect of construction, which a test has no business doing to the working + tree, so the four attributes it sets are set here instead. + + """ + vendor = self.vendor_module.Packet.__new__(self.vendor_module.Packet) + vendor.NAME = self.vendor_module.Packet.__name__ + vendor.DOCS = self.vendor_module.Packet.__doc__ + data = vendor.request() + vendor.record = vendor.count(data) + return vendor + + def test_link_is_none_so_nothing_is_fetched(self) -> None: + # The retirement, stated as an assertion: with no LINK, + # Vendor._request() short-circuits to Packet.request() and never reaches + # requests.get() -- which is what used to take the 403. + self.assertIsNone(self.vendor_module.Packet.LINK) + + def test_request_makes_no_network_call(self) -> None: + import requests + + def explode(*args: 'Any', **kwargs: 'Any') -> 'Any': + raise AssertionError(f'the crawler made a network call: {args!r}') + + original_get, original_request = requests.get, requests.Session.request + requests.get = explode # type: ignore[assignment] + requests.Session.request = explode # type: ignore[assignment,method-assign] + try: + vendor = self.vendor_module.Packet.__new__(self.vendor_module.Packet) + vendor.NAME = self.vendor_module.Packet.__name__ + vendor.DOCS = self.vendor_module.Packet.__doc__ + self.assertIs(vendor._request(), self.vendor_module.DATA) # pylint: disable=protected-access + finally: + requests.get = original_get # type: ignore[assignment] + requests.Session.request = original_request # type: ignore[method-assign] + + def test_regeneration_reproduces_the_committed_constant_file(self) -> None: + # The guard that makes the hand-maintained table trustworthy: running the + # crawler must be a no-op against what is checked in, so an edit to DATA + # that was never regenerated shows up here. + vendor = self._vendor() + generated = _normalize(vendor.context(vendor.request())) + committed = (ROOT / 'pcapkit' / 'const' / 'ipx' / 'packet.py').read_text(encoding='utf-8') + self.assertEqual(generated, committed, + 'regenerating pcapkit/const/ipx/packet.py would change it; run ' + '`python -m pcapkit.vendor.ipx.packet` and commit the result') + + def test_no_packet_type_is_lost(self) -> None: + members = tuple((member.name, int(member.value)) for member in self.const_module.Packet) + self.assertEqual(members, EXPECTED_MEMBERS) + + def test_registry_and_enumeration_agree(self) -> None: + # Every hand-maintained row reaches the enumeration, and nothing in the + # enumeration came from anywhere else. + vendor = self._vendor() + from_data = tuple( + (vendor.rename(name, str(code)), code) + for code, (name, _) in self.vendor_module.DATA.items() + ) + self.assertEqual(from_data, EXPECTED_MEMBERS) + + def test_broadcast_footnote_artefact_is_preserved(self) -> None: + # ``Broadcast_4`` is a public member whose name came from Wikipedia's + # footnote marker for its RFC 1132 citation. Renaming it would be a + # breaking change, and it is what makes byte identity reachable, so both + # the member and the DATA row it comes from are pinned. + self.assertEqual(self.const_module.Packet(20), self.const_module.Packet.Broadcast_4) + self.assertEqual(self.vendor_module.DATA[20], ('Broadcast[4]', 'Broadcast[4]')) + + def test_unknown_packet_type_survives_the_retirement(self) -> None: + # 0 is both the scraped table's "Unknown" row and IPX's own default for + # the type field, so it must be present whatever the registry says. + self.assertEqual(self.const_module.Packet(0), self.const_module.Packet.Unknown) + self.assertEqual(self.const_module.Packet(0).value, 0) + self.assertIn(0, self.vendor_module.DATA) + + def test_unlisted_packet_types_still_resolve(self) -> None: + # ``_missing_`` has to cover the whole octet, so no legal wire value + # raises. Sampled at the bounds and either side of every listed value. + for value in (0, 1, 5, 6, 16, 17, 18, 19, 20, 21, 127, 128, 254, 255): + with self.subTest(packet=value): + self.assertEqual(int(self.const_module.Packet(value)), value) + + def test_out_of_range_packet_types_are_rejected(self) -> None: + for value in (-1, 256): + with self.subTest(packet=value): + with self.assertRaises(ValueError): + self.const_module.Packet(value) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/vendor/test_user_agent_unit.py b/tests/vendor/test_user_agent_unit.py new file mode 100644 index 0000000000..d3fa0f0d72 --- /dev/null +++ b/tests/vendor/test_user_agent_unit.py @@ -0,0 +1,384 @@ +# -*- coding: utf-8 -*- +"""Regression tests for the ``User-Agent`` the vendor crawlers send. + +GitHub issue #518: :meth:`pcapkit.vendor.default.Vendor._request` used to call +``requests.get(self.LINK)`` with no headers at all, in both its direct and its +proxy branch. Wikimedia rejects |requests|_' default ``python-requests/`` +agent outright -- measured 2026-09-19 as HTTP 403 and 126 bytes of robot-policy +text, against 200 for a descriptive agent, reproduced on four separate articles +with nothing but the header changing -- so four crawlers could not fetch their +registry at all, and burned ``MAX_RETRY`` attempts against a refusal that no +amount of retrying lifts. + +.. |requests| replace:: ``requests`` +.. _requests: https://requests.readthedocs.io + +What is worth pinning is narrower than "a header is sent", and there are three +parts to it: + +* **Both** call sites send it. The proxy branch is the one that gets forgotten, + because it only runs when the direct fetch has already raised, so a test that + exercises only the happy path would have passed on the unfixed code's sibling + bug. :meth:`test_the_proxy_branch_sends_it_too` drives that branch deliberately. +* The agent is **descriptive, not a browser spoof**. Wikimedia's policy asks for + an agent that identifies the tool and offers a contact address; sending + ``Mozilla/5.0 …`` would satisfy the 403 and misrepresent the client, so + :meth:`test_the_agent_is_not_a_browser_spoof` fails on browser tokens. +* It is **composed from package metadata**, so it tracks + :data:`pcapkit.__version__` instead of going stale as a literal that nobody + remembers to bump. + +No test here makes a network call: ``requests.get`` is replaced by a recorder for +the whole of each case, and :meth:`test_no_request_goes_out_without_the_header` +asserts that every call it saw carried one. + +The suite is unit-tier (see :mod:`tests._tiers`): it reads no capture. + +""" +from __future__ import annotations + +import contextlib +import importlib.util +import os +import pathlib +import unittest +from typing import TYPE_CHECKING +from unittest import mock + +from tests._support import purge_modules + +if TYPE_CHECKING: + from typing import Any, Iterator + +#: Repository root, i.e. the grandparent of the directory holding this file. +ROOT = pathlib.Path(__file__).resolve().parents[2] + +#: Every distribution importing :mod:`pcapkit.vendor` needs. ``requests`` is the +#: obvious one -- ``pcapkit.vendor.default`` imports it at module scope -- but it +#: is not sufficient: importing ``pcapkit.vendor.default`` imports the +#: ``pcapkit.vendor`` package first, and its :file:`__init__.py` pulls in all +#: seventeen subpackages, seven of which ``import bs4`` at module scope. So a +#: guard on ``requests`` alone lets the suite error instead of skipping on a +#: machine that happens to have ``requests`` and not ``beautifulsoup4``. +VENDOR_DEPS = ('requests', 'bs4', 'html5lib') + +#: Whether the crawlers are importable at all. They ship in the ``vendor`` extra +#: (:file:`pyproject.toml`), not ``test``, and CI installs ``.[test]`` -- so these +#: tests skip in CI as things stand. Guarded the same way +#: :file:`tests/protocols/test_dispatch_registry_unit.py` guards its own optional +#: runtime dependencies, rather than making the whole unit tier depend on the +#: crawlers' requirements. See #518. +HAS_VENDOR_DEPS = all(importlib.util.find_spec(name) is not None for name in VENDOR_DEPS) + +#: Tokens that only appear in a browser's ``User-Agent``. The point of the fix is +#: an agent that says what the client actually is, so any of these appearing in +#: it means the fix has been replaced by a spoof. ``Gecko`` covers both the real +#: token and the ``like Gecko`` every Chromium agent carries. +BROWSER_TOKENS = ('Mozilla', 'AppleWebKit', 'Chrome', 'Chromium', 'Safari', + 'Gecko', 'Edg/', 'OPR/', 'Opera', 'Trident', 'Firefox') + + +#: Repository URL the faked metadata advertises. Deliberately unlike +#: :data:`pcapkit.vendor.default.PROJECT_URL`, so that a ``get_user_agent`` which +#: silently fell back to the constant is distinguishable from one that read the +#: metadata it was given. +_MOVED_URL = 'https://example.invalid/moved-repo' + +#: Homepage URL the faked metadata also advertises, listed *before* the repository +#: one. Picking this up would mean the label match had degenerated into "first +#: Project-URL wins". +_HOMEPAGE_URL = 'https://example.invalid/home' + + +class _Response: + """The parts of :class:`requests.Response` that ``_request`` looks at.""" + + def __init__(self, text: 'str' = 'ok', + ok: 'bool' = True) -> 'None': + self.ok = ok + self.text = text + + +class _FakeMetadata: + """The parts of :class:`importlib.metadata.PackageMetadata` the agent reads. + + Args: + name: Value to return for the ``Name`` field. + repository_label: Label to file :data:`_MOVED_URL` under. The whole point + of the parameter is that the caller chooses its *case*, which no real + installed distribution lets a test vary. + + """ + + def __init__(self, name: 'str', repository_label: 'str') -> 'None': + self.name = name + self.repository_label = repository_label + + def get(self, key: 'str', default: 'Any' = None) -> 'Any': + return {'Name': self.name}.get(key, default) + + def get_all(self, key: 'str') -> 'Any': + if key != 'Project-URL': + return [] + # ``homepage`` first, so that order alone cannot produce a pass. + return [f'homepage, {_HOMEPAGE_URL}', + f'{self.repository_label}, {_MOVED_URL}', + 'changelog, https://example.invalid/changes'] + + +@unittest.skipUnless(HAS_VENDOR_DEPS, f'vendor extra not installed ({", ".join(VENDOR_DEPS)})') +class VendorUserAgentTests(unittest.TestCase): + """The descriptive agent, and that both fetch paths actually send it.""" + + if TYPE_CHECKING: + default: 'Any' + requests: 'Any' + + def setUp(self) -> None: + purge_modules(['pcapkit']) + + import requests + + import pcapkit + import pcapkit.vendor.default as default + + # The module has to come from this checkout for the assertions to mean + # anything: a copy imported from an installed distribution elsewhere + # would be tested instead of the one being changed. That is an + # environment mismatch rather than a defect, hence a skip. + resolved = pathlib.Path(default.__file__).resolve() + if ROOT not in resolved.parents: + self.skipTest(f'{default.__name__} was imported from {resolved}, which is outside ' + f'{ROOT}; install this checkout with `pip install -e .` to run this ' + f'suite against it') + + self.default = default + self.requests = requests + self.pcapkit = pcapkit + + def _crawler(self, link: 'str' = 'https://example.invalid/registry') -> 'Any': + """A throwaway crawler with a ``LINK``, built without touching the disk. + + ``Vendor.__init__`` fetches *and writes a constant file* as a side effect + of construction, which a test has no business doing to the working tree, + so the two attributes it sets that ``_request`` needs are set by hand. + ``request`` is reduced to the identity so that the fetched text comes + straight back and can be asserted on. + + """ + class _Crawler(self.default.Vendor): # type: ignore[name-defined,misc] + FLAG = 'isinstance(value, int)' + LINK = link + + def count(self, data: 'Any') -> 'Any': + import collections + return collections.Counter() + + def request(self, text: 'str') -> 'str': # type: ignore[override] + return text + + def process(self, data: 'Any') -> 'Any': + return [], [] + + crawler = _Crawler.__new__(_Crawler) + crawler.NAME = _Crawler.__name__ + crawler.DOCS = 'throwaway' + return crawler + + def _agent_from_metadata(self, metadata: 'Any') -> 'str': + """Build the agent as if the distribution metadata were ``metadata``. + + ``get_user_agent`` is :func:`~functools.lru_cache`\\ d, so the cache is + cleared on both sides of the call: once so the faked metadata is actually + consulted rather than a cached real answer returned, and once afterwards so + the fake does not leak into any later case. + + """ + import importlib.metadata as md + + self.default.get_user_agent.cache_clear() + try: + with mock.patch.object(md, 'metadata', return_value=metadata): + return self.default.get_user_agent() + finally: + self.default.get_user_agent.cache_clear() + + @contextlib.contextmanager + def _recording_get(self, *responses: 'Any') -> 'Iterator[list[tuple[Any, Any]]]': + """Replace ``requests.get`` with a recorder, and hand back its log. + + Each element of ``responses`` is returned by the corresponding call, or + raised if it is an exception. Running out of them is an error rather than + a silent repeat, so a test that provokes more fetches than it accounted + for fails here instead of hanging in ``_request``'s retry loop. + + """ + calls = [] # type: list[tuple[Any, Any]] + queue = list(responses) + + def fake_get(url: 'Any' = None, **kwargs: 'Any') -> 'Any': + calls.append((url, kwargs)) + if not queue: + raise AssertionError(f'unexpected fetch #{len(calls)} of {url!r}') + reply = queue.pop(0) + if isinstance(reply, BaseException): + raise reply + return reply + + original = self.requests.get + self.requests.get = fake_get + try: + yield calls + finally: + self.requests.get = original + + # -- the agent itself --------------------------------------------------- + + def test_agent_names_the_package_its_version_and_a_contact_url(self) -> None: + agent = self.default.get_user_agent() + + self.assertIsInstance(agent, str) + self.assertIn(self.pcapkit.__version__, agent, + 'the agent must carry the package version, so it tracks releases') + self.assertIn('https://', agent, + "Wikimedia's policy asks for a contact address in the agent") + self.assertIn(self.default.PROJECT_URL.rstrip('/').rsplit('/', 1)[-1], agent, + 'the agent must name the project, so an operator can be identified') + + def test_the_agent_is_not_a_browser_spoof(self) -> None: + # Wikimedia asks to be told what the client is. Passing the 403 by + # pretending to be a browser would work and would be a lie, so it is + # pinned against rather than left to judgement. + agent = self.default.get_user_agent() + for token in BROWSER_TOKENS: + with self.subTest(token=token): + self.assertNotIn(token, agent) + + def test_agent_is_composed_from_metadata_not_hardcoded(self) -> None: + # The version has to come from pcapkit.__version__ rather than a literal: + # pointing that name at something else must move the agent with it. + with mock.patch.object(self.default, '__version__', '99.98.97'): + self.default.get_user_agent.cache_clear() + try: + self.assertIn('99.98.97', self.default.get_user_agent()) + finally: + self.default.get_user_agent.cache_clear() + + def test_agent_survives_missing_distribution_metadata(self) -> None: + # Running from a source checkout that was never installed must not raise; + # the fallback constants stand in for the metadata. + import importlib.metadata as md + + self.default.get_user_agent.cache_clear() + with mock.patch.object(md, 'metadata', side_effect=md.PackageNotFoundError): + try: + agent = self.default.get_user_agent() + finally: + self.default.get_user_agent.cache_clear() + self.assertIn(self.default.DISTRIBUTION, agent) + self.assertIn(self.default.PROJECT_URL, agent) + self.assertIn(self.pcapkit.__version__, agent) + + def test_metadata_is_actually_read_rather_than_hardcoded(self) -> None: + # The installed metadata happens to agree with DISTRIBUTION and + # PROJECT_URL, so a get_user_agent() that ignored the metadata entirely + # would produce the identical string and pass every other case here. + # Faking it to disagree is the only way to tell the two apart. + agent = self._agent_from_metadata(_FakeMetadata('RenamedDist', 'repository')) + + self.assertIn('RenamedDist/', agent) + self.assertIn(_MOVED_URL, agent) + self.assertNotIn(_HOMEPAGE_URL, agent, + 'homepage was picked up instead of repository') + + def test_project_url_label_is_matched_case_insensitively(self) -> None: + # ``get_user_agent`` lowercases the label before comparing it, and the + # docstring promises as much -- but every real distribution writes + # ``repository`` in lower case already, so dropping the ``.casefold()`` + # breaks nothing that the installed metadata can reveal. Core metadata + # does not constrain the case of a Project-URL label, and a wheel built + # from a ``pyproject.toml`` that spells it ``Repository`` is perfectly + # legal, so the promise is pinned here with labels that exercise it. + for label in ('Repository', 'REPOSITORY', 'RePoSiToRy'): + with self.subTest(label=label): + agent = self._agent_from_metadata(_FakeMetadata('RenamedDist', label)) + self.assertIn(_MOVED_URL, agent, + f'a Project-URL labelled {label!r} was not recognised; ' + f'the label comparison is case-sensitive') + self.assertNotIn(self.default.PROJECT_URL, agent, + 'fell back to PROJECT_URL rather than reading the metadata') + + def test_unrelated_project_url_labels_are_ignored(self) -> None: + # The flip side: matching must stay anchored to ``repository`` rather than + # becoming "any label that looks close enough". + agent = self._agent_from_metadata(_FakeMetadata('RenamedDist', 'repository-mirror')) + + self.assertNotIn(_MOVED_URL, agent) + self.assertIn(self.default.PROJECT_URL, agent, + 'no label matched, so the fallback URL should have been used') + + # -- that it is actually sent ------------------------------------------- + + def test_direct_branch_sends_the_user_agent(self) -> None: + crawler = self._crawler() + with self._recording_get(_Response()) as calls: + crawler._request() # pylint: disable=protected-access + + self.assertEqual(len(calls), 1) + _, kwargs = calls[0] + self.assertIn('headers', kwargs, 'the direct fetch sent no headers at all') + self.assertEqual(kwargs['headers'].get('User-Agent'), + self.default.get_user_agent()) + + def test_the_proxy_branch_sends_it_too(self) -> None: + # The bug this guards is sending the header on one path and not the + # other. Make the direct fetch raise so the proxy branch runs, and give + # get_proxies() something to find so it is not skipped. + crawler = self._crawler() + boom = self.requests.exceptions.RequestException('no route') + + with mock.patch.dict(os.environ, {'PCAPKIT_HTTP_PROXY': 'http://127.0.0.1:9', + 'PCAPKIT_HTTPS_PROXY': 'http://127.0.0.1:9'}): + with self._recording_get(boom, _Response()) as calls: + with self.assertWarns(Warning): + crawler._request() # pylint: disable=protected-access + + self.assertEqual(len(calls), 2, 'expected a direct attempt then a proxied one') + _, proxied = calls[1] + self.assertIn('proxies', proxied, 'the second call was not the proxy branch') + self.assertIn('headers', proxied, 'the proxy fetch sent no headers at all') + self.assertEqual(proxied['headers'].get('User-Agent'), + self.default.get_user_agent()) + + def test_no_request_goes_out_without_the_header(self) -> None: + # Belt and braces over the two cases above: whatever path _request takes, + # every fetch it makes carries the agent. Driven through a retry so that + # the loop's second and third attempts are covered too, not just the + # first -- a header computed inside the loop could regress on one of them. + crawler = self._crawler() + with self._recording_get(_Response(ok=False), _Response(text=''), _Response()) as calls: + with self.assertWarns(Warning): + crawler._request() # pylint: disable=protected-access + + self.assertEqual(len(calls), 3) + for index, (_, kwargs) in enumerate(calls): + with self.subTest(fetch=index): + self.assertEqual(kwargs.get('headers', {}).get('User-Agent'), + self.default.get_user_agent()) + + def test_link_none_still_short_circuits(self) -> None: + # The retired crawlers (#507, #518) rely on this: no LINK means + # Vendor.request() is called directly and nothing is fetched. Adding the + # header must not have moved the short-circuit. + crawler = self._crawler() + type(crawler).LINK = None + + sentinel = object() + with mock.patch.object(type(crawler), 'request', return_value=sentinel): + with self._recording_get() as calls: + self.assertIs(crawler._request(), sentinel) # pylint: disable=protected-access + self.assertEqual(calls, [], 'a crawler with no LINK must not fetch anything') + + +if __name__ == '__main__': + unittest.main() From 1240093d2d142705054fa4f7d820d5e3d7ead4af Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Sat, 19 Sep 2026 20:40:27 -0400 Subject: [PATCH 2/2] docs: render the pcapng -03 rationale and retarget the dead -02 URL The reason all three scraping crawlers read draft revision -03 lived only in a `#` comment block, and the docs pull in nothing but autoclass output, so Sphinx never rendered any of it. Meanwhile the rendered pages still cited the dead -02 staging URL, so the docs pointed at a 404 while the code fetched -03. - add a note to the pcapng vendor docs stating once that BlockType, OptionType and RecordType all read -03, that the -02 staging URL is dead, and that the table-1 through table-10 ids they select on first exist in -03 - retarget staging/draft-tuexen-opsawg-pcapng-02.html at archive/id/draft-tuexen-opsawg-pcapng-03.html in all 31 places that cited it across the vendor, const and protocols docs and sources; all 8 anchor fragments were confirmed present in -03 - leave 3 deliberately historical -02 references intact: the "which is dead" URL in block_type.py, and two draft-ietf-opsawg-pcapng-02 section citations in protocols/misc/pcapng.py that quote that revision's own text - trim the block_type.py comment to the implementer-facing measurements, the reader-facing rationale now being in the rendered note Sphinx 9.1.0 builds clean: 83 warnings, warning set byte-identical to the parent commit, and no clickable -02 link left anywhere in the output. Refs #518. --- docs/source/pcapkit/const/pcapng.rst | 14 ++++----- docs/source/pcapkit/protocols/misc/pcapng.rst | 2 +- docs/source/pcapkit/vendor/pcapng.rst | 29 ++++++++++++++----- pcapkit/const/pcapng/__init__.py | 14 ++++----- pcapkit/protocols/misc/pcapng.py | 2 +- pcapkit/vendor/pcapng/__init__.py | 14 ++++----- pcapkit/vendor/pcapng/block_type.py | 27 ++++++++--------- pcapkit/vendor/pcapng/filter_type.py | 2 +- 8 files changed, 58 insertions(+), 46 deletions(-) diff --git a/docs/source/pcapkit/const/pcapng.rst b/docs/source/pcapkit/const/pcapng.rst index ff0c146ad9..c044996f22 100644 --- a/docs/source/pcapkit/const/pcapng.rst +++ b/docs/source/pcapkit/const/pcapng.rst @@ -120,10 +120,10 @@ which is automatically generated from :class:`pcapkit.vendor.pcapng.filter_type.
-.. [*] https://www.ietf.org/staging/draft-tuexen-opsawg-pcapng-02.html#name-standardized-block-type-cod -.. [*] https://www.ietf.org/staging/draft-tuexen-opsawg-pcapng-02.html#name-options -.. [*] https://www.ietf.org/staging/draft-tuexen-opsawg-pcapng-02.html#name-enhanced-packet-block-flags -.. [*] https://www.ietf.org/staging/draft-tuexen-opsawg-pcapng-02.html#name-enhanced-packet-block -.. [*] https://www.ietf.org/staging/draft-tuexen-opsawg-pcapng-02.html#name-name-resolution-block -.. [*] https://www.ietf.org/staging/draft-tuexen-opsawg-pcapng-02.html#name-decryption-secrets-block -.. [*] https://www.ietf.org/staging/draft-tuexen-opsawg-pcapng-02.html#name-interface-description-block +.. [*] https://www.ietf.org/archive/id/draft-tuexen-opsawg-pcapng-03.html#name-standardized-block-type-cod +.. [*] https://www.ietf.org/archive/id/draft-tuexen-opsawg-pcapng-03.html#name-options +.. [*] https://www.ietf.org/archive/id/draft-tuexen-opsawg-pcapng-03.html#name-enhanced-packet-block-flags +.. [*] https://www.ietf.org/archive/id/draft-tuexen-opsawg-pcapng-03.html#name-enhanced-packet-block +.. [*] https://www.ietf.org/archive/id/draft-tuexen-opsawg-pcapng-03.html#name-name-resolution-block +.. [*] https://www.ietf.org/archive/id/draft-tuexen-opsawg-pcapng-03.html#name-decryption-secrets-block +.. [*] https://www.ietf.org/archive/id/draft-tuexen-opsawg-pcapng-03.html#name-interface-description-block diff --git a/docs/source/pcapkit/protocols/misc/pcapng.rst b/docs/source/pcapkit/protocols/misc/pcapng.rst index 311f726d92..07f1f69268 100644 --- a/docs/source/pcapkit/protocols/misc/pcapng.rst +++ b/docs/source/pcapkit/protocols/misc/pcapng.rst @@ -770,4 +770,4 @@ Data Models .. rubric:: Footnotes -.. [*] https://www.ietf.org/staging/draft-tuexen-opsawg-pcapng-02.html +.. [*] https://www.ietf.org/archive/id/draft-tuexen-opsawg-pcapng-03.html diff --git a/docs/source/pcapkit/vendor/pcapng.rst b/docs/source/pcapkit/vendor/pcapng.rst index 611ae2cc11..110016f13a 100644 --- a/docs/source/pcapkit/vendor/pcapng.rst +++ b/docs/source/pcapkit/vendor/pcapng.rst @@ -25,6 +25,21 @@ vendor crawlers include: * - :class:`PCAPNG_FilterType ` - Filter Types [*]_ +.. note:: + + The three crawlers that scrape a registry table -- + :class:`BlockType `, + :class:`OptionType ` and + :class:`RecordType ` -- all read + revision ``-03`` of the draft, as each one's ``LINK`` below shows. They used to + read ``https://www.ietf.org/staging/draft-tuexen-opsawg-pcapng-02.html``, which + is dead: measured 2026-09-19 as a 404 under any User-Agent. The revision number + was wrong as well as the path -- ``-02`` renders its registries as ASCII art + inside ``
`` and carries no registry ```` at all, so the ``table-1``
+   through ``table-10`` ids the three crawlers select on first exist in ``-03``,
+   where the registries became real tables. The footnotes below point at ``-03``
+   for the same reason. See #518.
+
 Block Types
 ===========
 
@@ -111,10 +126,10 @@ which is automatically generating :class:`pcapkit.const.pcapng.filter_type.Filte
 
 .. rubric:: Footnotes
 
-.. [*] https://www.ietf.org/staging/draft-tuexen-opsawg-pcapng-02.html#name-standardized-block-type-cod
-.. [*] https://www.ietf.org/staging/draft-tuexen-opsawg-pcapng-02.html#name-options
-.. [*] https://www.ietf.org/staging/draft-tuexen-opsawg-pcapng-02.html#name-enhanced-packet-block-flags
-.. [*] https://www.ietf.org/staging/draft-tuexen-opsawg-pcapng-02.html#name-enhanced-packet-block
-.. [*] https://www.ietf.org/staging/draft-tuexen-opsawg-pcapng-02.html#name-name-resolution-block
-.. [*] https://www.ietf.org/staging/draft-tuexen-opsawg-pcapng-02.html#name-decryption-secrets-block
-.. [*] https://www.ietf.org/staging/draft-tuexen-opsawg-pcapng-02.html#name-interface-description-block
+.. [*] https://www.ietf.org/archive/id/draft-tuexen-opsawg-pcapng-03.html#name-standardized-block-type-cod
+.. [*] https://www.ietf.org/archive/id/draft-tuexen-opsawg-pcapng-03.html#name-options
+.. [*] https://www.ietf.org/archive/id/draft-tuexen-opsawg-pcapng-03.html#name-enhanced-packet-block-flags
+.. [*] https://www.ietf.org/archive/id/draft-tuexen-opsawg-pcapng-03.html#name-enhanced-packet-block
+.. [*] https://www.ietf.org/archive/id/draft-tuexen-opsawg-pcapng-03.html#name-name-resolution-block
+.. [*] https://www.ietf.org/archive/id/draft-tuexen-opsawg-pcapng-03.html#name-decryption-secrets-block
+.. [*] https://www.ietf.org/archive/id/draft-tuexen-opsawg-pcapng-03.html#name-interface-description-block
diff --git a/pcapkit/const/pcapng/__init__.py b/pcapkit/const/pcapng/__init__.py
index c68c11706a..790c1f7e07 100644
--- a/pcapkit/const/pcapng/__init__.py
+++ b/pcapkit/const/pcapng/__init__.py
@@ -26,13 +26,13 @@
    * - :class:`PCAPNG_FilterType `
      - Filter Types [*]_
 
-.. [*] https://www.ietf.org/staging/draft-tuexen-opsawg-pcapng-02.html#name-standardized-block-type-cod
-.. [*] https://www.ietf.org/staging/draft-tuexen-opsawg-pcapng-02.html#name-options
-.. [*] https://www.ietf.org/staging/draft-tuexen-opsawg-pcapng-02.html#name-enhanced-packet-block-flags
-.. [*] https://www.ietf.org/staging/draft-tuexen-opsawg-pcapng-02.html#name-enhanced-packet-block
-.. [*] https://www.ietf.org/staging/draft-tuexen-opsawg-pcapng-02.html#name-name-resolution-block
-.. [*] https://www.ietf.org/staging/draft-tuexen-opsawg-pcapng-02.html#name-decryption-secrets-block
-.. [*] https://www.ietf.org/staging/draft-tuexen-opsawg-pcapng-02.html#name-interface-description-block
+.. [*] https://www.ietf.org/archive/id/draft-tuexen-opsawg-pcapng-03.html#name-standardized-block-type-cod
+.. [*] https://www.ietf.org/archive/id/draft-tuexen-opsawg-pcapng-03.html#name-options
+.. [*] https://www.ietf.org/archive/id/draft-tuexen-opsawg-pcapng-03.html#name-enhanced-packet-block-flags
+.. [*] https://www.ietf.org/archive/id/draft-tuexen-opsawg-pcapng-03.html#name-enhanced-packet-block
+.. [*] https://www.ietf.org/archive/id/draft-tuexen-opsawg-pcapng-03.html#name-name-resolution-block
+.. [*] https://www.ietf.org/archive/id/draft-tuexen-opsawg-pcapng-03.html#name-decryption-secrets-block
+.. [*] https://www.ietf.org/archive/id/draft-tuexen-opsawg-pcapng-03.html#name-interface-description-block
 
 """
 
diff --git a/pcapkit/protocols/misc/pcapng.py b/pcapkit/protocols/misc/pcapng.py
index 9a57b91517..6994249c53 100644
--- a/pcapkit/protocols/misc/pcapng.py
+++ b/pcapkit/protocols/misc/pcapng.py
@@ -9,7 +9,7 @@
 :class:`~pcapkit.protocols.misc.pcapng.PCAPNG` only,
 which implements extractor for PCAP-NG file format [*]_.
 
-.. [*] https://www.ietf.org/staging/draft-tuexen-opsawg-pcapng-02.html
+.. [*] https://www.ietf.org/archive/id/draft-tuexen-opsawg-pcapng-03.html
 
 """
 import base64
diff --git a/pcapkit/vendor/pcapng/__init__.py b/pcapkit/vendor/pcapng/__init__.py
index 3d500327e8..6d628e00bd 100644
--- a/pcapkit/vendor/pcapng/__init__.py
+++ b/pcapkit/vendor/pcapng/__init__.py
@@ -26,13 +26,13 @@
    * - :class:`PCAPNG_FilterType `
      - Filter Types [*]_
 
-.. [*] https://www.ietf.org/staging/draft-tuexen-opsawg-pcapng-02.html#name-standardized-block-type-cod
-.. [*] https://www.ietf.org/staging/draft-tuexen-opsawg-pcapng-02.html#name-options
-.. [*] https://www.ietf.org/staging/draft-tuexen-opsawg-pcapng-02.html#name-enhanced-packet-block-flags
-.. [*] https://www.ietf.org/staging/draft-tuexen-opsawg-pcapng-02.html#name-enhanced-packet-block
-.. [*] https://www.ietf.org/staging/draft-tuexen-opsawg-pcapng-02.html#name-name-resolution-block
-.. [*] https://www.ietf.org/staging/draft-tuexen-opsawg-pcapng-02.html#name-decryption-secrets-block
-.. [*] https://www.ietf.org/staging/draft-tuexen-opsawg-pcapng-02.html#name-interface-description-block
+.. [*] https://www.ietf.org/archive/id/draft-tuexen-opsawg-pcapng-03.html#name-standardized-block-type-cod
+.. [*] https://www.ietf.org/archive/id/draft-tuexen-opsawg-pcapng-03.html#name-options
+.. [*] https://www.ietf.org/archive/id/draft-tuexen-opsawg-pcapng-03.html#name-enhanced-packet-block-flags
+.. [*] https://www.ietf.org/archive/id/draft-tuexen-opsawg-pcapng-03.html#name-enhanced-packet-block
+.. [*] https://www.ietf.org/archive/id/draft-tuexen-opsawg-pcapng-03.html#name-name-resolution-block
+.. [*] https://www.ietf.org/archive/id/draft-tuexen-opsawg-pcapng-03.html#name-decryption-secrets-block
+.. [*] https://www.ietf.org/archive/id/draft-tuexen-opsawg-pcapng-03.html#name-interface-description-block
 
 """
 
diff --git a/pcapkit/vendor/pcapng/block_type.py b/pcapkit/vendor/pcapng/block_type.py
index 8feea75f52..d125c58e2f 100644
--- a/pcapkit/vendor/pcapng/block_type.py
+++ b/pcapkit/vendor/pcapng/block_type.py
@@ -27,22 +27,19 @@
 ###############################################################################
 # NOTE: on the registry URL, which this module and its two siblings
 # (:mod:`~pcapkit.vendor.pcapng.option_type`,
-# :mod:`~pcapkit.vendor.pcapng.record_type`) share; see #518.
+# :mod:`~pcapkit.vendor.pcapng.record_type`) share. Why all three read ``-03`` is
+# stated for readers in the note in ``docs/source/pcapkit/vendor/pcapng.rst``,
+# which renders; what follows is the measurement detail behind it. See #518.
 #
-# All three used to point at
-# https://www.ietf.org/staging/draft-tuexen-opsawg-pcapng-02.html, which is dead
-# -- measured 2026-09-19 as 404 with any User-Agent, serving a 77968-byte HTML
-# error page, so ``Vendor._request`` rejects it on ``page.ok`` and retries
-# MAX_RETRY times against a page that will never come back.
-#
-# The revision number in that URL was wrong as well as the path, which is why
-# this is now ``-03`` and not ``-02``. The ``-02`` draft renders its registries
-# as ASCII art inside ``
``, and carries exactly one HTML ``
`` -- the -# running-header one -- so ``soup.select('table#table-9')`` below finds nothing -# and raises ``IndexError``. The ``table-1`` .. ``table-10`` ids the three -# crawlers select on first exist in ``-03``, where the registries became real -# tables. Measured across every published revision, ``-03`` is also the only one -# that reproduces all three committed constant files byte for byte. +# The dead ``-02`` URL, +# https://www.ietf.org/staging/draft-tuexen-opsawg-pcapng-02.html, serves a +# 77968-byte HTML error page rather than a clean refusal, so ``Vendor._request`` +# rejects it on ``page.ok`` and retries MAX_RETRY times against a page that will +# never come back. On ``-02`` the registries are ASCII art inside ``
`` and
+# the only HTML ``
`` is the running-header one, so +# ``soup.select('table#table-9')`` below finds nothing and raises ``IndexError``. +# Measured across every published revision, ``-03`` is the only one that +# reproduces all three committed constant files byte for byte. # # Two renderings of ``-03`` were compared, and both reproduce the three constant # files byte-identically, so the choice is about exposure rather than data: diff --git a/pcapkit/vendor/pcapng/filter_type.py b/pcapkit/vendor/pcapng/filter_type.py index db6cb76cd2..2be7b8829e 100644 --- a/pcapkit/vendor/pcapng/filter_type.py +++ b/pcapkit/vendor/pcapng/filter_type.py @@ -22,7 +22,7 @@ #: Filter type registry. DATA = { - # TODO: https://www.ietf.org/staging/draft-tuexen-opsawg-pcapng-02.html#section-4.2-28.2.1 + # TODO: https://www.ietf.org/archive/id/draft-tuexen-opsawg-pcapng-03.html#section-4.2-28.2.1 } # type: dict[int, str]