diff --git a/docs/source/pcapkit/vendor/ipx.rst b/docs/source/pcapkit/vendor/ipx.rst index 0fe5aa024..8cb0fd0b8 100644 --- a/docs/source/pcapkit/vendor/ipx.rst +++ b/docs/source/pcapkit/vendor/ipx.rst @@ -35,11 +35,21 @@ Socket Types This module contains the vendor crawler for **Socket Types**, which is automatically generating :class:`pcapkit.const.ipx.socket.Socket`. +.. note:: + + This crawler no longer crawls. The table it used to scrape was deleted from + the Wikipedia article on 2026-08-25, and the registry itself is closed, so + the data is now maintained by hand in the ``DATA`` and ``RANGES`` mappings of + :mod:`pcapkit.vendor.ipx.socket`, and the class defines no ``LINK``. The + footnote below points at the last revision that still carried the table -- + what the hand-maintained registry was transcribed from, not a page the + crawler fetches. See #507. + .. autoclass:: pcapkit.vendor.ipx.socket.Socket - :members: FLAG, LINK + :members: FLAG :show-inheritance: .. rubric:: Footnotes .. [*] https://en.wikipedia.org/wiki/Internetwork_Packet_Exchange#IPX_packet_structure -.. [*] https://en.wikipedia.org/wiki/Internetwork_Packet_Exchange#Socket_number +.. [*] https://en.wikipedia.org/w/index.php?title=Internetwork_Packet_Exchange&oldid=1368657333 diff --git a/pcapkit/vendor/ipx/socket.py b/pcapkit/vendor/ipx/socket.py index 7a47f3708..6caaefae7 100644 --- a/pcapkit/vendor/ipx/socket.py +++ b/pcapkit/vendor/ipx/socket.py @@ -18,115 +18,290 @@ ############################################################################### import collections -import re from typing import TYPE_CHECKING -import bs4 - from pcapkit.vendor.default import Vendor if TYPE_CHECKING: from collections import Counter - from bs4 import BeautifulSoup - ############################################################################### sys.path.insert(0, path) ############################################################################### __all__ = ['Socket'] +############################################################################### +# NOTE: this crawler no longer crawls, and the registry below is maintained by +# hand; see #507. +# +# It used to scrape the well-known-socket table out of +# https://en.wikipedia.org/wiki/Internetwork_Packet_Exchange#Socket_number, +# and two separate things broke that. Wikipedia now answers ``requests``' +# default ``python-requests/`` User-Agent with HTTP 403 (measured +# 2026-09-19: 403 and 126 bytes of robot-policy text for the default agent, +# 200 and 114117 bytes for a descriptive one), so ``Vendor._request`` could not +# fetch the page at all. And the table itself was deleted from the article on +# 2026-08-25, in revision 1371327031 ("deleted some irrelevant technical +# tables"), leaving the page with a single ``wikitable`` -- the IPX header +# format one -- so that ``find_all('table', class_='wikitable')[3]`` raises +# ``IndexError`` even once the fetch is fixed. +# +# Pointing ``LINK`` at the last revision that still carries the table +# (``oldid=1368657333``) would have worked, but it buys none of what a crawler +# is for: a network fetch pinned to a frozen snapshot is all of the fragility +# with none of the freshness, and it would still be selecting the table by +# index out of a document that nothing stops from changing shape again. The +# registry itself is closed, which is what settles it -- IPX socket numbers +# were Novell's to assign, and Novell's own documentation still ends the list +# with "software developers writing NetWare applications can contact Novell to +# reserve well-known sockets". There is no longer a Novell to contact, and IPX +# will not gain new socket numbers, so there is nothing for a crawler to pick +# up on its next run. +# +# The table below is therefore transcribed from that last revision that carried +# it, row for row, and each entry names the primary source for the assignment +# where one exists. +############################################################################### + +#: Socket number registry, transcribed from the last revision of the Wikipedia +#: article that still carried the table (2026-08-10), +#: https://en.wikipedia.org/w/index.php?title=Internetwork_Packet_Exchange&oldid=1368657333 +#: +#: Maps a socket number to the enumeration name it takes and the comment +#: rendered above it. Names go through :meth:`~Vendor.rename`, so the generated +#: member is the :meth:`~Vendor.safe_name` of the first element. +DATA = { + # NOTE: 0x0000 was never listed as a well-known socket in the scraped + # registry table, but it is the IPX protocol's own default for the + # ``dst``/``src`` socket field (an ordinary "unspecified socket"), so it + # must be present regardless of what the registry says; see #492, #503. + # + # That is a statement about the *scraped table*, not about the number being + # unsourced. XSIS 028112 Appendix D (cited under RANGES below) reserves it + # in the primary standard: "The socket numbers zero and all ones are + # reserved to mean unknown and all, respectively." The same sentence + # reserves all-ones -- 0xFFFF, to mean "all" -- and this table has no entry + # for that; ``_missing_`` resolves 0xFFFF through the "Dynamically Assigned" + # range instead. Adding it would change the generated file, so it is left as + # a follow-up rather than done here. + 0x0000: ('Unspecified', "Unspecified socket; this is IPX's own default for the dst/src socket field."), + + # Sockets in the "Registered by Xerox" range below. These are formally + # XNS/IDP well-known sockets that IPX inherited, and the *values* have both + # a live registry and a primary standard behind them: + # + # * IANA, "Xerox Network System (XNS) Protocol Types", sub-registry + # "Assigned well-known socket numbers" -- + # https://www.iana.org/assignments/xns-protocol-types -- lists 1 Routing + # Information, 2 Echo and 3 Router Error, each referenced to "Xerox System + # Integration Standard: Internet Transport Protocols. XSIS 028112, + # December 1981". IANA publishes the registry but does not own it: its + # registration procedure reads "Not assigned by IANA". + # * That standard itself, Appendix D, "Assigned well-known socket numbers", + # scanned at http://www.bitsavers.org/pdf/xerox/xns/standards/XSIS_028112-Internet_Transport_Protocols_198112.pdf + # -- the same three functions, under a column headed "Well-Known Socket + # (octal)". + # + # The *names* are a different matter, and are the archived revision's own. + # Both sources call socket 3 "Router Error" rather than "Error Handling + # Packet", and neither appends "Packet" to any of the three. That suffix + # looks like the adjacent packet-type registry bleeding in -- XSIS Appendix + # E, "Assigned internet packet types", and the second table on the same IANA + # page, which do use 1 Routing Information / 2 Echo / 3 Error as *packet* + # types. The names are kept as transcribed anyway, for the same reason + # ``LLC_4`` is kept below: changing one renames a public member. + 0x0001: ('Routing Information Packet', 'Routing Information Packet'), + 0x0002: ('Echo Protocol Packet', 'Echo Protocol Packet'), + 0x0003: ('Error Handling Packet', 'Error Handling Packet'), + + # Novell, *IPX Addressing*, "Table 2. NetWare Socket Numbers and + # Processes", https://www.novell.com/documentation/nw6p/ipx_enu/data/hvvqznoa.html + # -- still live. That page is also the citation the article itself used. + # + # NOTE: Table 2 has **seven** rows and only five of them are below. In full + # it reads 0x451 NCP, 0x452 SAP, 0x453 RIP, 0x455 Novell NetBIOS, 0x456 + # Diagnostics, 0x9001 NLSP, 0x9004 "IPXWAN(TM) protocol". The last two have + # never been in this enumeration, and 0x9004 is independently reserved by + # :rfc:`1634` line 552 -- "The socket number 0x9004 is a Novell reserved + # socket number for exclusive use with IPX WAN protocol exchange" -- and by + # the two RFCs it obsoletes: :rfc:`1551` line 532 word for word, and + # :rfc:`1362` line 253 with "information exchange" for "protocol exchange". + # So they are known omissions with good sourcing, not rows nobody noticed. + # Adding them would add members to the generated file and so give up the + # byte-identity proof this change rests on, which makes them a follow-up + # rather than part of #507. + # + # Two of the five below also depart from Novell's own wording: Novell writes + # "Novell NetBIOS" and "Diagnostics" where this table says "NetBIOS" and + # "Diagnostic Packet". Those are the archived revision's strings, kept for + # the same member-renaming reason as everything else here. + 0x0451: ('NetWare Core Protocol', 'NetWare Core Protocol, NCP – used by Novell NetWare servers'), + 0x0452: ('Service Advertising Protocol', 'Service Advertising Protocol, SAP'), + 0x0453: ('Routing Information Protocol', 'Routing Information Protocol, RIP'), + 0x0455: ('NetBIOS', 'NetBIOS'), + 0x0456: ('Diagnostic Packet', 'Diagnostic Packet'), + + # These two are the only rows the archived revision alone supplies: absent + # from Novell's Table 2, absent from the Xerox registry above, and absent + # from every RFC swept. The sweep covered RFCs 1132, 1362, 1377, 1551, 1552, + # 1553, 1634, 1791, 1973 and 2043, looking for the five codes 0x0001, 0x0002, + # 0x0003, 0x0457 and 0x4003, and found 0 hits. (The first three of those five + # turned out to be sourced after all, just not from an RFC -- see the Xerox + # note above.) + # + # "0 hits" is a claim about those five codes and nothing more. Three of the + # ten RFCs do assign an IPX socket, 0x9004, which this table lacks; see the + # note on Novell's Table 2 above. + # + # 0x4003 is the weaker of the two: it falls inside Novell's own + # 0x4000-0x7FFF *dynamic* range, which is assigned to workstations on + # demand, so a fixed well-known name for it is inherently low-confidence. + 0x0457: ('Serialization Packet', 'Serialization Packet, used for NCP as well'), + 0x4003: ('Used by Novell NetWare Client', 'Used by Novell NetWare Client'), + + # NOTE: :rfc:`1132`, "A Standard for the Transmission of 802.2 Packets over + # IPX Networks", reserves this one -- "The IPX socket 0x8060 has been + # reserved by Novell for the implementation of this protocol." + # + # The archived table renders the row as ``LLC[4]``, where ``[4]`` is + # Wikipedia's own footnote marker for its citation of that RFC rather than + # any part of the protocol name -- the cell markup is an ```` reading + # "LLC" followed by ````. The spaces in + # ``LLC [ 4 ]`` are not in the page at all: they are an artefact of the + # scrape, which joined the cell's strings with a space and so picked up the + # ```` wrappers around the ``[`` and ``]`` as + # separate tokens. The value is right and the explanation of it was not. + # Both the name and the comment are kept verbatim so that + # regenerating the constant file stays a no-op against the last scraped + # output: renaming the member would break + # :attr:`pcapkit.const.ipx.socket.Socket.LLC_4` for anyone using it, which + # is a call for the maintainer rather than for #507. + 0x8060: ('LLC_4', 'LLC [ 4 ]'), + + # :rfc:`1791`, "TCP And UDP Over IPX Networks With Fixed Path MTU" -- "The + # IPX socket number 0x9091 is reserved for the TCP", "UDP must send and + # receive the packets on IPX/IPXF socket 0x9092", and "IPXF fragments are + # received by IPXF on the IPX socket 0x9093". + # + # NOTE: the name ``TCP over IPXF`` is wrong, and it is the archived + # revision's wording rather than the RFC's. RFC 1791 reserves 0x9091 for TCP + # over plain **IPX**: the section that assigns it is s3, "Running TCP Over + # IPX", which opens "Unlike UDP, TCP runs directly over IPX", and the same + # section ends "Hence, running TCP over IPXF is not recommended." So the + # member name describes the one arrangement the cited RFC advises against. + # Only the name is wrong; 0x9091 is the right value for TCP. It is kept for + # the member-renaming reason set out above, and is the strongest candidate in + # this table for a rename with a deprecated alias. + # + # ``UDP over IPXF`` for 0x9092, by contrast, is accurate: "UDP must run on + # IPXF rather than directly on IPX", and the socket is named as + # "IPX/IPXF socket 0x9092". + # + # One more scruple about wording: "reserved" is the RFC's own word for + # 0x9091 only. For 0x9092 and 0x9093 it says packets "must" be sent and + # received on them, which amounts to the same reservation in practice but is + # an inference rather than a quote. + 0x9091: ('TCP over IPXF', 'TCP over IPXF'), + 0x9092: ('UDP over IPXF', 'UDP over IPXF'), + 0x9093: ('IPXF', 'IPXF, IPX Fragmentation Protocol'), +} # type: dict[int, tuple[str, str]] + +#: Socket number ranges, transcribed from the same revision as +#: :data:`~pcapkit.vendor.ipx.socket.DATA`, as ``(start, stop, name)``. +#: +#: Three of the five have a primary source, and it is not the archived +#: revision. Xerox, *Internet Transport Protocols*, XSIS 028112, December 1981, +#: Appendix D: "The socket numbers zero and all ones are reserved to mean +#: unknown and all, respectively. Well-known socket numbers have the range 1 to +#: 3000 decimal. All other socket numbers are ephemeral, that is, they may be +#: dynamically assigned and reused." Its table then gives "Experimental 40-77" +#: under a column header that the scan OCRs as ``goctalf`` and which reads +#: "Well-Known Socket (octal)". +#: +#: That reproduces the first three ranges exactly, and the arithmetic confirms +#: the radix without having to trust that OCR: 3000 decimal is 0x0BB8, 3001 is +#: 0x0BB9, and octal 40-77 is 0x0020-0x003F -- all three as transcribed. Read as +#: decimal instead, 40-77 would be 0x0028-0x004D, which is not what the table +#: says. So the octal reading is settled by the numbers themselves. +#: +#: Novell's *IPX Addressing* bears on the last two, and in opposite directions. +#: ``(0x8000, 0xFFFF)`` matches it exactly -- "Socket numbers between 0x8000 and +#: 0xFFFF are well-known sockets; these are assigned by Novell to specific +#: processes." ``(0x4000, 0x4FFF)`` is *contradicted*, not merely rounded: the +#: preceding sentence reads "Socket numbers between 0x4000 and 0x7FFF are +#: dynamic sockets", so the upper bound is 0x7FFF. The transcribed 0x4FFF is +#: kept because widening it would change the generated ``_missing_``; the +#: archived table is what is wrong here, and this range is masked by +#: ``(0x0BB9, 0xFFFF)`` in any case (see the note below), so the bound has no +#: observable effect today. +RANGES = [ + # NOTE: order is significant, and is the order the rows appeared in. The + # generated ``_missing_`` tests these in sequence and returns on the first + # match, so the wide ranges here mask the narrow ones that follow them -- + # reordering the list silently changes which name an unlisted socket gets. + (0x0001, 0x0BB8, 'Registered by Xerox'), + (0x0020, 0x003F, 'Experimental'), + (0x0BB9, 0xFFFF, 'Dynamically Assigned'), + (0x4000, 0x4FFF, 'Dynamically Assigned Socket Numbers'), + (0x8000, 0xFFFF, 'Statically Assigned Socket Numbers'), +] # type: list[tuple[int, int, str]] + class Socket(Vendor): """Socket Types""" #: Value limit checker. FLAG = 'isinstance(value, int) and 0x0000 <= value <= 0xFFFF' - #: Link to registry. - LINK = 'https://en.wikipedia.org/wiki/Internetwork_Packet_Exchange#Socket_number' - def count(self, data: 'BeautifulSoup') -> 'Counter[str]': - """Count field records. - - Args: - data: Registry data. + def request(self) -> 'dict[int, tuple[str, str]]': # type: ignore[override] # pylint: disable=arguments-differ + """Fetch registry data. Returns: - Field recordings. + Registry data (:data:`~pcapkit.vendor.ipx.socket.DATA`). """ - return collections.Counter() + return DATA - def request(self, text: 'str') -> 'BeautifulSoup': # type: ignore[override] # pylint: disable=signature-differs - """Fetch HTML source. + def count(self, data: 'dict[int, tuple[str, str]]') -> 'Counter[str]': # type: ignore[override] + """Count field records. Args: - text: Context from :attr:`~Vendor.LINK`. + data: Registry data. Returns: - Parsed HTML source. + Field recordings. """ - return bs4.BeautifulSoup(text, 'html5lib') + return collections.Counter(self.safe_name(name) for name, _ in data.values()) - def process(self, soup: 'BeautifulSoup') -> 'tuple[list[str], list[str]]': # pylint: disable=arguments-differ,arguments-renamed - """Process HTML source. + def process(self, data: 'dict[int, tuple[str, str]]') -> 'tuple[list[str], list[str]]': # type: ignore[override] + """Process registry data. Args: - data: Parsed HTML source. + data: Registry data. Returns: Enumeration fields and missing fields. """ - table = soup.find_all('table', class_='wikitable')[3] - content = filter(lambda item: isinstance(item, bs4.element.Tag), table.tbody) - next(content) # header - enum = [] # type: list[str] miss = [] # type: list[str] - # NOTE: 0x0000 is not listed as a well-known socket in the Wikipedia - # registry table scraped below, but it is the IPX protocol's own - # default for the ``dst``/``src`` socket field (an ordinary - # "unspecified socket"), so it must be present regardless of what the - # scraped table contains; see #492. - desc = self.wrap_comment("Unspecified socket; this is IPX's own default for the dst/src socket field.") - enum.append(f'#: {desc}\n Unspecified = 0x0000') - - for item in content: - line = item.find_all('td') - - pval = ' '.join(line[0].stripped_strings) - dscp = ' '.join(line[1].stripped_strings) - - data = list(filter(None, map(lambda s: s.strip(), re.split(r'\W*,|\(|\)\W*', dscp)))) - if len(data) == 2: - name, desc = data - else: - name, desc = dscp, '' - renm = self.safe_name(name) - - tmp1 = f', {desc}' if desc else '' - desc = self.wrap_comment(f'{name}{tmp1}') - - try: - code, _ = pval, int(pval, base=16) - - pres = f"{renm} = {code}" - sufs = f'#: {desc}' - - # if len(pres) > 74: - # sufs = f"\n{' '*80}{sufs}" + for code, (name, desc) in data.items(): + pval = f'0x{code:04X}' + renm = self.rename(name, pval) - # enum.append(f'{pres.ljust(76)}{sufs}') - enum.append(f'{sufs}\n {pres}') - except ValueError: - start, stop = pval.split('–') + enum.append(f'#: {self.wrap_comment(desc)}\n {renm} = {pval}') - miss.append(f'if {start} <= value <= {stop}:') - miss.append(f' #: {desc}') - miss.append(f" return extend_enum(cls, '{name}_0x%s' % hex(value)[2:].upper().zfill(4), value)") + # NOTE: the range names are emitted raw, spaces and all, rather than + # through ``safe_name``, because that is what the scrape did and what + # the shipped constant file contains; sanitising them here would rename + # every member ``_missing_`` extends the enumeration with; see #507. + for start, stop, name in RANGES: + miss.append(f'if 0x{start:04X} <= value <= 0x{stop:04X}:') + miss.append(f' #: {self.wrap_comment(name)}') + miss.append(f" return extend_enum(cls, '{name}_0x%s' % hex(value)[2:].upper().zfill(4), value)") return enum, miss diff --git a/pyproject.toml b/pyproject.toml index 3f160dd70..2d69ea4ee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -231,6 +231,27 @@ docs = [ test = [ "pytest>=8", "typing-extensions", + # Enough of the ``vendor`` extra for a test to import a vendor crawler. + # Without these, ``tests/vendor/test_ipx_socket_unit.py`` reports "9 skipped" + # rather than failing, so the byte-for-byte regeneration check that is the + # only guard on the hand-maintained IPX socket registry never runs on any CI + # path: the ``unit-tests.yml`` jobs install ``.[test]``, and + # ``cron-vendor.yml`` installs ``.[all]`` but never invokes pytest. + # + # Both are needed, not just ``requests``. ``pcapkit.vendor.default`` imports + # ``requests`` at module scope, and ``pcapkit.vendor.__init__`` then imports + # every crawler unconditionally -- seven of which import ``bs4`` at module + # scope. Its ``try: import bs4`` guard suppresses only the warning, not the + # ``ModuleNotFoundError`` that the crawler imports below it raise anyway. + # + # Plain, without the ``vendor`` extra's ``[socks]`` and ``[html5lib]``: + # nothing in the test suite goes through a proxy, and no unit-tier test + # parses HTML (none of them may -- the tier makes no network call), so + # neither parser nor proxy support is reachable from here. Their absence + # makes :mod:`pcapkit.vendor` log two ``VendorWarning``s at import, which + # pytest captures and shows only on failure; they change no result. See #507. + "requests", + "beautifulsoup4", ] # setuptools specific diff --git a/tests/vendor/__init__.py b/tests/vendor/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/vendor/test_ipx_socket_unit.py b/tests/vendor/test_ipx_socket_unit.py new file mode 100644 index 000000000..5a7ea26ee --- /dev/null +++ b/tests/vendor/test_ipx_socket_unit.py @@ -0,0 +1,274 @@ +# -*- coding: utf-8 -*- +"""Regression tests for the retired :mod:`pcapkit.vendor.ipx.socket` scrape. + +GitHub issue #507: the crawler used to scrape the well-known-socket table out of +the Wikipedia *Internetwork Packet Exchange* article, and both halves of that +stopped working. Wikipedia answers ``requests``' default User-Agent with HTTP +403, so :meth:`pcapkit.vendor.default.Vendor._request` could not fetch the page; +and the table was deleted from the article on 2026-08-25, so there was nothing +left at ``find_all('table', class_='wikitable')[3]`` to parse even with the fetch +fixed. The scrape is retired in favour of the hand-maintained +:data:`pcapkit.vendor.ipx.socket.DATA`. + +What that leaves worth pinning is the failure mode the issue is actually about: +a regeneration that quietly drops sockets. :data:`EXPECTED_MEMBERS` spells out +the enumeration in full, so losing one fails here rather than shipping. The +value ``0x0000`` in it is the member GitHub issue #503 added for GitHub issue +#492 -- IPX's own default for the ``dst``/``src`` socket field -- which the old +crawler had to prepend unconditionally *because* the scrape yielded nothing, and +which a hand-maintained table has no excuse to lose. + +The suite is unit-tier (see :mod:`tests._tiers`): it reads no capture and, by +the whole point of the change, makes no network call. + +""" +from __future__ import annotations + +import importlib.util +import pathlib +import unittest +from typing import TYPE_CHECKING + +from tests._support import purge_modules + +if TYPE_CHECKING: + from typing import Any + +#: Repository root, i.e. the grandparent of the directory holding this file. +ROOT = pathlib.Path(__file__).resolve().parents[2] + +#: Whether the crawler machinery is importable at all. Two packages are needed +#: and neither is optional to this file: ``pcapkit.vendor.default`` imports +#: :mod:`requests` at module scope, and :mod:`pcapkit.vendor` then imports every +#: crawler unconditionally, seven of which import :mod:`bs4` at module scope -- +#: so importing *one* crawler needs both. (:mod:`pcapkit.vendor`'s own +#: ``try: import bs4`` suppresses the warning, not the ``ModuleNotFoundError`` +#: raised by the crawler imports below it.) +#: +#: Both now ship in the ``test`` extra as well as ``vendor`` +#: (:file:`pyproject.toml`), because CI installs ``.[test]`` and a skipped pin +#: guards nothing. The guard is kept as belt-and-braces for an environment that +#: lacks them anyway, so this file skips rather than erroring -- the same way +#: :file:`tests/protocols/test_dispatch_registry_unit.py` guards its own optional +#: dependencies. See #507. +HAS_CRAWLER_DEPS = all(importlib.util.find_spec(name) is not None + for name in ('requests', 'bs4')) + +#: Every member the generated :class:`pcapkit.const.ipx.socket.Socket` is +#: expected to carry, as ``(name, value)`` in definition order. Spelled out +#: rather than derived so that a regeneration which loses a socket -- the exact +#: failure GitHub issue #507 describes -- fails this test instead of passing a +#: comparison against its own output. +EXPECTED_MEMBERS = ( + ('Unspecified', 0x0000), + ('Routing_Information_Packet', 0x0001), + ('Echo_Protocol_Packet', 0x0002), + ('Error_Handling_Packet', 0x0003), + ('NetWare_Core_Protocol', 0x0451), + ('Service_Advertising_Protocol', 0x0452), + ('Routing_Information_Protocol', 0x0453), + ('NetBIOS', 0x0455), + ('Diagnostic_Packet', 0x0456), + ('Serialization_Packet', 0x0457), + ('Used_by_Novell_NetWare_Client', 0x4003), + ('LLC_4', 0x8060), + ('TCP_over_IPXF', 0x9091), + ('UDP_over_IPXF', 0x9092), + ('IPXF', 0x9093), +) + +#: The socket number ranges, in the order :meth:`Socket.process` emits them and +#: therefore the order the generated ``_missing_`` tests them in. Pinned because +#: that order decides which name an unlisted socket is given: the wide ranges +#: mask the narrow ones after them, so reordering the list changes behaviour +#: without changing any member. +EXPECTED_RANGES = ( + (0x0001, 0x0BB8, 'Registered by Xerox'), + (0x0020, 0x003F, 'Experimental'), + (0x0BB9, 0xFFFF, 'Dynamically Assigned'), + (0x4000, 0x4FFF, 'Dynamically Assigned Socket Numbers'), + (0x8000, 0xFFFF, 'Statically Assigned Socket Numbers'), +) + +#: Sampled sockets and the member name each is expected to resolve to, at the +#: bounds of every range plus the gaps between. The *name* is pinned as well as +#: the value because the value alone cannot tell a live range branch from a +#: masked one: every branch of the generated ``_missing_`` returns ``value``, so +#: only the name says which branch ran, and asserting the value alone would pass +#: just as happily with three of the five branches deleted. +#: +#: Three of them are in fact unreachable -- ``(0x0001, 0x0BB8)`` masks +#: ``(0x0020, 0x003F)``, and ``(0x0BB9, 0xFFFF)`` masks both +#: ``(0x4000, 0x4FFF)`` and ``(0x8000, 0xFFFF)``. That is preserved scrape +#: behaviour rather than a defect this change introduces, and it is documented on +#: :data:`pcapkit.vendor.ipx.socket.RANGES`; pinning the names is what makes that +#: documentation fail here if it ever stops being true. +EXPECTED_MISSING_NAMES = { + 0x0000: 'Unspecified', # a defined member + 0x0001: 'Routing_Information_Packet', # a defined member + 0x0004: 'Registered by Xerox_0x0004', + 0x0020: 'Registered by Xerox_0x0020', # masks 'Experimental' + 0x003F: 'Registered by Xerox_0x003F', # masks 'Experimental' + 0x0BB8: 'Registered by Xerox_0x0BB8', + 0x0BB9: 'Dynamically Assigned_0x0BB9', + 0x4000: 'Dynamically Assigned_0x4000', # masks 'Dynamically Assigned Socket Numbers' + 0x4FFF: 'Dynamically Assigned_0x4FFF', # masks 'Dynamically Assigned Socket Numbers' + 0x7FFF: 'Dynamically Assigned_0x7FFF', + 0x8000: 'Dynamically Assigned_0x8000', # masks 'Statically Assigned Socket Numbers' + 0x8061: 'Dynamically Assigned_0x8061', # masks 'Statically Assigned Socket Numbers' + 0x9094: 'Dynamically Assigned_0x9094', # masks 'Statically Assigned Socket Numbers' + 0xFFFF: 'Dynamically Assigned_0xFFFF', # masks 'Statically Assigned Socket Numbers' +} + + +def _normalize(context: 'str') -> 'str': + """Apply the whitespace normalisation the generator writes files through. + + :meth:`pcapkit.vendor.default.Vendor.__init__` does not write what + :meth:`~pcapkit.vendor.default.Vendor.context` returns verbatim: it strips + trailing whitespace from every non-blank line, drops whitespace-only lines + outright, and ends the file with a newline courtesy of :func:`print`. Byte + comparison against the committed file has to do the same, or it reports a + difference that regeneration would not actually produce. + + Args: + context: Return value of :meth:`~pcapkit.vendor.default.Vendor.context`. + + Returns: + The text as the generator would have written it to disk. + + """ + lines = [] # type: list[str] + for line in context.splitlines(): + if line: + if line.strip(): + lines.append(line.rstrip()) + else: + lines.append(line) + return '\n'.join(lines) + '\n' + + +@unittest.skipUnless(HAS_CRAWLER_DEPS, 'requests and/or beautifulsoup4 not installed') +class IPXSocketVendorTests(unittest.TestCase): + """The hand-maintained registry, and the crawler that no longer crawls.""" + + if TYPE_CHECKING: + vendor_module: 'Any' + const_module: 'Any' + + def setUp(self) -> None: + purge_modules(['pcapkit']) + + import pcapkit.const.ipx.socket as const_module + import pcapkit.vendor.ipx.socket as vendor_module + + # Both modules have to come from this checkout for any of the + # comparisons below to mean anything: the generated text is compared + # against this repository's constant file, so a vendor module imported + # from an installed copy elsewhere would be comparing two trees. That is + # an environment mismatch rather than a defect, hence a skip. + for module in (vendor_module, const_module): + resolved = pathlib.Path(module.__file__).resolve() + if ROOT not in resolved.parents: + self.skipTest(f'{module.__name__} was imported from {resolved}, which is outside ' + f'{ROOT}; install this checkout with `pip install -e .` to run this ' + f'suite against it') + + self.vendor_module = vendor_module + self.const_module = const_module + + def _vendor(self) -> 'Any': + """A crawler instance with the attributes ``__init__`` would have set. + + ``Vendor.__init__`` regenerates and *writes* the constant file as a side + effect of construction, which a test has no business doing to the + working tree, so the three attributes it sets -- ``NAME``, ``DOCS`` and + ``record``, at :file:`pcapkit/vendor/default.py` lines 338, 340 and 343 + -- are set here instead. + + """ + vendor = self.vendor_module.Socket.__new__(self.vendor_module.Socket) + vendor.NAME = self.vendor_module.Socket.__name__ + vendor.DOCS = self.vendor_module.Socket.__doc__ + data = vendor.request() + vendor.record = vendor.count(data) + return vendor + + def test_link_is_none_so_nothing_is_fetched(self) -> None: + # The retirement, stated as an assertion: with no LINK, + # Vendor._request() short-circuits to Socket.request() and never + # reaches requests.get() -- which is what used to take the 403. + self.assertIsNone(self.vendor_module.Socket.LINK) + + def test_request_makes_no_network_call(self) -> None: + import requests + + def explode(*args: 'Any', **kwargs: 'Any') -> 'Any': + raise AssertionError(f'the crawler made a network call: {args!r}') + + original_get, original_request = requests.get, requests.Session.request + requests.get = explode # type: ignore[assignment] + requests.Session.request = explode # type: ignore[assignment,method-assign] + try: + vendor = self.vendor_module.Socket.__new__(self.vendor_module.Socket) + vendor.NAME = self.vendor_module.Socket.__name__ + vendor.DOCS = self.vendor_module.Socket.__doc__ + self.assertIs(vendor._request(), self.vendor_module.DATA) # pylint: disable=protected-access + finally: + requests.get = original_get # type: ignore[assignment] + requests.Session.request = original_request # type: ignore[method-assign] + + def test_regeneration_reproduces_the_committed_constant_file(self) -> None: + # The guard that makes the hand-maintained table trustworthy: running + # the crawler must be a no-op against what is checked in, so an edit to + # DATA that was never regenerated shows up here. + vendor = self._vendor() + generated = _normalize(vendor.context(vendor.request())) + committed = (ROOT / 'pcapkit' / 'const' / 'ipx' / 'socket.py').read_text(encoding='utf-8') + self.assertEqual(generated, committed, + 'regenerating pcapkit/const/ipx/socket.py would change it; run ' + '`python -m pcapkit.vendor.ipx.socket` and commit the result') + + def test_no_socket_is_lost(self) -> None: + members = tuple((member.name, int(member.value)) for member in self.const_module.Socket) + self.assertEqual(members, EXPECTED_MEMBERS) + + def test_registry_and_enumeration_agree(self) -> None: + # Every hand-maintained row reaches the enumeration, and nothing in the + # enumeration came from anywhere else. + vendor = self._vendor() + from_data = tuple( + (vendor.rename(name, f'0x{code:04X}'), code) + for code, (name, _) in self.vendor_module.DATA.items() + ) + self.assertEqual(from_data, EXPECTED_MEMBERS) + + def test_unspecified_socket_survives_the_retirement(self) -> None: + # GitHub issue #503's member, for GitHub issue #492: 0x0000 is IPX's own + # default for the dst/src socket field, and was never in the scraped + # table at all. + self.assertEqual(self.const_module.Socket(0x0000), self.const_module.Socket.Unspecified) + self.assertEqual(self.const_module.Socket(0x0000).value, 0) + self.assertIn(0x0000, self.vendor_module.DATA) + + def test_range_order_is_preserved(self) -> None: + self.assertEqual(tuple(self.vendor_module.RANGES), EXPECTED_RANGES) + + def test_unlisted_sockets_still_resolve(self) -> None: + # _missing_ has to cover the whole 16-bit space, so no legal wire value + # raises -- and it has to reach the branch it looks like it reaches. + for value, name in EXPECTED_MISSING_NAMES.items(): + with self.subTest(socket=f'0x{value:04X}'): + member = self.const_module.Socket(value) + self.assertEqual(int(member), value) + self.assertEqual(member.name, name) + + def test_out_of_range_sockets_are_rejected(self) -> None: + for value in (-1, 0x10000): + with self.subTest(socket=value): + with self.assertRaises(ValueError): + self.const_module.Socket(value) + + +if __name__ == '__main__': + unittest.main()