fix(vendor): send a descriptive User-Agent and retarget the dead pcapng draft

docs/source/pcapkit/const/pcapng.rst

@@ -120,10 +120,10 @@ which is automatically generated from :class:`pcapkit.vendor.pcapng.filter_type.
 
    <br />
 
-.. [*] https://www.ietf.org/staging/draft-tuexen-opsawg-pcapng-02.html#name-standardized-block-type-cod
-.. [*] https://www.ietf.org/staging/draft-tuexen-opsawg-pcapng-02.html#name-options
-.. [*] https://www.ietf.org/staging/draft-tuexen-opsawg-pcapng-02.html#name-enhanced-packet-block-flags
-.. [*] https://www.ietf.org/staging/draft-tuexen-opsawg-pcapng-02.html#name-enhanced-packet-block
-.. [*] https://www.ietf.org/staging/draft-tuexen-opsawg-pcapng-02.html#name-name-resolution-block
-.. [*] https://www.ietf.org/staging/draft-tuexen-opsawg-pcapng-02.html#name-decryption-secrets-block
-.. [*] https://www.ietf.org/staging/draft-tuexen-opsawg-pcapng-02.html#name-interface-description-block
+.. [*] https://www.ietf.org/archive/id/draft-tuexen-opsawg-pcapng-03.html#name-standardized-block-type-cod
+.. [*] https://www.ietf.org/archive/id/draft-tuexen-opsawg-pcapng-03.html#name-options
+.. [*] https://www.ietf.org/archive/id/draft-tuexen-opsawg-pcapng-03.html#name-enhanced-packet-block-flags
+.. [*] https://www.ietf.org/archive/id/draft-tuexen-opsawg-pcapng-03.html#name-enhanced-packet-block
+.. [*] https://www.ietf.org/archive/id/draft-tuexen-opsawg-pcapng-03.html#name-name-resolution-block
+.. [*] https://www.ietf.org/archive/id/draft-tuexen-opsawg-pcapng-03.html#name-decryption-secrets-block
+.. [*] https://www.ietf.org/archive/id/draft-tuexen-opsawg-pcapng-03.html#name-interface-description-block

docs/source/pcapkit/protocols/misc/pcapng.rst

@@ -770,4 +770,4 @@ Data Models
 
 .. rubric:: Footnotes
 
-.. [*] https://www.ietf.org/staging/draft-tuexen-opsawg-pcapng-02.html
+.. [*] https://www.ietf.org/archive/id/draft-tuexen-opsawg-pcapng-03.html

docs/source/pcapkit/vendor/pcapng.rst

@@ -25,6 +25,21 @@ vendor crawlers include:
    * - :class:`PCAPNG_FilterType <pcapkit.vendor.pcapng.filter_type.FilterType>`
      - Filter Types [*]_
 
+.. note::
+
+   The three crawlers that scrape a registry table --
+   :class:`BlockType <pcapkit.vendor.pcapng.block_type.BlockType>`,
+   :class:`OptionType <pcapkit.vendor.pcapng.option_type.OptionType>` and
+   :class:`RecordType <pcapkit.vendor.pcapng.record_type.RecordType>` -- all read
+   revision ``-03`` of the draft, as each one's ``LINK`` below shows. They used to
+   read ``https://www.ietf.org/staging/draft-tuexen-opsawg-pcapng-02.html``, which
+   is dead: measured 2026-09-19 as a 404 under any User-Agent. The revision number
+   was wrong as well as the path -- ``-02`` renders its registries as ASCII art
+   inside ``<pre>`` and carries no registry ``<table>`` at all, so the ``table-1``
+   through ``table-10`` ids the three crawlers select on first exist in ``-03``,
+   where the registries became real tables. The footnotes below point at ``-03``
+   for the same reason. See #518.
+
 Block Types
 ===========
 
@@ -111,10 +126,10 @@ which is automatically generating :class:`pcapkit.const.pcapng.filter_type.Filte
 
 .. rubric:: Footnotes
 
-.. [*] https://www.ietf.org/staging/draft-tuexen-opsawg-pcapng-02.html#name-standardized-block-type-cod
-.. [*] https://www.ietf.org/staging/draft-tuexen-opsawg-pcapng-02.html#name-options
-.. [*] https://www.ietf.org/staging/draft-tuexen-opsawg-pcapng-02.html#name-enhanced-packet-block-flags
-.. [*] https://www.ietf.org/staging/draft-tuexen-opsawg-pcapng-02.html#name-enhanced-packet-block
-.. [*] https://www.ietf.org/staging/draft-tuexen-opsawg-pcapng-02.html#name-name-resolution-block
-.. [*] https://www.ietf.org/staging/draft-tuexen-opsawg-pcapng-02.html#name-decryption-secrets-block
-.. [*] https://www.ietf.org/staging/draft-tuexen-opsawg-pcapng-02.html#name-interface-description-block
+.. [*] https://www.ietf.org/archive/id/draft-tuexen-opsawg-pcapng-03.html#name-standardized-block-type-cod
+.. [*] https://www.ietf.org/archive/id/draft-tuexen-opsawg-pcapng-03.html#name-options
+.. [*] https://www.ietf.org/archive/id/draft-tuexen-opsawg-pcapng-03.html#name-enhanced-packet-block-flags
+.. [*] https://www.ietf.org/archive/id/draft-tuexen-opsawg-pcapng-03.html#name-enhanced-packet-block
+.. [*] https://www.ietf.org/archive/id/draft-tuexen-opsawg-pcapng-03.html#name-name-resolution-block
+.. [*] https://www.ietf.org/archive/id/draft-tuexen-opsawg-pcapng-03.html#name-decryption-secrets-block
+.. [*] https://www.ietf.org/archive/id/draft-tuexen-opsawg-pcapng-03.html#name-interface-description-block

pcapkit/const/ftp/return_code.py

@@ -159,7 +159,7 @@ def __str__(self) -> 'str':
     #: Entering Long Passive Mode (long address, port).
     CODE_228: 'ReturnCode' = 228, 'Entering Long Passive Mode.'
 
-    #: Entering Extended Passive Mode (|||port|).
+    #: Entering Extended Passive Mode ( |||port| ).
     CODE_229: 'ReturnCode' = 229, 'Entering Extended Passive Mode.'
 
     #: User logged in, proceed.
@@ -189,6 +189,10 @@ def __str__(self) -> 'str':
     #: data needs to be exchanged.
     CODE_334: 'ReturnCode' = 334, 'Server accepts the security mechanism specified by the client; some security data needs to be exchanged.'
 
+    #: Server accepts the security data given by the client; more security data
+    #: needs to be exchanged.
+    CODE_335: 'ReturnCode' = 335, 'Server accepts the security data given by the client; more security data needs to be exchanged.'
+
     #: Username okay, password okay. Challenge is ". . . . ".
     CODE_336: 'ReturnCode' = 336, 'Username okay, password okay.'
 
@@ -211,14 +215,13 @@ def __str__(self) -> 'str':
     #: Requested host unavailable.
     CODE_434: 'ReturnCode' = 434, 'Requested host unavailable.'
 
-    #: Requested file action not taken.
+    #: Requested file action not taken. File unavailable (e.g., file busy).
     CODE_450: 'ReturnCode' = 450, 'Requested file action not taken.'
 
     #: Requested action aborted. Local error in processing.
     CODE_451: 'ReturnCode' = 451, 'Requested action aborted.'
 
-    #: Requested action not taken. Insufficient storage space in system. File
-    #: unavailable (e.g., file busy).
+    #: Requested action not taken. Insufficient storage space in system.
     CODE_452: 'ReturnCode' = 452, 'Requested action not taken.'
 
     #: Syntax error in parameters or arguments.

pcapkit/const/pcapng/__init__.py

@@ -26,13 +26,13 @@
    * - :class:`PCAPNG_FilterType <pcapkit.const.pcapng.filter_type.FilterType>`
      - Filter Types [*]_
 
-.. [*] https://www.ietf.org/staging/draft-tuexen-opsawg-pcapng-02.html#name-standardized-block-type-cod
-.. [*] https://www.ietf.org/staging/draft-tuexen-opsawg-pcapng-02.html#name-options
-.. [*] https://www.ietf.org/staging/draft-tuexen-opsawg-pcapng-02.html#name-enhanced-packet-block-flags
-.. [*] https://www.ietf.org/staging/draft-tuexen-opsawg-pcapng-02.html#name-enhanced-packet-block
-.. [*] https://www.ietf.org/staging/draft-tuexen-opsawg-pcapng-02.html#name-name-resolution-block
-.. [*] https://www.ietf.org/staging/draft-tuexen-opsawg-pcapng-02.html#name-decryption-secrets-block
-.. [*] https://www.ietf.org/staging/draft-tuexen-opsawg-pcapng-02.html#name-interface-description-block
+.. [*] https://www.ietf.org/archive/id/draft-tuexen-opsawg-pcapng-03.html#name-standardized-block-type-cod
+.. [*] https://www.ietf.org/archive/id/draft-tuexen-opsawg-pcapng-03.html#name-options
+.. [*] https://www.ietf.org/archive/id/draft-tuexen-opsawg-pcapng-03.html#name-enhanced-packet-block-flags
+.. [*] https://www.ietf.org/archive/id/draft-tuexen-opsawg-pcapng-03.html#name-enhanced-packet-block
+.. [*] https://www.ietf.org/archive/id/draft-tuexen-opsawg-pcapng-03.html#name-name-resolution-block
+.. [*] https://www.ietf.org/archive/id/draft-tuexen-opsawg-pcapng-03.html#name-decryption-secrets-block
+.. [*] https://www.ietf.org/archive/id/draft-tuexen-opsawg-pcapng-03.html#name-interface-description-block
 
 """
 

pcapkit/protocols/misc/pcapng.py

@@ -9,7 +9,7 @@
 :class:`~pcapkit.protocols.misc.pcapng.PCAPNG` only,
 which implements extractor for PCAP-NG file format [*]_.
 
-.. [*] https://www.ietf.org/staging/draft-tuexen-opsawg-pcapng-02.html
+.. [*] https://www.ietf.org/archive/id/draft-tuexen-opsawg-pcapng-03.html
 
 """
 import base64

pcapkit/vendor/default.py

@@ -12,6 +12,8 @@
 import collections
 import contextlib
 import csv
+import functools
+import importlib.metadata
 import inspect
 import os
 import re
@@ -22,6 +24,7 @@
 
 import requests
 
+from pcapkit import __version__
 from pcapkit.utilities.exceptions import VendorNotImplemented
 from pcapkit.utilities.logging import BOOLEAN_STATES
 from pcapkit.utilities.warnings import VendorRequestWarning, warn
@@ -35,6 +38,16 @@
 MAX_RETRY = int(os.environ.get('PCAPKIT_VENDOR_RETRY', 5)) or 1
 CI_MODE = BOOLEAN_STATES.get(os.environ.get('PCAPKIT_CI_MODE', 'false').casefold(), False)
 
+#: Distribution name of this package, i.e. the name its metadata is registered
+#: under, which is not the import name (:mod:`pcapkit`).
+DISTRIBUTION = 'pypcapkit'
+
+#: Project URL used as the contact address in :func:`get_user_agent` when the
+#: distribution metadata cannot be read, i.e. when running straight from a
+#: source checkout that was never installed. Kept in step with ``repository``
+#: under ``[project.urls]`` in :file:`pyproject.toml`.
+PROJECT_URL = 'https://github.com/JarryShaw/PyPCAPKit'
+
 #: Default constant template of enumerate registry from IANA CSV.
 LINE = lambda NAME, DOCS, FLAG, ENUM, MISS, MODL: f'''\
 # -*- coding: utf-8 -*-
@@ -114,6 +127,50 @@ def get_proxies() -> 'dict[str, str]':
     return PROXIES
 
 
+@functools.lru_cache(maxsize=1)
+def get_user_agent() -> 'str':
+    """Get the ``User-Agent`` header the crawlers identify themselves with.
+
+    Many :attr:`~Vendor.LINK` registries are Wikipedia articles, and the
+    Wikimedia Foundation's User-Agent policy refuses |requests|_' default
+    ``python-requests/<version>`` agent outright -- HTTP 403 with a body reading
+    *"Please set a user-agent and respect our robot policy"*. What the policy
+    asks for is an agent that names the tool and gives a contact address, so that
+    a misbehaving client can be reached instead of simply blocked. It does
+    **not** ask for a browser agent, and sending one would misrepresent what is
+    making the request, so this deliberately identifies the crawler as itself.
+
+    The string is composed from the package's own metadata -- distribution name,
+    :data:`pcapkit.__version__` and the ``repository`` project URL -- rather than
+    written out as a literal, so that it follows the package instead of going
+    stale. Where the distribution metadata cannot be read, i.e. when running from
+    a source checkout that was never installed, :data:`DISTRIBUTION` and
+    :data:`PROJECT_URL` stand in for it.
+
+    Returns:
+        Value for the ``User-Agent`` request header.
+
+    See Also:
+        `Wikimedia Foundation User-Agent policy
+        <https://foundation.wikimedia.org/wiki/Policy:Wikimedia_Foundation_User-Agent_Policy>`__
+
+    """
+    name = DISTRIBUTION
+    url = PROJECT_URL
+
+    with contextlib.suppress(importlib.metadata.PackageNotFoundError):
+        metadata = importlib.metadata.metadata(DISTRIBUTION)
+
+        name = metadata.get('Name') or name
+        for entry in metadata.get_all('Project-URL') or []:
+            label, _, value = entry.partition(',')
+            if label.strip().casefold() == 'repository' and value.strip():
+                url = value.strip()
+                break
+
+    return f'{name}/{__version__} (+{url}) python-requests/{requests.__version__}'
+
+
 class VendorMeta(abc.ABCMeta):
     """Meta class to add dynamic support to :class:`Vendor`.
 
@@ -395,13 +452,19 @@ def _request(self) -> 'list[str]':
         if self.LINK is None:
             return self.request()  # type: ignore[unreachable]
 
+        # NOTE: both branches below send this. Wikimedia rejects ``requests``'
+        # default agent with HTTP 403, so a crawler without it fetches 126 bytes
+        # of robot-policy text and retries MAX_RETRY times against a refusal
+        # that no amount of retrying will lift; see #518.
+        headers = {'User-Agent': get_user_agent()}
+
         try:
             counter = 1
             while True:
                 if counter > MAX_RETRY:
                     raise requests.exceptions.RequestException
 
-                page = requests.get(self.LINK)  # nosec: B113
+                page = requests.get(self.LINK, headers=headers)  # nosec: B113
                 if not page.ok or not page.text:
                     warn(f'Connection failed; retry for {counter}/{MAX_RETRY}...',
                          VendorRequestWarning, stacklevel=2)
@@ -423,7 +486,7 @@ def _request(self) -> 'list[str]':
                     if counter > MAX_RETRY:
                         raise
 
-                    page = requests.get(self.LINK, proxies=proxies)  # nosec: B113
+                    page = requests.get(self.LINK, headers=headers, proxies=proxies)  # nosec: B113
                     if not page.ok or not page.text:
                         warn(f'Connection failed; retry with proxy for {counter}/{MAX_RETRY}...',
                              VendorRequestWarning, stacklevel=2)

pcapkit/vendor/ftp/return_code.py

@@ -218,6 +218,17 @@ def process(self, soup: 'BeautifulSoup') -> 'list[str]':  # type: ignore[overrid
         for item in content:
             line = item.find_all('td')
 
+            # NOTE: MediaWiki renders a trailing ``|-`` row separator in the
+            # wikitext as an empty ``<tr class="mw-empty-elt">``, which revision
+            # 1354125851 of the article carries at the end of this table. It has
+            # no cells at all, so reading ``line[0]`` raised ``IndexError`` and
+            # took the whole crawler down. Skipping cell-less rows is the fix
+            # rather than indexing defensively further down, because a row that
+            # cannot supply both a code and an explanation has nothing to
+            # contribute either way; see #518.
+            if len(line) < 2:
+                continue
+
             code = ' '.join(line[0].stripped_strings)
             if len(code) != 3:
                 continue