From 6eee9bd9c69133dbe977af63d37ef44f5fe8b214 Mon Sep 17 00:00:00 2001 From: Mingwei Zhang Date: Wed, 5 Aug 2026 13:54:26 -0700 Subject: [PATCH 1/4] feat: add Cisco sh ip bgp text dump parser with unified MRT/text API Port the fixed-width text RIB dump parser from monocle (#143, #146) into bgpkit-parser so it can be used programmatically. The new parser::text_dump module handles PCH daily snapshots and route-views oix-full-snapshot files, extracting column offsets from the table header, handling multipath continuations and wrapped prefixes (including IPv6), and tolerating missing preambles (route-views sentinel peer identity). Add unified BgpkitParser constructors so text dumps integrate into the standard for-elem-in-parser loop: - new_text(path) / from_text_reader(r): parse a known text dump - new_auto(path) / from_auto_reader(r): peek first bytes, auto-dispatch text vs MRT Text-dump elements are materialized at construction (RIB snapshots are full snapshots, not streaming). All filter methods work on both paths. The default new(path) constructor remains MRT-only. 22 unit tests + 5 constructor integration tests cover detection, header parsing, continuations, wrapped prefixes, origin codes, route-views sentinel behavior, timestamp inference, auto-detection, and filter integration. Three examples run against real data: - parse_text_dump_pch: PCH daily snapshot via new_text - parse_text_dump_routeviews: route-views oix snapshot via from_text_reader - parse_text_dump_auto: new_auto on both text dump and MRT file Closes #320 --- CHANGELOG.md | 2 + examples/parse_text_dump_auto.rs | 28 + examples/parse_text_dump_pch.rs | 26 + examples/parse_text_dump_routeviews.rs | 40 ++ src/parser/iters/default.rs | 10 + src/parser/iters/fallible.rs | 10 + src/parser/mod.rs | 210 ++++++- src/parser/text_dump.rs | 821 +++++++++++++++++++++++++ 8 files changed, 1144 insertions(+), 3 deletions(-) create mode 100644 examples/parse_text_dump_auto.rs create mode 100644 examples/parse_text_dump_pch.rs create mode 100644 examples/parse_text_dump_routeviews.rs create mode 100644 src/parser/text_dump.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index d2526091..21475dc9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,8 @@ All notable changes to this project will be documented in this file. * **Early RIPE RIS MRT support**: Parse deprecated MRT Type 5 BGP OPEN, UPDATE, NOTIFY, KEEPALIVE, and STATE_CHANGE records, along with historical TABLE_DUMP v1 records that batch multiple entries and declare their physical length four bytes short. Record iteration preserves each physical TABLE_DUMP batch while element, update, and route iteration expands its entries. * **Historical RIPE regression fixtures**: Added original RRC00 update and bview gzip files from 1999 and January 2000 as repository-only, offline integration fixtures. * **RFC 10005 Link Bandwidth Extended Community**: Typed parsing and encoding for the BGP Link Bandwidth Extended Community in both transitive (`0x00`) and non-transitive (`0x40`) forms ([#299](https://github.com/bgpkit/bgpkit-parser/issues/299)). Exposes the Global Administrator, bandwidth in bytes per second, and transitivity, and preserves the wire type on encode. +* **Cisco `sh ip bgp` text dump parsing** ([#320](https://github.com/bgpkit/bgpkit-parser/issues/320)): new `parser::text_dump` module parses fixed-width text RIB dumps published by PCH (daily routing table snapshots) and route-views (`oix-full-snapshot-*.bz2`) into `BgpElem`s. Column offsets are derived from the table header so blank numeric columns (Metric, LocPrf, Weight) stay distinct from AS-path data; multipath continuation lines and wrapped prefixes (including IPv6) are supported. Route-views dumps without the `BGP table version` / `local AS` preamble parse with sentinel peer identity (`0.0.0.0` / AS0). Also provides `detect_text_dump` for format sniffing and `infer_timestamp_from_path` for PCH (`YYYY.MM.DD`) and route-views (`YYYY-MM-DD-HHMM`) file-name timestamps. Ported from monocle ([#143](https://github.com/bgpkit/monocle/pull/143), [#146](https://github.com/bgpkit/monocle/pull/146)). +* **Unified MRT/text-dump parser API**: `BgpkitParser` gains three new constructor groups that integrate text dumps into the standard `for elem in parser` iteration loop. `new_text(path)` / `from_text_reader(r)` parse a known text dump; `new_auto(path)` / `from_auto_reader(r)` peek the first bytes and auto-dispatch to the text or MRT path. Text-dump elements are fully materialized at construction (RIB snapshots are not streaming); all existing filter methods (`add_filter`, `with_filters`, etc.) work on both paths. The default `new(path)` constructor remains MRT-only. ### Fixed diff --git a/examples/parse_text_dump_auto.rs b/examples/parse_text_dump_auto.rs new file mode 100644 index 00000000..deae74f0 --- /dev/null +++ b/examples/parse_text_dump_auto.rs @@ -0,0 +1,28 @@ +use bgpkit_parser::BgpkitParser; + +/// This example demonstrates `BgpkitParser::new_auto`, which auto-detects +/// whether the input is an MRT file or a Cisco `sh ip bgp` text dump and +/// parses accordingly — no need to know the format in advance. +/// +/// The same `for elem in parser` loop works for both MRT and text dumps. +fn main() { + env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init(); + + // A Cisco `sh ip bgp` text dump from PCH — new_auto detects and parses it. + let text_url = "https://downloads.pch.net/files/Routing_Data/IPv4_daily_snapshots/2026/07/route-collector.bom2.pch.net/route-collector.bom2.pch.net-ipv4_bgp_routes.2026.07.01.gz"; + log::info!("auto-detecting and parsing text dump: {text_url}"); + let count = BgpkitParser::new_auto(text_url) + .unwrap() + .into_elem_iter() + .count(); + log::info!("text dump: {count} elements"); + + // A standard MRT RIB file — new_auto treats it as MRT (lazy streaming). + let mrt_url = "https://spaces.bgpkit.org/parser/update-example.gz"; + log::info!("auto-detecting and parsing MRT file: {mrt_url}"); + let count = BgpkitParser::new_auto(mrt_url) + .unwrap() + .into_elem_iter() + .count(); + log::info!("MRT file: {count} elements"); +} diff --git a/examples/parse_text_dump_pch.rs b/examples/parse_text_dump_pch.rs new file mode 100644 index 00000000..e8b37dbc --- /dev/null +++ b/examples/parse_text_dump_pch.rs @@ -0,0 +1,26 @@ +use bgpkit_parser::BgpkitParser; + +/// This example parses a PCH daily routing table snapshot, which is a Cisco +/// `sh ip bgp` fixed-width text dump (gzip-compressed), into `BgpElem`s. +/// +/// `BgpkitParser::new_text` auto-detects decompression, infers the snapshot +/// timestamp from the file name, and returns elements through the same +/// `for elem in parser` interface used for MRT files. +fn main() { + env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init(); + + let url = "https://downloads.pch.net/files/Routing_Data/IPv4_daily_snapshots/2026/07/route-collector.bom2.pch.net/route-collector.bom2.pch.net-ipv4_bgp_routes.2026.07.01.gz"; + + log::info!("opening {url}"); + let parser = BgpkitParser::new_text(url).unwrap(); + log::info!("parsing text dump"); + + let mut count = 0; + for elem in parser { + if count < 5 { + println!("{elem}"); + } + count += 1; + } + log::info!("parsed {count} elements"); +} diff --git a/examples/parse_text_dump_routeviews.rs b/examples/parse_text_dump_routeviews.rs new file mode 100644 index 00000000..b27555e0 --- /dev/null +++ b/examples/parse_text_dump_routeviews.rs @@ -0,0 +1,40 @@ +use bgpkit_parser::BgpkitParser; +use std::io::Read; + +/// This example parses a route-views `sh ip bgp` snapshot +/// (`oix-full-snapshot-*.bz2`) into `BgpElem`s. +/// +/// `BgpkitParser::new_text` handles the bzip2 decompression and timestamp +/// inference. Route-views snapshots omit the `BGP table version` / `local AS` +/// preamble, so parsed elements carry the sentinel peer identity `0.0.0.0` / +/// AS0. +/// +/// The full snapshot is the entire global routing table (gigabytes of text), +/// so this example caps the input at ~20 MB via a manual reader. Remove the +/// cap to parse the complete file. +fn main() { + env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init(); + + let url = "https://archive.routeviews.org/oix-route-views/2026.07/oix-full-snapshot-2026-07-01-0000.bz2"; + + log::info!("opening {url} (first 20 MB)"); + let mut reader = oneio::get_reader(url).unwrap(); + // Cap the input for this demonstration: read at most 20 MB of decompressed + // data, then feed it into the text-dump parser. + let mut buf = Vec::with_capacity(20 * 1024 * 1024); + let _ = reader.by_ref().take(20 * 1024 * 1024).read_to_end(&mut buf); + let timestamp = bgpkit_parser::parser::text_dump::infer_timestamp_from_path(url).unwrap_or(0.0); + let parser = + BgpkitParser::from_text_reader_with_timestamp(std::io::Cursor::new(buf), timestamp) + .unwrap(); + log::info!("parsing text dump (timestamp={timestamp})"); + + let mut count = 0; + for elem in parser { + if count < 5 { + println!("{elem}"); + } + count += 1; + } + log::info!("parsed {count} elements"); +} diff --git a/src/parser/iters/default.rs b/src/parser/iters/default.rs index 12191418..ccba498a 100644 --- a/src/parser/iters/default.rs +++ b/src/parser/iters/default.rs @@ -139,6 +139,16 @@ impl Iterator for ElemIterator { self.count += 1; loop { + // Fast path: drain pre-parsed text-dump elems directly, with filter support. + if let Some(elems) = &mut self.record_iter.parser.text_dump_elems { + while let Some(elem) = elems.pop_front() { + if elem.match_filters(&self.record_iter.parser.filters) { + return Some(elem); + } + } + return None; + } + if self.cache_elems.is_empty() { // refill cache elems loop { diff --git a/src/parser/iters/fallible.rs b/src/parser/iters/fallible.rs index d5edb152..e2fa7581 100644 --- a/src/parser/iters/fallible.rs +++ b/src/parser/iters/fallible.rs @@ -96,6 +96,16 @@ impl Iterator for FallibleElemIterator { fn next(&mut self) -> Option { loop { + // Fast path: drain pre-parsed text-dump elems directly, with filter support. + if let Some(elems) = &mut self.record_iter.parser.text_dump_elems { + while let Some(elem) = elems.pop_front() { + if elem.match_filters(&self.record_iter.parser.filters) { + return Some(Ok(elem)); + } + } + return None; + } + // First check if we have cached elements if !self.cache_elems.is_empty() { if let Some(elem) = self.cache_elems.pop() { diff --git a/src/parser/mod.rs b/src/parser/mod.rs index b11a7978..bd586947 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -1,7 +1,11 @@ /*! parser module maintains the main logic for processing BGP and MRT messages. */ -use std::io::Read; +use crate::models::{BgpElem, MrtRecord}; +use log::warn; +use std::collections::VecDeque; +use std::io::{BufReader, Cursor, Read}; +pub use text_dump::{detect_text_dump, infer_timestamp_from_path, parse_text_dump_with_timestamp}; #[macro_use] pub mod utils; @@ -11,14 +15,13 @@ pub mod filter; pub mod iters; pub mod mrt; pub mod rpki; +pub mod text_dump; #[cfg(feature = "rislive")] pub mod rislive; pub(crate) use self::utils::*; -use crate::models::MrtRecord; -use log::warn; pub use mrt::mrt_elem::{BgpUpdateElemIter, ElemError, Elementor, RecordElemIter}; #[cfg(feature = "oneio")] use oneio::{get_cache_reader, get_reader}; @@ -43,6 +46,9 @@ pub struct BgpkitParser { core_dump: bool, filters: Vec, options: ParserOptions, + /// Pre-parsed [`BgpElem`]s from a text dump (PCH / route-views). `None` for + /// MRT input, which is parsed lazily through [`Self::next_record`]. + text_dump_elems: Option>, } pub(crate) struct ParserOptions { @@ -68,6 +74,7 @@ impl BgpkitParser> { core_dump: false, filters: vec![], options: ParserOptions::default(), + text_dump_elems: None, }) } @@ -88,8 +95,58 @@ impl BgpkitParser> { core_dump: false, filters: vec![], options: ParserOptions::default(), + text_dump_elems: None, }) } + + /// Create a parser for a Cisco `sh ip bgp` text dump (PCH daily snapshots + /// or route-views `oix-full-snapshot-*` files). + /// + /// The file is auto-decompressed by oneio. The timestamp for all elements + /// is inferred from the file name when possible; pass + /// [`infer_timestamp_from_path`] yourself to override. The resulting parser + /// iterates over [`BgpElem`]s — calling [`into_record_iter`](Self::into_record_iter) + /// or [`next_record`](Self::next_record) on a text-dump parser panics, since + /// text dumps have no MRT-record representation. + /// + /// # Example + /// + /// ```no_run + /// use bgpkit_parser::BgpkitParser; + /// + /// let url = "https://downloads.pch.net/files/Routing_Data/IPv4_daily_snapshots/2026/07/route-collector.bom2.pch.net/route-collector.bom2.pch.net-ipv4_bgp_routes.2026.07.01.gz"; + /// for elem in BgpkitParser::new_text(url).unwrap() { + /// println!("{elem}"); + /// } + /// ``` + pub fn new_text(path: &str) -> Result { + let timestamp = infer_timestamp_from_path(path).unwrap_or(0.0); + let reader = get_reader(path)?; + Self::from_text_reader_with_timestamp(reader, timestamp) + } + + /// Create a parser that auto-detects whether the input is an MRT file or a + /// Cisco `sh ip bgp` text dump, parsing accordingly. + /// + /// Peeks the first 256 bytes: if they look like a Cisco text dump the file + /// is parsed as one (timestamp inferred from the path); otherwise it is + /// treated as MRT and parsed lazily as usual. This is the most convenient + /// constructor when the input type is unknown. + /// + /// # Example + /// + /// ```no_run + /// use bgpkit_parser::BgpkitParser; + /// + /// // works for either MRT or text dumps + /// for elem in BgpkitParser::new_auto("https://downloads.pch.net/files/Routing_Data/IPv4_daily_snapshots/2026/07/route-collector.bom2.pch.net/route-collector.bom2.pch.net-ipv4_bgp_routes.2026.07.01.gz").unwrap() { + /// println!("{elem}"); + /// } + /// ``` + pub fn new_auto(path: &str) -> Result { + let reader = get_reader(path)?; + Self::from_auto_reader_with_timestamp(reader, infer_timestamp_from_path(path)) + } } #[cfg(feature = "oneio")] @@ -114,11 +171,19 @@ impl BgpkitParser { core_dump: false, filters: vec![], options: ParserOptions::default(), + text_dump_elems: None, } } /// This is used in for loop `for item in parser{}` pub fn next_record(&mut self) -> Result { + if self.text_dump_elems.is_some() { + return Err(ParserError::Unsupported( + "text-dump parsers have no MRT record representation; iterate elements instead" + .to_string(), + ) + .into()); + } let (record, used_zebra_compat) = mrt::mrt_record::parse_mrt_record_with_zebra_compat(&mut self.reader)?; if used_zebra_compat { @@ -128,6 +193,75 @@ impl BgpkitParser { } } +impl BgpkitParser> { + /// Create a text-dump parser from any reader, with timestamp `0.0`. + /// Prefer [`BgpkitParser::new_text`] when you have a file path or URL, + /// as it will infer the timestamp automatically. + pub fn from_text_reader( + reader: impl Read + Send + 'static, + ) -> Result { + Self::from_text_reader_with_timestamp(reader, 0.0) + } + + /// Create a text-dump parser from a reader with an explicit element + /// timestamp. The reader is fully consumed up front; the resulting parser + /// iterates over [`BgpElem`]s but has no MRT-record representation. + pub fn from_text_reader_with_timestamp( + reader: impl Read + Send + 'static, + timestamp: f64, + ) -> Result { + let mut buf_reader = BufReader::new(reader); + let elems = parse_text_dump_with_timestamp(&mut buf_reader, timestamp) + .map_err(ParserError::from)?; + Ok(BgpkitParser { + reader: Box::new(std::io::empty()), + core_dump: false, + filters: vec![], + options: ParserOptions::default(), + text_dump_elems: Some(elems.into()), + }) + } + + /// Create a parser from any reader, auto-detecting MRT vs text dump by + /// sniffing the first bytes. Timestamp defaults to `0.0` for text dumps. + pub fn from_auto_reader( + reader: impl Read + Send + 'static, + ) -> Result { + Self::from_auto_reader_with_timestamp(reader, None) + } + + /// Create a parser from any reader, auto-detecting MRT vs text dump. + /// When text is detected, `timestamp` overrides the inferred value + /// (`None` → `0.0`). + pub fn from_auto_reader_with_timestamp( + reader: impl Read + Send + 'static, + timestamp: Option, + ) -> Result { + let mut buf_reader = BufReader::new(reader); + let (is_text, head) = detect_text_dump(&mut buf_reader).map_err(ParserError::from)?; + if is_text { + let chained = BufReader::new(Cursor::new(head).chain(buf_reader)); + let ts = timestamp.unwrap_or(0.0); + let elems = parse_text_dump_with_timestamp(chained, ts).map_err(ParserError::from)?; + Ok(BgpkitParser { + reader: Box::new(std::io::empty()), + core_dump: false, + filters: vec![], + options: ParserOptions::default(), + text_dump_elems: Some(elems.into()), + }) + } else { + Ok(BgpkitParser { + reader: Box::new(Cursor::new(head).chain(buf_reader)), + core_dump: false, + filters: vec![], + options: ParserOptions::default(), + text_dump_elems: None, + }) + } + } +} + impl BgpkitParser { pub(crate) fn warn_zebra_compat_once(&mut self) { if self.options.show_warnings && !self.options.warned_zebra_compat { @@ -144,6 +278,7 @@ impl BgpkitParser { core_dump: true, filters: self.filters, options: self.options, + text_dump_elems: self.text_dump_elems, } } @@ -155,6 +290,7 @@ impl BgpkitParser { core_dump: self.core_dump, filters: self.filters, options, + text_dump_elems: self.text_dump_elems, } } @@ -233,6 +369,7 @@ impl BgpkitParser { core_dump: self.core_dump, filters, options: self.options, + text_dump_elems: self.text_dump_elems, }) } @@ -296,6 +433,7 @@ impl BgpkitParser { #[cfg(test)] mod tests { use super::*; + use crate::models::Asn; #[test] fn test_new_with_reader() { @@ -468,4 +606,70 @@ mod tests { assert_eq!(count1, 132); assert_eq!(count2, 132); } + + #[test] + fn test_from_text_reader_inline() { + let dump = "BGP table version is 1, local router ID is 1.2.3.4, vrf id 0\n\ +Default local pref 100, local AS 65001\n\n\ + Network Next Hop Metric LocPrf Weight Path\n\ + *> 1.0.0.0/24 10.0.0.1 0 0 13335 i\n"; + let parser = + BgpkitParser::from_text_reader(dump.as_bytes()).expect("inline text-dump parse"); + let elems: Vec<_> = parser.into_elem_iter().collect(); + assert_eq!(elems.len(), 1); + assert_eq!(elems[0].prefix.prefix.to_string(), "1.0.0.0/24"); + assert_eq!(elems[0].peer_ip.to_string(), "1.2.3.4"); + assert_eq!(u32::from(elems[0].peer_asn), 65001); + assert_eq!(elems[0].origin_asns, Some(vec![Asn::from(13335u32)])); + } + + #[test] + fn test_from_auto_reader_detects_text() { + let dump = "BGP table version is 1, local router ID is 1.2.3.4, vrf id 0\n\ +Default local pref 100, local AS 65001\n\n\ + Network Next Hop Metric LocPrf Weight Path\n\ + *> 1.0.0.0/24 10.0.0.1 0 0 13335 i\n"; + let parser = BgpkitParser::from_auto_reader(dump.as_bytes()).expect("auto-detect parse"); + let elems: Vec<_> = parser.into_elem_iter().collect(); + assert_eq!(elems.len(), 1); + assert_eq!(elems[0].origin_asns, Some(vec![Asn::from(13335u32)])); + } + + #[test] + fn test_from_auto_reader_detects_mrt() { + // A few zero bytes — not a text dump, so auto-detect should fall + // through to the MRT path (which will then hit EOF cleanly). + let data: Vec = vec![0x00u8; 16]; + let parser = + BgpkitParser::from_auto_reader(std::io::Cursor::new(data)).expect("auto-detect parse"); + let count = parser.into_elem_iter().count(); + assert_eq!(count, 0); + } + + #[test] + fn test_text_dump_parser_with_filter() { + let dump = "BGP table version is 1, local router ID is 1.2.3.4, vrf id 0\n\ +Default local pref 100, local AS 65001\n\n\ + Network Next Hop Metric LocPrf Weight Path\n\ + *> 1.0.0.0/24 10.0.0.1 0 0 13335 i\n\ + *> 8.8.8.0/24 10.0.0.2 0 0 15169 i\n"; + let parser = BgpkitParser::from_text_reader(dump.as_bytes()) + .unwrap() + .add_filter("origin_asn", "13335") + .unwrap(); + let elems: Vec<_> = parser.into_elem_iter().collect(); + assert_eq!(elems.len(), 1); + assert_eq!(elems[0].prefix.prefix.to_string(), "1.0.0.0/24"); + } + + #[test] + fn test_text_dump_next_record_errors() { + let dump = "BGP table version is 1, local router ID is 1.2.3.4, vrf id 0\n\ +Default local pref 100, local AS 65001\n\n\ + Network Next Hop Metric LocPrf Weight Path\n\ + *> 1.0.0.0/24 10.0.0.1 0 0 13335 i\n"; + let mut parser = + BgpkitParser::from_text_reader(dump.as_bytes()).expect("inline text-dump parse"); + assert!(parser.next_record().is_err()); + } } diff --git a/src/parser/text_dump.rs b/src/parser/text_dump.rs new file mode 100644 index 00000000..508c668b --- /dev/null +++ b/src/parser/text_dump.rs @@ -0,0 +1,821 @@ +//! Cisco `sh ip bgp` text dump parser. +//! +//! Parses the fixed-width column format produced by Cisco IOS routers, as +//! published by PCH (daily routing table snapshots) and route-views +//! (`oix-full-snapshot-*.bz2`). Field boundaries are extracted from the table +//! header so that numeric attributes do not become indistinguishable from +//! numeric AS-path segments. +//! +//! # Format +//! +//! ```text +//! BGP table version is N, local router ID is X.X.X.X, vrf id 0 +//! Default local pref 100, local AS NNNN +//! ... +//! Network Next Hop Metric LocPrf Weight Path +//! *> 1.0.0.0/24 103.77.108.118 0 0 13335 i +//! *= 103.77.108.11 0 0 13335 i +//! ``` +//! +//! Route-views `sh ip bgp` snapshots (e.g. `oix-full-snapshot-*.bz2`) use the +//! same fixed-width layout but omit the `BGP table version` / `local AS` +//! preamble. When the preamble is absent, `peer_ip` and `peer_asn` default to +//! the unspecified sentinel values `0.0.0.0` and AS0. +//! +//! # Example +//! +//! ```no_run +//! use bgpkit_parser::parser::text_dump::{infer_timestamp_from_path, parse_text_dump_with_timestamp}; +//! +//! let url = "https://downloads.pch.net/files/Routing_Data/IPv4_daily_snapshots/2026/07/route-collector.bom2.pch.net/route-collector.bom2.pch.net-ipv4_bgp_routes.2026.07.01.gz"; +//! let reader = oneio::get_reader(url).unwrap(); +//! let mut reader = std::io::BufReader::new(reader); +//! let timestamp = infer_timestamp_from_path(url).unwrap_or(0.0); +//! let elems = parse_text_dump_with_timestamp(&mut reader, timestamp).unwrap(); +//! println!("parsed {} elements", elems.len()); +//! ``` + +use crate::models::*; +use ipnet::IpNet; +use std::io::{BufRead, Read}; +use std::net::IpAddr; + +/// Byte offsets for the Cisco `Next Hop`, `Metric`, `LocPrf`, `Weight`, and +/// `Path` columns, respectively. +type ColumnPositions = (usize, usize, usize, usize, usize); + +/// Header metadata extracted from the preamble of a Cisco `sh ip bgp` dump. +#[derive(Debug, Clone)] +pub struct TextDumpHeader { + /// BGP table version, when the preamble carries one (absent in route-views dumps). + pub table_version: Option, + /// Local router ID, used as the `peer_ip` of parsed elements (absent in route-views dumps). + pub router_id: Option, + /// Local AS number, used as the `peer_asn` of parsed elements (absent in route-views dumps). + pub local_as: Option, + column_positions: Option, +} + +// ── Detection ────────────────────────────────────────────────────── + +/// Detect whether a reader contains a Cisco `sh ip bgp` text dump. +/// +/// Reads up to 256 bytes and returns them alongside the detection result so +/// callers can chain the buffered bytes into the actual parser. +pub fn detect_text_dump(mut reader: R) -> std::io::Result<(bool, Vec)> { + let mut buf = vec![0u8; 256]; + let n = reader.read(&mut buf)?; + buf.truncate(n); + + let is_text = n > 0 + && buf[0].is_ascii_graphic() + && std::str::from_utf8(&buf[..n.min(128)]).is_ok_and(|s| { + s.contains("BGP table") || s.contains("Next Hop") || s.contains("Status codes:") + }); + + Ok((is_text, buf)) +} + +// ── Header parsing ───────────────────────────────────────────────── + +/// Extract column offsets from the Cisco table header using whitespace split. +/// +/// Splits the header line into whitespace-delimited tokens, merges known +/// multi-word field names (e.g. "Next" + "Hop" → "Next Hop"), and records +/// the byte position of each column in the original header line. +fn parse_column_header(line: &str) -> Option { + // Tokenize by whitespace: record (byte_position, token) for each word. + let bytes = line.as_bytes(); + let mut tokens: Vec<(usize, &str)> = Vec::new(); + let mut i = 0; + + while i < bytes.len() { + // Skip leading whitespace. + while i < bytes.len() && bytes[i].is_ascii_whitespace() { + i += 1; + } + if i >= bytes.len() { + break; + } + let start = i; + // Consume non-whitespace characters. + while i < bytes.len() && !bytes[i].is_ascii_whitespace() { + i += 1; + } + let token = std::str::from_utf8(&bytes[start..i]).ok()?; + tokens.push((start, token)); + } + + if tokens.is_empty() { + return None; + } + + // Merge known multi-word field names. + // "Next" immediately followed by "Hop" → "Next Hop" starting at "Next". + let mut columns: Vec<(usize, String)> = Vec::new(); + let mut skip = false; + for idx in 0..tokens.len() { + if skip { + skip = false; + continue; + } + let (pos, token) = tokens[idx]; + if token == "Next" && idx + 1 < tokens.len() && tokens[idx + 1].1 == "Hop" { + columns.push((pos, "Next Hop".to_string())); + skip = true; + } else { + columns.push((pos, token.to_string())); + } + } + + // Extract positions for the five columns we depend on. + let target_names = ["Next Hop", "Metric", "LocPrf", "Weight", "Path"]; + let mut positions: [Option; 5] = [None; 5]; + + for (pos, name) in &columns { + for (i, target) in target_names.iter().enumerate() { + if name == *target { + positions[i] = Some(*pos); + } + } + } + + let next_hop = positions[0]?; + let metric = positions[1]?; + let local_pref = positions[2]?; + let weight = positions[3]?; + let path = positions[4]?; + + if next_hop < metric && metric < local_pref && local_pref < weight && weight < path { + Some((next_hop, metric, local_pref, weight, path)) + } else { + None + } +} + +/// Parse the preamble to extract router metadata and fixed-width column offsets. +/// +/// Consumes lines up to and including the column header line +/// (`Network / Next Hop / Metric / LocPrf / Weight / Path`); the reader is left +/// positioned at the first route line. +pub fn parse_header(reader: &mut R) -> std::io::Result { + let mut header = TextDumpHeader { + table_version: None, + router_id: None, + local_as: None, + column_positions: None, + }; + let mut buf = String::new(); + + for _ in 0..64 { + buf.clear(); + if reader.read_line(&mut buf)? == 0 { + break; + } + let line = buf.trim_end(); + + if line.starts_with("BGP table version") { + if let Some(rest) = line.strip_prefix("BGP table version is ") { + if let Some((ver_str, rest)) = rest.split_once(',') { + header.table_version = ver_str.trim().parse().ok(); + if let Some(rid_part) = rest.split("local router ID is ").nth(1) { + if let Some((rid_str, _)) = rid_part.split_once(',') { + header.router_id = rid_str.trim().parse().ok(); + } + } + } + } + } + + if line.starts_with("Default local pref") { + if let Some(rest) = line.split("local AS ").nth(1) { + header.local_as = rest.trim().parse().ok(); + } + } + + if let Some(column_positions) = parse_column_header(line) { + header.column_positions = Some(column_positions); + break; + } + } + Ok(header) +} + +// ── Fixed-width route parsing ────────────────────────────────────── + +/// A parsed route entry from a single line. +#[derive(Debug, Clone)] +struct RouteEntry { + prefix: String, + next_hop: String, + metric: Option, + local_pref: Option, + as_path: Vec, + origin: Option, +} + +fn parse_u32_column(line: &str, start: usize, end: usize) -> Option { + line.get(start..end)?.trim().parse().ok() +} + +fn shift_columns_left(columns: ColumnPositions) -> Option { + Some(( + columns.0.checked_sub(1)?, + columns.1.checked_sub(1)?, + columns.2.checked_sub(1)?, + columns.3.checked_sub(1)?, + columns.4.checked_sub(1)?, + )) +} + +/// Detect a "wrapped" prefix-only line (no route data, just a prefix after +/// the status flags). The next-hop column position is used to distinguish +/// prefix-only lines from full route lines: if the line segment at the +/// next-hop position is empty, the line carries only a prefix. +fn parse_wrapped_prefix_line(line: &str, next_hop_start: usize) -> Option { + // If the line extends into the next-hop column, check whether the + // characters there form route data (digits/IP) or are just whitespace. + if line.len() > next_hop_start { + let rest = line[next_hop_start..].trim(); + if !rest.is_empty() { + return None; // has route data → regular line, not a wrapped prefix + } + } + + let prefix = if line.len() > next_hop_start { + line.get(3..next_hop_start)? + } else { + line.get(3..)? + } + .trim(); + prefix.parse::().ok().map(|_| prefix.to_string()) +} + +/// Parse a single Cisco route line using positions extracted from its column header. +fn parse_route_line(line: &str, columns: ColumnPositions) -> Option { + let line = line.trim_end(); + if line.trim().is_empty() || line.trim_start().starts_with("Displayed") { + return None; + } + + let (next_hop_start, metric_start, local_pref_start, weight_start, path_start) = columns; + let next_hop = line.get(next_hop_start..metric_start)?.trim(); + if next_hop.parse::().is_err() { + return None; + } + + let prefix = line + .get(3..next_hop_start) + .unwrap_or_default() + .trim() + .to_string(); + let metric = parse_u32_column(line, metric_start, local_pref_start); + let local_pref = parse_u32_column(line, local_pref_start, weight_start); + // Cisco weight is a router-local attribute with no `BgpElem` representation. + let _weight = parse_u32_column(line, weight_start, path_start); + + let path_tokens: Vec<&str> = line + .get(path_start..) + .unwrap_or_default() + .split_whitespace() + .collect(); + let origin = match path_tokens.last().copied() { + Some("i") => Some(Origin::IGP), + Some("e") => Some(Origin::EGP), + Some("?") => Some(Origin::INCOMPLETE), + _ => None, + }; + let path_end = path_tokens.len() - usize::from(origin.is_some()); + let as_path = path_tokens[..path_end] + .iter() + .map(|token| (*token).to_string()) + .collect(); + + Some(RouteEntry { + prefix, + next_hop: next_hop.to_string(), + metric, + local_pref, + as_path, + origin, + }) +} + +// ── BgpElem construction ─────────────────────────────────────────── + +/// Convert path tokens into an AsPath. +/// +/// AS-set delimiters (`{` / `}`) are silently dropped, flattening AS-sets +/// into plain AS-sequences. This is intentional: the parser aims to recover +/// the AS-level propagation path, and set membership is not preserved. +fn as_path_from_tokens(tokens: &[String]) -> AsPath { + let mut asns: Vec = Vec::new(); + for token in tokens { + if token == "{" || token == "}" { + continue; + } + if let Ok(asn) = token.parse::() { + asns.push(Asn::from(asn)); + } + } + if asns.is_empty() { + return AsPath { + segments: vec![AsPathSegment::AsSequence(Default::default())].into(), + }; + } + AsPath { + segments: vec![AsPathSegment::AsSequence(asns.into())].into(), + } +} + +fn entry_to_elem( + entry: &RouteEntry, + prefix_str: &str, + peer_ip: IpAddr, + peer_asn: u32, + timestamp: f64, +) -> Option { + let prefix: IpNet = match prefix_str.parse() { + Ok(p) => p, + Err(_) => return None, + }; + + let network_prefix = NetworkPrefix::new(prefix, None); + let next_hop: Option = entry.next_hop.parse().ok(); + let as_path = Some(as_path_from_tokens(&entry.as_path)); + let origin = entry.origin; + let local_pref = entry.local_pref; + let med = entry.metric; + + let origin_asns: Option> = as_path.as_ref().and_then(|ap| { + ap.segments + .last() + .and_then(|seg| match seg { + AsPathSegment::AsSequence(asns) => asns.last().copied(), + _ => None, + }) + .map(|asn| vec![asn]) + }); + + Some(BgpElem { + timestamp, + elem_type: ElemType::ANNOUNCE, + peer_ip, + peer_asn: Asn::from(peer_asn), + prefix: network_prefix, + next_hop, + as_path, + origin_asns, + origin, + local_pref, + med, + communities: None, + atomic: false, + aggr_asn: None, + aggr_ip: None, + only_to_customer: None, + unknown: None, + deprecated: None, + peer_bgp_id: None, + }) +} + +// ── Timestamp inference ──────────────────────────────────────────── + +/// Try to extract a Unix timestamp from a file path or URL. +/// +/// Recognises route-views-style timestamps like +/// `oix-full-snapshot-2026-07-01-0000.bz2` (`YYYY-MM-DD-HHMM`, using the +/// embedded time of day) and PCH-style date components like +/// `...2026.07.01.gz` (`YYYY.MM.DD`, noon UTC). Returns `None` when no +/// recognisable date is found. +pub fn infer_timestamp_from_path(path: &str) -> Option { + let bytes = path.as_bytes(); + + // Scan for YYYY-MM-DD-HHMM pattern (route-views snapshot filenames). + for (idx, window) in bytes.windows(15).enumerate() { + if window[0].is_ascii_digit() + && window[1].is_ascii_digit() + && window[2].is_ascii_digit() + && window[3].is_ascii_digit() + && window[4] == b'-' + && window[5].is_ascii_digit() + && window[6].is_ascii_digit() + && window[7] == b'-' + && window[8].is_ascii_digit() + && window[9].is_ascii_digit() + && window[10] == b'-' + && window[11].is_ascii_digit() + && window[12].is_ascii_digit() + && window[13].is_ascii_digit() + && window[14].is_ascii_digit() + { + let y: Option = std::str::from_utf8(&window[0..4]) + .ok() + .and_then(|s| s.parse().ok()); + let m: Option = std::str::from_utf8(&window[5..7]) + .ok() + .and_then(|s| s.parse().ok()); + let d: Option = std::str::from_utf8(&window[8..10]) + .ok() + .and_then(|s| s.parse().ok()); + let hh: Option = std::str::from_utf8(&window[11..13]) + .ok() + .and_then(|s| s.parse().ok()); + let mm: Option = std::str::from_utf8(&window[13..15]) + .ok() + .and_then(|s| s.parse().ok()); + // Require a separator before the date to avoid matching digit + // runs inside longer numbers. + if idx > 0 && bytes[idx - 1].is_ascii_digit() { + continue; + } + if let (Some(y), Some(m), Some(d), Some(hh), Some(mm)) = (y, m, d, hh, mm) { + if let Some(dt) = chrono::NaiveDate::from_ymd_opt(y, m, d) + .and_then(|date| date.and_hms_opt(hh, mm, 0)) + { + return Some(dt.and_utc().timestamp() as f64); + } + } + } + } + + // Scan for YYYY.MM.DD pattern (PCH file URLs). + for window in bytes.windows(10) { + if window.len() == 10 + && window[0].is_ascii_digit() + && window[1].is_ascii_digit() + && window[2].is_ascii_digit() + && window[3].is_ascii_digit() + && window[4] == b'.' + && window[5].is_ascii_digit() + && window[6].is_ascii_digit() + && window[7] == b'.' + && window[8].is_ascii_digit() + && window[9].is_ascii_digit() + { + let y: i32 = std::str::from_utf8(&window[0..4]).ok()?.parse().ok()?; + let m: u32 = std::str::from_utf8(&window[5..7]).ok()?.parse().ok()?; + let d: u32 = std::str::from_utf8(&window[8..10]).ok()?.parse().ok()?; + // Use noon UTC to avoid DST boundary issues. + let dt = chrono::NaiveDate::from_ymd_opt(y, m, d)?.and_hms_opt(12, 0, 0)?; + return Some(dt.and_utc().timestamp() as f64); + } + } + None +} + +// ── Top-level parse ──────────────────────────────────────────────── + +/// Parse a complete Cisco `sh ip bgp` text dump into [`BgpElem`]s with +/// timestamp `0.0`. +/// +/// See [`parse_text_dump_with_timestamp`] for details. +pub fn parse_text_dump(reader: R) -> std::io::Result> { + parse_text_dump_with_timestamp(reader, 0.0) +} + +/// Parse a complete Cisco `sh ip bgp` text dump into [`BgpElem`]s. +/// +/// All parsed elements share the given `timestamp`; use +/// [`infer_timestamp_from_path`] to derive one from a PCH or route-views +/// file name when available. +/// +/// Route-views style snapshots omit the `BGP table version` / `local AS` +/// preamble. The parser falls back to the unspecified sentinels rather than +/// rejecting the dump: `0.0.0.0` and AS0 carry no peer identity. +/// +/// Lines that do not parse as route entries (banner text, the trailing +/// `Displayed ...` summary, malformed rows) are skipped silently. +pub fn parse_text_dump_with_timestamp( + mut reader: R, + timestamp: f64, +) -> std::io::Result> { + let header = parse_header(&mut reader)?; + let column_positions = match header.column_positions { + Some(positions) => positions, + None => { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "missing Cisco BGP table column header", + )); + } + }; + let peer_ip = header + .router_id + .unwrap_or_else(|| IpAddr::from([0, 0, 0, 0])); + let peer_asn = header.local_as.unwrap_or(0); + + let wrapped_column_positions = shift_columns_left(column_positions); + + let mut buf = String::new(); + let mut current_prefix = String::new(); + let mut entries: Vec<(String, RouteEntry)> = Vec::new(); + + while reader.read_line(&mut buf)? > 0 { + let line = buf.trim_end().to_string(); + buf.clear(); + + if line.is_empty() { + continue; + } + + let entry = parse_route_line(&line, column_positions).or_else(|| { + wrapped_column_positions.and_then(|positions| parse_route_line(&line, positions)) + }); + if let Some(entry) = entry { + if !entry.prefix.is_empty() { + current_prefix = entry.prefix.clone(); + } + entries.push((current_prefix.clone(), entry)); + continue; + } + + if let Some(prefix) = parse_wrapped_prefix_line(&line, column_positions.0) { + current_prefix = prefix; + continue; + } + } + + let mut elems: Vec = Vec::with_capacity(entries.len()); + for (prefix, entry) in &entries { + if prefix.is_empty() { + continue; + } + if let Some(elem) = entry_to_elem(entry, prefix, peer_ip, peer_asn, timestamp) { + elems.push(elem); + } + } + + Ok(elems) +} + +// ── Tests ────────────────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + use super::*; + + const TEST_COLUMNS: ColumnPositions = (21, 41, 48, 55, 62); + const TABLE_HEADER: &str = " Network Next Hop Metric LocPrf Weight Path"; + + fn fixed_width_route( + prefix: &str, + next_hop: &str, + metric: &str, + local_pref: &str, + weight: &str, + path: &str, + ) -> String { + format!( + " *> {:<17}{:<20}{:>7}{:>7}{:>7}{}", + prefix, next_hop, metric, local_pref, weight, path + ) + } + + fn parsed_route(line: &str) -> RouteEntry { + match parse_route_line(line, TEST_COLUMNS) { + Some(entry) => entry, + None => panic!("expected a valid fixed-width route line"), + } + } + + #[test] + fn test_detect_text_dump_positive() { + let data = b"BGP table version is 123, local router ID is 1.2.3.4, vrf id 0\n"; + let (is_text, buf) = match detect_text_dump(&data[..]) { + Ok(result) => result, + Err(error) => panic!("text dump detection failed: {error}"), + }; + assert!(is_text); + assert!(!buf.is_empty()); + } + + #[test] + fn test_detect_text_dump_negative() { + let data = [0x00, 0x00, 0x00, 0x00, 0x00, 0x0d, 0x00, 0x01]; + let (is_text, _buf) = match detect_text_dump(&data[..]) { + Ok(result) => result, + Err(error) => panic!("text dump detection failed: {error}"), + }; + assert!(!is_text); + } + + #[test] + fn test_parse_header() { + let preamble = "\ +BGP table version is 1350657, local router ID is 45.112.180.132, vrf id 0 +Default local pref 100, local AS 3856 +Status codes: s suppressed, d damped, h history, * valid, > best, = multipath, + i internal, r RIB-failure, S Stale, R Removed +Nexthop codes: @NNN nexthop's vrf id, < announce-nh-self +Origin codes: i - IGP, e - EGP, ? - incomplete +RPKI validation codes: V valid, I invalid, N Not found + + Network Next Hop Metric LocPrf Weight Path +"; + let header = match parse_header(&mut preamble.as_bytes()) { + Ok(header) => header, + Err(error) => panic!("header parsing failed: {error}"), + }; + let expected_router_id = match "45.112.180.132".parse() { + Ok(router_id) => router_id, + Err(error) => panic!("invalid expected router ID: {error}"), + }; + assert_eq!(header.table_version, Some(1350657)); + assert_eq!(header.router_id, Some(expected_router_id)); + assert_eq!(header.local_as, Some(3856)); + assert_eq!(header.column_positions, Some(TEST_COLUMNS)); + } + + #[test] + fn test_parse_route_line_basic() { + let line = fixed_width_route("1.0.0.0/24", "103.77.108.11", "0", "", "0", "13335 i"); + let entry = parsed_route(&line); + assert_eq!(entry.prefix, "1.0.0.0/24"); + assert_eq!(entry.next_hop, "103.77.108.11"); + assert_eq!(entry.metric, Some(0)); + assert_eq!(entry.local_pref, None); + assert_eq!(entry.origin, Some(Origin::IGP)); + assert_eq!(entry.as_path, vec!["13335"]); + } + + #[test] + fn test_parse_route_line_continuation() { + let line = fixed_width_route("", "103.77.108.118", "0", "", "0", "13335 i"); + let entry = parsed_route(&line); + assert!(entry.prefix.is_empty()); + assert_eq!(entry.next_hop, "103.77.108.118"); + assert_eq!(entry.as_path, vec!["13335"]); + } + + #[test] + fn test_parse_route_line_uses_fixed_width_attributes() { + let line = fixed_width_route("0.0.0.0/0", "103.77.108.116", "0", "", "0", "134942 4755 i"); + let entry = parsed_route(&line); + assert_eq!(entry.metric, Some(0)); + assert_eq!(entry.local_pref, None); + assert_eq!(entry.as_path, vec!["134942", "4755"]); + } + + #[test] + fn test_parse_route_line_multi_asn() { + let line = fixed_width_route( + "1.0.0.0/24", + "103.77.108.116", + "10", + "200", + "0", + "134942 4755 i", + ); + let entry = parsed_route(&line); + assert_eq!(entry.metric, Some(10)); + assert_eq!(entry.local_pref, Some(200)); + assert_eq!(entry.as_path, vec!["134942", "4755"]); + assert_eq!(entry.origin, Some(Origin::IGP)); + } + + #[test] + fn test_parse_route_line_origin_codes() { + let prefix = "1.0.0.0/24"; + let next_hop = "10.0.0.1"; + assert_eq!( + parsed_route(&fixed_width_route(prefix, next_hop, "0", "", "0", "100 i")).origin, + Some(Origin::IGP) + ); + assert_eq!( + parsed_route(&fixed_width_route(prefix, next_hop, "0", "", "0", "100 e")).origin, + Some(Origin::EGP) + ); + assert_eq!( + parsed_route(&fixed_width_route(prefix, next_hop, "0", "", "0", "100 ?")).origin, + Some(Origin::INCOMPLETE) + ); + } + + #[test] + fn test_parse_route_line_no_numeric_attrs() { + let line = fixed_width_route("1.0.0.0/24", "10.0.0.1", "", "", "", "100 i"); + let entry = parsed_route(&line); + assert_eq!(entry.next_hop, "10.0.0.1"); + assert_eq!(entry.as_path, vec!["100"]); + assert!(entry.metric.is_none()); + assert!(entry.local_pref.is_none()); + } + + #[test] + fn test_parse_text_dump_preserves_continuation_prefix() { + let first = fixed_width_route("0.0.0.0/0", "103.77.108.116", "0", "", "0", "134942 4755 i"); + let continuation = fixed_width_route("", "103.77.108.118", "0", "", "0", "13335 i"); + let dump = format!( + "BGP table version is 1350657, local router ID is 45.112.180.132, vrf id 0\nDefault local pref 100, local AS 3856\n\n{TABLE_HEADER}\n{first}\n{continuation}\nDisplayed 1 routes and 2 total paths\n" + ); + let elems = match parse_text_dump(dump.as_bytes()) { + Ok(elems) => elems, + Err(error) => panic!("text dump parsing failed: {error}"), + }; + assert_eq!(elems.len(), 2); + assert_eq!(elems[0].prefix, elems[1].prefix); + assert_eq!(elems[0].med, Some(0)); + assert_eq!(elems[0].local_pref, None); + assert_eq!(elems[0].origin_asns, Some(vec![Asn::from(4755u32)])); + assert_eq!(elems[1].origin_asns, Some(vec![Asn::from(13335u32)])); + } + + #[test] + fn test_parse_text_dump_supports_wrapped_prefix() { + let dump = format!( + "BGP table version is 1350657, local router ID is 45.112.180.132, vrf id 0\nDefault local pref 100, local AS 3856\n\n{TABLE_HEADER}\n * 103.85.157.176/29\n 103.77.108.116 0 0 134942 58715 152125 i\nDisplayed 1 routes and 1 total paths\n" + ); + let elems = match parse_text_dump(dump.as_bytes()) { + Ok(elems) => elems, + Err(error) => panic!("text dump parsing failed: {error}"), + }; + assert_eq!(elems.len(), 1); + assert_eq!(elems[0].origin_asns, Some(vec![Asn::from(152125u32)])); + } + + #[test] + fn test_entry_to_elem() { + let entry = RouteEntry { + prefix: "1.0.0.0/24".into(), + next_hop: "103.77.108.11".into(), + metric: Some(0), + local_pref: Some(0), + as_path: vec!["13335".into()], + origin: Some(Origin::IGP), + }; + let peer_ip = match "45.112.180.132".parse() { + Ok(peer_ip) => peer_ip, + Err(error) => panic!("invalid expected peer IP: {error}"), + }; + let elem = match entry_to_elem(&entry, "1.0.0.0/24", peer_ip, 3856, 0.0) { + Some(elem) => elem, + None => panic!("BgpElem conversion failed"), + }; + assert_eq!(elem.peer_ip.to_string(), "45.112.180.132"); + assert_eq!(u32::from(elem.peer_asn), 3856); + assert_eq!(elem.origin, Some(Origin::IGP)); + assert_eq!(elem.origin_asns, Some(vec![Asn::from(13335u32)])); + } + + const ROUTE_VIEWS_HEADER: &str = + " Network Next Hop Metric LocPrf Weight Path"; + + #[test] + fn test_detect_text_dump_route_views() { + let data = b"Status codes: s suppressed, d damped, h history, * valid, > best, i - internal,\n r RIB-failure, S Stale\nOrigin codes: i - IGP, e - EGP, ? - incomplete\n"; + let (is_text, _buf) = match detect_text_dump(&data[..]) { + Ok(result) => result, + Err(error) => panic!("text dump detection failed: {error}"), + }; + assert!(is_text); + } + + #[test] + fn test_parse_route_views_dump() { + let dump = format!( + "Status codes: s suppressed, d damped, h history, * valid, > best, i - internal,\n r RIB-failure, S Stale\nOrigin codes: i - IGP, e - EGP, ? - incomplete\n\n{ROUTE_VIEWS_HEADER}\n* 0.0.0.0/0 147.28.0.3 0 0 0 3130 174 i\n* 1.0.0.0/24 12.0.1.63 0 0 0 7018 13335 i\n* 1.0.0.0/24 129.250.1.71 2001 0 0 2914 13335 i\n" + ); + let elems = match parse_text_dump(dump.as_bytes()) { + Ok(elems) => elems, + Err(error) => panic!("route-views dump parsing failed: {error}"), + }; + assert_eq!(elems.len(), 3); + // No router ID / local AS preamble: unspecified sentinel values. + assert_eq!(elems[0].peer_ip.to_string(), "0.0.0.0"); + assert_eq!(u32::from(elems[0].peer_asn), 0); + + assert_eq!(elems[0].prefix.prefix.to_string(), "0.0.0.0/0"); + assert_eq!(elems[0].med, Some(0)); + assert_eq!(elems[0].origin_asns, Some(vec![Asn::from(174u32)])); + + assert_eq!(elems[2].prefix.prefix.to_string(), "1.0.0.0/24"); + assert_eq!(elems[2].med, Some(2001)); + assert_eq!(elems[2].origin, Some(Origin::IGP)); + assert_eq!(elems[2].origin_asns, Some(vec![Asn::from(13335u32)])); + } + + #[test] + fn test_infer_timestamp_route_views_path() { + let ts = infer_timestamp_from_path( + "https://archive.routeviews.org/oix-route-views/2026.07/oix-full-snapshot-2026-07-01-0000.bz2", + ); + // 2026-07-01 00:00:00 UTC + assert_eq!(ts, Some(1782864000.0)); + } + + #[test] + fn test_infer_timestamp_pch_path() { + let ts = infer_timestamp_from_path( + "https://www.pch.net/resources/data/routing-tables/2026/2026.07/rib-ipv4.2026.07.01.gz", + ); + // 2026-07-01 12:00:00 UTC (noon convention for date-only paths) + assert_eq!(ts, Some(1782907200.0)); + } + + #[test] + fn test_infer_timestamp_no_date() { + assert_eq!(infer_timestamp_from_path("/tmp/some-file.gz"), None); + } +} From 18f751800462bb3e639f64c7081ed5fd3abed9e7 Mon Sep 17 00:00:00 2001 From: Mingwei Zhang Date: Wed, 5 Aug 2026 14:06:58 -0700 Subject: [PATCH 2/4] refactor: stream text dump elements lazily (constant memory) Replace the collect-all parse_text_dump_with_timestamp approach with a streaming TextDumpElemIterator that yields one BgpElem per route line. new_text, from_text_reader, new_auto, and from_auto_reader now store the iterator directly in BgpkitParser instead of materializing a Vec at construction. ElemIterator and FallibleElemIterator drain it lazily with filter support. This cuts peak memory from O(all elems) to O(1) for text dumps. The full route-views oix snapshot (~900k prefixes, gigabytes of text) now streams without buffering. parse_text_dump / parse_text_dump_with_timestamp are kept as convenience wrappers that collect the iterator. Remove the 20 MB demo cap from the route-views example since streaming makes full-dump processing practical. --- CHANGELOG.md | 2 +- examples/parse_text_dump_routeviews.rs | 24 +--- src/parser/iters/default.rs | 6 +- src/parser/iters/fallible.rs | 6 +- src/parser/mod.rs | 47 ++++--- src/parser/text_dump.rs | 173 ++++++++++++++++--------- 6 files changed, 150 insertions(+), 108 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 21475dc9..ba6667e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,7 +36,7 @@ All notable changes to this project will be documented in this file. * **Historical RIPE regression fixtures**: Added original RRC00 update and bview gzip files from 1999 and January 2000 as repository-only, offline integration fixtures. * **RFC 10005 Link Bandwidth Extended Community**: Typed parsing and encoding for the BGP Link Bandwidth Extended Community in both transitive (`0x00`) and non-transitive (`0x40`) forms ([#299](https://github.com/bgpkit/bgpkit-parser/issues/299)). Exposes the Global Administrator, bandwidth in bytes per second, and transitivity, and preserves the wire type on encode. * **Cisco `sh ip bgp` text dump parsing** ([#320](https://github.com/bgpkit/bgpkit-parser/issues/320)): new `parser::text_dump` module parses fixed-width text RIB dumps published by PCH (daily routing table snapshots) and route-views (`oix-full-snapshot-*.bz2`) into `BgpElem`s. Column offsets are derived from the table header so blank numeric columns (Metric, LocPrf, Weight) stay distinct from AS-path data; multipath continuation lines and wrapped prefixes (including IPv6) are supported. Route-views dumps without the `BGP table version` / `local AS` preamble parse with sentinel peer identity (`0.0.0.0` / AS0). Also provides `detect_text_dump` for format sniffing and `infer_timestamp_from_path` for PCH (`YYYY.MM.DD`) and route-views (`YYYY-MM-DD-HHMM`) file-name timestamps. Ported from monocle ([#143](https://github.com/bgpkit/monocle/pull/143), [#146](https://github.com/bgpkit/monocle/pull/146)). -* **Unified MRT/text-dump parser API**: `BgpkitParser` gains three new constructor groups that integrate text dumps into the standard `for elem in parser` iteration loop. `new_text(path)` / `from_text_reader(r)` parse a known text dump; `new_auto(path)` / `from_auto_reader(r)` peek the first bytes and auto-dispatch to the text or MRT path. Text-dump elements are fully materialized at construction (RIB snapshots are not streaming); all existing filter methods (`add_filter`, `with_filters`, etc.) work on both paths. The default `new(path)` constructor remains MRT-only. +* **Unified MRT/text-dump parser API**: `BgpkitParser` gains three new constructor groups that integrate text dumps into the standard `for elem in parser` iteration loop. `new_text(path)` / `from_text_reader(r)` parse a known text dump; `new_auto(path)` / `from_auto_reader(r)` peek the first bytes and auto-dispatch to the text or MRT path. Both text-dump paths **stream elements lazily** — one route line at a time, constant memory — via a new `TextDumpElemIterator`. All existing filter methods (`add_filter`, `with_filters`, etc.) work on both paths. The default `new(path)` constructor remains MRT-only. ### Fixed diff --git a/examples/parse_text_dump_routeviews.rs b/examples/parse_text_dump_routeviews.rs index b27555e0..1282d0e3 100644 --- a/examples/parse_text_dump_routeviews.rs +++ b/examples/parse_text_dump_routeviews.rs @@ -1,33 +1,21 @@ use bgpkit_parser::BgpkitParser; -use std::io::Read; /// This example parses a route-views `sh ip bgp` snapshot /// (`oix-full-snapshot-*.bz2`) into `BgpElem`s. /// /// `BgpkitParser::new_text` handles the bzip2 decompression and timestamp -/// inference. Route-views snapshots omit the `BGP table version` / `local AS` -/// preamble, so parsed elements carry the sentinel peer identity `0.0.0.0` / -/// AS0. +/// inference, and streams elements one route line at a time (constant memory). /// -/// The full snapshot is the entire global routing table (gigabytes of text), -/// so this example caps the input at ~20 MB via a manual reader. Remove the -/// cap to parse the complete file. +/// Route-views snapshots omit the `BGP table version` / `local AS` preamble, +/// so parsed elements carry the sentinel peer identity `0.0.0.0` / AS0. fn main() { env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init(); let url = "https://archive.routeviews.org/oix-route-views/2026.07/oix-full-snapshot-2026-07-01-0000.bz2"; - log::info!("opening {url} (first 20 MB)"); - let mut reader = oneio::get_reader(url).unwrap(); - // Cap the input for this demonstration: read at most 20 MB of decompressed - // data, then feed it into the text-dump parser. - let mut buf = Vec::with_capacity(20 * 1024 * 1024); - let _ = reader.by_ref().take(20 * 1024 * 1024).read_to_end(&mut buf); - let timestamp = bgpkit_parser::parser::text_dump::infer_timestamp_from_path(url).unwrap_or(0.0); - let parser = - BgpkitParser::from_text_reader_with_timestamp(std::io::Cursor::new(buf), timestamp) - .unwrap(); - log::info!("parsing text dump (timestamp={timestamp})"); + log::info!("opening {url}"); + let parser = BgpkitParser::new_text(url).unwrap(); + log::info!("streaming text dump"); let mut count = 0; for elem in parser { diff --git a/src/parser/iters/default.rs b/src/parser/iters/default.rs index ccba498a..c8563b22 100644 --- a/src/parser/iters/default.rs +++ b/src/parser/iters/default.rs @@ -139,9 +139,9 @@ impl Iterator for ElemIterator { self.count += 1; loop { - // Fast path: drain pre-parsed text-dump elems directly, with filter support. - if let Some(elems) = &mut self.record_iter.parser.text_dump_elems { - while let Some(elem) = elems.pop_front() { + // Fast path: drain streaming text-dump elems directly, with filter support. + if let Some(iter) = &mut self.record_iter.parser.text_dump_iter { + for elem in iter.by_ref() { if elem.match_filters(&self.record_iter.parser.filters) { return Some(elem); } diff --git a/src/parser/iters/fallible.rs b/src/parser/iters/fallible.rs index e2fa7581..d549defa 100644 --- a/src/parser/iters/fallible.rs +++ b/src/parser/iters/fallible.rs @@ -96,9 +96,9 @@ impl Iterator for FallibleElemIterator { fn next(&mut self) -> Option { loop { - // Fast path: drain pre-parsed text-dump elems directly, with filter support. - if let Some(elems) = &mut self.record_iter.parser.text_dump_elems { - while let Some(elem) = elems.pop_front() { + // Fast path: drain streaming text-dump elems directly, with filter support. + if let Some(iter) = &mut self.record_iter.parser.text_dump_iter { + for elem in iter.by_ref() { if elem.match_filters(&self.record_iter.parser.filters) { return Some(Ok(elem)); } diff --git a/src/parser/mod.rs b/src/parser/mod.rs index bd586947..d3e3f1d6 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -3,9 +3,8 @@ parser module maintains the main logic for processing BGP and MRT messages. */ use crate::models::{BgpElem, MrtRecord}; use log::warn; -use std::collections::VecDeque; use std::io::{BufReader, Cursor, Read}; -pub use text_dump::{detect_text_dump, infer_timestamp_from_path, parse_text_dump_with_timestamp}; +pub use text_dump::{detect_text_dump, infer_timestamp_from_path, TextDumpElemIterator}; #[macro_use] pub mod utils; @@ -46,9 +45,9 @@ pub struct BgpkitParser { core_dump: bool, filters: Vec, options: ParserOptions, - /// Pre-parsed [`BgpElem`]s from a text dump (PCH / route-views). `None` for - /// MRT input, which is parsed lazily through [`Self::next_record`]. - text_dump_elems: Option>, + /// Streaming element iterator for text dumps. `None` for MRT input, + /// which is parsed lazily through [`Self::next_record`]. + text_dump_iter: Option + Send>>, } pub(crate) struct ParserOptions { @@ -74,7 +73,7 @@ impl BgpkitParser> { core_dump: false, filters: vec![], options: ParserOptions::default(), - text_dump_elems: None, + text_dump_iter: None, }) } @@ -95,7 +94,7 @@ impl BgpkitParser> { core_dump: false, filters: vec![], options: ParserOptions::default(), - text_dump_elems: None, + text_dump_iter: None, }) } @@ -105,9 +104,10 @@ impl BgpkitParser> { /// The file is auto-decompressed by oneio. The timestamp for all elements /// is inferred from the file name when possible; pass /// [`infer_timestamp_from_path`] yourself to override. The resulting parser - /// iterates over [`BgpElem`]s — calling [`into_record_iter`](Self::into_record_iter) - /// or [`next_record`](Self::next_record) on a text-dump parser panics, since - /// text dumps have no MRT-record representation. + /// streams [`BgpElem`]s lazily — one route line at a time, constant memory. + /// Calling [`into_record_iter`](Self::into_record_iter) or + /// [`next_record`](Self::next_record) on a text-dump parser returns an + /// error, since text dumps have no MRT-record representation. /// /// # Example /// @@ -171,13 +171,13 @@ impl BgpkitParser { core_dump: false, filters: vec![], options: ParserOptions::default(), - text_dump_elems: None, + text_dump_iter: None, } } /// This is used in for loop `for item in parser{}` pub fn next_record(&mut self) -> Result { - if self.text_dump_elems.is_some() { + if self.text_dump_iter.is_some() { return Err(ParserError::Unsupported( "text-dump parsers have no MRT record representation; iterate elements instead" .to_string(), @@ -204,21 +204,20 @@ impl BgpkitParser> { } /// Create a text-dump parser from a reader with an explicit element - /// timestamp. The reader is fully consumed up front; the resulting parser - /// iterates over [`BgpElem`]s but has no MRT-record representation. + /// timestamp. The parser streams elements lazily — one route line at a + /// time, constant memory. It has no MRT-record representation. pub fn from_text_reader_with_timestamp( reader: impl Read + Send + 'static, timestamp: f64, ) -> Result { - let mut buf_reader = BufReader::new(reader); - let elems = parse_text_dump_with_timestamp(&mut buf_reader, timestamp) - .map_err(ParserError::from)?; + let buf_reader = BufReader::new(reader); + let iter = TextDumpElemIterator::new(buf_reader, timestamp).map_err(ParserError::from)?; Ok(BgpkitParser { reader: Box::new(std::io::empty()), core_dump: false, filters: vec![], options: ParserOptions::default(), - text_dump_elems: Some(elems.into()), + text_dump_iter: Some(Box::new(iter)), }) } @@ -242,13 +241,13 @@ impl BgpkitParser> { if is_text { let chained = BufReader::new(Cursor::new(head).chain(buf_reader)); let ts = timestamp.unwrap_or(0.0); - let elems = parse_text_dump_with_timestamp(chained, ts).map_err(ParserError::from)?; + let iter = TextDumpElemIterator::new(chained, ts).map_err(ParserError::from)?; Ok(BgpkitParser { reader: Box::new(std::io::empty()), core_dump: false, filters: vec![], options: ParserOptions::default(), - text_dump_elems: Some(elems.into()), + text_dump_iter: Some(Box::new(iter)), }) } else { Ok(BgpkitParser { @@ -256,7 +255,7 @@ impl BgpkitParser> { core_dump: false, filters: vec![], options: ParserOptions::default(), - text_dump_elems: None, + text_dump_iter: None, }) } } @@ -278,7 +277,7 @@ impl BgpkitParser { core_dump: true, filters: self.filters, options: self.options, - text_dump_elems: self.text_dump_elems, + text_dump_iter: self.text_dump_iter, } } @@ -290,7 +289,7 @@ impl BgpkitParser { core_dump: self.core_dump, filters: self.filters, options, - text_dump_elems: self.text_dump_elems, + text_dump_iter: self.text_dump_iter, } } @@ -369,7 +368,7 @@ impl BgpkitParser { core_dump: self.core_dump, filters, options: self.options, - text_dump_elems: self.text_dump_elems, + text_dump_iter: self.text_dump_iter, }) } diff --git a/src/parser/text_dump.rs b/src/parser/text_dump.rs index 508c668b..1d4b23c0 100644 --- a/src/parser/text_dump.rs +++ b/src/parser/text_dump.rs @@ -465,7 +465,114 @@ pub fn infer_timestamp_from_path(path: &str) -> Option { None } -// ── Top-level parse ──────────────────────────────────────────────── +// ── Streaming iterator ───────────────────────────────────────────── + +/// A streaming iterator that yields [`BgpElem`]s from a Cisco `sh ip bgp` +/// text dump, one route line at a time. +/// +/// Created by [`TextDumpElemIterator::new`], which consumes the preamble +/// (header + column definitions) and leaves the reader positioned at the +/// first route line. Each call to [`Iterator::next`] reads at most one line, +/// so peak memory is O(1) regardless of dump size. +/// +/// Continuation lines (multipath entries with empty prefix) reuse the +/// most-recently-seen prefix, and wrapped-prefix lines update that prefix +/// without yielding an element — both are handled in-stream. +pub struct TextDumpElemIterator { + reader: R, + column_positions: ColumnPositions, + wrapped_column_positions: Option, + peer_ip: IpAddr, + peer_asn: u32, + timestamp: f64, + current_prefix: String, + buf: String, +} + +impl TextDumpElemIterator { + /// Create a streaming text-dump element iterator. + /// + /// Reads and discards the preamble (up to and including the column header + /// line). All yielded elements carry the given `timestamp`. + pub fn new(mut reader: R, timestamp: f64) -> std::io::Result { + let header = parse_header(&mut reader)?; + let column_positions = match header.column_positions { + Some(positions) => positions, + None => { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "missing Cisco BGP table column header", + )); + } + }; + let peer_ip = header + .router_id + .unwrap_or_else(|| IpAddr::from([0, 0, 0, 0])); + let peer_asn = header.local_as.unwrap_or(0); + + Ok(TextDumpElemIterator { + reader, + column_positions, + wrapped_column_positions: shift_columns_left(column_positions), + peer_ip, + peer_asn, + timestamp, + current_prefix: String::new(), + buf: String::new(), + }) + } +} + +impl Iterator for TextDumpElemIterator { + type Item = BgpElem; + + fn next(&mut self) -> Option { + loop { + self.buf.clear(); + match self.reader.read_line(&mut self.buf) { + Ok(0) => return None, // EOF + Ok(_) => {} + Err(_) => return None, + } + + let line = self.buf.trim_end(); + if line.is_empty() { + continue; + } + + // Try parsing as a fixed-width route line (standard or shifted columns). + let entry = parse_route_line(line, self.column_positions).or_else(|| { + self.wrapped_column_positions + .and_then(|positions| parse_route_line(line, positions)) + }); + if let Some(entry) = entry { + if !entry.prefix.is_empty() { + self.current_prefix = entry.prefix.clone(); + } + if self.current_prefix.is_empty() { + continue; + } + if let Some(elem) = entry_to_elem( + &entry, + &self.current_prefix, + self.peer_ip, + self.peer_asn, + self.timestamp, + ) { + return Some(elem); + } + continue; + } + + // Wrapped-prefix-only line: update prefix, no element yielded. + if let Some(prefix) = parse_wrapped_prefix_line(line, self.column_positions.0) { + self.current_prefix = prefix; + } + } + } +} + +// ── Convenience: collect-all wrappers ────────────────────────────── /// Parse a complete Cisco `sh ip bgp` text dump into [`BgpElem`]s with /// timestamp `0.0`. @@ -481,6 +588,9 @@ pub fn parse_text_dump(reader: R) -> std::io::Result> { /// [`infer_timestamp_from_path`] to derive one from a PCH or route-views /// file name when available. /// +/// Internally this uses [`TextDumpElemIterator`] and collects the results. +/// For streaming (constant-memory) usage, construct the iterator directly. +/// /// Route-views style snapshots omit the `BGP table version` / `local AS` /// preamble. The parser falls back to the unspecified sentinels rather than /// rejecting the dump: `0.0.0.0` and AS0 carry no peer identity. @@ -488,66 +598,11 @@ pub fn parse_text_dump(reader: R) -> std::io::Result> { /// Lines that do not parse as route entries (banner text, the trailing /// `Displayed ...` summary, malformed rows) are skipped silently. pub fn parse_text_dump_with_timestamp( - mut reader: R, + reader: R, timestamp: f64, ) -> std::io::Result> { - let header = parse_header(&mut reader)?; - let column_positions = match header.column_positions { - Some(positions) => positions, - None => { - return Err(std::io::Error::new( - std::io::ErrorKind::InvalidData, - "missing Cisco BGP table column header", - )); - } - }; - let peer_ip = header - .router_id - .unwrap_or_else(|| IpAddr::from([0, 0, 0, 0])); - let peer_asn = header.local_as.unwrap_or(0); - - let wrapped_column_positions = shift_columns_left(column_positions); - - let mut buf = String::new(); - let mut current_prefix = String::new(); - let mut entries: Vec<(String, RouteEntry)> = Vec::new(); - - while reader.read_line(&mut buf)? > 0 { - let line = buf.trim_end().to_string(); - buf.clear(); - - if line.is_empty() { - continue; - } - - let entry = parse_route_line(&line, column_positions).or_else(|| { - wrapped_column_positions.and_then(|positions| parse_route_line(&line, positions)) - }); - if let Some(entry) = entry { - if !entry.prefix.is_empty() { - current_prefix = entry.prefix.clone(); - } - entries.push((current_prefix.clone(), entry)); - continue; - } - - if let Some(prefix) = parse_wrapped_prefix_line(&line, column_positions.0) { - current_prefix = prefix; - continue; - } - } - - let mut elems: Vec = Vec::with_capacity(entries.len()); - for (prefix, entry) in &entries { - if prefix.is_empty() { - continue; - } - if let Some(elem) = entry_to_elem(entry, prefix, peer_ip, peer_asn, timestamp) { - elems.push(elem); - } - } - - Ok(elems) + let iter = TextDumpElemIterator::new(reader, timestamp)?; + Ok(iter.collect()) } // ── Tests ────────────────────────────────────────────────────────── From 9b2f80acf5e9fa3d168a0653c1c85800ac3f55cd Mon Sep 17 00:00:00 2001 From: Mingwei Zhang Date: Wed, 5 Aug 2026 14:10:22 -0700 Subject: [PATCH 3/4] fix: short-circuit MRT iterators on text-dump parsers Address Copilot review: RecordIterator, FallibleRecordIterator, UpdateIterator, FallibleUpdateIterator, RawRecordIterator, RouteIterator, and FallibleRouteIterator all treated ParserError::Unsupported as a recoverable warning and continued, which caused into_record_iter() and into_update_iter() on text-dump parsers to spin forever. Add an early text_dump_iter.is_some() check to each iterator's next() that returns None immediately. Also fix the new_text doc comment to point to from_text_reader_with_timestamp for timestamp override and clarify the behavior of record-based iterators on text dumps. --- src/parser/iters/default.rs | 5 +++++ src/parser/iters/fallible.rs | 5 +++++ src/parser/iters/raw.rs | 4 ++++ src/parser/iters/route.rs | 8 ++++++++ src/parser/iters/update.rs | 9 +++++++++ src/parser/mod.rs | 26 ++++++++++++++++++++------ 6 files changed, 51 insertions(+), 6 deletions(-) diff --git a/src/parser/iters/default.rs b/src/parser/iters/default.rs index c8563b22..a10334cb 100644 --- a/src/parser/iters/default.rs +++ b/src/parser/iters/default.rs @@ -33,6 +33,11 @@ impl Iterator for RecordIterator { type Item = MrtRecord; fn next(&mut self) -> Option { + // Text-dump parsers have no MRT-record representation; short-circuit + // instead of spinning forever on Unsupported errors from next_record(). + if self.parser.text_dump_iter.is_some() { + return None; + } self.count += 1; loop { return match self.parser.next_record() { diff --git a/src/parser/iters/fallible.rs b/src/parser/iters/fallible.rs index d549defa..4009634f 100644 --- a/src/parser/iters/fallible.rs +++ b/src/parser/iters/fallible.rs @@ -33,6 +33,11 @@ impl Iterator for FallibleRecordIterator { type Item = Result; fn next(&mut self) -> Option { + // Text-dump parsers have no MRT-record representation; short-circuit + // instead of repeatedly returning Unsupported errors from next_record(). + if self.parser.text_dump_iter.is_some() { + return None; + } loop { match self.parser.next_record() { Ok(record) => { diff --git a/src/parser/iters/raw.rs b/src/parser/iters/raw.rs index 89e40833..80ed3a82 100644 --- a/src/parser/iters/raw.rs +++ b/src/parser/iters/raw.rs @@ -31,6 +31,10 @@ impl Iterator for RawRecordIterator { type Item = RawMrtRecord; fn next(&mut self) -> Option { + // Text-dump parsers have no MRT-record representation; short-circuit. + if self.parser.text_dump_iter.is_some() { + return None; + } self.count += 1; loop { match chunk_mrt_record(&mut self.parser.reader) { diff --git a/src/parser/iters/route.rs b/src/parser/iters/route.rs index f4f5645e..cc2937e8 100644 --- a/src/parser/iters/route.rs +++ b/src/parser/iters/route.rs @@ -681,6 +681,10 @@ impl Iterator for RouteIterator { type Item = BgpRouteElem; fn next(&mut self) -> Option { + // Text-dump parsers have no MRT-record representation; short-circuit. + if self.parser.text_dump_iter.is_some() { + return None; + } loop { match self.pending_routes.next_route() { Ok(Some(route)) => { @@ -789,6 +793,10 @@ impl Iterator for FallibleRouteIterator { type Item = Result; fn next(&mut self) -> Option { + // Text-dump parsers have no MRT-record representation; short-circuit. + if self.parser.text_dump_iter.is_some() { + return None; + } loop { match self.pending_routes.next_route() { Ok(Some(route)) => { diff --git a/src/parser/iters/update.rs b/src/parser/iters/update.rs index 6ab63843..8d77f15b 100644 --- a/src/parser/iters/update.rs +++ b/src/parser/iters/update.rs @@ -168,6 +168,11 @@ impl Iterator for UpdateIterator { type Item = MrtUpdate; fn next(&mut self) -> Option { + // Text-dump parsers have no MRT-record representation; short-circuit + // instead of spinning forever on Unsupported errors from next_record(). + if self.parser.text_dump_iter.is_some() { + return None; + } loop { if let Some(message) = self.pending_table_dump.pop() { return Some(MrtUpdate::TableDumpMessage(message)); @@ -316,6 +321,10 @@ impl Iterator for FallibleUpdateIterator { type Item = Result; fn next(&mut self) -> Option { + // Text-dump parsers have no MRT-record representation; short-circuit. + if self.parser.text_dump_iter.is_some() { + return None; + } loop { if let Some(message) = self.pending_table_dump.pop() { return Some(Ok(MrtUpdate::TableDumpMessage(message))); diff --git a/src/parser/mod.rs b/src/parser/mod.rs index d3e3f1d6..5336d6a3 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -102,12 +102,13 @@ impl BgpkitParser> { /// or route-views `oix-full-snapshot-*` files). /// /// The file is auto-decompressed by oneio. The timestamp for all elements - /// is inferred from the file name when possible; pass - /// [`infer_timestamp_from_path`] yourself to override. The resulting parser - /// streams [`BgpElem`]s lazily — one route line at a time, constant memory. - /// Calling [`into_record_iter`](Self::into_record_iter) or - /// [`next_record`](Self::next_record) on a text-dump parser returns an - /// error, since text dumps have no MRT-record representation. + /// is inferred from the file name when possible. To override the timestamp, + /// use [`from_text_reader_with_timestamp`](Self::from_text_reader_with_timestamp) + /// directly. The resulting parser streams [`BgpElem`]s lazily — one route + /// line at a time, constant memory. Calling [`into_record_iter`](Self::into_record_iter) + /// or [`next_record`](Self::next_record) on a text-dump parser returns no + /// records (text dumps have no MRT-record representation); use + /// [`into_elem_iter`](Self::into_elem_iter) or the `for elem in parser` loop instead. /// /// # Example /// @@ -671,4 +672,17 @@ Default local pref 100, local AS 65001\n\n\ BgpkitParser::from_text_reader(dump.as_bytes()).expect("inline text-dump parse"); assert!(parser.next_record().is_err()); } + + #[test] + fn test_text_dump_record_iter_terminates() { + // Calling into_record_iter on a text-dump parser should yield 0 + // records immediately, not spin forever on Unsupported errors. + let dump = "BGP table version is 1, local router ID is 1.2.3.4, vrf id 0\n\ +Default local pref 100, local AS 65001\n\n\ + Network Next Hop Metric LocPrf Weight Path\n\ + *> 1.0.0.0/24 10.0.0.1 0 0 13335 i\n"; + let parser = + BgpkitParser::from_text_reader(dump.as_bytes()).expect("inline text-dump parse"); + assert_eq!(parser.into_record_iter().count(), 0); + } } From 7d8a219a8ff88960e89661fb23d5e99261ebf568 Mon Sep 17 00:00:00 2001 From: Mingwei Zhang Date: Wed, 5 Aug 2026 14:23:32 -0700 Subject: [PATCH 4/4] =?UTF-8?q?fix:=20address=20Copilot=20review=20?= =?UTF-8?q?=E2=80=94=20log=20read=20errors,=20fix=20doc=20comments?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - TextDumpElemIterator: log read_line I/O errors instead of silently treating them as EOF, so partial reads are visible in logs - from_auto_reader_with_timestamp: fix doc to clarify that timestamp sets (not overrides) the text-dump timestamp, since this method has no path to infer from; point to new_auto for filename-based inference - as_path_from_tokens: document why bare u32 parsing is correct (Cisco sh ip bgp output never uses AS{n} notation) --- src/parser/mod.rs | 4 ++-- src/parser/text_dump.rs | 15 ++++++++++++++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 5336d6a3..1f03cbfe 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -231,8 +231,8 @@ impl BgpkitParser> { } /// Create a parser from any reader, auto-detecting MRT vs text dump. - /// When text is detected, `timestamp` overrides the inferred value - /// (`None` → `0.0`). + /// `timestamp` sets the element timestamp for text dumps (`None` → `0.0`); + /// for filename-based inference, use [`new_auto`](Self::new_auto) instead. pub fn from_auto_reader_with_timestamp( reader: impl Read + Send + 'static, timestamp: Option, diff --git a/src/parser/text_dump.rs b/src/parser/text_dump.rs index 1d4b23c0..60feb4b1 100644 --- a/src/parser/text_dump.rs +++ b/src/parser/text_dump.rs @@ -308,6 +308,10 @@ fn parse_route_line(line: &str, columns: ColumnPositions) -> Option /// AS-set delimiters (`{` / `}`) are silently dropped, flattening AS-sets /// into plain AS-sequences. This is intentional: the parser aims to recover /// the AS-level propagation path, and set membership is not preserved. +/// +/// Only bare integer tokens are recognized (e.g. `13335`, `4755`). Cisco +/// `sh ip bgp` output never uses the `AS{n}` notation, so `Asn`'s `FromStr` +/// (which handles that syntax) is not needed here. fn as_path_from_tokens(tokens: &[String]) -> AsPath { let mut asns: Vec = Vec::new(); for token in tokens { @@ -532,7 +536,16 @@ impl Iterator for TextDumpElemIterator { match self.reader.read_line(&mut self.buf) { Ok(0) => return None, // EOF Ok(_) => {} - Err(_) => return None, + Err(e) => { + // Iterator::next cannot propagate io::Error, so log the + // failure rather than silently truncating. Note that the + // collect-all wrapper (parse_text_dump_with_timestamp) + // also loses the error since it uses this iterator + // internally; callers needing error propagation should + // construct the iterator and inspect logs. + log::warn!("text-dump read error, stopping iteration: {e}"); + return None; + } } let line = self.buf.trim_end();