diff --git a/crawl4ai/deep_crawling/bfs_strategy.py b/crawl4ai/deep_crawling/bfs_strategy.py index 2b00a6cda..8353038cb 100644 --- a/crawl4ai/deep_crawling/bfs_strategy.py +++ b/crawl4ai/deep_crawling/bfs_strategy.py @@ -150,7 +150,7 @@ async def link_discovery( return # If we've reached the max pages limit, don't discover new links - remaining_capacity = self.max_pages - self._pages_crawled + remaining_capacity = self.max_pages - self._pages_crawled - len(next_level) if remaining_capacity <= 0: self.logger.info(f"Max pages limit ({self.max_pages}) reached, stopping link discovery") return diff --git a/tests/deep_crawling/test_deep_crawl_resume.py b/tests/deep_crawling/test_deep_crawl_resume.py index c6d84180d..b191c42f7 100644 --- a/tests/deep_crawling/test_deep_crawl_resume.py +++ b/tests/deep_crawling/test_deep_crawl_resume.py @@ -672,6 +672,34 @@ async def test_max_pages_limit_respected(self): # Should stop at max_pages assert strategy._pages_crawled <= 3 + @pytest.mark.asyncio + async def test_link_discovery_counts_already_queued_pages(self): + """Parents on one level share the remaining page budget.""" + strategy = BFSDeepCrawlStrategy(max_depth=2, max_pages=5) + strategy._pages_crawled = 3 + visited = set() + next_level = [] + depths = {} + + for parent in ("first", "second"): + result = MagicMock() + result.links = { + "internal": [ + {"href": f"https://example.com/{parent}-{index}"} + for index in range(3) + ] + } + await strategy.link_discovery( + result, + f"https://example.com/{parent}", + 1, + visited, + next_level, + depths, + ) + + assert len(next_level) == 2 + @pytest.mark.asyncio async def test_max_depth_limit_respected(self): """max_depth limit still enforced."""