fix: enforce BFS max_pages across each level

crawl4ai/deep_crawling/bfs_strategy.py

@@ -150,7 +150,7 @@ async def link_discovery(
             return
 
         # If we've reached the max pages limit, don't discover new links
-        remaining_capacity = self.max_pages - self._pages_crawled
+        remaining_capacity = self.max_pages - self._pages_crawled - len(next_level)
         if remaining_capacity <= 0:
             self.logger.info(f"Max pages limit ({self.max_pages}) reached, stopping link discovery")
             return

tests/deep_crawling/test_deep_crawl_resume.py

@@ -672,6 +672,34 @@ async def test_max_pages_limit_respected(self):
         # Should stop at max_pages
         assert strategy._pages_crawled <= 3
 
+    @pytest.mark.asyncio
+    async def test_link_discovery_counts_already_queued_pages(self):
+        """Parents on one level share the remaining page budget."""
+        strategy = BFSDeepCrawlStrategy(max_depth=2, max_pages=5)
+        strategy._pages_crawled = 3
+        visited = set()
+        next_level = []
+        depths = {}
+
+        for parent in ("first", "second"):
+            result = MagicMock()
+            result.links = {
+                "internal": [
+                    {"href": f"https://example.com/{parent}-{index}"}
+                    for index in range(3)
+                ]
+            }
+            await strategy.link_discovery(
+                result,
+                f"https://example.com/{parent}",
+                1,
+                visited,
+                next_level,
+                depths,
+            )
+
+        assert len(next_level) == 2
+
     @pytest.mark.asyncio
     async def test_max_depth_limit_respected(self):
         """max_depth limit still enforced."""