feat: add FileDownloadCrawler

docs/examples/code_examples/file_download.py

@@ -0,0 +1,43 @@
+import asyncio
+
+from yarl import URL
+
+from crawlee.crawlers import FileDownloadCrawler, FileDownloadCrawlingContext
+
+
+async def main() -> None:
+    # FileDownloadCrawler downloads files with plain HTTP requests and accepts
+    # any content type.
+    crawler = FileDownloadCrawler(
+        # Limit the crawl to max requests. Remove or increase it for crawling all links.
+        max_requests_per_crawl=10,
+    )
+
+    # Define the default request handler, which will be called for every request.
+    @crawler.router.default_handler
+    async def request_handler(context: FileDownloadCrawlingContext) -> None:
+        context.log.info(f'Downloading {context.request.url} ...')
+
+        # Read the whole file into memory.
+        content = await context.http_response.read()
+
+        # Save the file to the default key-value store with the server's content type.
+        kvs = await context.get_key_value_store()
+        file_name = URL(context.request.url).name
+        await kvs.set_value(
+            key=file_name,
+            value=content,
+            content_type=context.http_response.headers.get('content-type'),
+        )
+
+    # Run the crawler with the list of files to download.
+    await crawler.run(
+        [
+            'https://pdfobject.com/pdf/sample.pdf',
+            'https://crawlee.dev/assets/images/gradcracker-scraper-caefb62d1c150c4209a6e564c052fa41.webp',
+        ]
+    )
+
+
+if __name__ == '__main__':
+    asyncio.run(main())

docs/examples/code_examples/file_download_stream.py

@@ -0,0 +1,49 @@
+import asyncio
+from datetime import timedelta
+from pathlib import Path
+
+from yarl import URL
+
+from crawlee.crawlers import FileDownloadCrawler, FileDownloadCrawlingContext
+
+DOWNLOAD_DIR = Path('downloads')
+
+
+async def main() -> None:
+    # With stream=True, the request handler receives a response whose body has not
+    # been read yet.
+    crawler = FileDownloadCrawler(
+        stream=True,
+        # Bounds establishing the connection and receiving the response headers.
+        navigation_timeout=timedelta(minutes=5),
+        # The body is downloaded inside the handler, so this bounds the transfer itself.
+        request_handler_timeout=timedelta(minutes=5),
+        # Limit the crawl to max requests. Remove or increase it for crawling all links.
+        max_requests_per_crawl=10,
+    )
+
+    # Define the default request handler, which will be called for every request.
+    @crawler.router.default_handler
+    async def request_handler(context: FileDownloadCrawlingContext) -> None:
+        context.log.info(f'Downloading {context.request.url} ...')
+
+        file_name = URL(context.request.url).name
+
+        # Write each chunk to disk as it arrives, without buffering the whole file.
+        with (DOWNLOAD_DIR / file_name).open('wb') as file:
+            async for chunk in context.http_response.read_stream():
+                file.write(chunk)
+
+        context.log.info(f'Saved {file_name}')
+
+    # Run the crawler with the list of files to download.
+    await crawler.run(
+        [
+            'https://samplelib.com/mp4/sample-15s-720p.mp4',
+        ]
+    )
+
+
+if __name__ == '__main__':
+    DOWNLOAD_DIR.mkdir(exist_ok=True)
+    asyncio.run(main())

docs/examples/file_download.mdx

@@ -0,0 +1,25 @@
+---
+id: file-download
+title: Download files
+---
+
+import ApiLink from '@site/src/components/ApiLink';
+import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock';
+import CodeBlock from '@theme/CodeBlock';
+
+import FileDownloadExample from '!!raw-loader!roa-loader!./code_examples/file_download.py';
+import FileDownloadStreamExample from '!!raw-loader!./code_examples/file_download_stream.py';
+
+This example demonstrates how to use <ApiLink to="class/FileDownloadCrawler">`FileDownloadCrawler`</ApiLink> to download files such as PDFs, images or videos with plain HTTP requests. The crawler doesn't parse the response body, so any content type is accepted. Each downloaded file is saved to the default <ApiLink to="class/KeyValueStore">`KeyValueStore`</ApiLink> together with the content type reported by the server.
+
+<RunnableCodeBlock className="language-python" language="python">
+    {FileDownloadExample}
+</RunnableCodeBlock>
+
+## Streaming large files
+
+Buffering a whole file in memory doesn't scale to large downloads. Construct the crawler with `stream=True` and the request handler receives a response whose body hasn't been read yet. Consume it in chunks with <ApiLink to="class/HttpResponse#read_stream">`read_stream()`</ApiLink> and write each chunk to disk as it arrives.
+
+<CodeBlock className="language-python" language="python">
+    {FileDownloadStreamExample}
+</CodeBlock>

docs/guides/architecture_overview.mdx

@@ -51,6 +51,8 @@ class BeautifulSoupCrawler
 
 class PydanticAiCrawler
 
+class FileDownloadCrawler
+
 class PlaywrightCrawler
 
 class AdaptivePlaywrightCrawler
@@ -68,19 +70,21 @@ AbstractHttpCrawler --|> HttpCrawler
 AbstractHttpCrawler --|> ParselCrawler
 AbstractHttpCrawler --|> BeautifulSoupCrawler
 AbstractHttpCrawler --|> PydanticAiCrawler
+AbstractHttpCrawler --|> FileDownloadCrawler
 PlaywrightCrawler --|> StagehandCrawler
 ```
 
 ### HTTP crawlers
 
 HTTP crawlers use HTTP clients to fetch pages and parse them with HTML parsing libraries. They are fast and efficient for sites that do not require JavaScript rendering. HTTP clients are Crawlee components that wrap around HTTP libraries like [httpx](https://www.python-httpx.org/), [curl-impersonate](https://github.com/lwthiker/curl-impersonate) or [impit](https://github.com/apify/impit) and handle HTTP communication for requests and responses. You can learn more about them in the [HTTP clients guide](./http-clients).
 
-HTTP crawlers inherit from <ApiLink to="class/AbstractHttpCrawler">`AbstractHttpCrawler`</ApiLink> and there are four crawlers that belong to this category:
+HTTP crawlers inherit from <ApiLink to="class/AbstractHttpCrawler">`AbstractHttpCrawler`</ApiLink> and there are five crawlers that belong to this category:
 
 - <ApiLink to="class/BeautifulSoupCrawler">`BeautifulSoupCrawler`</ApiLink> utilizes the [BeautifulSoup](https://www.crummy.com/software/BeautifulSoup/) HTML parser.
 - <ApiLink to="class/ParselCrawler">`ParselCrawler`</ApiLink> utilizes [Parsel](https://github.com/scrapy/parsel) for parsing HTML.
 - <ApiLink to="class/HttpCrawler">`HttpCrawler`</ApiLink> does not parse HTTP responses at all and is used when no content parsing is required.
 - <ApiLink to="class/PydanticAiCrawler">`PydanticAiCrawler`</ApiLink> parses HTML with Parsel and uses an LLM to extract structured data into a validated Pydantic model.
+- <ApiLink to="class/FileDownloadCrawler">`FileDownloadCrawler`</ApiLink> downloads files of any content type, optionally streaming large bodies in chunks.
 
 You can learn more about HTTP crawlers in the [HTTP crawlers guide](./http-crawlers).
 
@@ -126,6 +130,8 @@ class BeautifulSoupCrawlingContext
 
 class PydanticAiCrawlingContext
 
+class FileDownloadCrawlingContext
+
 class PlaywrightPreNavCrawlingContext
 
 class PlaywrightCrawlingContext
@@ -156,6 +162,8 @@ ParsedHttpCrawlingContext --|> BeautifulSoupCrawlingContext
 
 ParselCrawlingContext --|> PydanticAiCrawlingContext
 
+HttpCrawlingContext --|> FileDownloadCrawlingContext
+
 BasicCrawlingContext --|> PlaywrightPreNavCrawlingContext
 
 PlaywrightPreNavCrawlingContext --|> PlaywrightCrawlingContext
@@ -178,6 +186,7 @@ They have a similar inheritance structure as the crawlers, with the base class b
 - <ApiLink to="class/ParselCrawlingContext">`ParselCrawlingContext`</ApiLink> for HTTP crawlers that use [Parsel](https://github.com/scrapy/parsel) for parsing.
 - <ApiLink to="class/BeautifulSoupCrawlingContext">`BeautifulSoupCrawlingContext`</ApiLink> for HTTP crawlers that use [BeautifulSoup](https://www.crummy.com/software/BeautifulSoup/) for parsing.
 - <ApiLink to="class/PydanticAiCrawlingContext">`PydanticAiCrawlingContext`</ApiLink> for the AI crawler, extending the Parsel context with an `extract` helper.
+- <ApiLink to="class/FileDownloadCrawlingContext">`FileDownloadCrawlingContext`</ApiLink> for the file download crawler.
 - <ApiLink to="class/PlaywrightPreNavCrawlingContext">`PlaywrightPreNavCrawlingContext`</ApiLink> for Playwright crawlers before the page is navigated.
 - <ApiLink to="class/PlaywrightCrawlingContext">`PlaywrightCrawlingContext`</ApiLink> for Playwright crawlers.
 - <ApiLink to="class/AdaptivePlaywrightPreNavCrawlingContext">`AdaptivePlaywrightPreNavCrawlingContext`</ApiLink> for Adaptive Playwright crawlers before the page is navigated.

docs/guides/http_crawlers.mdx

@@ -30,7 +30,7 @@ HTTP crawlers are ideal for extracting data from server-rendered websites that d
 
 ## Overview
 
-All HTTP crawlers share a common architecture built around the <ApiLink to="class/AbstractHttpCrawler">`AbstractHttpCrawler`</ApiLink> base class. The main differences lie in the parsing strategy and the context provided to request handlers. There are <ApiLink to="class/BeautifulSoupCrawler">`BeautifulSoupCrawler`</ApiLink>, <ApiLink to="class/ParselCrawler">`ParselCrawler`</ApiLink>, and <ApiLink to="class/HttpCrawler">`HttpCrawler`</ApiLink>. It can also be extended to create custom crawlers with specialized parsing requirements. They use HTTP clients to fetch page content and parsing libraries to extract data from the HTML, check out the [HTTP clients guide](./http-clients) to learn about the HTTP clients used by these crawlers, how to switch between them, and how to create custom HTTP clients tailored to your specific requirements.
+All HTTP crawlers share a common architecture built around the <ApiLink to="class/AbstractHttpCrawler">`AbstractHttpCrawler`</ApiLink> base class. The main differences lie in the parsing strategy and the context provided to request handlers. There are <ApiLink to="class/BeautifulSoupCrawler">`BeautifulSoupCrawler`</ApiLink>, <ApiLink to="class/ParselCrawler">`ParselCrawler`</ApiLink>, <ApiLink to="class/HttpCrawler">`HttpCrawler`</ApiLink>, and <ApiLink to="class/FileDownloadCrawler">`FileDownloadCrawler`</ApiLink>. It can also be extended to create custom crawlers with specialized parsing requirements. They use HTTP clients to fetch page content and parsing libraries to extract data from the HTML, check out the [HTTP clients guide](./http-clients) to learn about the HTTP clients used by these crawlers, how to switch between them, and how to create custom HTTP clients tailored to your specific requirements.
 
 ```mermaid
 ---
@@ -63,6 +63,8 @@ class ParselCrawler
 
 class BeautifulSoupCrawler
 
+class FileDownloadCrawler
+
 %% ========================
 %% Inheritance arrows
 %% ========================
@@ -71,6 +73,7 @@ BasicCrawler --|> AbstractHttpCrawler
 AbstractHttpCrawler --|> HttpCrawler
 AbstractHttpCrawler --|> ParselCrawler
 AbstractHttpCrawler --|> BeautifulSoupCrawler
+AbstractHttpCrawler --|> FileDownloadCrawler
 ```
 
 ## BeautifulSoupCrawler
@@ -131,6 +134,10 @@ The following examples demonstrate how to integrate with several popular parsing
     </TabItem>
 </Tabs>
 
+## FileDownloadCrawler
+
+The <ApiLink to="class/FileDownloadCrawler">`FileDownloadCrawler`</ApiLink> downloads files instead of scraping pages. It accepts any content type without parsing and gives the request handler direct access to the response body. By default the whole file is buffered in memory. For large files, construct the crawler with `stream=True` and consume the body in chunks via <ApiLink to="class/HttpResponse#read_stream">`read_stream()`</ApiLink>. For usage, see the [Download files](../examples/file-download) example.
+
 ## Custom HTTP crawler
 
 While the built-in crawlers cover most use cases, you might need a custom HTTP crawler for specialized parsing requirements. To create a custom HTTP crawler, inherit directly from <ApiLink to="class/AbstractHttpCrawler">`AbstractHttpCrawler`</ApiLink>. This approach requires implementing:

src/crawlee/crawlers/__init__.py

@@ -3,6 +3,7 @@
 
 from ._abstract_http import AbstractHttpCrawler, AbstractHttpParser, HttpCrawlerOptions, ParsedHttpCrawlingContext
 from ._basic import BasicCrawler, BasicCrawlerOptions, BasicCrawlingContext, ContextPipeline
+from ._file_download import FileDownloadCrawler, FileDownloadCrawlingContext
 from ._http import HttpCrawler, HttpCrawlingContext, HttpCrawlingResult
 
 _install_import_hook(__name__)
@@ -113,6 +114,8 @@
     'BeautifulSoupCrawlingContext',
     'BeautifulSoupParserType',
     'ContextPipeline',
+    'FileDownloadCrawler',
+    'FileDownloadCrawlingContext',
     'HttpCrawler',
     'HttpCrawlerOptions',
     'HttpCrawlingContext',

src/crawlee/crawlers/_abstract_http/_abstract_http_crawler.py

@@ -30,7 +30,7 @@
 
     from ._abstract_http_parser import AbstractHttpParser
 
-TCrawlingContext = TypeVar('TCrawlingContext', bound=ParsedHttpCrawlingContext)
+TCrawlingContext = TypeVar('TCrawlingContext', bound=HttpCrawlingContext)
 TStatisticsState = TypeVar('TStatisticsState', bound=StatisticsState, default=StatisticsState)
 
 

src/crawlee/crawlers/_file_download/__init__.py

@@ -0,0 +1,7 @@
+from ._file_download_crawler import FileDownloadCrawler
+from ._file_download_crawling_context import FileDownloadCrawlingContext
+
+__all__ = [
+    'FileDownloadCrawler',
+    'FileDownloadCrawlingContext',
+]