From c5c4c9e7084ccc838e340697a25ca31bc2dd7d7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Richard=20Sol=C3=A1r?= Date: Fri, 18 Sep 2026 15:53:19 +0200 Subject: [PATCH 1/2] fix(scrapy): pass extra_name to try_import so apify.scrapy imports on crawlee 1.10.1 crawlee 1.10.1 made extra_name a required keyword-only argument of crawlee._utils.try_import.try_import. Every call site in apify.scrapy omitted it, so importing the package raised TypeError at module load and broke every fresh apify[scrapy] install. Bump crawlee in uv.lock to 1.10.1 as well, so CI installs the version that exposes the break instead of the pinned 1.10.0. Co-Authored-By: Claude Opus 5 --- src/apify/scrapy/__init__.py | 10 +++++----- uv.lock | 8 ++++---- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/apify/scrapy/__init__.py b/src/apify/scrapy/__init__.py index dfab99436..5f966faa2 100644 --- a/src/apify/scrapy/__init__.py +++ b/src/apify/scrapy/__init__.py @@ -5,19 +5,19 @@ # The following imports use try_import to handle optional dependencies, as they may not always be available. -with _try_import(__name__, 'run_scrapy_actor'): +with _try_import(__name__, 'run_scrapy_actor', extra_name='scrapy'): from ._actor_runner import run_scrapy_actor -with _try_import(__name__, 'initialize_logging'): +with _try_import(__name__, 'initialize_logging', extra_name='scrapy'): from ._logging_config import initialize_logging -with _try_import(__name__, 'to_apify_request', 'to_scrapy_request'): +with _try_import(__name__, 'to_apify_request', 'to_scrapy_request', extra_name='scrapy'): from .requests import to_apify_request, to_scrapy_request -with _try_import(__name__, 'ApifyScheduler'): +with _try_import(__name__, 'ApifyScheduler', extra_name='scrapy'): from .scheduler import ApifyScheduler -with _try_import(__name__, 'apply_apify_settings', 'get_basic_auth_header'): +with _try_import(__name__, 'apply_apify_settings', 'get_basic_auth_header', extra_name='scrapy'): from .utils import apply_apify_settings, get_basic_auth_header diff --git a/uv.lock b/uv.lock index fd6ee70c6..a7065f117 100644 --- a/uv.lock +++ b/uv.lock @@ -12,8 +12,8 @@ exclude-newer-span = "PT24H" [options.exclude-newer-package] apify-client = false -apify-fingerprint-datapoints = false crawlee = false +apify-fingerprint-datapoints = false [[package]] name = "aiohappyeyeballs" @@ -925,7 +925,7 @@ toml = [ [[package]] name = "crawlee" -version = "1.10.0" +version = "1.10.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "async-timeout" }, @@ -941,9 +941,9 @@ dependencies = [ { name = "typing-extensions" }, { name = "yarl" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/7d/f8/cc9deb758524bab531e302c843e594813dccd331fe5ec1c000d2fe8a6d74/crawlee-1.10.0.tar.gz", hash = "sha256:3f83a24258034aa11ab946cc528dd72c6307467e6d9d416966a011f7c549bf6c", size = 334721, upload-time = "2026-08-31T07:49:54.003Z" } +sdist = { url = "https://files.pythonhosted.org/packages/30/c4/9e077f31165b8b8e55bae61021925477c979081dddb8faf15c71a7cfd417/crawlee-1.10.1.tar.gz", hash = "sha256:86e40a8dedf16698765738f2b1f0df21cba3a418f5f60442ccdb15c3da14c24f", size = 336270, upload-time = "2026-09-16T12:48:20.325Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/fd/11/d50bfcd0938b3a8eb83a8c75cf4c8cb6894d8c030dcdf8412af03f67aa82/crawlee-1.10.0-py3-none-any.whl", hash = "sha256:cf496f56b2d59b9416227169882e33c441b99dcf1127d3917a2ab7f891e35597", size = 420571, upload-time = "2026-08-31T07:49:52.435Z" }, + { url = "https://files.pythonhosted.org/packages/33/1f/88cbfbc665649e4e8c143ceb7c4bfe24b1780845e1f2a8fdfcc991d48936/crawlee-1.10.1-py3-none-any.whl", hash = "sha256:27204c4e505d8e28c8ac432963732648267b0c353acddac04675aedb9dae9253", size = 421821, upload-time = "2026-09-16T12:48:18.769Z" }, ] [package.optional-dependencies] From faed4461c34cec36081bf76d9c86a8f6ee539ff0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Richard=20Sol=C3=A1r?= Date: Fri, 18 Sep 2026 16:02:13 +0200 Subject: [PATCH 2/2] fix(scrapy): own the import guards so the missing-extra hint names apify[scrapy] Reusing crawlee's private try_import made the hint tell users to run pip install 'crawlee[scrapy]'. crawlee has no scrapy extra, so that command installs nothing and leaves the user stuck. Move the import-guard machinery into apify._try_import, which names the apify distribution. This also stops a signature change in a private crawlee module from breaking apify.scrapy at import time, as 1.10.1 did. Co-Authored-By: Claude Opus 5 --- src/apify/_try_import.py | 68 ++++++++++++++++++++++++++++++++++ src/apify/scrapy/__init__.py | 4 +- tests/unit/test_try_import.py | 70 +++++++++++++++++++++++++++++++++++ 3 files changed, 140 insertions(+), 2 deletions(-) create mode 100644 src/apify/_try_import.py create mode 100644 tests/unit/test_try_import.py diff --git a/src/apify/_try_import.py b/src/apify/_try_import.py new file mode 100644 index 000000000..6846cbe84 --- /dev/null +++ b/src/apify/_try_import.py @@ -0,0 +1,68 @@ +from __future__ import annotations + +import sys +from contextlib import contextmanager +from dataclasses import dataclass +from types import ModuleType +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Iterator + from typing import Any + +_DISTRIBUTION_NAME = 'apify' + + +@contextmanager +def try_import(module_name: str, *symbol_names: str, extra_name: str | list[str]) -> Iterator[None]: + """Context manager to attempt importing symbols into a module. + + If an `ImportError` is raised during the import, the symbols are replaced with `FailedImport` objects. When the + error is a `ModuleNotFoundError`, the message also names the optional extra (or one of several) that installs the + missing dependency. Other import errors, including those raised by a nested guard, keep their message as is. + """ + try: + yield + except ImportError as e: + message = e.args[0] + if isinstance(e, ModuleNotFoundError): + message = f'{message}. {_get_install_hint(extra_name)}' + for symbol_name in symbol_names: + setattr(sys.modules[module_name], symbol_name, FailedImport(message)) + + +def _get_install_hint(extra_name: str | list[str]) -> str: + """Build the sentence telling the user which extra installs the missing optional dependency.""" + if isinstance(extra_name, str): + return f"Install the optional '{extra_name}' extra to use it: pip install '{_DISTRIBUTION_NAME}[{extra_name}]'" + + extras = ', '.join(f"'{name}'" for name in extra_name) + return ( + f'Install one of the optional extras {extras} to use it, e.g. ' + f"pip install '{_DISTRIBUTION_NAME}[{extra_name[0]}]'" + ) + + +def install_import_hook(module_name: str) -> None: + """Install an import hook for a specified module.""" + sys.modules[module_name].__class__ = ImportWrapper + + +@dataclass +class FailedImport: + """Represent a placeholder for a failed import.""" + + message: str + """The error message associated with the failed import.""" + + +class ImportWrapper(ModuleType): + """A wrapper class for modules to handle attribute access for failed imports.""" + + def __getattribute__(self, name: str) -> Any: + result = super().__getattribute__(name) + + if isinstance(result, FailedImport): + raise ImportError(result.message) # noqa: TRY004 + + return result diff --git a/src/apify/scrapy/__init__.py b/src/apify/scrapy/__init__.py index 5f966faa2..0cbfca005 100644 --- a/src/apify/scrapy/__init__.py +++ b/src/apify/scrapy/__init__.py @@ -1,5 +1,5 @@ -from crawlee._utils.try_import import install_import_hook as _install_import_hook -from crawlee._utils.try_import import try_import as _try_import +from apify._try_import import install_import_hook as _install_import_hook +from apify._try_import import try_import as _try_import _install_import_hook(__name__) diff --git a/tests/unit/test_try_import.py b/tests/unit/test_try_import.py new file mode 100644 index 000000000..64f43875f --- /dev/null +++ b/tests/unit/test_try_import.py @@ -0,0 +1,70 @@ +from __future__ import annotations + +import sys +from types import ModuleType +from typing import TYPE_CHECKING, Any + +import pytest + +from apify._try_import import FailedImport, install_import_hook, try_import + +if TYPE_CHECKING: + from collections.abc import Iterator + + +@pytest.fixture +def module() -> Iterator[Any]: + """Register a throwaway module that the import guards can write their placeholders into.""" + mod = ModuleType('apify_test_try_import_target') + sys.modules[mod.__name__] = mod + install_import_hook(mod.__name__) + yield mod + del sys.modules[mod.__name__] + + +def test_successful_import_is_left_alone(module: Any) -> None: + with try_import(module.__name__, 'symbol', extra_name='scrapy'): + module.symbol = 'value' + + assert module.symbol == 'value' + + +def test_missing_module_names_the_apify_extra(module: Any) -> None: + with try_import(module.__name__, 'symbol', extra_name='scrapy'): + raise ModuleNotFoundError("No module named 'scrapy'") + + with pytest.raises(ImportError) as exc_info: + _ = module.symbol + + assert str(exc_info.value) == ( + "No module named 'scrapy'. Install the optional 'scrapy' extra to use it: pip install 'apify[scrapy]'" + ) + + +def test_missing_module_names_one_of_several_extras(module: Any) -> None: + with try_import(module.__name__, 'symbol', extra_name=['scrapy', 'other']): + raise ModuleNotFoundError("No module named 'scrapy'") + + with pytest.raises(ImportError) as exc_info: + _ = module.symbol + + assert str(exc_info.value) == ( + "No module named 'scrapy'. Install one of the optional extras 'scrapy', 'other' to use it, " + "e.g. pip install 'apify[scrapy]'" + ) + + +def test_other_import_errors_keep_their_message(module: Any) -> None: + with try_import(module.__name__, 'symbol', extra_name='scrapy'): + raise ImportError('cannot import name X') + + with pytest.raises(ImportError, match=r'^cannot import name X$'): + _ = module.symbol + + +def test_all_guarded_symbols_are_replaced(module: Any) -> None: + with try_import(module.__name__, 'first', 'second', extra_name='scrapy'): + raise ModuleNotFoundError("No module named 'scrapy'") + + for name in ('first', 'second'): + assert isinstance(object.__getattribute__(module, name), FailedImport)