diff --git a/.circleci/config.yml b/.circleci/config.yml index 5fc6d40897..887bee554a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -17,6 +17,20 @@ jobs: - run: name: Check dependencies are compatible command: pixi run --as-is pip check + - run: + name: Generate pre-commit cache key + command: date '+%Y-%W' | tee pre_commit_cache_key.txt + - restore_cache: + key: pre-commit-{{ .Branch }}-{{ checksum "pre_commit_cache_key.txt" }} + - run: + name: Check code quality + command: pixi run --as-is pre-commit run -a + - save_cache: + key: pre-commit-{{ .Branch }}-{{ checksum "pre_commit_cache_key.txt" }} + paths: + - ~/.cache/pre-commit + - ./.mypy_cache + - ./.ruff_cache - run: name: Run tests command: | diff --git a/.github/dependabot.yml b/.github/dependabot.yml index b8a7c258bc..446682c200 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -14,6 +14,24 @@ updates: interval: "weekly" cooldown: default-days: 7 + groups: + gha: + patterns: + - "*" + labels: + - dependencies + - github_actions + - automatedPR + - package-ecosystem: "pre-commit" + directory: "/" + schedule: + interval: "weekly" + cooldown: + default-days: 7 + groups: + pre-commit: + patterns: + - "*" labels: - dependencies - github_actions diff --git a/.github/workflows/zizmor.yml b/.github/workflows/zizmor.yml new file mode 100644 index 0000000000..021560f489 --- /dev/null +++ b/.github/workflows/zizmor.yml @@ -0,0 +1,23 @@ +name: GitHub Actions Security Analysis with zizmor 🌈 + +on: + push: + branches: ["main"] + pull_request: + branches: ["**"] + +permissions: {} + +jobs: + zizmor: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Run zizmor 🌈 + uses: zizmorcore/zizmor-action@192e21d79ab29983730a13d1382995c2307fbcaa # v0.5.7 + with: + advanced-security: false diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 39f1308947..02dfd4946d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -25,20 +25,35 @@ repos: - id: end-of-file-fixer - id: trailing-whitespace args: [--markdown-linebreak-ext=md] - - repo: https://github.com/astral-sh/ruff-pre-commit - rev: "v0.15.22" + - repo: local hooks: - id: ruff-check - args: [--fix] + name: ruff check + entry: "pixi run --locked ruff check --force-exclude --fix" + additional_dependencies: ['pixi'] + language: python # system + require_serial: true + types_or: + - python + - jupyter - id: ruff-format - - repo: https://github.com/pre-commit/mirrors-mypy - rev: "v2.3.0" - hooks: + name: ruff format + entry: "pixi run --locked ruff format --force-exclude" + additional_dependencies: ['pixi'] + language: python # system + require_serial: true + types_or: + - python + - jupyter - id: mypy - additional_dependencies: - - "types-PyYAML" - - "types-requests" - - "numpy" + name: mypy + entry: "pixi run --locked mypy" + additional_dependencies: ['pixi'] + language: python # system + types: + - python + pass_filenames: false + require_serial: true - repo: https://github.com/adrienverge/yamllint rev: "v1.38.0" hooks: @@ -53,8 +68,3 @@ repos: - id: check-citation-file-format - id: check-github-workflows - id: check-circle-ci - - repo: https://github.com/zizmorcore/zizmor-pre-commit - rev: v1.27.0 - hooks: - - id: zizmor - args: [--fix=safe] diff --git a/doc/contributing.rst b/doc/contributing.rst index a6b1701fa6..8cba49cd2c 100644 --- a/doc/contributing.rst +++ b/doc/contributing.rst @@ -39,8 +39,8 @@ whenever you read it. It is recommended that you open a `draft pull request `__ -early, as this will cause :ref:`CircleCI to run the unit tests `, -:ref:`pre-commit.ci to analyse your code `, and +early, as this will cause CircleCI to :ref:`run the tests ` and +:ref:`analyse the code `, and :ref:`readthedocs to build the documentation `. It's also easier to get help from other developers if your code is visible in a pull request. @@ -249,7 +249,7 @@ should be correct. Both `ruff`_ and `mypy`_ are automatically run by pre-commit. When you make a pull request, adherence to the Python development best practices -is checked by `pre-commit.ci `_. +is checked by running the pre-commit hooks on CircleCI. YAML ~~~~ diff --git a/esmvalcore/_task.py b/esmvalcore/_task.py index 5e50ae847f..432e05c425 100644 --- a/esmvalcore/_task.py +++ b/esmvalcore/_task.py @@ -1006,7 +1006,7 @@ def _run_task( ) -> tuple[list[str], set[TrackedFile]]: """Run task and return the result.""" if scheduler_address is None: - client = contextlib.nullcontext() + client: contextlib.AbstractContextManager = contextlib.nullcontext() else: client = Client(scheduler_address) diff --git a/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py b/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py index 9cc483ba11..18ded9688a 100644 --- a/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py +++ b/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py @@ -327,7 +327,7 @@ def _use_standard_lambert_conformal_grid( lon_coord.units = "degrees_east" lat_coord.units = "degrees_north" lon_coord.points, lat_coord.points = transformer.transform( - *np.meshgrid(x_coord.points, y_coord.points), + *np.meshgrid(x_coord.points, y_coord.points), # type: ignore[call-overload] errcheck=True, ) diff --git a/esmvalcore/io/intake_esgf.py b/esmvalcore/io/intake_esgf.py index eb1bffc4bc..6cf8f0eba9 100644 --- a/esmvalcore/io/intake_esgf.py +++ b/esmvalcore/io/intake_esgf.py @@ -186,7 +186,7 @@ def find_data(self, **facets: FacetValue) -> list[IntakeESGFDataset]: if not any(_isglob(v) for v in values) } # Translate "our" facets to ESGF facets and "our" values to ESGF values. - query = { + query: dict[str, Any] = { their_facet: [ self.values.get(our_facet, {}).get(v, v) for v in non_glob_facets[our_facet] diff --git a/esmvalcore/preprocessor/_area.py b/esmvalcore/preprocessor/_area.py index 34ceacad0c..801de835b7 100644 --- a/esmvalcore/preprocessor/_area.py +++ b/esmvalcore/preprocessor/_area.py @@ -16,6 +16,7 @@ import numpy as np import shapely import shapely.ops +import shapely.vectorized from dask import array as da from iris.coords import AuxCoord from iris.cube import CubeList @@ -47,6 +48,8 @@ logger = logging.getLogger(__name__) +type ExtractShapeMethod = Literal["contains", "representative"] + SHAPE_ID_KEYS: tuple[str, ...] = ("name", "NAME", "Name", "id", "ID") @@ -597,7 +600,7 @@ def _get_masks_from_geometries( geometries: dict[str, dict], lon: np.ndarray, lat: np.ndarray, - method: str = "contains", + method: ExtractShapeMethod = "contains", decomposed: bool = False, ) -> dict[str, np.ndarray]: """Get cube masks from requested regions.""" @@ -643,7 +646,7 @@ def _get_bounds( def _get_single_mask( lon: np.ndarray, lat: np.ndarray, - method: str, + method: ExtractShapeMethod, geometry: dict, ) -> np.ndarray: """Get single mask from one region.""" @@ -750,7 +753,7 @@ def _update_shapefile_path( def extract_shape( cube: Cube, shapefile: str | Path, - method: str = "contains", + method: ExtractShapeMethod = "contains", crop: bool = True, decomposed: bool = False, ids: list | dict | None = None, diff --git a/esmvalcore/preprocessor/_compare_with_refs.py b/esmvalcore/preprocessor/_compare_with_refs.py index 4e6fc6c180..a77c3b9ce5 100644 --- a/esmvalcore/preprocessor/_compare_with_refs.py +++ b/esmvalcore/preprocessor/_compare_with_refs.py @@ -592,7 +592,7 @@ def _get_emd( arr: np.ndarray, ref_arr: np.ndarray, bin_centers: np.ndarray, -) -> np.ndarray: +) -> float | np.ma.core.MaskedConstant: """Calculate Earth mover's distance (non-lazy).""" if np.ma.is_masked(arr) or np.ma.is_masked(ref_arr): return np.ma.masked # this is safe because PMFs will be masked arrays diff --git a/esmvalcore/preprocessor/_io.py b/esmvalcore/preprocessor/_io.py index 34b3838120..2b78838f79 100644 --- a/esmvalcore/preprocessor/_io.py +++ b/esmvalcore/preprocessor/_io.py @@ -25,7 +25,7 @@ from esmvalcore.iris_helpers import dataset_to_iris if TYPE_CHECKING: - from collections.abc import Sequence + from collections.abc import Mapping, MutableMapping, Sequence from dask.delayed import Delayed @@ -163,7 +163,7 @@ def load( def _load_zarr( file: str | Path, ignore_warnings: list[dict[str, Any]] | None = None, - backend_kwargs: dict[str, Any] | None = None, + backend_kwargs: Mapping[str, Any] | None = None, ) -> CubeList: # note on ``chunks`` kwarg to ``xr.open_dataset()`` # docs.xarray.dev/en/stable/generated/xarray.open_dataset.html @@ -177,7 +177,7 @@ def _load_zarr( # https://github.com/pp-mo/ncdata/issues/139 time_coder = xr.coders.CFDatetimeCoder(use_cftime=True) - open_kwargs = { + open_kwargs: MutableMapping[str, Any] = { "consolidated": False, "decode_times": time_coder, "engine": "zarr", diff --git a/pyproject.toml b/pyproject.toml index e77eb18e9d..bdd047dd41 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -137,10 +137,15 @@ exclude_lines = [ [tool.mypy] # See https://mypy.readthedocs.io/en/stable/config_file.html +packages = "esmvalcore" ignore_missing_imports = true enable_error_code = [ "truthy-bool", ] +exclude = [ + "^doc/conf.py$", + "esmvalcore/cmor/tables/", +] # Configure linters @@ -296,8 +301,16 @@ test-r = { features = ["test-r"], solve-group = "test-r" } # Other development tools. [tool.pixi.feature.dev.dependencies] "jupyterlab" = "*" +"mypy" = "*" "pre-commit" = "*" "py-spy" = "*" +"ruff" = "*" +# Mypy type stubs +"pandas-stubs" = "*" +"scipy-stubs" = "*" +"types-PyYAML" = "*" +"types-psutil" = "*" +"types-shapely" = "*" # Feature for running tests with Python 3.12. [tool.pixi.feature.py312.dependencies]