From ed36fc21b98a294eb2efb0b334dd3dfecde544e8 Mon Sep 17 00:00:00 2001 From: GianmarcoLattaruolo <87989501+GianmarcoLattaruolo@users.noreply.github.com> Date: Thu, 24 Sep 2026 12:06:18 +0200 Subject: [PATCH 1/9] extend the entrypoint function of the module to accept year chunking. --- xarray_ecmwf/client_common.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/xarray_ecmwf/client_common.py b/xarray_ecmwf/client_common.py index 99ca461..b8c6e83 100644 --- a/xarray_ecmwf/client_common.py +++ b/xarray_ecmwf/client_common.py @@ -189,12 +189,14 @@ def build_chunk_ymd_requests( assert len(time_chunk_keys) == 1 time_chunk_key = time_chunk_keys[0] - assert time_chunk_key in set(["day", "month"]) + assert time_chunk_key in set(["day", "month", "year"]) # this check is redundant if time_chunk_key == "month": out = build_chunk_ymd_month_requests(request, request_chunks) elif time_chunk_key == "day": out = build_chunk_ymd_day_requests(request, request_chunks) + elif time_chunk_key == "year": + out = build_chunk_ymd_year_requests(request, request_chunks) return out From fbca5034b98068628a255d278f96b890fd0ff1c3 Mon Sep 17 00:00:00 2001 From: GianmarcoLattaruolo <87989501+GianmarcoLattaruolo@users.noreply.github.com> Date: Thu, 24 Sep 2026 12:09:06 +0200 Subject: [PATCH 2/9] proposed function for year chunking. --- xarray_ecmwf/client_common.py | 41 +++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/xarray_ecmwf/client_common.py b/xarray_ecmwf/client_common.py index b8c6e83..57f2ffc 100644 --- a/xarray_ecmwf/client_common.py +++ b/xarray_ecmwf/client_common.py @@ -232,6 +232,47 @@ def build_chunk_ymd_day_requests( return np.array(times), len(request["time"]), chunk_requests +def build_chunk_ymd_year_requests( + request: dict[str, Any], request_chunks: dict[str, int] +) -> tuple[ + np.typing.NDArray[np.datetime64], + int | tuple[int, ...], + list[tuple[int, dict[str, Any]]], +]: + year_chunk_size = request_chunks["year"] + if year_chunk_size < 1: + raise ValueError("split on year values < 1 not supported") + + datetimes: list[np.datetime64] = [] + chunk_requests : list[tuple[int, dict[str, Any]]] = [] + chunks : list[int] = [] + years = request["year"] + index_start = 0 + while index_start < len(years): + index_stop = min(index_start + year_chunk_size, len(years)) + year_values = years[index_start:index_stop] + start = len(datetimes) + chunk = 0 + for year in year_values: + assert len(year) == 4 + for month in request["month"]: + assert len(month) == 2 + ndays = calendar.monthrange(int(year), int(month))[1] + for day in request["day"]: + assert len(day) == 2 + if int(day) > ndays: + break + for time in request["time"]: + assert len(time) == 5 + chunk += 1 + datetime = np.datetime64(f"{year}-{month}-{day}T{time}", "ns") + datetimes.append(datetime) + chunks.append(chunk) + chunk_requests.append((start, {"year": year_values})) + index_start += year_chunk_size + return np.array(datetimes), tuple(chunks), chunk_requests + + def build_time_chunk_requests( request: dict[str, Any], request_chunks: dict[str, int], sep: str = "/" ) -> tuple[ From c52edcdea34944ade686179a1423bf8bdcaeb2ae Mon Sep 17 00:00:00 2001 From: GianmarcoLattaruolo <87989501+GianmarcoLattaruolo@users.noreply.github.com> Date: Thu, 24 Sep 2026 12:09:50 +0200 Subject: [PATCH 3/9] typo correction. --- xarray_ecmwf/client_common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray_ecmwf/client_common.py b/xarray_ecmwf/client_common.py index 57f2ffc..3a5fa89 100644 --- a/xarray_ecmwf/client_common.py +++ b/xarray_ecmwf/client_common.py @@ -147,7 +147,7 @@ def build_chunk_ymd_month_requests( list[tuple[int, dict[str, Any]]], ]: if request_chunks["month"] != 1: - raise ValueError("split on day values != 1 not supported") + raise ValueError("split on month values != 1 not supported") datetimes: list[np.datetime64] = [] chunk_requests = [] From bc017d33b98a31e61352940f87995f6f9207f6b5 Mon Sep 17 00:00:00 2001 From: GianmarcoLattaruolo <87989501+GianmarcoLattaruolo@users.noreply.github.com> Date: Thu, 24 Sep 2026 12:12:05 +0200 Subject: [PATCH 4/9] correction to be uniform with the rest of the module. --- xarray_ecmwf/client_common.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/xarray_ecmwf/client_common.py b/xarray_ecmwf/client_common.py index 3a5fa89..8441808 100644 --- a/xarray_ecmwf/client_common.py +++ b/xarray_ecmwf/client_common.py @@ -247,10 +247,10 @@ def build_chunk_ymd_year_requests( chunk_requests : list[tuple[int, dict[str, Any]]] = [] chunks : list[int] = [] years = request["year"] - index_start = 0 - while index_start < len(years): - index_stop = min(index_start + year_chunk_size, len(years)) - year_values = years[index_start:index_stop] + istart = 0 + while istart < len(years): + istop = min(istart + year_chunk_size, len(years)) + year_values = years[istart:istop] start = len(datetimes) chunk = 0 for year in year_values: @@ -269,7 +269,7 @@ def build_chunk_ymd_year_requests( datetimes.append(datetime) chunks.append(chunk) chunk_requests.append((start, {"year": year_values})) - index_start += year_chunk_size + istart += year_chunk_size return np.array(datetimes), tuple(chunks), chunk_requests From de69244d121346b07710497c188b7ce702afbc25 Mon Sep 17 00:00:00 2001 From: GianmarcoLattaruolo <87989501+GianmarcoLattaruolo@users.noreply.github.com> Date: Mon, 28 Sep 2026 14:29:36 +0200 Subject: [PATCH 5/9] unit test for new function. --- tests/test_10_client_common.py | 72 ++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/tests/test_10_client_common.py b/tests/test_10_client_common.py index a60c55d..df20a0b 100644 --- a/tests/test_10_client_common.py +++ b/tests/test_10_client_common.py @@ -106,6 +106,78 @@ def test_build_chunk_ymd_requests( client_common.build_chunk_ymd_requests(request, request_chunks) +@pytest.mark.parametrize( + "years, months, days, year_chunk", + [ + (["2022"], ALL_MONTHS, ALL_DAYS, 1), + (["2020"], ALL_MONTHS, ALL_DAYS, 1), # leap year + ( + [str(y) for y in range(2015, 2020)], + ALL_MONTHS, + ALL_DAYS, + 2, + ), # consecutive years, leftover chunk + (["2010", "2015", "2020"], ALL_MONTHS, ALL_DAYS, 1), + (["2010", "2015", "2020"], ALL_MONTHS, ALL_DAYS, 2), + (["2010", "2015", "2020"], ALL_MONTHS, ALL_DAYS, 5), # N larger than n years + (["2010", "2015", "2020"], ["01", "02", "09"], ALL_DAYS, 2), + (["2010", "2015", "2020"], ["01", "02", "09"], ["01", "29", "30", "31"], 2), + ], +) +def test_build_chunk_ymd_year_requests( + years: list[str], months: list[str], days: list[str], year_chunk: int +) -> None: + request_chunks = {"year": year_chunk} + request = { + k: v for k, v in [("year", years), ("month", months), ("day", days)] if v + } + request["time"] = ALL_TIMES + + ( + time, + time_chunk, + time_chunk_requests, + ) = client_common.build_chunk_ymd_requests(request, request_chunks) + total_days = 0 + for year, month, day in itertools.product( + map(int, request["year"]), + map(int, request["month"]), + map(int, request["day"]), + ): + if day <= calendar.monthrange(year, month)[1]: + total_days += 1 + + expected_n_chunks = (len(years) + year_chunk - 1) // year_chunk + expected_chunks = [] + for chunk_index in range(expected_n_chunks): + years_in_chunk = years[ + chunk_index * year_chunk : (chunk_index + 1) * year_chunk + ] + n_times = 0 + for year, month, day in itertools.product( + map(int, years_in_chunk), + map(int, request["month"]), + map(int, request["day"]), + ): + if day <= calendar.monthrange(year, month)[1]: + n_times += 24 + expected_chunks.append(n_times) + + assert len(time) == 24 * total_days # 24h default value + assert time_chunk == tuple(expected_chunks) + assert len(time_chunk_requests) == expected_n_chunks + + for chunk_index, (start, chunk_request) in enumerate(time_chunk_requests): + expected_years = years[ + chunk_index * year_chunk : (chunk_index + 1) * year_chunk + ] + assert chunk_request == {"year": expected_years} + assert start == sum(time_chunk[:chunk_index]) + + with pytest.raises(ValueError): + client_common.build_chunk_ymd_requests(request, {"year": 0}) + + def test_build_chunk_request() -> None: coord, chunk, chunk_request = client_common.build_chunks_header_requests( dim="x", From df8dd0f2cc7a2dbeb70a261bd3cc1007837cc9a0 Mon Sep 17 00:00:00 2001 From: GianmarcoLattaruolo <87989501+GianmarcoLattaruolo@users.noreply.github.com> Date: Mon, 28 Sep 2026 15:07:52 +0200 Subject: [PATCH 6/9] ruff correction. --- xarray_ecmwf/client_common.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xarray_ecmwf/client_common.py b/xarray_ecmwf/client_common.py index 8441808..37519c0 100644 --- a/xarray_ecmwf/client_common.py +++ b/xarray_ecmwf/client_common.py @@ -244,8 +244,8 @@ def build_chunk_ymd_year_requests( raise ValueError("split on year values < 1 not supported") datetimes: list[np.datetime64] = [] - chunk_requests : list[tuple[int, dict[str, Any]]] = [] - chunks : list[int] = [] + chunk_requests: list[tuple[int, dict[str, Any]]] = [] + chunks: list[int] = [] years = request["year"] istart = 0 while istart < len(years): From 03a691e52a5c7f5cc69f48371741140f531321ae Mon Sep 17 00:00:00 2001 From: GianmarcoLattaruolo <87989501+GianmarcoLattaruolo@users.noreply.github.com> Date: Thu, 1 Oct 2026 16:21:57 +0200 Subject: [PATCH 7/9] adding integration tests for new fucntion. --- ...egration_test_10_cdsapi_era5_reanalysis.py | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/tests/integration_test_10_cdsapi_era5_reanalysis.py b/tests/integration_test_10_cdsapi_era5_reanalysis.py index d2704d9..cfeca68 100644 --- a/tests/integration_test_10_cdsapi_era5_reanalysis.py +++ b/tests/integration_test_10_cdsapi_era5_reanalysis.py @@ -14,6 +14,16 @@ "time": ["00:00", "12:00"], } +REQUEST_YEARS = { + "dataset": "reanalysis-era5-single-levels", + "product_type": ["reanalysis"], + "variable": ["2m_temperature"], + "year": ["2021", "2022", "2023"], + "month": ["01"], + "day": ["01"], + "time": ["00:00", "12:00"], +} + def test_open_dataset() -> None: res = xr.open_dataset(REQUEST, engine="ecmwf") # type: ignore @@ -195,6 +205,53 @@ def test_cds_era5_big_slice_time_month() -> None: assert res.size == 1 +def test_compare_chunked_no_chunked_year() -> None: + ds = xr.open_dataset( + REQUEST, # type: ignore + engine="ecmwf", + request_chunks={"year": 1}, + chunks={}, + ) + da = ds.data_vars["t2m"] + + res = da.sel(time=slice("2022-07-01", "2022-07-16")).mean().compute() + + assert isinstance(res, xr.DataArray) + assert res.size == 1 + + +def test_compare_chunked_no_chunked_year_n() -> None: + # no year chunking + ds0 = xr.open_dataset(REQUEST_YEARS, engine="ecmwf", chunks={}) # type: ignore + assert ds0.chunks["time"] == (6,) + res0 = ds0.data_vars["t2m"].load() + + # 1 year chunking + ds1 = xr.open_dataset( + REQUEST_YEARS, # type: ignore + engine="ecmwf", + request_chunks={"year": 1}, + chunks={}, + ) + assert ds1.chunks["time"] == (2, 2, 2) + res1 = ds1.data_vars["t2m"].load() + + # 2 years chunking + ds2 = xr.open_dataset( + REQUEST_YEARS, # type: ignore + engine="ecmwf", + request_chunks={"year": 2}, + chunks={}, + ) + assert ds2.chunks["time"] == (4, 2) + res2 = ds2.data_vars["t2m"].load() + + assert (res0 - res1).shape == res0.shape + assert (res0 == res1).all() + assert (res0 - res2).shape == res0.shape + assert (res0 == res2).all() + + def test_cds_era5_small_slice_time_longitute() -> None: ds = xr.open_dataset( REQUEST, # type: ignore From d81a73c7e72fa3d18a1ce241cd28f31d742839a0 Mon Sep 17 00:00:00 2001 From: GianmarcoLattaruolo <87989501+GianmarcoLattaruolo@users.noreply.github.com> Date: Tue, 6 Oct 2026 10:30:30 +0200 Subject: [PATCH 8/9] semplify the unit test for the new function. More atomic unit tests instead of complex one. --- tests/test_10_client_common.py | 98 ++++++++++++++++++++-------------- 1 file changed, 59 insertions(+), 39 deletions(-) diff --git a/tests/test_10_client_common.py b/tests/test_10_client_common.py index df20a0b..ab687bc 100644 --- a/tests/test_10_client_common.py +++ b/tests/test_10_client_common.py @@ -79,7 +79,7 @@ def test_build_chunk_ymd_requests( k: v for k, v in filter( lambda x: x[1], [("year", years), ("month", months), ("day", days)] - ) + ) # why this filter if they are never empty? } request["time"] = ALL_TIMES @@ -107,31 +107,30 @@ def test_build_chunk_ymd_requests( @pytest.mark.parametrize( - "years, months, days, year_chunk", + "years, months, days", [ - (["2022"], ALL_MONTHS, ALL_DAYS, 1), - (["2020"], ALL_MONTHS, ALL_DAYS, 1), # leap year + (["2022"], ALL_MONTHS, ALL_DAYS), + (["2020"], ALL_MONTHS, ALL_DAYS), # leap year ( [str(y) for y in range(2015, 2020)], ALL_MONTHS, ALL_DAYS, - 2, - ), # consecutive years, leftover chunk - (["2010", "2015", "2020"], ALL_MONTHS, ALL_DAYS, 1), - (["2010", "2015", "2020"], ALL_MONTHS, ALL_DAYS, 2), - (["2010", "2015", "2020"], ALL_MONTHS, ALL_DAYS, 5), # N larger than n years - (["2010", "2015", "2020"], ["01", "02", "09"], ALL_DAYS, 2), - (["2010", "2015", "2020"], ["01", "02", "09"], ["01", "29", "30", "31"], 2), + ), # consecutive years + (["2010", "2015", "2020"], ALL_MONTHS, ALL_DAYS), + (["2010", "2015", "2020"], ["01", "02", "09"], ALL_DAYS), + (["2010", "2015", "2020"], ["01", "02", "09"], ["01", "29", "30", "31"]), ], ) def test_build_chunk_ymd_year_requests( - years: list[str], months: list[str], days: list[str], year_chunk: int + years: list[str], months: list[str], days: list[str] ) -> None: - request_chunks = {"year": year_chunk} + request_chunks = {"year": 1} request = { - k: v for k, v in [("year", years), ("month", months), ("day", days)] if v + "year": years, + "month": months, + "day": days, + "time": ALL_TIMES, } - request["time"] = ALL_TIMES ( time, @@ -147,33 +146,54 @@ def test_build_chunk_ymd_year_requests( if day <= calendar.monthrange(year, month)[1]: total_days += 1 - expected_n_chunks = (len(years) + year_chunk - 1) // year_chunk - expected_chunks = [] - for chunk_index in range(expected_n_chunks): - years_in_chunk = years[ - chunk_index * year_chunk : (chunk_index + 1) * year_chunk - ] - n_times = 0 - for year, month, day in itertools.product( - map(int, years_in_chunk), - map(int, request["month"]), - map(int, request["day"]), - ): - if day <= calendar.monthrange(year, month)[1]: - n_times += 24 - expected_chunks.append(n_times) - assert len(time) == 24 * total_days # 24h default value - assert time_chunk == tuple(expected_chunks) - assert len(time_chunk_requests) == expected_n_chunks + assert len(time_chunk_requests) == len(years) + assert sum(time_chunk) == len(time) + + offset = 0 + for (start, chunk_request), year, size in zip( + time_chunk_requests, years, time_chunk + ): + assert start == offset + assert chunk_request == {"year": [year]} + offset += size - for chunk_index, (start, chunk_request) in enumerate(time_chunk_requests): - expected_years = years[ - chunk_index * year_chunk : (chunk_index + 1) * year_chunk - ] - assert chunk_request == {"year": expected_years} - assert start == sum(time_chunk[:chunk_index]) +@pytest.mark.parametrize( + "year_chunk, expected_year_groups", + [ + (1, [["2015"], ["2016"], ["2017"]]), + (2, [["2015", "2016"], ["2017"]]), # leftover chunk + (5, [["2015", "2016", "2017"]]), # N larger than n years + ], +) +def test_build_chunk_ymd_year_requests_groups_years( + year_chunk: int, expected_year_groups: list[list[str]] +) -> None: + request = { + "year": ["2015", "2016", "2017"], + "month": ["01"], + "day": ["01"], + "time": ["00:00"], + } # one time stamp per year + request_chunks = {"year": year_chunk} + time, time_chunk, time_chunk_requests = client_common.build_chunk_ymd_requests( + request, request_chunks + ) + assert len(time) == 3 + assert [chunk_request["year"] for _, chunk_request in time_chunk_requests] == ( + expected_year_groups + ) + assert time_chunk == tuple(len(group) for group in expected_year_groups) + + +def test_build_chunk_ymd_year_requests_invalid() -> None: + request = { + "year": ["2022"], + "month": ["01"], + "day": ["01"], + "time": ["00:00"], + } with pytest.raises(ValueError): client_common.build_chunk_ymd_requests(request, {"year": 0}) From adf83493728b9541441eff3864d8724045021bb9 Mon Sep 17 00:00:00 2001 From: GianmarcoLattaruolo <87989501+GianmarcoLattaruolo@users.noreply.github.com> Date: Tue, 6 Oct 2026 10:54:59 +0200 Subject: [PATCH 9/9] small type checks corrections. --- tests/test_10_client_common.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_10_client_common.py b/tests/test_10_client_common.py index ab687bc..07c80b7 100644 --- a/tests/test_10_client_common.py +++ b/tests/test_10_client_common.py @@ -148,14 +148,15 @@ def test_build_chunk_ymd_year_requests( assert len(time) == 24 * total_days # 24h default value assert len(time_chunk_requests) == len(years) + assert isinstance(time_chunk, tuple) assert sum(time_chunk) == len(time) offset = 0 - for (start, chunk_request), year, size in zip( + for (start, chunk_request), year_str, size in zip( time_chunk_requests, years, time_chunk ): assert start == offset - assert chunk_request == {"year": [year]} + assert chunk_request == {"year": [year_str]} offset += size