Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7,769 changes: 24 additions & 7,745 deletions docs/user-guide/structured.ipynb

Large diffs are not rendered by default.

39 changes: 39 additions & 0 deletions test/core/test_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -564,6 +564,45 @@ def test_open_dataset_with_fallback_chains_both_engine_errors(tmp_path):
)


@pytest.fixture
def _small_structured_dataset():
"""Return a small structured dataset for testing."""
return xr.Dataset(
{
"var": (("lat", "lon"), np.arange(4*8).reshape(4, 8)),
},
coords={
"lat": ("lat", np.linspace(-80, 80, 4), {"standard_name": "latitude"}),
"lon": ("lon", np.linspace(0, 350, 8), {"standard_name": "longitude"}),
},
)


def test_open_grid_fails_for_structured_inputs(_small_structured_dataset):
"""open_grid() must crash with GridStructureError when provided structured data."""
ds = _small_structured_dataset
with pytest.raises(ux.errors.GridStructureError, match=r"open_grid\(\) is not intended for structured"):
uxgrid = ux.open_grid(ds)


def test_open_dataset_fails_for_structured_inputs(_small_structured_dataset):
"""open_dataset() must crash with GridStructureError when provided structured data."""
ds = _small_structured_dataset
with pytest.raises(ux.errors.GridStructureError, match=r"open_dataset\(\) is not intended for structured"):
uxds = ux.open_dataset(ds, ds) # grid, data


def test_open_mfdataset_warns_for_structured_inputs(tmp_path, _small_structured_dataset):
"""open_mfdataset() must raise a GridStructureError when provided structured data."""
ds = _small_structured_dataset
# Write the dataset to a temporary netCDF file
temp_file = tmp_path / "temp.nc"
ds.to_netcdf(temp_file)

with pytest.raises(ux.errors.GridStructureError, match=r"open_mfdataset\(\) is not intended for structured"):
uxds = ux.open_mfdataset(ds, [temp_file]) # grid, [paths to data files]


def test_list_grid_names_multigrid(gridpath):
"""List grids from an OASIS-style multi-grid file."""
grid_file = gridpath("scrip", "oasis", "grids.nc")
Expand Down
40 changes: 8 additions & 32 deletions test/io/test_structured.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,42 +39,18 @@ def test_read_structured_uxds_from_ds(ds_name):

assert uxds.uxgrid.validate()


@pytest.mark.parametrize("ds_name", ["air_temperature"])
def test_from_xarray_with_grid_from_latlon(ds_name):
"""Regression test for GH #1410: a Grid built via ``from_structured(lon=, lat=)``
must record its source dimensions so ``UxDataset.from_xarray`` flattens the
structured (lon, lat) data variables onto ``n_face``."""
ds = xr.tutorial.open_dataset(ds_name)
def test_from_xarray_fails_with_grid_from_latlon():
"""from_xarray must crash upon receiving structured data; see PR #1887."""
ds = xr.tutorial.open_dataset("air_temperature")

uxgrid = ux.Grid.from_structured(lon=ds.lon, lat=ds.lat)

# The grid must carry the structured spec and the source-dim mapping.
assert uxgrid.source_grid_spec == "Structured"
assert uxgrid._source_dims_dict == {"n_face": (ds.lon.dims[0], ds.lat.dims[0])}

uxds = ux.UxDataset.from_xarray(ds, uxgrid=uxgrid)
with pytest.raises(ux.errors.DimensionError, match=r"UxDataset.from_xarray\(\) is not intended for structured"):
ux.UxDataset.from_xarray(ds, uxgrid=uxgrid)

# Data must now be mapped onto n_face, not left on lon/lat.
assert "n_face" in uxds.dims
assert "lon" not in uxds.dims
assert "lat" not in uxds.dims
assert uxds["air"].sizes["n_face"] == ds.sizes["lon"] * ds.sizes["lat"]

# The flatten must preserve data order: each face value must equal the
# original (lat, lon) cell value at that face. n_face is stacked as
# (lat, lon) C-order, so face k corresponds to (k // n_lon, k % n_lon).
n_lon = ds.sizes["lon"]
original = ds["air"].isel(time=0).values # (lat, lon)
flattened = uxds["air"].isel(time=0).values # (n_face,)
for k in (0, n_lon + 1, flattened.size - 1): # first, an interior, last
i, j = k // n_lon, k % n_lon
assert flattened[k] == original[i, j]

# End-to-end: the mapped data must be subsettable (the symptom in #1410).
subset = uxds["air"].isel(time=0).subset.bounding_circle((-100.0, 40.0), 5)
assert "n_face" in subset.dims
assert subset.sizes["n_face"] > 0
arr = ds.air
with pytest.raises(ux.errors.DimensionError, match=r"UxDataArray.from_xarray\(\) is not intended for structured"):
ux.UxDataArray.from_xarray(arr, uxgrid=uxgrid)


def test_global_structured_grid_merges_poles_and_seam():
Expand Down
116 changes: 47 additions & 69 deletions uxarray/core/api.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,12 @@
from uxarray.core.dataarray import UxDataArray
from uxarray.core.dataset import UxDataset
from uxarray.core.utils import (
_get_grid,
_map_dataset_to_ugrid,
_open_dataset_with_fallback,
match_chunks_to_ugrid,
)
from uxarray.errors import GridInvalidError, GridsMismatchError
from uxarray.errors import GridInvalidError, GridsMismatchError, GridStructureError
from uxarray.grid import Grid
from uxarray.io._scrip import (
_detect_multigrid,
Expand Down Expand Up @@ -137,6 +138,13 @@ def open_grid(
)
grid = Grid.from_dataset(grid_ds, use_dual=use_dual)

if grid.source_grid_spec == "Structured":
raise GridStructureError(
f"open_grid() is not intended for structured (lat/lon) grids or data, but "
f"received a grid with source_grid_spec={grid.source_grid_spec!r}. "
f"For structured grids, consider ux.Grid.from_structured() instead."
)

# Return the grid (and chunks, if requested) in a consistent manner.
if return_chunks:
return grid, chunks
Expand Down Expand Up @@ -380,22 +388,25 @@ def open_dataset(
Parameters
----------
grid_filename_or_obj : str | os.PathLike[Any] | dict | xr.Dataset | Grid
Grid information for the ``UxDataset``. Strings and Path objects are interpreted as a path to a grid
file. An ``xarray.Dataset`` containing grid information in a supported unstructured format is used to
create a ``ux.Grid``. Similarly, a dictionary containing UGRID variables can be used to create a
``ux.Grid``. A path to a directory containing grid files (e.g. a FESOM2 ASCII grid) is also accepted,
but only when ``filename_or_obj`` is provided; directory input is not supported in the single-argument
form below. An existing ``ux.Grid`` can be used directly, allowing its
topology and cached properties to be shared across datasets.
Specifies the unstructured grid toplogy.

- Strings and Path objects are interpreted as a path to a grid file.
A path to a directory containing grid files (e.g. a FESOM2 ASCII grid) is also accepted.
but only when ``filename_or_obj`` is also provided to separately specify the dataset data.
- A dict must specify UGRID variables ``node_lon``, ``node_lat``, and ``face_node_connectivity``,
as per :meth:`uxarray.Grid.from_topology`.
- An ``xarray.Dataset`` must contain grid information in any supported unstructured format.
- An existing ``ux.Grid`` is attached directly without reconstructing or modifying it,
allowing its cached properties to be shared across datasets.

Also specifies the dataset data if ``filename_or_obj`` is not provided. This single-argument
form requires ``grid_filename_or_obj`` to be a string, Path, or ``xarray.Dataset``, and causes
the resulting dataset to be populated with all variables it contains, including grid variables.
filename_or_obj : str | os.PathLike[Any] | xr.Dataset, optional
String or Path object as a path to a netCDF file or an OpenDAP URL that
stores the actual data set, or an already-open ``xarray.Dataset``. It
is the same ``filename_or_obj`` in ``xarray.open_dataset``. If omitted,
``grid_filename_or_obj`` is also used as the data source, allowing a
combined grid-and-data file or ``xarray.Dataset`` to be opened with a
single argument. In this single-argument form the file is used to build
the grid and also treated as data; all variables it contains, including
grid variables, will be included as data.
``grid_filename_or_obj`` is also used as the data source instead.
chunks : int, dict, 'auto' or None, default: None
If provided, used to load file-based data into dask arrays. For an
``xarray.Dataset`` data input, this parameter is applied when
Expand Down Expand Up @@ -433,7 +444,6 @@ def open_dataset(

Returns
-------

uxds : uxarray.UxDataset
Dataset with linked `uxgrid` property of type `Grid`.

Expand Down Expand Up @@ -491,12 +501,19 @@ def open_dataset(
"in ux.open_dataset(grid_filename_or_obj, filename_or_obj=None)."
)

uxgrid, _ = _get_grid(ds, chunks, chunk_grid, use_dual, grid_kwargs)
uxgrid, _ = _get_grid(
ds, chunks, chunk_grid, use_dual, grid_kwargs, _fname="ux.open_dataset"
)
filename_or_obj = grid_filename_or_obj
else:
# Construct a Grid, validate parameters, and correct chunks
uxgrid, corrected_chunks = _get_grid(
grid_filename_or_obj, chunks, chunk_grid, use_dual, grid_kwargs
grid_filename_or_obj,
chunks,
chunk_grid,
use_dual,
grid_kwargs,
_fname="ux.open_dataset",
)

# Load the data as a Xarray Dataset
Expand All @@ -515,7 +532,7 @@ def open_dataset(
)

# Map original dimensions to the UGRID conventions
ds = _map_dataset_to_ugrid(ds, uxgrid)
ds = _map_dataset_to_ugrid(ds, uxgrid, _fname="ux.open_dataset")

# Create a UXarray Dataset by linking the Xarray Dataset with a UXarray Grid
source_datasets = (
Expand All @@ -534,16 +551,21 @@ def open_mfdataset(
grid_kwargs: dict[str, Any] | None = None,
**kwargs: dict[str, Any],
) -> UxDataset:
"""Wraps ``xarray.open_dataset()`` to support reading in a grid and
"""Wraps ``xarray.open_mfdataset()`` to support reading in a grid and
multiple data files together.

Parameters
----------
grid_filename_or_obj : str | os.PathLike[Any] | dict | xr.Dataset | Grid
Strings and Path objects are interpreted as a path to a grid file. An ``xarray.Dataset`` containing
grid information in a supported unstructured format is used to create a ``ux.Grid``. Similarly, a
dictionary containing UGRID variables can be used to create a ``ux.Grid``. An existing
``ux.Grid`` is attached directly without reconstructing or modifying it.
Specifies the unstructured grid topology.

- Strings and Path objects are interpreted as a path to a grid file.
- A dict must specify UGRID variables ``node_lon``, ``node_lat``, and ``face_node_connectivity``,
as per :meth:`uxarray.Grid.from_topology`.
- An ``xarray.Dataset`` must contain grid information in any supported unstructured format.
- An existing ``ux.Grid`` is attached directly without reconstructing or modifying it,
allowing its cached properties to be shared across datasets.

paths : string, required
Either a string glob in the form ``"path/to/my/files/*.nc"`` or an explicit
list of files to open. It is the same ``paths`` in ``xarray.open_mfdataset``.
Expand Down Expand Up @@ -580,13 +602,11 @@ def open_mfdataset(

Returns
-------

object : uxarray.UxDataset
uxds : uxarray.UxDataset
Dataset with the unstructured grid.

Examples
--------

Open grid file along with multiple data files (two or more)

>>> import uxarray as ux
Expand Down Expand Up @@ -616,61 +636,19 @@ def open_mfdataset(
chunk_grid,
use_dual,
grid_kwargs,
_fname="ux.open_mfdataset",
)

# Load the data as a Xarray Dataset
ds = xr.open_mfdataset(paths, chunks=corrected_chunks, **kwargs)

# Map original dimensions to the UGRID conventions
ds = _map_dataset_to_ugrid(ds, uxgrid)
ds = _map_dataset_to_ugrid(ds, uxgrid, _fname="ux.open_mfdataset")

# Create a UXarray Dataset by linking the Xarray Dataset with a UXarray Grid
return UxDataset(ds, uxgrid=uxgrid, source_datasets=str(paths))


def _get_grid(
grid_filename_or_obj,
chunks,
chunk_grid,
use_dual,
grid_kwargs,
):
"""Validate input parameters and return the ``(grid, chunks)`` tuple."""
if isinstance(grid_filename_or_obj, Grid):
if use_dual:
raise ValueError(
"use_dual=True is not supported with an existing Grid. "
"Select the desired mesh when constructing the Grid."
)
if grid_kwargs:
raise ValueError(
"grid_kwargs must be empty when passing an existing Grid. "
"Apply these options when constructing the Grid."
)
if chunks is not None and chunk_grid:
raise ValueError(
"Providing Grid, non-None chunks, and chunk_grid=True simultaneously "
"is not supported. Call Grid.chunk() explicitly to change the grid's "
"chunking, or set chunk_grid=False to chunk only the data."
)
if isinstance(chunks, dict):
# Keep the caller's options and the shared Grid unchanged. Match
# UGRID chunk keys to the data's source dimensions without grid I/O.
requested_chunks = chunks
chunks = chunks.copy()
for source_dim, ugrid_dim in grid_filename_or_obj._source_dims_dict.items():
if isinstance(ugrid_dim, str) and ugrid_dim in requested_chunks:
chunks[source_dim] = requested_chunks[ugrid_dim]
return grid_filename_or_obj, chunks

grid_kwargs["data_chunks"] = chunks
grid_kwargs["chunk_grid"] = chunk_grid
grid_kwargs["return_chunks"] = True

# Create a Grid
return open_grid(grid_filename_or_obj, use_dual=use_dual, **grid_kwargs)


def concat(
objs: Iterable[UxDataArray | UxDataset],
dim: Hashable | xr.Variable | xr.DataArray | pd.Index,
Expand Down
8 changes: 5 additions & 3 deletions uxarray/core/dataarray.py
Original file line number Diff line number Diff line change
Expand Up @@ -2412,7 +2412,7 @@ def sel(
@classmethod
def from_xarray(cls, da: xr.DataArray, uxgrid: Grid, ugrid_dims: dict = None):
"""
Converts a ``xarray.DataArray`` into a ``uxarray.UxDataset`` paired with a user-defined ``Grid``
Converts a ``xarray.DataArray`` into a ``uxarray.UxDataArray`` paired with a user-defined ``Grid``

Parameters
----------
Expand All @@ -2432,9 +2432,11 @@ def from_xarray(cls, da: xr.DataArray, uxgrid: Grid, ugrid_dims: dict = None):
ugrid_dims = uxgrid._source_dims_dict

# map each dimension to its UGRID equivalent
ds = _map_dims_to_ugrid(da, ugrid_dims, uxgrid)
da = _map_dims_to_ugrid(
da, ugrid_dims, uxgrid, _fname="UxDataArray.from_xarray"
)

return cls(ds, uxgrid=uxgrid)
return cls(da, uxgrid=uxgrid)

@classmethod
def from_healpix(
Expand Down
7 changes: 5 additions & 2 deletions uxarray/core/dataset.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
from uxarray.core.dataarray import UxDataArray
from uxarray.core.utils import (
_map_dims_to_ugrid,
_map_structured_dims_to_ugrid,
_open_dataset_with_fallback,
_resolve_coordinate_labels_to_indices,
_validate_indexers,
Expand Down Expand Up @@ -328,7 +329,9 @@ def from_structured(cls, ds: xr.Dataset):

uxgrid = Grid.from_dataset(ds)

ds = _map_dims_to_ugrid(ds, uxgrid._source_dims_dict, uxgrid)
ds = _map_structured_dims_to_ugrid(
ds, uxgrid._source_dims_dict, uxgrid, _fname="UxDataset.from_structured"
)

# Drop spatial coordinates
coords_to_drop = [
Expand Down Expand Up @@ -369,7 +372,7 @@ def from_xarray(cls, ds: xr.Dataset, uxgrid: Grid = None, ugrid_dims: dict = Non
ugrid_dims = uxgrid._source_dims_dict

# map each dimension to its UGRID equivalent
ds = _map_dims_to_ugrid(ds, ugrid_dims, uxgrid)
ds = _map_dims_to_ugrid(ds, ugrid_dims, uxgrid, _fname="UxDataset.from_xarray")

return cls(ds, uxgrid=uxgrid)

Expand Down
Loading
Loading