Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
222 changes: 222 additions & 0 deletions test/grid/grid/test_non_grid_data.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,222 @@
"""Tests for dropping non-grid data during ``Grid`` construction (#1444, #1720)."""

import numpy as np
import pytest
import xarray as xr

import uxarray as ux
from uxarray.conventions.ugrid import DIM_NAMES
from uxarray.grid.utils import _drop_non_grid_data

TIMES = np.array(["2018-01-01", "2020-01-01"], dtype="datetime64[ns]")
STRAY_NAMES = {"time", "stray", "temp", "time_since_epoch", "special_index", "z"}

# the shapes listed in #1720
STRAY_CASES = {
"scalar_coord": lambda ds: ds.assign_coords(time=np.datetime64("2016-01-01")),
"dimension_coord": lambda ds: ds.assign_coords(time=("time", TIMES)),
"bare_dimension": lambda ds: ds.assign(stray=("time", [0.0, 1.0])),
"data_var_on_time_and_face": lambda ds: ds.assign(
temp=(("time", "n_face"), np.ones((2, ds.sizes["n_face"])))
).assign_coords(time=("time", TIMES)),
"non_dimension_coord": lambda ds: ds.assign_coords(
time=("time", TIMES), time_since_epoch=("time", [0, 1])
),
"coord_on_two_non_grid_dims": lambda ds: ds.assign_coords(
special_index=(("time", "elevation"), np.arange(6).reshape(2, 3))
),
"coord_mixing_grid_and_non_grid_dims": lambda ds: ds.assign_coords(
z=(("n_node", "elevation"), np.zeros((ds.sizes["n_node"], 3)))
),
}


@pytest.fixture
def quad_hexagon_ds(gridpath):
"""Parsed quad-hexagon grid dataset (UGRID names and dims)."""
return ux.open_grid(gridpath("ugrid", "quad-hexagon", "grid.nc"))._ds


@pytest.mark.parametrize("case", STRAY_CASES)
def test_stray_variables_are_dropped(quad_hexagon_ds, case):
"""Each stray ``time`` shape listed in #1720 should be gone after
``Grid.from_dataset``, with the grid itself unchanged."""
uxgrid = ux.Grid.from_dataset(STRAY_CASES[case](quad_hexagon_ds))

assert not STRAY_NAMES & set(uxgrid._ds.variables)
assert set(uxgrid._ds.dims) <= set(DIM_NAMES)
assert uxgrid.n_face == quad_hexagon_ds.sizes["n_face"]


def test_bounding_box_with_stray_grid_time(gridpath):
"""Subsetting must work when the grid was built from a source that carried a stray
scalar ``time`` coordinate. Ensures issue #1444 has been fixed.
"""
# grid built from a source carrying a stray scalar `time`
grid_ds = xr.open_dataset(gridpath("ugrid", "quad-hexagon", "grid.nc"))
uxgrid = ux.open_grid(grid_ds.assign_coords(time=np.datetime64("2016-01-01")))

# the stray coordinate is dropped during grid construction
assert "time" not in uxgrid._ds.coords

# a face-centered variable carrying a *different* time coordinate
times = np.array(["2018-01-01", "2020-01-01"], dtype="datetime64[ns]")
uxda = ux.UxDataArray(
data=np.ones((times.size, uxgrid.n_face)),
dims=("time", "n_face"),
coords={"time": times},
uxgrid=uxgrid,
)

# previously raised: "IndexError: dimension coordinate 'time' conflicts ..."
res = uxda.subset.bounding_box(lon_bounds=(-10, 10), lat_bounds=(-10, 10))

assert isinstance(res, ux.UxDataArray)
assert res.sizes["time"] == times.size
assert "n_face" in res.dims


def test_scalar_time_data_var_does_not_block_subsetting(gridpath):
"""A scalar ``time`` stored as a data variable stays on the grid, but it isn't
attached to the subset indexer, so subsetting keeps the data's own ``time``."""
grid_ds = xr.open_dataset(gridpath("ugrid", "quad-hexagon", "grid.nc"))
uxgrid = ux.open_grid(
grid_ds.assign(time=xr.DataArray(np.datetime64("2016-01-01")))
)
uxda = ux.UxDataArray(
np.ones((2, uxgrid.n_face)),
dims=("time", "n_face"),
coords={"time": TIMES},
uxgrid=uxgrid,
)

res = uxda.subset.bounding_box((-10, 10), (-10, 10))

np.testing.assert_array_equal(res["time"].values, TIMES)


def test_directly_built_grid_with_stray_time_can_be_subset(quad_hexagon_ds):
"""``ux.Grid(ds)`` doesn't filter, so a stray scalar ``time`` stays on the grid;
subsetting data with a different ``time`` must still work."""
ds = quad_hexagon_ds.assign_coords(time=np.datetime64("2016-01-01"))
uxgrid = ux.Grid(ds, source_grid_spec="UGRID")
uxda = ux.UxDataArray(
np.ones((2, uxgrid.n_face)),
dims=("time", "n_face"),
coords={"time": TIMES},
uxgrid=uxgrid,
)

res = uxda.subset.bounding_box((-10, 10), (-10, 10))

np.testing.assert_array_equal(res["time"].values, TIMES)


def test_grid_coordinates_not_attached_to_data_subset(gridpath):
"""ICON's ``clon``/``clat`` are coordinates on the grid; slicing data shouldn't
copy them onto the result."""
uxgrid = ux.open_grid(gridpath("icon", "R02B04", "icon_grid_0010_R02B04_G.nc"))
uxda = ux.UxDataArray(np.ones(uxgrid.n_face), dims=("n_face",), uxgrid=uxgrid)

res = uxda.isel(n_face=[0, 1, 2])

assert not {"clon", "clat"} & set(res.coords)


def test_single_file_with_grid_and_time_varying_data(gridpath, tmp_path):
"""A file holding the grid and data along ``time``: the grid gets neither."""
raw = xr.open_dataset(gridpath("ugrid", "quad-hexagon", "grid.nc")).load()
single = raw.assign(
temp=(("time", "n_face"), np.ones((TIMES.size, raw.sizes["n_face"])))
).assign_coords(time=("time", TIMES))
path = tmp_path / "grid_and_data.nc"
single.to_netcdf(path)

uxds = ux.open_dataset(path)

assert "temp" not in uxds.uxgrid._ds
assert "time" not in uxds.uxgrid._ds.dims
# the data keeps its own time axis
assert "time" in uxds["temp"].dims

sub = (
uxds["temp"]
.sel(time=TIMES[:1])
.subset.bounding_box(lon_bounds=(-10, 10), lat_bounds=(-10, 10))
)
assert sub.sizes["time"] == 1
uxds.close()


@pytest.mark.parametrize(
"path, kept, dropped",
[
(("mpas", "QU", "oQU480.231010.nc"), ["boundary_node_indices"], []),
(
("icon", "R02B04", "icon_grid_0010_R02B04_G.nc"),
["clon", "clat", "vlon", "vlat", "elon", "elat"],
[],
),
(
("ugrid", "fesom", "fesom.mesh.diag.nc"),
["edge_node_connectivity", "grid_topology"],
["nod_area", "nod_in_elem2D", "edge_cross_dxdy", "nz", "nz1"],
),
],
ids=["mpas", "icon", "fesom"],
)
def test_grid_files(gridpath, path, kept, dropped):
"""Real grid files keep what uxarray reads from them (MPAS boundary nodes, ICON
coordinates, FESOM2 connectivity); FESOM2's extras on non-grid dims are dropped."""
uxgrid = ux.open_grid(gridpath(*path))

assert set(kept) <= set(uxgrid._ds.variables)
assert not set(dropped) & set(uxgrid._ds.variables)


def test_known_variable_on_misnamed_dim_kept(quad_hexagon_ds):

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Non-blocking: Fine to leave the keep-by-name for coordinates and connectivity as it is in this PR. The proper fix for the n2 shape is #1717, already fixed in #1736 (FESOM2 now gets two). In my prototype nothing in test/meshfiles needs the name list, so a follow-up can remove it.

"""A known variable on a non-standard dim (the #1717 ``n2`` shape) is kept."""
uxgrid = ux.Grid(quad_hexagon_ds, source_grid_spec="UGRID")
ds = uxgrid._ds.assign(
edge_node_connectivity=uxgrid.edge_node_connectivity.rename(two="n2")
)

cleaned = _drop_non_grid_data(ds)

assert cleaned["edge_node_connectivity"].dims == ("n_edge", "n2")


def test_cached_values_kept_on_reread(gridpath):
"""Ensure cached data like ``bounds`` and ``max_face_radius`` survives writing the
grid out with ``to_xarray()`` and reading it back, so it isn't recomputed."""
uxgrid = ux.open_grid(gridpath("ugrid", "outCSne30", "outCSne30.ug"))
uxgrid.bounds
uxgrid.face_bounds_lon
uxgrid.face_bounds_lat
uxgrid.max_face_radius
cached = {"bounds", "face_bounds_lon", "face_bounds_lat", "max_face_radius"}

reread = ux.open_grid(uxgrid.to_xarray())

assert cached <= set(reread._ds.variables)


@pytest.mark.parametrize("as_coord", [False, True], ids=["data_var", "scalar_coord"])
def test_crs_variable_kept_for_projected_detection(gridpath, tmp_path, as_coord):
"""A CRS variable named by ``grid_mapping`` is kept, stored either as a scalar
data variable (``crs``) or as a scalar coordinate (``spatial_ref``, as rioxarray
writes it), so projected-grid detection still sees it."""
crs_name = "spatial_ref" if as_coord else "crs"
ds = xr.open_dataset(gridpath("ugrid", "quad-hexagon", "grid.nc")).load()
for name in ("node_lon", "node_lat"):
ds[name].attrs = {"grid_mapping": crs_name}
crs = xr.DataArray(0, attrs={"grid_mapping_name": "lambert_conformal_conic"})
ds = ds.assign_coords({crs_name: crs}) if as_coord else ds.assign({crs_name: crs})
path = tmp_path / "projected.nc"
ds.to_netcdf(path)

with pytest.warns(UserWarning, match="Projected"):
uxgrid = ux.open_grid(path)

assert crs_name in uxgrid._ds
uxgrid._ds.close()
8 changes: 5 additions & 3 deletions uxarray/core/dataarray.py
Original file line number Diff line number Diff line change
Expand Up @@ -2488,19 +2488,21 @@ def _slice_from_grid(self, sliced_grid):
"""Slices a ``UxDataArray`` from a sliced ``Grid``, using cached
indices to correctly slice the data variable."""

# index with the underlying arrays, so coordinates on the sliced grid
# (e.g. a stray ``time``) aren't aligned against, or attached to, the data
if self._face_centered():
da_sliced = self.isel(
n_face=sliced_grid._ds["_subgrid_face_indices"], ignore_grid=True
n_face=sliced_grid._ds["_subgrid_face_indices"].data, ignore_grid=True
)

elif self._edge_centered():
da_sliced = self.isel(
n_edge=sliced_grid._ds["_subgrid_edge_indices"], ignore_grid=True
n_edge=sliced_grid._ds["_subgrid_edge_indices"].data, ignore_grid=True
)

elif self._node_centered():
da_sliced = self.isel(
n_node=sliced_grid._ds["_subgrid_node_indices"], ignore_grid=True
n_node=sliced_grid._ds["_subgrid_node_indices"].data, ignore_grid=True
)

else:
Expand Down
6 changes: 5 additions & 1 deletion uxarray/grid/grid.py
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@
_populate_edge_node_distances,
)
from uxarray.grid.point_in_face import _point_in_face_query
from uxarray.grid.utils import make_setter
from uxarray.grid.utils import _drop_non_grid_data, make_setter
from uxarray.grid.validation import (
_check_area,
_check_connectivity,
Expand Down Expand Up @@ -338,6 +338,10 @@ def from_dataset(cls, dataset, use_dual: bool = False, **kwargs):
raise GridInvalidError(
f"Unsupported source_grid_spec={source_grid_spec!r}, in {cls.__name__}.from_dataset()"
)
# drop variables that don't belong on the grid (e.g. a `time` carried in
# from a file holding both grid and data), so they can't collide during
# subsetting.
grid_ds = _drop_non_grid_data(grid_ds)
else:
# custom source grid spec is provided
source_grid_spec = kwargs.get("source_grid_spec", None)
Expand Down
42 changes: 42 additions & 0 deletions uxarray/grid/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
from numba import njit, prange

from uxarray.constants import EDGE_NODE_SORT_THRESHOLD, INT_DTYPE, INT_FILL_VALUE
from uxarray.conventions import ugrid
from uxarray.conventions.descriptors import DESCRIPTOR_NAMES
from uxarray.utils.numba_math import (
_numba_add3,
_numba_cross3,
Expand All @@ -14,6 +16,46 @@
from uxarray.utils.parallel import parallel_njit


def _drop_non_grid_data(ds):
"""Drop variables that don't belong on a ``Grid`` (e.g. a ``time`` carried in
from a file that holds both the grid and its data).

A variable or coordinate is dropped when any of its dimensions is not a grid
dimension (``ugrid.DIM_NAMES``, plus ``lat_lon`` and ``min_max`` used by the
cached face bounds). Scalar coordinates are dropped too; scalar data variables
are kept.

Grid coordinates, connectivity, descriptors (including MPAS
``boundary_node_indices``) and any CRS variable named by a ``grid_mapping``
attribute (read by projected-grid detection) are always kept.
"""
grid_dims = set(ugrid.DIM_NAMES) | {"lat_lon", "min_max"}

keep = (
set(ugrid.SPHERICAL_COORD_NAMES)
| set(ugrid.CARTESIAN_COORD_NAMES)
| set(ugrid.CONNECTIVITY_NAMES)
| set(DESCRIPTOR_NAMES)
Comment on lines +35 to +38

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What is the point of whitelisting these explicitly? Aren't they all being kept automatically, because they all correspond to variables with only grid dimensions?

If it is possible to claim the simpler invariant of "only keep variables with only grid dims, plus a few exceptions" by dropping these from keep, instead of needing to have code like "only keep [long list of whitelisted variables], plus variables with only grid dims, plus a few exceptions", I think that would be preferable.

)
# CRS variables, referenced by name from a ``grid_mapping`` attribute
# (CF also allows the extended form "crs: x y")
for var in ds.variables.values():
grid_mapping = var.attrs.get("grid_mapping")
if isinstance(grid_mapping, str):
keep.update(token.rstrip(":") for token in grid_mapping.split())

drop = [
name
for name, var in ds.variables.items()
if name not in keep
and (
any(dim not in grid_dims for dim in var.dims)
or (var.ndim == 0 and name in ds.coords)
)
]
return ds.drop_vars(drop)


@njit(cache=True)
def _small_angle_of_2_vectors(u, v):
"""
Expand Down
Loading