Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 57 additions & 0 deletions test/core/test_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -424,6 +424,63 @@ def test_open_grid(gridpath, mesh_constants):

nt.assert_almost_equal(uxgrid.calculate_total_face_area(), mesh_constants['MESH30_AREA'], decimal=3)


def test_open_grid_raises_when_file_not_found():
"""open_grid() must raise FileNotFoundError for nonexistent files
(not a pyogrio.errors.DataSourceError)
"""
for ext in ["", ".nc", ".ug", ".shp", ".SHP", ".geojson", ".GEOJSON"]:
filepath = f"nonexistent_file{ext}"
assert not os.path.exists(filepath)
with pytest.raises(FileNotFoundError, match=r"No such file or directory"):
ux.open_grid(filepath)


def test_open_grid_geopandas(gridpath):
"""Calling open_grid with a geopandas file must not crash (see #1754),
and should be equivalent to Grid.from_file(..., backend="geopandas").
Additionally, passing other kwargs along with a geopandas file should raise TypeError.
"""
path = gridpath("shp", "cb_2018_us_nation_20m", "cb_2018_us_nation_20m.shp")
uxgridA = ux.open_grid(path)
uxgridB = ux.Grid.from_file(path, backend="geopandas")
assert uxgridA == uxgridB

## passing kwargs -- special cases which do not crash:
# (merge_coincident_nodes is silently ignored for geopandas files)
for val in [True, False, None, "a completely unexpected value"]:
grid = ux.open_grid(path, merge_coincident_nodes=val)
assert grid == uxgridA
# (chunks must be ignored whenever chunk_grid=False)
chunk_vals = [{}, "auto", -1, {"n_face": 10}]
for val in chunk_vals + [None]:
grid = ux.open_grid(path, chunks=val, chunk_grid=False)
assert grid == uxgridA

## passing kwargs -- other cases should crash:
for val in chunk_vals:
with pytest.raises(TypeError, match=r"geopandas .+ got unexpected keyword argument"):
ux.open_grid(path, chunks=val)
with pytest.raises(TypeError, match=r"geopandas .+ got unexpected keyword argument"):
ux.open_grid(path, cache=False)


def test_open_grid_geopandas_uppercase_extension(gridpath, tmp_path):
"""Shapefiles with uppercase extensions (e.g. from older GIS tools) must
also be dispatched to geopandas, rather than failing in xarray as in #1754.
"""
src_dir = Path(gridpath("shp", "cb_2018_us_nation_20m", "cb_2018_us_nation_20m.shp")).parent
# A shapefile is several sidecar files; rename all of them to uppercase.
for ext in (".shp", ".shx", ".dbf", ".prj", ".cpg"):
src = src_dir / f"cb_2018_us_nation_20m{ext}"
(tmp_path / f"CB_2018_US_NATION_20M{ext.upper()}").write_bytes(src.read_bytes())
upper_path = tmp_path / "CB_2018_US_NATION_20M.SHP"

uxgrid = ux.open_grid(upper_path)
assert uxgrid == ux.Grid.from_file(str(src_dir / "cb_2018_us_nation_20m.shp"))
assert uxgrid.source_grid_spec == "Shapefile"


def test_copy_dataset(gridpath, datasetpath):
"""Loads a single dataset with its grid topology file using uxarray's
open_dataset call and make a copy of the object."""
Expand Down
7 changes: 0 additions & 7 deletions test/grid/grid/test_initialization.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,13 +48,6 @@ def test_grid_init_verts_fill_values():
assert uxgrid.n_node > 0


def test_read_shpfile(test_data_dir):
"""Reads a shape file and write ugrid file."""
shp_filename = test_data_dir / "shp" / "grid_fire.shp"
with pytest.raises((ValueError, FileNotFoundError, OSError)):
grid_shp = ux.open_grid(shp_filename)


def test_class_methods_from_face_vertices():
"""Test class methods for creating grids from face vertices."""
single_face_latlon = [(0.0, 90.0), (-180, 0.0), (0.0, -90)]
Expand Down
29 changes: 28 additions & 1 deletion test/io/test_geopandas.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import geopandas as gpd
import numpy as np
import pytest
from shapely.geometry import Point, Polygon
from shapely.geometry import MultiPolygon, Point, Polygon

import uxarray as ux
from uxarray.io._geopandas import _extract_geometry_info, _gpd_read, _set_crs
Expand All @@ -11,6 +11,9 @@ def test_read_shpfile(test_data_dir):
uxgrid = ux.Grid.from_file(str(test_data_dir / "shp" / "cb_2018_us_nation_20m" / "cb_2018_us_nation_20m.shp"))
assert uxgrid.validate()

# tiny regression test for #1754: since this file has just the U.S., expect all latitudes>17.
assert np.all(uxgrid.node_lat.values > 17)

def test_read_shpfile_multi(test_data_dir):
"""Read a shapefile that consists of multipolygons."""
uxgrid = ux.Grid.from_file(str(test_data_dir / "shp" / "multipoly" / "multipoly.shp"))
Expand Down Expand Up @@ -115,3 +118,27 @@ def test_unsupported_geometry_is_reported():

# Only the polygon contributes a face; the point is skipped.
assert connectivity.shape[0] == 1


def test_multipolygon_preserves_lon_lat():
"""Every part of a MultiPolygon must keep its lon/lat order.

Regression test for issue #1754: _read_multipolygon unpacked
_read_polygon's (lat, lon) return as (lon, lat), so the lists traded
places after each part and every other part was read with lon/lat swapped.
"""
# Longitudes are well outside [-90, 90], so a swap can't be mistaken for valid data.
parts = [
Polygon([(100, 10), (110, 10), (110, 20), (100, 20)]),
Polygon([(120, 30), (130, 30), (130, 40), (120, 40)]),
Polygon([(140, 50), (150, 50), (150, 60)]),
]
gdf = gpd.GeoDataFrame(geometry=[MultiPolygon(parts)], crs="EPSG:4326")
max_nodes = max(len(p.exterior.coords) for p in parts)

node_lon, node_lat, connectivity = _extract_geometry_info(gdf, max_nodes)

expected = np.concatenate([np.asarray(p.exterior.coords)[:-1] for p in parts])
np.testing.assert_array_equal(node_lon, expected[:, 0])
np.testing.assert_array_equal(node_lat, expected[:, 1])
assert connectivity.shape[0] == len(parts)
56 changes: 46 additions & 10 deletions uxarray/core/api.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,9 +46,13 @@ def open_grid(
Parameters
----------
grid_filename_or_obj : str | os.PathLike[Any] | dict | xr.dataset
Strings and Path objects are interpreted as a path to a grid file. Xarray Datasets assume that
each member variable is in the UGRID conventions and will be used to create a ``ux.Grid``. Similarly, a dictionary
containing UGRID variables can be used to create a ``ux.Grid``
Specifies the unstructured grid topology.
Strings and Path objects are interpreted as a path to a grid file.
Most files are opened via ``xr.open_dataset(..., **kwargs)``.
Files with special extensions (".shp" or ".geojson") use ``geopandas.read_file`` instead,
and reject additional kwargs; see also ``Grid.from_file``.
Dictionaries and xarray Datasets assume that each variable is in the UGRID conventions,
and will be used to create a ``ux.Grid``.
chunks : int, dict, 'auto' or None, default: None
If provided, used to load the grid into dask arrays.

Expand All @@ -65,10 +69,10 @@ def open_grid(
use_dual : bool, optional
Specify whether to use the primal (use_dual=False) or dual (use_dual=True) mesh if the file type is MPAS
merge_coincident_nodes : bool, default=True
Passed through ``**kwargs`` for dataset and file inputs. Set to False to
preserve original node connectivity, including faces that would acquire
nonconsecutive repeated corners after merging. Dual construction still
rejects referenced duplicates.
Passed through ``**kwargs`` for dataset and file inputs, except for geopandas files
(where it is silently ignored). Set to False to preserve original node connectivity,
including faces that would acquire nonconsecutive repeated corners after merging.
Dual construction still rejects referenced duplicates.
**kwargs
Additional arguments passed on to ``xarray.open_dataset``.
Refer to the :func:`xarray docs <xarray.open_dataset>`
Expand All @@ -91,7 +95,32 @@ def open_grid(

>>> uxgrid = ux.open_grid("grid_filename.nc", chunks=-1)
"""
# Handle chunk-related kwargs first.
# geopandas bookkeeping (must handle this before match_chunks_to_ugrid(),
# which calls xr.open_dataset on the file and fails for geopandas files)
is_geopandas_file = (
isinstance(grid_filename_or_obj, (str, os.PathLike))
and os.path.splitext(grid_filename_or_obj)[1].lower()
in Grid._GEOPANDAS_FILE_EXTENSIONS
)
if is_geopandas_file:
# crash if any unsupported-for-geopandas kwargs were provided.
# (intentionally silently ignore merge_coincident_nodes; see docstring)
kwargs.pop("merge_coincident_nodes", None)
# special handling for chunks/chunk_grid; no need to crash whenever chunk_grid=False.
_got_chunk_grid_kw = ["chunk_grid"] if "chunk_grid" in kwargs else []
chunk_grid = kwargs.pop("chunk_grid", chunks is not None)
_got_chunk_kw = (["chunks"] + _got_chunk_grid_kw) if chunk_grid else []
# generic kw handling
_got_kw = list(kwargs.keys()) + _got_chunk_kw
if len(_got_kw) > 0:
raise TypeError(
f"open_grid() for geopandas files (extension one of: {Grid._GEOPANDAS_FILE_EXTENSIONS}) "
f"got unexpected keyword argument(s) {_got_kw!r}, when filepath={grid_filename_or_obj!r}."
)
# geopandas doesn't support chunking; force chunks=None to avoid match_chunks_to_ugrid() below.
chunks = None

# kwargs bookkeeping
data_chunks = kwargs.pop("data_chunks", None)
if data_chunks is not None:
chunks = match_chunks_to_ugrid(grid_filename_or_obj, data_chunks)
Expand All @@ -103,8 +132,6 @@ def open_grid(
grid_chunks = chunks if chunk_grid else None
merge_coincident_nodes = kwargs.pop("merge_coincident_nodes", True)

grid = None

# Special case for FESOM2 ASCII Dataset (stored as a directory)
if isinstance(grid_filename_or_obj, (str, os.PathLike)) and os.path.isdir(
grid_filename_or_obj
Expand All @@ -124,6 +151,15 @@ def open_grid(
f"got directory={os.path.abspath(grid_filename_or_obj)!r}, which is missing {' and '.join(_missing)}."
)

elif is_geopandas_file:
# Use geopandas backend to read from file
if not os.path.exists(grid_filename_or_obj):
# (explicit FileNotFoundError instead of pyogrio.errors.DataSourceError)
raise FileNotFoundError(
f"No such file or directory: {os.path.abspath(grid_filename_or_obj)!r}"
)
grid = Grid.from_file(grid_filename_or_obj, backend="geopandas")

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A missing .shp used to raise FileNotFoundError. It now raises pyogrio.errors.DataSourceError, a RuntimeError subclass. The removed test_read_shpfile in test_initialization.py was catching exactly this: restored on this branch, it fails with DataSourceError. That test was still working, since a missing file is its intended input. Could you raise FileNotFoundError here (e.g. an os.path.exists check) and keep the test?

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Added a new test, test_open_grid_raises_when_file_not_found, with a clearer docstring and purpose, instead of restoring the old test directly.


elif isinstance(grid_filename_or_obj, dict):
# Unpack the dictionary and construct a grid from topology
grid = Grid.from_topology(**grid_filename_or_obj)
Expand Down
12 changes: 10 additions & 2 deletions uxarray/grid/grid.py
Original file line number Diff line number Diff line change
Expand Up @@ -397,6 +397,9 @@ def from_file(
Path to grid file
backend : str, default='geopandas'
Backend to use to read the file, xarray or geopandas.
**kwargs
In geopandas backend, additional keyword arguments are silently discarded;
in xarray backend, they are passed to ``xarray.open_dataset``.

Examples
--------
Expand All @@ -411,9 +414,10 @@ def from_file(

# determine grid/mesh specification
if backend == "geopandas":
if str(filename).endswith(".shp"):
ext = os.path.splitext(filename)[1].lower()
if ext == ".shp":
source_grid_spec = "Shapefile"
elif str(filename).endswith(".geojson"):
elif ext == ".geojson":
source_grid_spec = "GeoJSON"
else:
source_grid_spec = "OtherGeoFormat"
Expand All @@ -432,6 +436,10 @@ def from_file(

return cls(grid_ds, source_grid_spec, source_dims_dict)

_GEOPANDAS_FILE_EXTENSIONS = {".shp", ".geojson"}
# Known geopandas file extensions, used by open_grid() to decide
# when to dispatch to Grid.from_file(..., backend='geopandas').

@classmethod
def from_points(
cls,
Expand Down
2 changes: 1 addition & 1 deletion uxarray/io/_geopandas.py
Original file line number Diff line number Diff line change
Expand Up @@ -216,7 +216,7 @@ def _read_multipolygon(geometry, node_lat, node_lon, connectivity, node_index):
Updated node index.
"""
for polygon in geometry.geoms:
node_lon, node_lat, connectivity, node_index = _read_polygon(
node_lat, node_lon, connectivity, node_index = _read_polygon(
polygon, node_lat, node_lon, connectivity, node_index
)
return np.array(node_lat), np.array(node_lon), connectivity, node_index
Expand Down
Loading