diff --git a/test/core/test_dataset.py b/test/core/test_dataset.py index 5259da764..20f061dcb 100644 --- a/test/core/test_dataset.py +++ b/test/core/test_dataset.py @@ -98,6 +98,43 @@ def test_get_dual(gridpath, datasetpath): assert len(uxds.data_vars) == len(dual.data_vars) +def test_get_dual_aligns_node_data_with_retained_primal_nodes(): + grid = ux.Grid.from_topology( + node_lon=np.array([-1.0, 0.0, 0.0, 1.0, 0.0]), + node_lat=np.array([0.0, -1.0, 0.0, 0.0, 1.0]), + face_node_connectivity=np.array( + [[2, 0, 1], [2, 1, 3], [2, 3, 4], [2, 4, 0]] + ), + ) + node_data = np.arange(2 * 5 * 3).reshape(2, 5, 3) + face_data = np.arange(8).reshape(2, 4) + unrelated_data = np.array([10, 20]) + uxds = UxDataset( + { + "node_data": (("time", "n_node", "level"), node_data), + "face_data": (("time", "n_face"), face_data), + "unrelated": (("time",), unrelated_data), + }, + uxgrid=grid, + ) + + with pytest.warns(Warning, match="mesh is partial"): + dual_array = uxds["node_data"].get_dual() + with pytest.warns(Warning, match="mesh is partial"): + dual_dataset = uxds.get_dual() + + assert dual_array.dims == ("time", "n_face", "level") + nt.assert_array_equal(dual_array.values, node_data[:, [2], :]) + assert dual_array.sizes["n_face"] == dual_array.uxgrid.n_face == 1 + + assert dual_dataset["node_data"].dims == ("time", "n_face", "level") + nt.assert_array_equal(dual_dataset["node_data"].values, node_data[:, [2], :]) + assert dual_dataset["face_data"].dims == ("time", "n_node") + nt.assert_array_equal(dual_dataset["face_data"].values, face_data) + assert dual_dataset["unrelated"].dims == ("time",) + nt.assert_array_equal(dual_dataset["unrelated"].values, unrelated_data) + + def _load_sel_subset(gridpath, datasetpath): grid_file = gridpath("ugrid", "outCSne30", "outCSne30.ug") data_file = datasetpath("ugrid", "outCSne30", "outCSne30_sel_timeseries.nc") diff --git a/test/grid/grid/test_core.py b/test/grid/grid/test_core.py index cc178390b..14a56c379 100644 --- a/test/grid/grid/test_core.py +++ b/test/grid/grid/test_core.py @@ -4,7 +4,17 @@ import xarray as xr import uxarray as ux -from uxarray.constants import ERROR_TOLERANCE, INT_FILL_VALUE +from uxarray.constants import INT_FILL_VALUE +from uxarray.grid.connectivity import ( + _merge_coincident_grid_ds_nodes, + _spec_guarantees_unique_nodes, +) +from uxarray.grid.validation import ( + _check_duplicate_nodes_indices, + _find_duplicate_node_map, + _find_duplicate_nodes, +) +from uxarray.errors import GridInvalidError def test_grid_with_holes(gridpath): @@ -146,7 +156,533 @@ def test_dual_mesh_mpas(gridpath): def test_dual_duplicate(gridpath): - """Test dual mesh creation with duplicate grids.""" - dataset = ux.open_dataset(gridpath("ugrid", "geoflow-small", "grid.nc"), gridpath("ugrid", "geoflow-small", "grid.nc")) - with pytest.raises(ux.errors.GridInvalidError): - dataset.get_dual() + """Test dual mesh creation on a grid whose source file has duplicate + (coincident) node indices, merged at construction time.""" + grid_path = gridpath("ugrid", "geoflow-small", "grid.nc") + grid = ux.open_grid(grid_path) + + # The source file really does contain duplicates: 6000 node coordinates for + # 3842 distinct locations, so 2158 indices are coincident with an earlier one. + # The count is hard-coded here only to guard against unexpected + # regressions in which duplicates are identified; it is not independently + # derived from the source file. For a correctness check of the duplicate + # identification itself, see test_duplicate_nodes_minimal_example. + duplicates = _find_duplicate_nodes(grid) + assert grid.n_node == 6000 + assert len(duplicates) == 2158 + + # Connectivity is canonicalized to a single index per coincident group, so no + # face references any duplicate index. + assert not _check_duplicate_nodes_indices(grid) + # duplicate coordinates are left in place by design, but connectivity is + # fully canonicalized, so validation passes + assert grid.validate() + + dual = grid.get_dual() + + assert dual.n_node == grid.n_face + + # One dual face per node that is a corner of at least three faces. After the + # merge, 3842 distinct nodes remain; every one is touched by at least three + # faces and therefore produces a dual cell. + face_nodes = grid.face_node_connectivity.values + faces_per_node = np.bincount( + face_nodes[face_nodes != INT_FILL_VALUE], minlength=grid.n_node + ) + assert grid.n_node - len(duplicates) == 3842 + assert (faces_per_node >= 3).sum() == 3842 + assert dual.n_face == 3842 + + dataset = ux.open_dataset(grid_path, grid_path) + dual_ds = dataset.get_dual() + assert dual_ds.uxgrid.n_face == dual.n_face + + +def test_dual_duplicate_geos_cs(gridpath): + """Test dual mesh creation on a cube-sphere grid with duplicate node + indices (issue #865).""" + grid_path = gridpath("geos-cs", "c12", "test-c12.native.nc4") + grid = ux.open_grid(grid_path) + + assert len(_find_duplicate_nodes(grid)) > 0 + assert not _check_duplicate_nodes_indices(grid) + + dual = grid.get_dual() + assert dual.n_node == grid.n_face + assert dual.n_face > 0 + + +def test_duplicate_nodes_minimal_example(): + """Two quads that share an edge, but whose shared corners are stored twice. + + Nodes 1 and 2 are repeated as nodes 6 and 7, so the file describes 8 nodes + at 6 distinct locations. Node 6 must canonicalize to node 1 and node 7 to + node 2, leaving the second face pointing at the first face's corners. + + 3-----2/7-----5 lat 1 + | | | + | f0 | f1 | + | | | + 0-----1/6-----4 lat 0 + lon 0 lon 1 lon 2 + + f0 = [0, 1, 2, 3] uses the shared edge as nodes 1, 2; f1 = [6, 4, 5, 7] + uses the duplicates 6, 7 for the same two corners. + """ + node_lon = np.array([0.0, 1.0, 1.0, 0.0, 2.0, 2.0, 1.0, 1.0]) + node_lat = np.array([0.0, 0.0, 1.0, 1.0, 0.0, 1.0, 0.0, 1.0]) + # left quad right quad, via the duplicates + face_node_connectivity = np.array([[0, 1, 2, 3], [6, 4, 5, 7]]) + + grid = ux.Grid.from_topology(node_lon, node_lat, face_node_connectivity) + + assert grid._duplicate_node_map == {6: 1, 7: 2} + duplicates = _find_duplicate_nodes(grid) + assert duplicates == {6: 1, 7: 2} + + # No face may still reference a duplicate index. + assert not _check_duplicate_nodes_indices(grid) + nt.assert_equal( + grid.face_node_connectivity.values, np.array([[0, 1, 2, 3], [1, 4, 5, 2]]) + ) + + +@pytest.mark.parametrize( + "lon_offset, merges", + [ + (1e-9, True), # 1.7e-11 chord, well inside + (1e-7, True), # 1.7e-09 chord, inside + (1e-5, False), # 1.7e-07 chord, outside + ], +) +def test_duplicate_nodes_tolerance(lon_offset, merges): + """Nodes merge only when they are coincident within the merge tolerance. + + Same topology as test_duplicate_nodes_minimal_example, but duplicate node 6 + is offset in lon. The tolerance is a chord length of ERROR_TOLERANCE (1e-8) + on the unit sphere, so the cutoff sits near 5.7e-7 degrees: the first two + offsets must still canonicalize 6 to 1, the third must not. + """ + node_lon = np.array([0.0, 1.0, 1.0, 0.0, 2.0, 2.0, 1.0 + lon_offset, 1.0]) + node_lat = np.array([0.0, 0.0, 1.0, 1.0, 0.0, 1.0, 0.0, 1.0]) + face_node_connectivity = np.array([[0, 1, 2, 3], [6, 4, 5, 7]]) + + grid = ux.Grid.from_topology(node_lon, node_lat, face_node_connectivity) + + duplicates = _find_duplicate_nodes(grid) + # Node 7 is an exact duplicate of node 2 in every case. + assert duplicates == ({6: 1, 7: 2} if merges else {7: 2}) + assert not _check_duplicate_nodes_indices(grid) + + # Face 1's first corner follows node 6: canonicalized to 1 when merged, + # left as 6 when the offset puts it outside the tolerance. + assert grid.face_node_connectivity.values[1][0] == (1 if merges else 6) + + +def test_merge_warns_when_node_locations_are_unknown(): + """Without node coordinates the merge cannot run; it must say so rather + than quietly return connectivity that may still hold duplicates.""" + grid_ds = xr.Dataset( + { + "face_node_connectivity": ( + ("n_face", "n_max_face_nodes"), + np.array([[0, 1, 2, 3], [6, 4, 5, 7]]), + ) + } + ) + + with pytest.warns(RuntimeWarning, match="Coincident nodes were not merged"): + out = _merge_coincident_grid_ds_nodes(grid_ds) + + nt.assert_equal( + out["face_node_connectivity"].values, + grid_ds["face_node_connectivity"].values, + ) + + +def test_merge_compacts_face_and_invalidates_edge_variables(): + grid_ds = xr.Dataset( + { + "node_lon": ("n_node", [0.0, 1.0, 0.0, 1.0]), + "node_lat": ("n_node", [0.0, 0.0, 1.0, 0.0]), + "face_node_connectivity": ( + ("n_face", "n_max_face_nodes"), + [[0, 1, 3, 2]], + ), + "edge_node_connectivity": (("n_edge", "two"), np.zeros((4, 2))), + "edge_data": ("n_edge", np.arange(4)), + "n_nodes_per_face": ("n_face", [4]), + } + ) + + merged = _merge_coincident_grid_ds_nodes(grid_ds) + + nt.assert_equal( + merged["face_node_connectivity"].values, + np.array([[0, 1, 2, INT_FILL_VALUE]]), + ) + assert "n_edge" not in merged.dims + assert "edge_node_connectivity" not in merged + assert "edge_data" not in merged + assert "n_nodes_per_face" not in merged + + grid = ux.Grid(merged, source_grid_spec="User Defined Topology") + nt.assert_equal(grid.n_nodes_per_face.values, [3]) + assert grid.n_edge == 3 + + +def test_merge_preserves_face_with_fewer_than_three_distinct_corners(): + grid_ds = xr.Dataset( + { + "node_lon": ("n_node", [0.0, 1.0, 1.0]), + "node_lat": ("n_node", [0.0, 0.0, 0.0]), + "face_node_connectivity": ( + ("n_face", "n_max_face_nodes"), + [[0, 1, 2]], + ), + } + ) + + merged = _merge_coincident_grid_ds_nodes(grid_ds) + nt.assert_equal(merged["face_node_connectivity"].values, [[0, 1, 1]]) + + +def test_merge_rejects_nonconsecutive_repeated_face_corner(): + grid_ds = xr.Dataset( + { + "node_lon": ("n_node", [0.0, 1.0, 0.0, 1.0]), + "node_lat": ("n_node", [0.0, 0.0, 1.0, 0.0]), + "face_node_connectivity": ( + ("n_face", "n_max_face_nodes"), + [[0, 1, 2, 3]], + ), + } + ) + + with pytest.raises(GridInvalidError, match="nonconsecutive repeated corners"): + _merge_coincident_grid_ds_nodes(grid_ds) + + +def test_bowtie_merge_opt_out_reaches_grid_constructor(tmp_path): + dataset = xr.Dataset( + { + "mesh": ((), 0, {"cf_role": "mesh_topology", "topology_dimension": 2, + "node_coordinates": "lon lat", "face_node_connectivity": "faces"}), + "lon": ("nodes", [0.0, 1.0, 0.0, 1.0], {"standard_name": "longitude"}), + "lat": ("nodes", [0.0, 0.0, 1.0, 0.0], {"standard_name": "latitude"}), + "faces": (("faces_dim", "corners"), [[0, 1, 2, 3]], {"start_index": 0}), + } + ) + path = tmp_path / "bowtie.nc" + dataset.to_netcdf(path) + with pytest.raises(GridInvalidError, match="nonconsecutive"): + ux.open_grid(path) + grids = [ + ux.Grid.from_dataset(dataset, merge_coincident_nodes=False), + ux.open_grid(dataset, merge_coincident_nodes=False), + ux.open_grid(path, merge_coincident_nodes=False), + ux.open_dataset(path, path, grid_kwargs={"merge_coincident_nodes": False}).uxgrid, + ] + for grid in grids: + nt.assert_equal(grid.face_node_connectivity.values, [[0, 1, 2, 3]]) + with pytest.raises(GridInvalidError, match="Duplicate nodes"): + grid.get_dual() + + +def test_chunked_dual_matches_eager_without_mutating_source(gridpath): + path = gridpath("ugrid", "geoflow-small", "grid.nc") + eager = ux.open_grid(path).get_dual() + lazy = ux.open_grid(path, chunks={}) + original = lazy.face_node_connectivity.data + dual = lazy.get_dual() + assert dual.n_face == eager.n_face == 3842 + nt.assert_equal(dual.face_node_connectivity.values, eager.face_node_connectivity.values) + assert lazy.face_node_connectivity.data is original + assert lazy.face_node_connectivity.chunks is not None + + +def test_chunked_data_dual_matches_eager(gridpath): + path = gridpath("ugrid", "geoflow-small", "grid.nc") + eager = ux.open_grid(path) + lazy = ux.open_grid(path, chunks={}) + values = np.arange(eager.n_node, dtype=float) + array = ux.UxDataArray(values, dims="n_node", uxgrid=lazy).chunk() + expected = ux.UxDataArray(values, dims="n_node", uxgrid=eager).get_dual() + result = array.get_dual() + nt.assert_equal(result.values, expected.values) + nt.assert_equal(result.uxgrid.face_node_connectivity.values, expected.uxgrid.face_node_connectivity.values) + assert result.chunks is not None + dataset = ux.UxDataset({"ids": array}, uxgrid=lazy).get_dual() + nt.assert_equal(dataset.ids.values, expected.values) + assert lazy.face_node_connectivity.chunks is not None + + +def test_repeated_face_dual_paths_agree(gridpath): + primal = ux.open_grid(gridpath("exodus", "outCSne8", "outCSne8.g")) + connectivity = np.vstack([ + primal.face_node_connectivity.values, + primal.face_node_connectivity.values[:1], + ]) + grid = ux.Grid.from_topology(primal.node_lon.values, primal.node_lat.values, connectivity) + dual = grid.get_dual() + ids = ux.UxDataArray(np.arange(grid.n_node), dims="n_node", uxgrid=grid) + array_dual = ids.get_dual() + dataset_dual = ux.UxDataset({"ids": ids}, uxgrid=grid).get_dual() + nt.assert_equal(dual.face_node_connectivity.values, array_dual.uxgrid.face_node_connectivity.values) + nt.assert_equal(dual.face_node_connectivity.values, dataset_dual.uxgrid.face_node_connectivity.values) + nt.assert_equal(array_dual.values, dataset_dual.ids.values) + assert dual.get_dual().n_face > 0 + + +@pytest.mark.parametrize("lazy", [False, True]) +@pytest.mark.parametrize("merge", [False, True]) +def test_dual_preserves_merge_setting(gridpath, lazy, merge): + primal = ux.open_grid(gridpath("exodus", "outCSne8", "outCSne8.g")) + lon = np.append(primal.node_lon.values, primal.node_lon.values[0]) + lat = np.append(primal.node_lat.values, primal.node_lat.values[0]) + faces = primal.face_node_connectivity.values.copy() + row, column = np.argwhere(faces == 0)[0] + faces[row, column] = lon.size - 1 + dataset = xr.Dataset( + { + "mesh": ((), 0, {"cf_role": "mesh_topology", "topology_dimension": 2, + "node_coordinates": "lon lat", "face_node_connectivity": "faces"}), + "lon": ("nodes", lon, {"standard_name": "longitude"}), + "lat": ("nodes", lat, {"standard_name": "latitude"}), + "faces": (("cells", "corners"), faces, {"start_index": 0}), + } + ) + grid = ux.open_grid(dataset.chunk() if lazy else dataset, merge_coincident_nodes=merge) + original = grid.face_node_connectivity.data + array = ux.UxDataArray(np.arange(grid.n_node), dims="n_node", uxgrid=grid) + data = ux.UxDataset({"ids": array}, uxgrid=grid) + for obj in (grid, array, data): + if merge: + dual = obj.get_dual() + dual_grid = dual if isinstance(dual, ux.Grid) else dual.uxgrid + assert dual_grid.n_face == 386 + else: + with pytest.raises(GridInvalidError, match="Duplicate nodes"): + obj.get_dual() + assert grid.face_node_connectivity.data is original + assert grid._merge_coincident_nodes is merge + + +def test_get_dual_rejects_faces_referencing_duplicate_nodes(): + """``construct_dual`` reads ``node_face_connectivity`` with no duplicate + handling, so a face still pointing at a dead duplicate index would yield a + degenerate dual face instead of an error. Merging at construction makes this + unreachable today; the guard keeps it that way.""" + node_lon = np.array([0.0, 1.0, 1.0, 0.0, 2.0, 2.0, 1.0, 1.0]) + node_lat = np.array([0.0, 0.0, 1.0, 1.0, 0.0, 1.0, 0.0, 1.0]) + unmerged = np.array([[0, 1, 2, 3], [6, 4, 5, 7]]) + + grid = ux.Grid.from_topology(node_lon, node_lat, unmerged) + # Construction canonicalized the connectivity; put the duplicates back. + grid.face_node_connectivity = xr.DataArray( + unmerged, dims=grid.face_node_connectivity.dims + ) + + assert _check_duplicate_nodes_indices(grid) + with pytest.raises(GridInvalidError): + grid.get_dual() + + +def test_pole_matching_merges_only_exact_xyz_rows(): + from uxarray.grid.validation import _coincident_node_canonical_indices + + z = 1.0 - 1e-6 + x = np.sqrt(1.0 - z * z) + + points_xyz = np.array( + [ + [0.0, 0.0, 1.0], + [0.0, 0.0, 1.0], # exact duplicate pole row + [np.finfo(np.float64).eps, 0.0, 1.0], # distinct pole representation + [x, 0.0, z], + [x, 0.0, z], + ] + ) + + canonical = _coincident_node_canonical_indices(points_xyz) + + nt.assert_equal(canonical, np.array([0, 0, 2, 3, 3])) + + +def test_merge_prefers_float64_lonlat_over_xyz(): + grid_ds = xr.Dataset( + { + "node_lon": ( + "n_node", np.array([0.0, 1.0, 1.0, 90.0], dtype=np.float32) + ), + "node_lat": ("n_node", np.zeros(4, dtype=np.float32)), + "node_x": ("n_node", np.array([1.0, 0.0, -1.0, 0.0])), + "node_y": ("n_node", np.array([0.0, 1.0, 0.0, -1.0])), + "node_z": ("n_node", np.zeros(4)), + "face_node_connectivity": ( + ("n_face", "n_max_face_nodes"), + np.array([[0, 1, 2, 3]]), + ), + } + ) + + merged = _merge_coincident_grid_ds_nodes(grid_ds) + + nt.assert_equal( + merged["face_node_connectivity"].values, + np.array([[0, 1, 3, INT_FILL_VALUE]]), + ) + + +def test_merge_normalizes_xyz_fallback(): + grid_ds = xr.Dataset( + { + "node_x": ("n_node", np.array([1.0, 2.0, 0.0, 0.0])), + "node_y": ("n_node", np.array([0.0, 0.0, 1.0, 0.0])), + "node_z": ("n_node", np.array([0.0, 0.0, 0.0, 1.0])), + "face_node_connectivity": ( + ("n_face", "n_max_face_nodes"), + np.array([[0, 1, 2, 3]]), + ), + } + ) + + merged = _merge_coincident_grid_ds_nodes(grid_ds) + + nt.assert_equal( + merged["face_node_connectivity"].values, + np.array([[0, 2, 3, INT_FILL_VALUE]]), + ) + + +@pytest.mark.parametrize("invalid", [np.nan, np.inf]) +def test_merge_preserves_unused_nonfinite_nodes(invalid): + grid = ux.Grid.from_topology( + np.array([0.0, 1.0, 1.0, 0.0, invalid]), + np.array([0.0, 0.0, 1.0, 1.0, invalid]), + np.array([[0, 1, 2, 3]]), + ) + assert grid.n_node == 5 + nt.assert_equal(grid.face_node_connectivity.values, [[0, 1, 2, 3]]) + assert _find_duplicate_nodes(grid) == {} + + +def test_merge_maps_finite_nodes_back_to_original_indices(): + grid = ux.Grid.from_topology( + np.array([np.nan, 0.0, 1.0, 1.0, 0.0, 1.0]), + np.array([np.nan, 0.0, 0.0, 1.0, 1.0, 0.0]), + np.array([[1, 5, 3, 4]]), + ) + nt.assert_equal(grid.face_node_connectivity.values, [[1, 2, 3, 4]]) + assert _find_duplicate_nodes(grid) == {5: 2} + + +@pytest.mark.parametrize("invalid", [np.nan, np.inf]) +def test_merge_rejects_referenced_nonfinite_lonlat(invalid): + with pytest.raises(ValueError, match="finite"): + ux.Grid.from_topology( + np.array([0.0, 1.0, invalid]), + np.array([0.0, 0.0, invalid]), + np.array([[0, 1, 2]]), + ) + + +@pytest.mark.parametrize("invalid", [np.nan, np.inf]) +def test_merge_rejects_invalid_xyz_fallback(invalid): + grid_ds = xr.Dataset( + { + "node_x": ("n_node", np.array([invalid, 0.0, 0.0])), + "node_y": ("n_node", np.array([0.0, 1.0, 0.0])), + "node_z": ("n_node", np.array([0.0, 0.0, 1.0])), + "face_node_connectivity": ( + ("n_face", "n_max_face_nodes"), + np.array([[0, 1, 2]]), + ), + } + ) + + with pytest.raises(ValueError, match="finite"): + _merge_coincident_grid_ds_nodes(grid_ds) + + +def test_coincident_prescreen_keeps_both_ends_of_a_run(): + """The x-sorted prescreen must mark both members of a close pair. + + It flags a point when the gap to its predecessor *or* its successor is within + tolerance. Dropping either half of that OR silently loses one node of every + coincident pair, so this places coincident pairs at both ends of the sorted + order, where only one of the two neighbour tests fires. + """ + from uxarray.grid.validation import _coincident_node_canonical_indices + + # x strictly increasing and far apart, except for the duplicated first and + # last points, which have no predecessor / no successor respectively. + points_xyz = np.array( + [ + [-1.0, 0.0, 0.0], # 0 + [-1.0, 0.0, 0.0], # 1, coincident with 0 -> first in sorted x + [0.25, np.sqrt(1.0 - 0.25**2), 0.0], # 2 + [0.5, np.sqrt(1.0 - 0.5**2), 0.0], # 3 + [1.0, 0.0, 0.0], # 4 + [1.0, 0.0, 0.0], # 5, coincident with 4 -> last in sorted x + ] + ) + + canonical = _coincident_node_canonical_indices(points_xyz) + + nt.assert_equal(canonical, np.array([0, 0, 2, 3, 4, 4])) + + +def test_no_duplicate_nodes_ne30pg3(gridpath): + """``esmf/ne30/ne30pg3.grid.nc`` no longer reproduces issue #865's + duplicate-node bug; this only checks the general fix is a safe no-op.""" + grid_path = gridpath("esmf", "ne30", "ne30pg3.grid.nc") + grid = ux.open_grid(grid_path) + + assert len(_find_duplicate_nodes(grid)) == 0 + + +def test_spec_guarantees_unique_nodes_membership(): + """Only formats whose node list is an already-unique vertex array may claim + the skip; per-face-corner formats must still be searched.""" + assert _spec_guarantees_unique_nodes("HEALPix") + + # File formats are always searched because their names alone cannot guarantee + # that a particular file has unique node coordinates. + for spec in ("MPAS", "Scrip", "Exodus", "GEOS-CS", "UGRID", "ESMF", "ICON"): + assert not _spec_guarantees_unique_nodes(spec) + + # An unrecognized or absent spec must fall back to searching. + assert not _spec_guarantees_unique_nodes(None) + assert not _spec_guarantees_unique_nodes("SomeFutureFormat") + + +def test_skipped_spec_really_has_no_coincident_nodes(): + """Only internally generated HEALPix nodes bypass the duplicate search.""" + healpix = ux.Grid.from_healpix(zoom=3) + assert _spec_guarantees_unique_nodes(healpix.source_grid_spec) + assert ( + _find_duplicate_node_map(healpix.node_lon.values, healpix.node_lat.values) == {} + ) + + +def test_mpas_label_does_not_skip_duplicate_merge(): + node_lon = np.array([0.0, 1.0, 1.0, 0.0, 2.0, 2.0, 1.0, 1.0]) + node_lat = np.array([0.0, 0.0, 1.0, 1.0, 0.0, 1.0, 0.0, 1.0]) + connectivity = np.array([[0, 1, 2, 3], [6, 4, 5, 7]]) + grid = ux.Grid.from_topology( + node_lon, node_lat, connectivity, source_grid_spec="MPAS" + ) + + nt.assert_equal( + grid.face_node_connectivity.values, + np.array([[0, 1, 2, 3], [1, 4, 5, 2]]), + ) + + +def test_duplicates_still_merged_for_non_skipped_specs(gridpath): + """The skip must not weaken the #865 fix for formats that do have duplicates.""" + grid = ux.open_grid(gridpath("ugrid", "geoflow-small", "grid.nc")) + assert grid.source_grid_spec == "UGRID" + assert not _spec_guarantees_unique_nodes(grid.source_grid_spec) + assert grid.get_dual().n_face == 3842 diff --git a/test/io/test_scrip_padding.py b/test/io/test_scrip_padding.py new file mode 100644 index 000000000..cd62c1c10 --- /dev/null +++ b/test/io/test_scrip_padding.py @@ -0,0 +1,37 @@ +import numpy as np +import pytest +import xarray as xr + +import uxarray as ux +from uxarray.constants import INT_FILL_VALUE + + +@pytest.mark.parametrize("chunks", [None, {}]) +def test_scrip_decoded_padding_is_not_a_face_corner(tmp_path, chunks): + fill = 1e20 + dataset = xr.Dataset( + { + "grid_center_lon": ("grid_size", [0.5, 1.33]), + "grid_center_lat": ("grid_size", [0.5, 0.33]), + "grid_corner_lon": ( + ("grid_size", "grid_corners"), + [[0.0, 1, 1, 0], [1, 2, 1, fill]], + {"units": "degrees", "_FillValue": fill}, + ), + "grid_corner_lat": ( + ("grid_size", "grid_corners"), + [[0.0, 0, 1, 1], [0, 0, 1, fill]], + {"units": "degrees", "_FillValue": fill}, + ), + "grid_imask": ("grid_size", np.array([1, 1], np.int32)), + "grid_dims": ("grid_rank", np.array([2], np.int32)), + } + ) + path = tmp_path / "padded.nc" + dataset.to_netcdf(path) + grid = ux.open_grid(path, chunks=chunks) + assert grid.face_node_connectivity.values[1, -1] == INT_FILL_VALUE + np.testing.assert_array_equal(grid.n_nodes_per_face.values, [4, 3]) + assert np.isfinite(grid.face_areas.values).all() + if chunks is not None: + assert grid.face_node_connectivity.chunks is not None diff --git a/test/test_subset.py b/test/test_subset.py index eb3df313e..e30babbd5 100644 --- a/test/test_subset.py +++ b/test/test_subset.py @@ -78,8 +78,12 @@ def test_grid_nn_subset(gridpath): for grid_path in GRID_PATHS: grid = ux.open_grid(grid_path) - # corner-nodes - ks = [1, 2, grid.n_node - 1] + # corner-nodes -- k is bounded by the number of *live* (non-duplicate) + # nodes, since the node search tree excludes dead coincident indices + from uxarray.grid.validation import _live_node_indices + + n_live_nodes = len(_live_node_indices(grid)) + ks = [1, 2, n_live_nodes - 1] for coord in coord_locs: for k in ks: grid_subset = grid.subset.nearest_neighbor(coord, diff --git a/uxarray/core/api.py b/uxarray/core/api.py index 9039aa5ae..c0a6336c1 100644 --- a/uxarray/core/api.py +++ b/uxarray/core/api.py @@ -64,6 +64,11 @@ def open_grid( See dask chunking for more details. use_dual : bool, optional Specify whether to use the primal (use_dual=False) or dual (use_dual=True) mesh if the file type is MPAS + merge_coincident_nodes : bool, default=True + Passed through ``**kwargs`` for dataset and file inputs. Set to False to + preserve original node connectivity, including faces that would acquire + nonconsecutive repeated corners after merging. Dual construction still + rejects referenced duplicates. **kwargs Additional arguments passed on to ``xarray.open_dataset``. Refer to the :func:`xarray docs ` @@ -96,6 +101,7 @@ def open_grid( return_chunks = kwargs.pop("return_chunks", False) chunk_grid = kwargs.pop("chunk_grid", chunks is not None) grid_chunks = chunks if chunk_grid else None + merge_coincident_nodes = kwargs.pop("merge_coincident_nodes", True) grid = None @@ -128,14 +134,22 @@ def open_grid( elif isinstance(grid_filename_or_obj, xr.Dataset): # Construct a grid from a dataset file - grid = Grid.from_dataset(grid_filename_or_obj, use_dual=use_dual) + grid = Grid.from_dataset( + grid_filename_or_obj, + use_dual=use_dual, + merge_coincident_nodes=merge_coincident_nodes, + ) else: # Attempt to use Xarray directly for remaining input types grid_ds = _open_dataset_with_fallback( grid_filename_or_obj, chunks=grid_chunks, **kwargs ) - grid = Grid.from_dataset(grid_ds, use_dual=use_dual) + grid = Grid.from_dataset( + grid_ds, + use_dual=use_dual, + merge_coincident_nodes=merge_coincident_nodes, + ) # Return the grid (and chunks, if requested) in a consistent manner. if return_chunks: diff --git a/uxarray/core/dataarray.py b/uxarray/core/dataarray.py index fb43a5220..dd3efc620 100644 --- a/uxarray/core/dataarray.py +++ b/uxarray/core/dataarray.py @@ -40,7 +40,7 @@ from uxarray.formatting_html import array_repr from uxarray.grid import Grid from uxarray.grid.area import _integration_face_areas -from uxarray.grid.dual import construct_dual +from uxarray.grid.dual import _dual_face_indices from uxarray.grid.neighbors import DataArrayNeighborhood, Neighborhood from uxarray.grid.validation import _check_duplicate_nodes_indices from uxarray.io._healpix import get_zoom_from_cells @@ -2519,37 +2519,41 @@ def get_dual(self): -------- dual : uxda Dual Mesh `uxda` constructed - """ - if _check_duplicate_nodes_indices(self.uxgrid): + Raises + ------ + GridInvalidError + If any face still references a coincident duplicate node. + """ + grid = self.uxgrid._prepare_for_dual() + if _check_duplicate_nodes_indices(grid): raise GridInvalidError( "Duplicate nodes found in UxDataArray's uxgrid; cannot get_dual()" ) - if self.uxgrid.partial_sphere_coverage: + if grid.partial_sphere_coverage: warn( "This mesh is partial, which could cause inconsistent results and data will be lost", Warning, ) - # Get dual mesh node face connectivity - dual_node_face_conn = construct_dual(grid=self.uxgrid) - - # Construct dual mesh - dual = self.uxgrid.from_topology( - self.uxgrid.face_lon.values, - self.uxgrid.face_lat.values, - dual_node_face_conn, - ) + # Get dual mesh node face connectivity and the primal nodes represented + # by its faces. + dual_face_indices = _dual_face_indices(grid) + dual = grid.get_dual() # Dictionary to swap dimensions dim_map = {"n_face": "n_node", "n_node": "n_face"} + data = self + if "n_node" in data.dims: + data = data.isel(n_node=dual_face_indices, ignore_grid=True) + # Get correct dimensions for the dual - dims = [dim_map.get(dim, dim) for dim in self.dims] + dims = [dim_map.get(dim, dim) for dim in data.dims] # Construct the new data array - uxda = type(self)(uxgrid=dual, data=self.data, dims=dims, name=self.name) + uxda = type(self)(uxgrid=dual, data=data.data, dims=dims, name=self.name) return uxda diff --git a/uxarray/core/dataset.py b/uxarray/core/dataset.py index b030aefbb..69c0b95c9 100644 --- a/uxarray/core/dataset.py +++ b/uxarray/core/dataset.py @@ -24,7 +24,7 @@ from uxarray.formatting_html import dataset_repr from uxarray.grid import Grid from uxarray.grid.area import _integration_face_areas -from uxarray.grid.dual import construct_dual +from uxarray.grid.dual import _dual_face_indices from uxarray.grid.neighbors import DatasetNeighborhood from uxarray.grid.validation import _check_duplicate_nodes_indices from uxarray.io._healpix import get_zoom_from_cells @@ -975,28 +975,28 @@ def get_dual(self): -------- dual : uxds Dual Mesh `uxds` constructed - """ - if _check_duplicate_nodes_indices(self.uxgrid): + Raises + ------ + GridInvalidError + If any face still references a coincident duplicate node. + """ + grid = self.uxgrid._prepare_for_dual() + if _check_duplicate_nodes_indices(grid): raise GridInvalidError( "Duplicate nodes found in UxDataset's uxgrid; cannot get_dual()" ) - if self.uxgrid.partial_sphere_coverage: + if grid.partial_sphere_coverage: warn( "This mesh is partial, which could cause inconsistent results and data will be lost", Warning, ) - # Get dual mesh node face connectivity - dual_node_face_conn = construct_dual(grid=self.uxgrid) - - # Construct dual mesh - dual = self.uxgrid.from_topology( - self.uxgrid.face_lon.values, - self.uxgrid.face_lat.values, - dual_node_face_conn, - ) + # Get dual mesh node face connectivity and the primal nodes represented + # by its faces. + dual_face_indices = _dual_face_indices(grid) + dual = grid.get_dual() # Initialize new dataset dataset = type(self)(uxgrid=dual) @@ -1006,11 +1006,15 @@ def get_dual(self): # For each data array in the dataset, reconstruct the data array with the dual mesh for var in self.data_vars: + data = self[var] + if "n_node" in data.dims: + data = data.isel(n_node=dual_face_indices, ignore_grid=True) + # Get correct dimensions for the dual - dims = [dim_map.get(dim, dim) for dim in self[var].dims] + dims = [dim_map.get(dim, dim) for dim in data.dims] # Construct the new data array - uxda = UxDataArray(uxgrid=dual, data=self[var].data, dims=dims, name=var) + uxda = UxDataArray(uxgrid=dual, data=data.data, dims=dims, name=var) # Add data array to dataset dataset[var] = uxda diff --git a/uxarray/grid/connectivity.py b/uxarray/grid/connectivity.py index 19ae21cd2..365a2a8f0 100644 --- a/uxarray/grid/connectivity.py +++ b/uxarray/grid/connectivity.py @@ -4,8 +4,9 @@ import xarray as xr from numba import njit, prange -from uxarray.constants import INT_DTYPE, INT_FILL_VALUE +from uxarray.constants import ERROR_TOLERANCE, INT_DTYPE, INT_FILL_VALUE from uxarray.conventions import ugrid +from uxarray.errors import GridInvalidError from uxarray.grid.utils import ( _build_pair_index, _count_unique_in_bucket, @@ -511,6 +512,269 @@ def _build_face_edge_connectivity( return face_edge_connectivity +def _remap_node_connectivity(connectivity, duplicate_node_map, n_node): + """Return a copy of connectivity with duplicate node indices canonicalized.""" + if not duplicate_node_map: + return connectivity + + lookup = np.arange(n_node, dtype=INT_DTYPE) + keys = np.fromiter( + duplicate_node_map.keys(), dtype=INT_DTYPE, count=len(duplicate_node_map) + ) + vals = np.fromiter( + duplicate_node_map.values(), dtype=INT_DTYPE, count=len(duplicate_node_map) + ) + lookup[keys] = vals + + remapped_connectivity = connectivity.copy() + valid = connectivity != INT_FILL_VALUE + remapped_connectivity[valid] = lookup[connectivity[valid]] + return remapped_connectivity + + +def _collapse_repeated_face_corners(face_node_connectivity, canonical_values): + """Collapse consecutive (cyclically) repeated corners in each face row. + + Remapping two originally-distinct, now-coincident corners of the same + face to a single canonical node can leave that node referenced twice in + a row, e.g. a quad (A, P, P, B) at a merged pole -- a triangle stored as + a 4-column row with one corner repeated. This pads it back down to + (A, P, B, FILL) so it is treated as the triangle it actually is. + Nonconsecutive repeats are invalid; collapsed pole rows are preserved. + + Only rows containing a value in ``canonical_values`` (nodes that + absorbed at least one duplicate) are inspected, since no other row can + have gained a repeat from the remap. + """ + if len(canonical_values) == 0: + return face_node_connectivity + + affected_rows = np.flatnonzero( + np.isin(face_node_connectivity, canonical_values).any(axis=1) + ) + if len(affected_rows) == 0: + return face_node_connectivity + + face_node_connectivity = face_node_connectivity.copy() + for row_index in affected_rows: + row = face_node_connectivity[row_index] + valid = row != INT_FILL_VALUE + n_valid = int(valid.sum()) + if n_valid <= 1: + continue + + corners = row[:n_valid] + keep = corners != np.roll(corners, 1) + compacted = corners[keep] + n_distinct = len(np.unique(compacted)) + if n_distinct < 3: + # Some SCRIP/OASIS grids intentionally encode collapsed pole rows. + # Leave those rows unchanged rather than turning a valid file into + # a construction error; downstream code already handles them. + continue + if n_distinct != len(compacted): + raise GridInvalidError( + f"Face {row_index} has nonconsecutive repeated corners after " + "coincident-node merging." + ) + if keep.all(): + continue + + new_row = np.full_like(row, INT_FILL_VALUE) + new_row[: len(compacted)] = compacted + face_node_connectivity[row_index] = new_row + + return face_node_connectivity + + +# node-index-valued connectivity: safe to remap element-wise in place +_NODE_INDEX_CONNECTIVITY_TO_REMAP = ("face_node_connectivity", "node_node_connectivity") + +# connectivity derived from (and referencing) node indices, but whose rows must stay +# unique (e.g. edge_node_connectivity) -- dropped rather than remapped, so the +# existing lazy `@property` getters rebuild them cleanly from the corrected +# face_node_connectivity instead of leaving phantom duplicate rows behind. +_DERIVED_CONNECTIVITY_TO_INVALIDATE = ( + "edge_node_connectivity", + "face_edge_connectivity", + "edge_face_connectivity", + "face_face_connectivity", + "node_edge_connectivity", + "node_face_connectivity", +) + +# Formats whose node list is generated internally as an already-unique set. +# File formats are not listed: even when their usual layout stores shared +# vertices once, malformed or nonstandard files can still contain coincidences. +_SPECS_WITHOUT_COINCIDENT_NODES = frozenset({"HEALPix"}) + + +def _spec_guarantees_unique_nodes(source_grid_spec): + """Whether ``source_grid_spec`` names a format that cannot have coincident + nodes, so the search may be skipped outright. + + Unknown or absent specs answer False: the search is the safe default, and a + format is only listed here once its node layout has been shown to make + duplicates impossible. + """ + return source_grid_spec in _SPECS_WITHOUT_COINCIDENT_NODES + + +def _merge_coincident_grid_ds_nodes( + grid_ds, tolerance=ERROR_TOLERANCE, return_duplicate_map=False +): + """Canonicalize coincident (within ``tolerance``) node indices in a raw grid + dataset's connectivity, before it is wrapped in a ``Grid``. + + Per issue #865, node coordinate/data arrays are left untouched -- only + connectivity references to coincident nodes are remapped to a single canonical + (lowest-indexed) node. Note this differs from TempestRemap and MOAB, which + delete the redundant nodes and renumber; keeping them preserves round-trip + fidelity and leaves node-centered data index-aligned, at the cost of leaving + unreferenced coordinates behind (see ``_live_node_indices``). + """ + from uxarray.grid.coordinates import _lonlat_rad_to_xyz + from uxarray.grid.validation import _coincident_node_canonical_indices + + if "face_node_connectivity" not in grid_ds: + return (grid_ds, {}) if return_duplicate_map else grid_ds + + required = ["face_node_connectivity"] + if "node_lon" in grid_ds and "node_lat" in grid_ds: + required.extend(["node_lon", "node_lat"]) + else: + required.extend(["node_x", "node_y", "node_z"]) + if any( + name in grid_ds and getattr(grid_ds[name].data, "chunks", None) is not None + for name in required + ): + # Preserve lazy readers such as chunked SCRIP. Canonicalization needs a + # global spatial search and is deferred until a duplicate-aware operation + # explicitly requests the map. + return (grid_ds, None) if return_duplicate_map else grid_ds + + if "node_lon" in grid_ds and "node_lat" in grid_ds: + node_lon = np.asarray(grid_ds["node_lon"].values, dtype=np.float64) + node_lat = np.asarray(grid_ds["node_lat"].values, dtype=np.float64) + points_xyz = np.column_stack( + _lonlat_rad_to_xyz(np.deg2rad(node_lon), np.deg2rad(node_lat)) + ) + elif {"node_x", "node_y", "node_z"} <= set(grid_ds.variables): + points_xyz = np.column_stack( + ( + grid_ds["node_x"].values, + grid_ds["node_y"].values, + grid_ds["node_z"].values, + ) + ).astype(np.float64, copy=False) + if not np.isfinite(points_xyz).all(): + raise ValueError("Node XYZ coordinates must contain only finite values.") + norms = np.hypot(np.hypot(points_xyz[:, 0], points_xyz[:, 1]), points_xyz[:, 2]) + if np.any(norms == 0.0): + # Cartesian face-vertex inputs may use the origin in a planar + # geometry, where spherical coincidence matching is not defined. + return (grid_ds, {}) if return_duplicate_map else grid_ds + points_xyz /= norms[:, np.newaxis] + else: + # Defensive: Grid construction validates minimum UGRID (node_lon, + # node_lat, face_node_connectivity) before reaching here, so this is + # unreachable on that path. Return unchanged rather than raising, so + # foreign datasets without coordinates round-trip untouched, but warn + # so a caller never mistakes "not merged" for "no duplicates". + warnings.warn( + "Coincident nodes were not merged: the dataset has neither " + "node_x/node_y/node_z nor node_lon/node_lat, so node locations are " + "unknown. Connectivity is returned unchanged and may still " + "reference coincident nodes.", + RuntimeWarning, + ) + return (grid_ds, {}) if return_duplicate_map else grid_ds + + n_node = points_xyz.shape[0] + finite = np.isfinite(points_xyz).all(axis=1) + if not finite.all(): + referenced = grid_ds["face_node_connectivity"].values + referenced = referenced[referenced != INT_FILL_VALUE] + if np.any(~finite[referenced]): + raise ValueError( + "Referenced node XYZ coordinates must contain only finite values." + ) + # Unused node coordinates are retained for data alignment, but must not enter + # the spatial search. Map its results back to the original node indices. + finite_indices = np.flatnonzero(finite) + canonical = np.arange(n_node, dtype=INT_DTYPE) + canonical[finite_indices] = finite_indices[ + _coincident_node_canonical_indices(points_xyz[finite], tolerance) + ] + duplicate_node_map = { + INT_DTYPE(index): INT_DTYPE(canonical[index]) + for index in np.flatnonzero(canonical != np.arange(n_node, dtype=INT_DTYPE)) + } + if not duplicate_node_map: + # No coincident nodes: return ``grid_ds`` untouched. Everything below -- + # the ``_NODE_INDEX_CONNECTIVITY_TO_REMAP`` remap, the face-ring collapse + # and the ``_DERIVED_CONNECTIVITY_TO_INVALIDATE`` drop -- only runs when + # duplicates were actually found. + return (grid_ds, {}) if return_duplicate_map else grid_ds + + grid_ds = grid_ds.copy() + + for name in _NODE_INDEX_CONNECTIVITY_TO_REMAP: + if name in grid_ds: + grid_ds[name] = grid_ds[name].copy( + data=_remap_node_connectivity( + grid_ds[name].values, duplicate_node_map, n_node + ) + ) + + if "face_node_connectivity" in grid_ds: + # Why only face_node_connectivity is collapsed. The remap can make two + # entries of a row identical wherever a row referenced both a duplicate + # and its canonical node. The three groups of node-index connectivity + # differ in what that means: + # + # 1. face_node_connectivity is a closed ring, so a repeat changes the + # geometry: a quad (A, P, P, B) at a merged pole is really the + # triangle (A, P, B) and must be stored as such, or area, bounds and + # edge construction all see a zero-length side. Hence the collapse. + # 2. node_node_connectivity is an unordered neighbor list padded with + # fill, not a ring. A repeat there is a redundant listing that names + # the same neighbor twice; it does not change what the row means, and + # cyclic collapsing would be the wrong operation for it (dedup would + # be). UXarray cannot construct this variable, so it only exists when + # read from a file, and nothing downstream consumes it. + # 3. Everything in _DERIVED_CONNECTIVITY_TO_INVALIDATE (edge_node, + # face_edge, edge_face, face_face, node_edge, node_face) is dropped a + # few lines below and rebuilt lazily from the corrected rings, so a + # repeat introduced by the remap never survives. + canonical_values = np.unique( + np.fromiter( + duplicate_node_map.values(), + dtype=INT_DTYPE, + count=len(duplicate_node_map), + ) + ) + grid_ds["face_node_connectivity"] = grid_ds["face_node_connectivity"].copy( + data=_collapse_repeated_face_corners( + grid_ds["face_node_connectivity"].values, canonical_values + ) + ) + + for name in _DERIVED_CONNECTIVITY_TO_INVALIDATE: + if name in grid_ds: + grid_ds = grid_ds.drop_vars(name) + + if "n_edge" in grid_ds.dims: + grid_ds = grid_ds.drop_dims("n_edge") + + if "n_nodes_per_face" in grid_ds: + grid_ds = grid_ds.drop_vars("n_nodes_per_face") + + if return_duplicate_map: + return grid_ds, duplicate_node_map + return grid_ds + + def _populate_node_face_connectivity(grid): """Constructs the UGRID connectivity variable (``node_face_connectivity``) and stores it within the internal (``Grid._ds``) and through the attribute diff --git a/uxarray/grid/dual.py b/uxarray/grid/dual.py index 4f7b7a622..fc82764e6 100644 --- a/uxarray/grid/dual.py +++ b/uxarray/grid/dual.py @@ -5,7 +5,14 @@ from uxarray.utils.parallel import parallel_njit -def construct_dual(grid): +def _dual_face_indices(grid): + """Return primal node indices that produce faces in the dual mesh.""" + node_face_connectivity = grid.node_face_connectivity.values + n_edges = np.sum(node_face_connectivity != INT_FILL_VALUE, axis=1) + return np.where(n_edges >= 3)[0] + + +def construct_dual(grid, valid_node_indices=None): """Constructs a dual mesh from a given grid, by connecting the face centers to make a grid centered over the primal mesh face centers, with the nodes of the primal mesh being the face centers of the dual mesh. @@ -14,6 +21,9 @@ def construct_dual(grid): ---------- grid: Grid Primal mesh to construct dual from + valid_node_indices: ndarray, optional + Primal node indices that will become dual faces. If omitted, nodes + incident to at least three faces are selected. Returns -------- @@ -38,7 +48,8 @@ def construct_dual(grid): max_edges = node_face_connectivity.shape[1] # Only nodes with 3+ edges can form valid dual faces - valid_node_indices = np.where(n_edges >= 3)[0] + if valid_node_indices is None: + valid_node_indices = _dual_face_indices(grid) construct_node_face_connectivity = np.full( (len(valid_node_indices), max_edges), INT_FILL_VALUE, dtype=INT_DTYPE diff --git a/uxarray/grid/grid.py b/uxarray/grid/grid.py index 1be9f0ed9..99039a882 100644 --- a/uxarray/grid/grid.py +++ b/uxarray/grid/grid.py @@ -29,6 +29,7 @@ _populate_face_bounds, ) from uxarray.grid.connectivity import ( + _merge_coincident_grid_ds_nodes, _populate_edge_face_connectivity, _populate_edge_node_connectivity, _populate_face_edge_connectivity, @@ -36,6 +37,7 @@ _populate_n_nodes_per_face, _populate_node_edge_connectivity, _populate_node_face_connectivity, + _spec_guarantees_unique_nodes, ) from uxarray.grid.coordinates import ( _populate_edge_centroids, @@ -174,6 +176,7 @@ def __init__( source_dims_dict: dict | None = None, is_subset: bool = False, inverse_indices: xr.Dataset | None = None, + merge_coincident_nodes: bool = True, ): # check if inputted dataset is a minimum representable 2D UGRID unstructured grid if source_grid_spec != "HEALPix": @@ -185,6 +188,22 @@ def __init__( ) # TODO: more checks for validate grid (lat/lon coords, etc) + # canonicalize coincident node indices in connectivity before this dataset + # is wrapped in a Grid, so every construction path benefits and no + # lazily-computed connectivity is ever built from stale indices. + # Users can opt out to retain the original connectivity, including face + # rings that would have nonconsecutive repeated corners after merging. + # Internally generated formats known to have unique vertices skip the + # search. File-backed formats are checked regardless of their usual layout. + duplicate_node_map = None + if merge_coincident_nodes: + if _spec_guarantees_unique_nodes(source_grid_spec): + duplicate_node_map = {} + else: + grid_ds, duplicate_node_map = _merge_coincident_grid_ds_nodes( + grid_ds, return_duplicate_map=True + ) + # mapping of ugrid dimensions and variables to source dataset's conventions self._source_dims_dict = source_dims_dict or {} @@ -202,6 +221,7 @@ def __init__( self._antimeridian_face_indices = None self._ds.assign_attrs({"source_grid_spec": self.source_grid_spec}) self._is_subset = is_subset + self._merge_coincident_nodes = merge_coincident_nodes self._inverse_indices = inverse_indices @@ -246,6 +266,10 @@ def __init__( # flag to track if coordinates are normalized self._normalized = None + # Cached map of coincident node indices to their canonical node. Reuse + # the construction-time search instead of repeating it in dual/tree paths. + self._duplicate_node_map = duplicate_node_map + # flag to ensure projected-grid warning fires only once per instance self._projected_warning_issued = False @@ -272,6 +296,10 @@ def from_dataset(cls, dataset, use_dual: bool = False, **kwargs): containing ASCII files represents a FESOM2 grid. use_dual : bool, default=False When reading in MPAS formatted datasets, indicates whether to use the Dual Mesh + merge_coincident_nodes : bool, default=True + Passed through ``**kwargs``. Set to False to preserve original node + connectivity, including faces that would acquire nonconsecutive repeated + corners after merging. Dual construction still rejects referenced duplicates. """ if isinstance(dataset, xr.Dataset): @@ -350,6 +378,7 @@ def from_dataset(cls, dataset, use_dual: bool = False, **kwargs): source_dims_dict, is_subset=kwargs.get("is_subset", False), inverse_indices=kwargs.get("inverse_indices"), + merge_coincident_nodes=kwargs.get("merge_coincident_nodes", True), ) @classmethod @@ -2774,30 +2803,72 @@ def to_linecollection( return copy.deepcopy(line_collection) - def get_dual(self, check_duplicate_nodes: bool = False): + def _prepare_for_dual(self): + """Canonicalize deferred lazy topology without modifying the source grid.""" + required = ["face_node_connectivity"] + if "node_lon" in self._ds and "node_lat" in self._ds: + required.extend(["node_lon", "node_lat"]) + else: + required.extend(["node_x", "node_y", "node_z"]) + if not any(self._ds[name].chunks is not None for name in required): + return self + grid_ds = self._ds.copy(deep=False) + computed = self._ds[required].compute() + for name in required: + grid_ds[name] = computed[name] + return type(self)( + grid_ds, + self.source_grid_spec, + source_dims_dict=self._source_dims_dict, + merge_coincident_nodes=self._merge_coincident_nodes, + ) + + def get_dual(self, check_duplicate_nodes: bool | None = None): """Compute the dual for a grid, which constructs a new grid centered around the nodes, where the nodes of the primal become the face centers of the dual, and the face centers of the primal become the nodes of the dual. Returns a new `Grid` object. + Parameters + ---------- + check_duplicate_nodes : bool, optional + Deprecated and ignored. Coincident nodes are merged at grid + construction, so the check below always runs and always passes. + Returns -------- dual : Grid Dual Mesh Grid constructed + + Raises + ------ + GridInvalidError + If any face still references a coincident duplicate node. The dual + reads ``node_face_connectivity`` directly, so a dead duplicate index + yields a degenerate dual face rather than an error. """ + if check_duplicate_nodes is not None: + warnings.warn( + "`check_duplicate_nodes` is deprecated and ignored; coincident " + "nodes are merged at grid construction and always checked here.", + DeprecationWarning, + stacklevel=2, + ) - if check_duplicate_nodes: - if _check_duplicate_nodes_indices(self): - # TODO: This is very slow - raise GridInvalidError("Duplicate nodes found, cannot construct dual") + grid = self._prepare_for_dual() + if _check_duplicate_nodes_indices(grid): + raise GridInvalidError("Duplicate nodes found, cannot construct dual") # Get dual mesh node face connectivity - dual_node_face_conn = construct_dual(grid=self) + dual_node_face_conn = construct_dual(grid=grid) - # Construct dual mesh - dual = self.from_topology( - self.face_lon.data, self.face_lat.data, dual_node_face_conn + # Primal face centers become dual nodes. Repeated primal faces can have + # coincident centers even when primal nodes are canonical, so the dual + # must also canonicalize its connectivity. + dual_ds = _read_topology( + grid.face_lon.data, grid.face_lat.data, dual_node_face_conn, None, 0 ) + dual = type(self)(dual_ds, "User Defined Topology") return dual diff --git a/uxarray/grid/neighbors.py b/uxarray/grid/neighbors.py index bd34ea61a..05464a6db 100644 --- a/uxarray/grid/neighbors.py +++ b/uxarray/grid/neighbors.py @@ -15,6 +15,7 @@ INT_FILL_VALUE, ) from uxarray.errors import DimensionError +from uxarray.grid.validation import _live_node_indices from uxarray.utils.parallel import parallel_njit @@ -65,11 +66,18 @@ def __init__( self._tree_from_nodes = None self._tree_from_face_centers = None self._tree_from_edge_centers = None + # maps node-tree-local index -> original grid node index, set only when + # the node tree excludes dead duplicate node indices (see _build_from_nodes) + self._node_index_map = None # Build the tree based on nodes, face centers, or edge centers if coordinates == "nodes": self._tree_from_nodes = self._build_from_nodes() - self._n_elements = self._source_grid.n_node + self._n_elements = ( + len(self._node_index_map) + if self._node_index_map is not None + else self._source_grid.n_node + ) elif coordinates == "face centers": self._tree_from_face_centers = self._build_from_face_centers() self._n_elements = self._source_grid.n_face @@ -112,6 +120,13 @@ def _build_from_nodes(self): f"Invalid coordinate_system. Expected 'cartesian' or 'spherical', got {self.coordinate_system!r}" ) + live_indices = _live_node_indices(self._source_grid) + if len(live_indices) < len(coords): + self._node_index_map = live_indices + coords = coords[live_indices] + else: + self._node_index_map = None + self._tree_from_nodes = SKKDTree(coords, metric=self.distance_metric) return self._tree_from_nodes @@ -274,6 +289,9 @@ def query( ind = np.asarray(ind, dtype=INT_DTYPE) + if self._coordinates == "nodes" and self._node_index_map is not None: + ind = self._node_index_map[ind] + if coords.shape[0] == 1: ind = ind.squeeze() @@ -294,6 +312,9 @@ def query( ind = np.asarray(ind, dtype=INT_DTYPE) + if self._coordinates == "nodes" and self._node_index_map is not None: + ind = self._node_index_map[ind] + if coords.shape[0] == 1: ind = ind.squeeze() return ind @@ -360,6 +381,8 @@ def query_radius( ) ind = [np.asarray(cur_ind, dtype=INT_DTYPE) for cur_ind in ind] + if self._coordinates == "nodes" and self._node_index_map is not None: + ind = [self._node_index_map[cur_ind] for cur_ind in ind] d = [np.asarray(cur_d) for cur_d in d] if coords.shape[0] == 1: @@ -376,6 +399,8 @@ def query_radius( ) ind = [np.asarray(cur_ind, dtype=INT_DTYPE) for cur_ind in ind] + if self._coordinates == "nodes" and self._node_index_map is not None: + ind = [self._node_index_map[cur_ind] for cur_ind in ind] if coords.shape[0] == 1: ind = ind[0] @@ -394,7 +419,11 @@ def coordinates(self, value): if self._coordinates == "nodes": if self._tree_from_nodes is None or self.reconstruct: self._tree_from_nodes = self._build_from_nodes() - self._n_elements = self._source_grid.n_node + self._n_elements = ( + len(self._node_index_map) + if self._node_index_map is not None + else self._source_grid.n_node + ) elif self._coordinates == "face centers": if self._tree_from_face_centers is None or self.reconstruct: self._tree_from_face_centers = self._build_from_face_centers() @@ -455,11 +484,18 @@ def __init__( self._tree_from_nodes = None self._tree_from_face_centers = None self._tree_from_edge_centers = None + # maps node-tree-local index -> original grid node index, set only when + # the node tree excludes dead duplicate node indices (see _build_from_nodes) + self._node_index_map = None # set up appropriate reference to tree if coordinates == "nodes": self._tree_from_nodes = self._build_from_nodes() - self._n_elements = self._source_grid.n_node + self._n_elements = ( + len(self._node_index_map) + if self._node_index_map is not None + else self._source_grid.n_node + ) elif coordinates == "face centers": self._tree_from_face_centers = self._build_from_face_centers() self._n_elements = self._source_grid.n_face @@ -531,6 +567,14 @@ def _build_from_nodes(self): ), axis=-1, ) + + live_indices = _live_node_indices(self._source_grid) + if len(live_indices) < len(coords): + self._node_index_map = live_indices + coords = coords[live_indices] + else: + self._node_index_map = None + self._tree_from_nodes = SKBallTree(coords, metric=self.distance_metric) return self._tree_from_nodes @@ -658,6 +702,9 @@ def query( ind = np.asarray(ind, dtype=INT_DTYPE) + if self._coordinates == "nodes" and self._node_index_map is not None: + ind = self._node_index_map[ind] + if coords.shape[0] == 1: ind = ind.squeeze() @@ -678,6 +725,9 @@ def query( ind = np.asarray(ind, dtype=INT_DTYPE) + if self._coordinates == "nodes" and self._node_index_map is not None: + ind = self._node_index_map[ind] + if coords.shape[0] == 1: ind = ind.squeeze() @@ -743,6 +793,8 @@ def query_radius( ) ind = [np.asarray(cur_ind, dtype=INT_DTYPE) for cur_ind in ind] + if self._coordinates == "nodes" and self._node_index_map is not None: + ind = [self._node_index_map[cur_ind] for cur_ind in ind] d = [np.asarray(cur_d) for cur_d in d] if coords.shape[0] == 1: @@ -759,6 +811,8 @@ def query_radius( ) ind = [np.asarray(cur_ind, dtype=INT_DTYPE) for cur_ind in ind] + if self._coordinates == "nodes" and self._node_index_map is not None: + ind = [self._node_index_map[cur_ind] for cur_ind in ind] if coords.shape[0] == 1: ind = ind[0] @@ -777,7 +831,11 @@ def coordinates(self, value): if self._coordinates == "nodes": if self._tree_from_nodes is None or self.reconstruct: self._tree_from_nodes = self._build_from_nodes() - self._n_elements = self._source_grid.n_node + self._n_elements = ( + len(self._node_index_map) + if self._node_index_map is not None + else self._source_grid.n_node + ) elif self._coordinates == "face centers": if self._tree_from_face_centers is None or self.reconstruct: self._tree_from_face_centers = self._build_from_face_centers() diff --git a/uxarray/grid/validation.py b/uxarray/grid/validation.py index 36c05b119..a59c7d375 100644 --- a/uxarray/grid/validation.py +++ b/uxarray/grid/validation.py @@ -4,10 +4,18 @@ import polars as pl from uxarray.constants import ERROR_TOLERANCE, INT_DTYPE +from uxarray.grid.coordinates import _lonlat_rad_to_xyz def _check_connectivity(grid): - """Check if all nodes are referenced by at least one element.""" + """Check if all nodes are referenced by at least one element. + + Node indices that are coincident duplicates of a node that *is* + referenced are expected to be unreferenced -- connectivity is + canonicalized to point at a single index per coincident group, while + the duplicate coordinates themselves are left in place (see + ``_find_duplicate_nodes``). + """ # Convert face_node_connectivity to a Polars Series and get unique values nodes_in_conn = pl.Series(grid.face_node_connectivity.values.flatten()).unique() @@ -15,12 +23,15 @@ def _check_connectivity(grid): # Filter out negative values nodes_in_conn = nodes_in_conn.filter(nodes_in_conn >= 0) - # Check if the size of unique nodes in connectivity is equal to the number of nodes - if len(nodes_in_conn) == grid.n_node: + n_duplicate_nodes = len(_find_duplicate_nodes(grid)) + + # Check if the size of unique nodes in connectivity is equal to the number of + # non-duplicate nodes + if len(nodes_in_conn) == grid.n_node - n_duplicate_nodes: return True else: warn( - f"Some nodes may not be referenced by any element. {len(nodes_in_conn)} and {grid.n_node}", + f"Some nodes may not be referenced by any element. {len(nodes_in_conn)} and {grid.n_node - n_duplicate_nodes}", RuntimeWarning, ) return False @@ -49,17 +60,17 @@ def _check_duplicate_nodes(grid): def _check_duplicate_nodes_indices(grid): - """Check if there are duplicate node indices, returns True if there are.""" + """Check if any face still references a duplicate node index, returns True if + it does.""" - # Create a duplication dictionary - duplicate_node_dict = _find_duplicate_nodes(grid) - - for face_nodes in grid.face_node_connectivity.values: - for node in face_nodes: - if node in duplicate_node_dict.keys(): - return True + duplicate_node_map = _find_duplicate_nodes(grid) + if not duplicate_node_map: + return False - return False + duplicate_indices = np.fromiter( + duplicate_node_map.keys(), dtype=INT_DTYPE, count=len(duplicate_node_map) + ) + return bool(np.isin(grid.face_node_connectivity.values, duplicate_indices).any()) def _check_area(grid): @@ -76,31 +87,140 @@ def _check_area(grid): return True -def _find_duplicate_nodes(grid): - # list of tuple indices - lonlat_t = [ - (lon, lat) for lon, lat in zip(grid.node_lon.values, grid.node_lat.values) +def _coincident_node_canonical_indices(points_xyz, tolerance=ERROR_TOLERANCE): + """For each point, find the lowest-indexed point within ``tolerance`` chordal + distance on the unit sphere (a point with no coincident neighbor maps to itself). + + ``tolerance`` is a chord length on the unit sphere, not an angle in degrees. + This differs from ``_read_structured_grid``, whose user-facing ``tol`` is in + degrees and is converted to a chord via ``2*sin(tol/2)`` before the KDTree + query; here the default ``ERROR_TOLERANCE`` is already that chord radius. + + At the geographic poles, only exactly equal XYZ rows are merged. Distinct + floating representations remain separate because they can encode a face's + locally meaningful pole longitude. + """ + from scipy.sparse import coo_matrix + from scipy.sparse.csgraph import connected_components + from scipy.spatial import KDTree + + n_points = len(points_xyz) + canonical = np.arange(n_points, dtype=INT_DTYPE) + + if not np.isfinite(points_xyz).all(): + raise ValueError("Node XYZ coordinates must contain only finite values.") + norms = np.hypot(np.hypot(points_xyz[:, 0], points_xyz[:, 1]), points_xyz[:, 2]) + if np.any(norms == 0.0): + raise ValueError("Node XYZ coordinates must be non-zero vectors.") + + # ``tolerance`` is a chord radius (see the ``query_pairs`` call below), so it + # cannot be used directly as a deviation of |z| from 1. For a point at + # colatitude t from the pole, 1 - |z| = 1 - cos(t) = 2*sin(t/2)**2 = chord**2/2. + pole_mask = np.isclose( + np.abs(points_xyz[:, 2]), 1.0, rtol=0.0, atol=tolerance**2 / 2 + ) + pole_indices = np.flatnonzero(pole_mask) + if len(pole_indices) > 1: + _, first_indices, inverse = np.unique( + points_xyz[pole_indices], axis=0, return_index=True, return_inverse=True + ) + canonical[pole_indices] = pole_indices[first_indices[inverse.ravel()]] + mergeable_indices = np.flatnonzero(~pole_mask) + + if len(mergeable_indices) < 2: + return canonical + + mergeable_xyz = points_xyz[mergeable_indices] + + # Prescreen on x before paying for a KDTree. Two points within a chord of + # ``tolerance`` differ by at most ``tolerance`` in x, so in x-sorted order every + # consecutive gap between them is also at most ``tolerance``. A point whose + # sorted neighbours are both further than that in x therefore cannot be + # coincident with anything and is dropped. This is exact -- no candidate pair is + # lost -- and on the common case of a grid with no coincident nodes it replaces + # the tree build entirely with one sort. + order = np.argsort(mergeable_xyz[:, 0], kind="stable") + gap_is_small = np.diff(mergeable_xyz[order, 0]) <= tolerance + is_candidate = np.zeros(len(order), dtype=bool) + is_candidate[:-1] |= gap_is_small + is_candidate[1:] |= gap_is_small + # kept ascending so that the first member of a connected component below is + # still the lowest-numbered node in it + candidates = np.sort(order[is_candidate]) + + if len(candidates) < 2: + return canonical + + tree = KDTree(mergeable_xyz[candidates]) + pairs = tree.query_pairs(r=tolerance, output_type="ndarray") + + if len(pairs) == 0: + return canonical + + rows = np.concatenate([pairs[:, 0], pairs[:, 1]]) + cols = np.concatenate([pairs[:, 1], pairs[:, 0]]) + n_candidates = len(candidates) + adj_matrix = coo_matrix( + (np.ones(len(rows)), (rows, cols)), shape=(n_candidates, n_candidates) + ) + _, labels = connected_components(csgraph=adj_matrix, directed=False) + + unique_labels, first_indices = np.unique(labels, return_index=True) + sub_canonical = first_indices[np.searchsorted(unique_labels, labels)] + canonical[mergeable_indices[candidates]] = mergeable_indices[ + candidates[sub_canonical] ] + return canonical - # # Dictionary to track first occurrence and subsequent indices - occurrences = {} - # Iterate through the list and track occurrences - for index, tpl in enumerate(lonlat_t): - if tpl in occurrences: - occurrences[tpl].append((INT_DTYPE(index))) - else: - occurrences[tpl] = [INT_DTYPE(index)] +def _find_duplicate_node_map(node_lon, node_lat, tolerance=ERROR_TOLERANCE): + """Map duplicate (within ``tolerance`` on the unit sphere) node indices to the + lowest-indexed node sharing their location.""" + node_lon = np.asarray(node_lon, dtype=np.float64) + node_lat = np.asarray(node_lat, dtype=np.float64) + points_xyz = np.column_stack( + _lonlat_rad_to_xyz(np.deg2rad(node_lon), np.deg2rad(node_lat)) + ) + canonical = _coincident_node_canonical_indices(points_xyz, tolerance) - duplicate_dict = {} + n_node = len(node_lon) + duplicate_indices = np.flatnonzero(canonical != np.arange(n_node, dtype=INT_DTYPE)) + return { + INT_DTYPE(index): INT_DTYPE(canonical[index]) for index in duplicate_indices + } - for tpl, indices in occurrences.items(): - if len(indices) > 1: - source_idx = indices[0] - for duplicate_idx in indices[1:]: - duplicate_dict[duplicate_idx] = source_idx - return duplicate_dict +def _find_duplicate_nodes(grid): + """Map duplicate node indices to the canonical (lowest-indexed) node sharing + their coordinates. + + Cached on the grid: the search is a KDTree query over every node, while the + node coordinates it reads are fixed once the grid is constructed. Callers + such as ``Grid.get_dual`` run this on every invocation, so recomputing it + would make an O(n_node) search part of the cost of each dual construction. + """ + if grid._duplicate_node_map is None: + grid._duplicate_node_map = _find_duplicate_node_map( + grid.node_lon.values, grid.node_lat.values + ) + return grid._duplicate_node_map + + +def _live_node_indices(grid): + """Node indices still referenced after connectivity is canonicalized. + + Duplicate node coordinates are left in the node arrays by design (see + ``_find_duplicate_nodes``), but a raw coordinate-space search (e.g. a + node KDTree/BallTree) can otherwise select a dead duplicate index that no + face references. Callers building such trees should restrict to this set. + """ + duplicate_map = _find_duplicate_nodes(grid) + if not duplicate_map: + return np.arange(grid.n_node, dtype=INT_DTYPE) + dead = np.fromiter(duplicate_map.keys(), dtype=INT_DTYPE, count=len(duplicate_map)) + return np.setdiff1d( + np.arange(grid.n_node, dtype=INT_DTYPE), dead, assume_unique=True + ) def _check_normalization(grid): diff --git a/uxarray/io/_scrip.py b/uxarray/io/_scrip.py index 91f60cd3c..59a7eb186 100644 --- a/uxarray/io/_scrip.py +++ b/uxarray/io/_scrip.py @@ -375,6 +375,9 @@ def _to_ugrid(in_ds, out_ds): corner_lon = np.asarray(corner_lon_raw) unq_lon, unq_lat, unq_inv = _dedup_scrip_nodes_eager(corner_lon, corner_lat) + # Decoded missing corners are padding, not nodes of the face ring. + valid_corners = np.isfinite(corner_lon_raw) & np.isfinite(corner_lat_raw) + unq_inv = np.where(valid_corners, unq_inv, -1) # Reshape face nodes array into original shape for use in 'face_node_connectivity' unq_inv = np.reshape(unq_inv, (len(in_ds.grid_size), len(in_ds.grid_corners)))