diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 2a60b61..cb09d18 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -11,7 +11,14 @@ permissions: jobs: build-and-check: - runs-on: macos-15 + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [macos-15, windows-latest] + defaults: + run: + shell: bash timeout-minutes: 20 steps: @@ -25,6 +32,11 @@ jobs: 7.0.x 10.0.x + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.13" + - name: Restore run: dotnet restore RhinoSpatial.sln @@ -46,8 +58,17 @@ jobs: - name: Check ATOM downloads and cache run: dotnet run --project RhinoSpatial.Sandbox.csproj -c Release --no-build -- lod2-atom-check + - name: Check source compatibility + run: dotnet run --project RhinoSpatial.Sandbox.csproj -c Release --no-build -- source-compatibility-check + + - name: Check medium-priority compatibility + run: dotnet run --project RhinoSpatial.Sandbox.csproj -c Release --no-build -- medium-compatibility-check + + - name: Check low-priority compatibility + run: dotnet run --project RhinoSpatial.Sandbox.csproj -c Release --no-build -- low-compatibility-check + - name: Check 3D Tiles selection run: dotnet run --project RhinoSpatial.Sandbox.csproj -c Release --no-build -- tiles-selection-check - name: Validate source catalogue JSON - run: python3 -m json.tool examples/sources.json >/dev/null + run: python -m json.tool examples/sources.json >/dev/null diff --git a/.gitignore b/.gitignore index 2dba788..7bc208a 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,6 @@ package/yak/*.yak package/food4rhino/*.zip RhinoSpatial/Resources/wfs_icons.psd RhinoSpatial/Resources/wfs_icons.pdf + +# Python example helper bytecode +__pycache__/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f257ae..3eebad3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,26 @@ RhinoSpatial improves its established source categories through real project use. Earlier releases remain documented on [GitHub Releases](https://github.com/PascalNun/RhinoSpatial/releases). +## Unreleased + +- Read remote classic-TIFF COG windows through bounded, validated HTTP byte ranges. Reuse LibTiff imagery/DEM codecs, usable overviews, masks and NoData; preserve affine placement, source elevations and the shared terrain baseline. Reject unsupported/range-refusing sources without whole-file fallback. Reference Source reads headers without decoding pixels. +- Add read-only local GeoPackage feature-layer discovery, declared EPSG CRS, bounded RTree/fallback spatial queries, attributes and standard WKB geometry through existing vector components. Preserve polygon-hole curves and geometry grouping; reject malformed/extended geometry without repairs. Catalog references do not scan feature geometry. +- Correct GDAL ASCII NoData tags that expose a count before their value. Add explicit WGS84 UTM north/south EPSG transformations needed by common COGs. +- Add Microsoft.Data.Sqlite.Core/platform SQLitePCLRaw managed providers, license texts and package inspection; no new native binary. Add 43 low-priority regression cases, macOS/Windows CI, [how-to/URLs](examples/LOW_COMPATIBILITY_WORKFLOWS.md) and synthetic COG/GeoPackage fixtures. Installed/public 0.4.0 remains unchanged. + +- Select one suitable CityJSON LoD per object, retaining parts at that level; support streamed CityJSONSeq and affine geometry templates through existing local/archive/download paths. Reject malformed indices or ambiguous LoDs instead of returning partial rings. +- Support embedded affine GeoTIFF georeferencing and local world-file TIFF/PNG/JPEG with declared CRS, preserving rotated footprints, UV clipping, pixel centers and terrain sampling. Sidecar edits invalidate caches; projected CRS keys take priority over geographic keys. +- Support WMTS layer discovery, reference metadata, bounded native tile selection and PNG/JPEG mosaics through existing WMS components. Honor axis order, matrix limits, dimensions and advertised REST/KVP endpoints; validate completed caches and fail required tile errors explicitly. +- Add the managed `StbImageNext` 1.0.0 decoder (no native or further package dependencies), update third-party notices, and save [medium-priority recipes and synthetic examples](examples/MEDIUM_COMPATIBILITY_WORKFLOWS.md). New native Rhino/Windows checks remain pending. + +- Follow OGC API next links and supported WFS paging with explicit user caps, deduplication, repeated-page protection and completeness diagnostics. Preserve raw GML for the building loader. +- Discover feature collections from OGC API landing pages, collection lists and advertised links using the existing layer-list/vector components. Layer selection is optional for a single collection and direct files. +- Load bounded, validated direct GeoJSON and Shapefile ZIP downloads, plus remote CityJSON/CityGML building files. Check companion files, CRS metadata, archive paths and expanded sizes; reuse validated downloads. +- Load terrain ATOM feeds through `Load Terrain`, using declared placement metadata and an aligned raster mosaic. Preserve source sample centers, NoData holes and the shared elevation baseline; reject incompatible grids or conflicting heights. +- Use ATOM and OGC API metadata as Spatial Context references without intentionally downloading feature data or archives. Existing saved selections retain priority. +- Correct projected Shapefile WKT authority detection so nested datum/ellipsoid EPSG codes cannot override the actual source CRS. Unsupported declared CRS metadata fails explicitly. +- Add deterministic compatibility checks to CI and save [copyable example workflows](examples/COMPATIBILITY_WORKFLOWS.md), source URLs and live probe results. New Rhino/Grasshopper visual checks remain pending; component GUIDs and input/output layouts are retained. No public release or installed-plugin replacement is included in this batch. + ## 0.4.0 — 2026-11-01 Version 0.4.0 is the recommended public release and is published as GitHub Latest. Component names, GUIDs and normal input/output connections are retained. Provider-specific compatibility continues to evolve. diff --git a/README.md b/README.md index 212a62b..ef4545b 100644 --- a/README.md +++ b/README.md @@ -96,6 +96,8 @@ See the [Component and Source Reference](docs/COMPONENT_REFERENCE.md) for suppor The toolkit has been exercised against a growing international set of public services and local formats. Individual providers, coordinate systems, geometry types, service versions, API access rules, and file metadata can still expose edge cases. For project work, prefer official or project-specific sources and keep an older working definition when upgrading. +Unreleased development adds [remote COG imagery/elevation and local GeoPackage vectors](examples/LOW_COMPATIBILITY_WORKFLOWS.md), following the earlier source and medium-priority compatibility batches. These features are not in the public 0.4.0 package; native Rhino validation remains before release. + The [changelog](CHANGELOG.md) records release changes. The [test source catalogue](docs/TEST_SOURCES.md) documents the current compatibility set and known provider limitations. ## Design principles diff --git a/RhinoSpatial.Core/CityJsonReader.cs b/RhinoSpatial.Core/CityJsonReader.cs index 0c77e82..dd5bab6 100644 --- a/RhinoSpatial.Core/CityJsonReader.cs +++ b/RhinoSpatial.Core/CityJsonReader.cs @@ -1,285 +1,228 @@ -using System; -using System.Collections.Generic; using System.Globalization; -using System.Linq; +using System.Text; using System.Text.Json; -namespace RhinoSpatial.Core +namespace RhinoSpatial.Core; + +public record CityJsonSourceMetadata(string SrsName, BoundingBox2D? BoundingBox); + +public static class CityJsonReader { - public record CityJsonSourceMetadata(string SrsName, BoundingBox2D? BoundingBox); + public static CityJsonSourceMetadata ReadSourceMetadata(string text) + { using var stream = new MemoryStream(Encoding.UTF8.GetBytes(text)); return ReadSourceMetadata(stream); } - public static class CityJsonReader + public static CityJsonSourceMetadata ReadSourceMetadata(Stream stream, CancellationToken cancellationToken = default) { - public static CityJsonSourceMetadata ReadSourceMetadata(string jsonText) + string srs = ""; BoundingBox2D? bounds = null; + foreach (var (root, header) in Records(stream, cancellationToken)) { - using var document = JsonDocument.Parse(jsonText); - var root = document.RootElement; - var vertices = ReadVertices(root); - var srsName = ReadSrsName(root); - return new CityJsonSourceMetadata(srsName, CalculateBounds(vertices)); + srs = ReadSrs(header); + if (header.TryGetProperty("metadata", out var metadata) && metadata.TryGetProperty("geographicalExtent", out var extent)) + { + var a = extent.EnumerateArray().Select(Number).ToArray(); + if (a.Length != 6 || a[0] > a[3] || a[1] > a[4]) throw new InvalidDataException("Invalid CityJSON geographicalExtent."); + return new(srs, new(a[0], a[1], a[3], a[4])); + } + var vertices = Vertices(root, header); + bounds = Union(bounds, Bounds(vertices)); + if (root.TryGetProperty("CityObjects", out var objects)) + foreach (var obj in objects.EnumerateObject()) + if (obj.Value.TryGetProperty("geometry", out var geometry)) + foreach (var g in geometry.EnumerateArray()) + if (Type(g) == "GeometryInstance") bounds = Union(bounds, Bounds(InstanceVertices(g, header, vertices))); } + return new(srs, bounds); + } - public static List ReadBuildings(string jsonText, string sourceLayerName, BoundingBox2D? filterBounds = null) - { - using var document = JsonDocument.Parse(jsonText); - var root = document.RootElement; - var vertices = ReadVertices(root); - var buildings = new List(); - - if (!root.TryGetProperty("CityObjects", out var cityObjects) || - cityObjects.ValueKind != JsonValueKind.Object) - { - return buildings; - } - - foreach (var cityObject in cityObjects.EnumerateObject()) - { - var value = cityObject.Value; - if (!IsBuildingObject(value)) - { - continue; - } + public static List ReadBuildings(string text, string layer, BoundingBox2D? filterBounds = null) + { using var stream = new MemoryStream(Encoding.UTF8.GetBytes(text)); return ReadBuildings(stream, layer, filterBounds); } + public static List ReadBuildings(Stream stream, string layer, BoundingBox2D? filterBounds = null, CancellationToken cancellationToken = default) + { + var result = new List(); var ids = new HashSet(StringComparer.Ordinal); + long pointCount = 0; + foreach (var (root, header) in Records(stream, cancellationToken)) + { + var vertices = Vertices(root, header); + if (!root.TryGetProperty("CityObjects", out var objects) || objects.ValueKind != JsonValueKind.Object) + throw new InvalidDataException("CityJSON requires a CityObjects object."); + foreach (var obj in objects.EnumerateObject()) + { + cancellationToken.ThrowIfCancellationRequested(); + if (Type(obj.Value) is not ("Building" or "BuildingPart" or "BuildingInstallation" or "BuildingConstructiveElement" or "BuildingRoom")) continue; + if (!ids.Add(obj.Name)) throw new InvalidDataException("Duplicate CityJSON building id: " + obj.Name); + if (!obj.Value.TryGetProperty("geometry", out var geometries)) continue; + var candidates = geometries.EnumerateArray().Select(g => (Geometry: g, Lod: ReadLod(g, header))).ToList(); + var lods = candidates.Where(g => g.Lod.HasValue).Select(g => g.Lod!.Value).Distinct().OrderBy(x => x).ToList(); + if (lods.Count > 0 && candidates.Any(c => c.Lod is null)) throw new InvalidDataException("CityJSON mixes declared and unspecified geometry LoDs; a complete representation cannot be chosen safely."); + decimal? selected = lods.Count == 0 ? null : lods.Where(l => l <= 2.2m).Select(l => (decimal?)l).LastOrDefault() ?? lods[0]; var surfaces = new List(); - if (value.TryGetProperty("geometry", out var geometries) && - geometries.ValueKind == JsonValueKind.Array) + foreach (var candidate in candidates.Where(c => c.Lod == selected)) { - foreach (var geometry in geometries.EnumerateArray()) + var geometry = candidate.Geometry; IReadOnlyList points = vertices; + if (Type(geometry) == "GeometryInstance") { - if (geometry.TryGetProperty("boundaries", out var boundaries)) - { - CollectSurfaces(boundaries, vertices, surfaces); - } + points = InstanceVertices(geometry, header, vertices); + geometry = Template(geometry, header); } + if (Type(geometry) is not ("MultiSurface" or "CompositeSurface" or "Solid" or "MultiSolid" or "CompositeSolid")) + throw new InvalidDataException("Unsupported CityJSON building primitive: " + Type(geometry)); + Collect(geometry.GetProperty("boundaries"), points, surfaces); } - - if (surfaces.Count == 0) - { - continue; - } - - var filteredSurfaces = filterBounds is null - ? surfaces - : surfaces.Where(surface => SurfaceIntersectsBounds(surface, filterBounds)).ToList(); - if (filteredSurfaces.Count == 0) - { - continue; - } - - buildings.Add(new Lod2Building( - cityObject.Name, - sourceLayerName, - filteredSurfaces, - new Dictionary - { - ["source_format"] = "CityJSON" - })); + pointCount += surfaces.Sum(s => (long)s.OuterPoints.Count + s.InnerRings.Sum(r => r.Count)); + if (pointCount > 4000000 || result.Count >= 100000) throw new InvalidDataException("CityJSON output exceeds bounded import limits; select a smaller dataset."); + if (filterBounds is not null) surfaces = surfaces.Where(s => Intersects(Bounds(s.OuterPoints), filterBounds)).ToList(); + if (surfaces.Count == 0) continue; + result.Add(new(obj.Name, layer, surfaces, new Dictionary { ["source_format"] = "CityJSON", ["source_lod"] = selected?.ToString(CultureInfo.InvariantCulture) ?? "unspecified" })); } - - return buildings; } + return result; + } - private static bool IsBuildingObject(JsonElement cityObject) + // Validation consumes every record, even when reference metadata can stop at a header extent. + public static bool Validate(Stream stream, bool requireCrs = false, CancellationToken cancellationToken = default) + { + var count = 0; + foreach (var (root, header) in Records(stream, cancellationToken, true)) { - if (!cityObject.TryGetProperty("type", out var typeElement) || - typeElement.ValueKind != JsonValueKind.String) - { - return false; - } - - var type = typeElement.GetString() ?? string.Empty; - return type.Contains("Building", StringComparison.OrdinalIgnoreCase); + count++; + if (requireCrs && string.IsNullOrWhiteSpace(ReadSrs(header))) throw new InvalidDataException("Downloaded CityJSON must declare metadata.referenceSystem."); + if (!root.TryGetProperty("CityObjects", out var objects) || objects.ValueKind != JsonValueKind.Object || !root.TryGetProperty("vertices", out _)) + throw new InvalidDataException("CityJSON requires CityObjects and vertices."); + _ = Vertices(root, header); } + return count > 0; + } - private static List ReadVertices(JsonElement root) - { - var scale = new[] { 1.0, 1.0, 1.0 }; - var translate = new[] { 0.0, 0.0, 0.0 }; - - if (root.TryGetProperty("transform", out var transform)) - { - if (transform.TryGetProperty("scale", out var scaleElement)) - { - ReadTriple(scaleElement, scale); - } - - if (transform.TryGetProperty("translate", out var translateElement)) - { - ReadTriple(translateElement, translate); - } - } - - var vertices = new List(); - if (!root.TryGetProperty("vertices", out var verticesElement) || - verticesElement.ValueKind != JsonValueKind.Array) - { - return vertices; - } - - foreach (var vertex in verticesElement.EnumerateArray()) - { - if (vertex.ValueKind != JsonValueKind.Array || vertex.GetArrayLength() < 3) - { - continue; - } - - vertices.Add(new Coordinate3D( - vertex[0].GetDouble() * scale[0] + translate[0], - vertex[1].GetDouble() * scale[1] + translate[1], - vertex[2].GetDouble() * scale[2] + translate[2])); - } - - return vertices; - } + public static bool IsCityJson(Stream stream) + { + try { using var e = Records(stream, default).GetEnumerator(); return e.MoveNext(); } + catch (JsonException) { return false; } + catch (InvalidDataException) { return false; } + } - private static void ReadTriple(JsonElement arrayElement, double[] target) + private static IEnumerable<(JsonElement Root, JsonElement Header)> Records(Stream stream, CancellationToken token, bool allowOtherRoot = false) + { + using var reader = new StreamReader(stream, new UTF8Encoding(false, true), true, 65536, leaveOpen: true); + JsonElement header = default; long total = 0; int count = 0; + while (true) { - if (arrayElement.ValueKind != JsonValueKind.Array) - { - return; - } - - for (var index = 0; index < Math.Min(3, arrayElement.GetArrayLength()); index++) - { - target[index] = arrayElement[index].GetDouble(); - } + int ch; do { ch = reader.Read(); } while (ch >= 0 && char.IsWhiteSpace((char)ch)); + if (ch < 0) yield break; + if (ch != '{') throw new InvalidDataException("Expected a CityJSON object or CityJSONSeq record."); + var text = new StringBuilder("{"); int depth = 1; bool quoted = false, escaped = false; + while (depth > 0) + { + ch = reader.Read(); if (ch < 0) throw new InvalidDataException("Truncated CityJSON record."); + var c = (char)ch; text.Append(c); + if (quoted) { if (escaped) escaped = false; else if (c == '\\') escaped = true; else if (c == '\"') quoted = false; } + else if (c == '\"') quoted = true; + else if (c is '{' or '[') depth++; + else if (c is '}' or ']') depth--; + if ((text.Length & 8191) == 0) token.ThrowIfCancellationRequested(); + if (text.Length > 64 * 1024 * 1024) throw new InvalidDataException("CityJSON record exceeds 64 MiB of text."); + } + total += text.Length; + if (total > 512L * 1024 * 1024 || ++count > 100001) throw new InvalidDataException("CityJSON stream exceeds import limits."); + using var doc = JsonDocument.Parse(text.ToString()); var root = doc.RootElement; + if (count == 1) + { + if (Type(root) != "CityJSON" && allowOtherRoot) yield break; + if (Type(root) != "CityJSON") throw new InvalidDataException("CityJSONSeq requires a CityJSON header before CityJSONFeature records."); + header = root.Clone(); + } + else + { + if (Type(root) != "CityJSONFeature") throw new InvalidDataException("Expected a CityJSONFeature sequence record."); + if (header.GetProperty("CityObjects").EnumerateObject().Any() || header.GetProperty("vertices").GetArrayLength() != 0) + throw new InvalidDataException("CityJSONSeq header CityObjects and vertices must be empty."); + if (root.TryGetProperty("transform", out _) || root.TryGetProperty("geometry-templates", out _) || root.TryGetProperty("metadata", out _)) + throw new InvalidDataException("CityJSONFeature inherits transform, templates and metadata from its header."); + } + token.ThrowIfCancellationRequested(); yield return (root, header); } + } - private static string ReadSrsName(JsonElement root) + private static string Type(JsonElement e) => e.TryGetProperty("type", out var t) && t.ValueKind == JsonValueKind.String ? t.GetString()! : ""; + private static double Number(JsonElement e) + { var n = e.GetDouble(); if (!double.IsFinite(n)) throw new InvalidDataException("CityJSON coordinates must be finite."); return n; } + private static string ReadSrs(JsonElement header) + { + if (!header.TryGetProperty("metadata", out var m) || !m.TryGetProperty("referenceSystem", out var r)) return ""; + var name = r.GetString() ?? ""; + var match = System.Text.RegularExpressions.Regex.Match(name, @"(?:EPSG[/,:]+(?:0[/,:]+)?)(\d+)$", System.Text.RegularExpressions.RegexOptions.IgnoreCase); + return match.Success ? "EPSG:" + match.Groups[1].Value : ""; + } + private static List Vertices(JsonElement root, JsonElement header) + { + var scale = new[] { 1d, 1d, 1d }; var translate = new[] { 0d, 0d, 0d }; + if (header.TryGetProperty("transform", out var t)) { - if (!root.TryGetProperty("metadata", out var metadata) || - metadata.ValueKind != JsonValueKind.Object || - !metadata.TryGetProperty("referenceSystem", out var referenceSystem) || - referenceSystem.ValueKind != JsonValueKind.String) - { - return string.Empty; - } - - var value = referenceSystem.GetString() ?? string.Empty; - var lastSlash = value.LastIndexOf('/'); - var epsgText = lastSlash >= 0 ? value[(lastSlash + 1)..] : value; - return int.TryParse(epsgText, NumberStyles.Integer, CultureInfo.InvariantCulture, out var epsg) - ? $"EPSG:{epsg}" - : string.Empty; + scale = t.GetProperty("scale").EnumerateArray().Select(Number).ToArray(); + translate = t.GetProperty("translate").EnumerateArray().Select(Number).ToArray(); + if (scale.Length != 3 || translate.Length != 3) throw new InvalidDataException("Invalid CityJSON transform."); } - - private static BoundingBox2D? CalculateBounds(IReadOnlyList vertices) + var result = new List(); + foreach (var v in root.GetProperty("vertices").EnumerateArray()) { - if (vertices.Count == 0) - { - return null; - } - - return new BoundingBox2D( - vertices.Min(point => point.X), - vertices.Min(point => point.Y), - vertices.Max(point => point.X), - vertices.Max(point => point.Y)); + var a = v.EnumerateArray().Select(Number).ToArray(); if (a.Length != 3) throw new InvalidDataException("CityJSON vertex must contain XYZ."); + var p = new Coordinate3D(a[0] * scale[0] + translate[0], a[1] * scale[1] + translate[1], a[2] * scale[2] + translate[2]); + if (!double.IsFinite(p.X) || !double.IsFinite(p.Y) || !double.IsFinite(p.Z)) throw new InvalidDataException("CityJSON transform overflow."); + result.Add(p); } - - private static void CollectSurfaces( - JsonElement element, - IReadOnlyList vertices, - List surfaces) + return result; + } + private static JsonElement Template(JsonElement g, JsonElement header) + { + var templates = header.GetProperty("geometry-templates").GetProperty("templates"); + var i = g.GetProperty("template").GetInt32(); + if (i < 0 || i >= templates.GetArrayLength()) throw new InvalidDataException("CityJSON template index out of range."); + return templates[i]; + } + private static decimal? ReadLod(JsonElement g, JsonElement header) + { + if (Type(g) == "GeometryInstance") g = Template(g, header); + if (!g.TryGetProperty("lod", out var lod)) return null; + var text = lod.ValueKind == JsonValueKind.String ? lod.GetString() : lod.GetRawText(); + if (!decimal.TryParse(text, NumberStyles.Float, CultureInfo.InvariantCulture, out var value) || value < 0 || value >= 5) + throw new InvalidDataException("Invalid CityJSON LoD: " + text); + return value; + } + private static List InstanceVertices(JsonElement g, JsonElement header, IReadOnlyList vertices) + { + var boundaries = g.GetProperty("boundaries"); if (boundaries.GetArrayLength() != 1) throw new InvalidDataException("GeometryInstance requires one anchor index."); + var index = boundaries[0].GetInt32(); if (index < 0 || index >= vertices.Count) throw new InvalidDataException("Invalid template anchor index."); + var anchor = vertices[index]; var m = g.GetProperty("transformationMatrix").EnumerateArray().Select(Number).ToArray(); + if (m.Length != 16 || m[12] != 0 || m[13] != 0 || m[14] != 0 || m[15] != 1) throw new InvalidDataException("GeometryInstance requires an affine row-major 4x4 matrix."); + var result = new List(); + foreach (var v in header.GetProperty("geometry-templates").GetProperty("vertices-templates").EnumerateArray()) { - if (TryReadSurface(element, vertices, out var surface)) - { - surfaces.Add(surface); - return; - } - - if (element.ValueKind != JsonValueKind.Array) - { - return; - } - - foreach (var child in element.EnumerateArray()) - { - CollectSurfaces(child, vertices, surfaces); - } + var a = v.EnumerateArray().Select(Number).ToArray(); if (a.Length != 3) throw new InvalidDataException("Invalid template vertex."); + var p = new Coordinate3D(m[0] * a[0] + m[1] * a[1] + m[2] * a[2] + m[3] + anchor.X, m[4] * a[0] + m[5] * a[1] + m[6] * a[2] + m[7] + anchor.Y, m[8] * a[0] + m[9] * a[1] + m[10] * a[2] + m[11] + anchor.Z); + if (!double.IsFinite(p.X) || !double.IsFinite(p.Y) || !double.IsFinite(p.Z)) throw new InvalidDataException("Template transform overflow."); + result.Add(p); } - - private static bool TryReadSurface( - JsonElement element, - IReadOnlyList vertices, - out SurfacePolygon3D surface) + return result; + } + private static void Collect(JsonElement e, IReadOnlyList vertices, List surfaces) + { + if (e.ValueKind != JsonValueKind.Array) throw new InvalidDataException("Invalid CityJSON boundaries."); + if (e.GetArrayLength() == 0) return; + if (e[0].ValueKind == JsonValueKind.Array && e[0].GetArrayLength() > 0 && e[0][0].ValueKind == JsonValueKind.Number) { - surface = new SurfacePolygon3D(new List(), new List>()); - if (element.ValueKind != JsonValueKind.Array || element.GetArrayLength() == 0) - { - return false; - } - - var firstRing = element[0]; - if (!IsIndexRing(firstRing)) - { - return false; - } - var rings = new List>(); - foreach (var ringElement in element.EnumerateArray()) - { - var ring = ReadRing(ringElement, vertices); - if (ring.Count >= 3) - { - rings.Add(ring); - } - } - - if (rings.Count == 0) + foreach (var ring in e.EnumerateArray()) { - return false; + if (ring.GetArrayLength() < 3) throw new InvalidDataException("CityJSON surface ring requires three vertices."); + var points = new List(); + foreach (var ix in ring.EnumerateArray()) { var i = ix.GetInt32(); if (i < 0 || i >= vertices.Count) throw new InvalidDataException("CityJSON vertex index out of range."); points.Add(vertices[i]); } + rings.Add(points); } - - surface = new SurfacePolygon3D( - rings[0], - rings.Skip(1).ToList()); - return true; - } - - private static bool IsIndexRing(JsonElement element) - { - return element.ValueKind == JsonValueKind.Array && - element.GetArrayLength() >= 3 && - element.EnumerateArray().All(item => item.ValueKind == JsonValueKind.Number); - } - - private static List ReadRing(JsonElement element, IReadOnlyList vertices) - { - var ring = new List(); - if (!IsIndexRing(element)) - { - return ring; - } - - foreach (var indexElement in element.EnumerateArray()) - { - var index = indexElement.GetInt32(); - if (index >= 0 && index < vertices.Count) - { - ring.Add(vertices[index]); - } - } - - return ring; - } - - private static bool SurfaceIntersectsBounds(SurfacePolygon3D surface, BoundingBox2D filterBounds) - { - if (surface.OuterPoints.Count == 0) - { - return false; - } - - var bounds = new BoundingBox2D( - surface.OuterPoints.Min(point => point.X), - surface.OuterPoints.Min(point => point.Y), - surface.OuterPoints.Max(point => point.X), - surface.OuterPoints.Max(point => point.Y)); - return bounds.MinX <= filterBounds.MaxX && - bounds.MaxX >= filterBounds.MinX && - bounds.MinY <= filterBounds.MaxY && - bounds.MaxY >= filterBounds.MinY; + surfaces.Add(new(rings[0], rings.Skip(1).ToList())); return; } + foreach (var child in e.EnumerateArray()) Collect(child, vertices, surfaces); } + private static BoundingBox2D? Bounds(IReadOnlyList p) => p.Count == 0 ? null : new(p.Min(v => v.X), p.Min(v => v.Y), p.Max(v => v.X), p.Max(v => v.Y)); + private static BoundingBox2D? Union(BoundingBox2D? a, BoundingBox2D? b) => a is null ? b : b is null ? a : new(Math.Min(a.MinX, b.MinX), Math.Min(a.MinY, b.MinY), Math.Max(a.MaxX, b.MaxX), Math.Max(a.MaxY, b.MaxY)); + private static bool Intersects(BoundingBox2D? a, BoundingBox2D b) => a is not null && a.MinX <= b.MaxX && a.MaxX >= b.MinX && a.MinY <= b.MaxY && a.MaxY >= b.MinY; } diff --git a/RhinoSpatial.Core/CogReader.cs b/RhinoSpatial.Core/CogReader.cs new file mode 100644 index 0000000..b1042b2 --- /dev/null +++ b/RhinoSpatial.Core/CogReader.cs @@ -0,0 +1,730 @@ +using System.Buffers.Binary; +using System.Globalization; +using System.Security.Cryptography; +using System.Text; +using BitMiracle.LibTiff.Classic; + +namespace RhinoSpatial.Core; + +public record CogImageResult( + GeoReferencedRasterInfo Image, + int RangeRequests, + long TransferredBytes, + int OverviewWidth +); + +public record CogTerrainResult( + TerrainRasterData Raster, + int RangeRequests, + long TransferredBytes, + int OverviewWidth +); + +/// Windowed classic-TIFF COG access using LibTiff codecs over bounded HTTP ranges. +public sealed class CogReader +{ + private static readonly HttpClient SharedHttp = new() { Timeout = Timeout.InfiniteTimeSpan }; + private readonly HttpClient _http; + + public CogReader(HttpClient? http = null) + { + _http = http ?? SharedHttp; + } + + public static bool LooksLike(string source) => + Uri.TryCreate(source, UriKind.Absolute, out var uri) + && uri.Scheme is "http" or "https" + && new[] { ".tif", ".tiff" }.Contains( + Path.GetExtension(uri.AbsolutePath), + StringComparer.OrdinalIgnoreCase + ); + + private sealed record ImageDirectory(long Offset, GeoReferencedRasterInfo Info, int Flags); + + private sealed record Window(int X, int Y, int Width, int Height); + + public async Task ReadMetadataAsync( + string source, + CancellationToken token = default + ) + { + using var stream = await HttpRangeStream + .OpenAsync(_http, source, token, 4 * 1024 * 1024, 64) + .ConfigureAwait(false); + using var tiff = Open(stream); + var info = GeoTiffReader.ReadImageInfo(tiff, source, stream.Length); + stream.ThrowIfFailed(); + return info; + } + + public async Task ReadImageAsync( + string source, + SpatialContext2D context, + CancellationToken token = default + ) + { + using var stream = await HttpRangeStream + .OpenAsync(_http, source, token) + .ConfigureAwait(false); + using var tiff = Open(stream); + var directories = Directories(tiff, source, stream); + var selected = Select(directories, context); + var image = selected.Image; + var window = selected.Window; + SetDirectory(tiff, image.Offset, stream); + var tile = TileDimensions(tiff); + var noData = + NoData(tiff) + ?? ( + image.Offset == directories[0].Offset + ? null + : ReadBaseNoData(tiff, directories[0], image, stream) + ); + var samples = Value(tiff, TiffTag.SAMPLESPERPIXEL, 1); + var bits = Value(tiff, TiffTag.BITSPERSAMPLE, 1); + var format = (SampleFormat)Value(tiff, TiffTag.SAMPLEFORMAT, (int)SampleFormat.UINT); + if ( + noData.HasValue + && ( + bits is not (8 or 16) + || Value(tiff, TiffTag.PLANARCONFIG, 1) != 1 + || Value(tiff, TiffTag.PHOTOMETRIC, 1) == (int)Photometric.YCBCR + ) + ) + throw new InvalidDataException( + "Imagery NoData requires contiguous 8/16-bit gray, palette or RGB samples." + ); + var mask = directories.SingleOrDefault(d => + (d.Flags & 4) != 0 + && d.Info.Width == image.Info.Width + && d.Info.Height == image.Info.Height + ); + if (directories.Any(d => (d.Flags & 4) != 0) && mask is null) + throw new InvalidDataException( + "No matching-resolution internal mask is available for this overview." + ); + var rgba = new byte[checked(window.Width * window.Height * 4)]; + var raster = new int[checked(tile.Width * tile.Height)]; + foreach (var (x, y) in Tiles(window, tile.Width, tile.Height)) + { + token.ThrowIfCancellationRequested(); + if (!tiff.ReadRGBATile(x, y, raster)) + { + stream.ThrowIfFailed(); + throw new InvalidDataException("The TIFF imagery tile could not be decoded."); + } + byte[]? raw = null; + if (noData.HasValue) + raw = ReadTile(tiff, x, y, stream); + CopyPixels( + window, + tile.Width, + tile.Height, + x, + y, + (lx, ly, destination) => + { + var pixel = raster[(tile.Height - 1 - ly) * tile.Width + lx]; + rgba[destination * 4] = (byte)Tiff.GetR(pixel); + rgba[destination * 4 + 1] = (byte)Tiff.GetG(pixel); + rgba[destination * 4 + 2] = (byte)Tiff.GetB(pixel); + rgba[destination * 4 + 3] = (byte)Tiff.GetA(pixel); + var alpha = rgba[destination * 4 + 3]; + if (alpha is > 0 and < 255) + for (var c = 0; c < 3; c++) + rgba[destination * 4 + c] = (byte) + Math.Min( + 255, + (rgba[destination * 4 + c] * 255 + alpha / 2) / alpha + ); + if (raw is not null) + { + var missing = true; + for (var c = 0; c < Math.Min(samples, 3); c++) + missing &= IsNoData( + TerrainRasterReader.ReadSample( + raw, + (ly * tile.Width + lx) * samples + c, + bits, + format + ), + noData!.Value + ); + if (missing) + rgba[destination * 4 + 3] = 0; + } + } + ); + } + if (mask is not null) + { + SetDirectory(tiff, mask.Offset, stream); + var mt = TileDimensions(tiff); + var mb = Value(tiff, TiffTag.BITSPERSAMPLE, 1); + if ( + mb is not (1 or 8) + || Value(tiff, TiffTag.SAMPLESPERPIXEL, 1) != 1 + || Value(tiff, TiffTag.FILLORDER, 1) != 1 + ) + throw new InvalidDataException( + "Unsupported internal mask encoding (requires one 1/8-bit, MSB-first band)." + ); + foreach (var (x, y) in Tiles(window, mt.Width, mt.Height)) + { + token.ThrowIfCancellationRequested(); + var raw = ReadTile(tiff, x, y, stream); + var stride = (mt.Width * mb + 7) / 8; + CopyPixels( + window, + mt.Width, + mt.Height, + x, + y, + (lx, ly, destination) => + { + var visible = + mb == 8 + ? raw[ly * stride + lx] != 0 + : (raw[ly * stride + lx / 8] & (128 >> (lx % 8))) != 0; + if (!visible) + rgba[destination * 4 + 3] = 0; + } + ); + } + } + stream.ThrowIfFailed(); + var cache = Path.Combine( + Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), + "RhinoSpatial", + "CogWindows" + ); + Directory.CreateDirectory(cache); + // The cache contains the selected PNG window only, never a full remote TIFF or a plaintext URL. + var signature = source + "\n" + stream.Validator + "\n" + image.Offset + "\n" + window; + var filename = + Convert + .ToHexString( + SHA256.HashData( + Encoding.UTF8.GetBytes( + signature + "\n" + Convert.ToHexString(SHA256.HashData(rgba)) + ) + ) + ) + .ToLowerInvariant() + ".png"; + var path = Path.Combine(cache, filename); + if (!File.Exists(path)) + { + var temporary = path + "." + Guid.NewGuid().ToString("N") + ".tmp"; + try + { + RasterImageCodec.WritePng(temporary, window.Width, window.Height, rgba); + File.Move(temporary, path, true); + } + finally + { + if (File.Exists(temporary)) + File.Delete(temporary); + } + } + return new( + Crop(image.Info, window, path), + stream.RequestCount, + stream.TransferredBytes, + image.Info.Width + ); + } + + public async Task ReadTerrainAsync( + string source, + SpatialContext2D context, + CancellationToken token = default + ) + { + using var stream = await HttpRangeStream + .OpenAsync(_http, source, token) + .ConfigureAwait(false); + using var tiff = Open(stream); + var directories = Directories(tiff, source, stream); + var selected = Select(directories, context); + var image = selected.Image; + var window = selected.Window; + SetDirectory(tiff, image.Offset, stream); + var tile = TileDimensions(tiff); + if ( + Value(tiff, TiffTag.SAMPLESPERPIXEL, 1) != 1 + || Value(tiff, TiffTag.PLANARCONFIG, 1) != 1 + ) + throw new InvalidDataException( + "Remote terrain requires one contiguous elevation band." + ); + var bits = Value(tiff, TiffTag.BITSPERSAMPLE, 1); + var format = (SampleFormat)Value(tiff, TiffTag.SAMPLEFORMAT, (int)SampleFormat.UINT); + if ( + bits is not (8 or 16 or 32 or 64) + || format is not (SampleFormat.UINT or SampleFormat.INT or SampleFormat.IEEEFP) + || bits == 64 && format != SampleFormat.IEEEFP + || format == SampleFormat.IEEEFP && bits is not (32 or 64) + ) + throw new InvalidDataException("Unsupported remote DEM sample type."); + var noData = + NoData(tiff) + ?? ( + image.Offset == directories[0].Offset + ? null + : ReadBaseNoData(tiff, directories[0], image, stream) + ); + var elevations = new float[checked(window.Width * window.Height)]; + foreach (var (x, y) in Tiles(window, tile.Width, tile.Height)) + { + token.ThrowIfCancellationRequested(); + var raw = ReadTile(tiff, x, y, stream); + CopyPixels( + window, + tile.Width, + tile.Height, + x, + y, + (lx, ly, destination) => + elevations[destination] = TerrainRasterReader.ReadSample( + raw, + ly * tile.Width + lx, + bits, + format + ) + ); + } + var mask = directories.SingleOrDefault(d => + (d.Flags & 4) != 0 + && d.Info.Width == image.Info.Width + && d.Info.Height == image.Info.Height + ); + if (directories.Any(d => (d.Flags & 4) != 0) && mask is null) + throw new InvalidDataException( + "No matching-resolution internal terrain mask is available." + ); + if (mask is not null) + { + SetDirectory(tiff, mask.Offset, stream); + var mt = TileDimensions(tiff); + var mb = Value(tiff, TiffTag.BITSPERSAMPLE, 1); + if ( + mb is not (1 or 8) + || Value(tiff, TiffTag.SAMPLESPERPIXEL, 1) != 1 + || Value(tiff, TiffTag.FILLORDER, 1) != 1 + ) + throw new InvalidDataException("Unsupported internal terrain mask encoding."); + foreach (var (x, y) in Tiles(window, mt.Width, mt.Height)) + { + token.ThrowIfCancellationRequested(); + var raw = ReadTile(tiff, x, y, stream); + var stride = (mt.Width * mb + 7) / 8; + CopyPixels( + window, + mt.Width, + mt.Height, + x, + y, + (lx, ly, destination) => + { + if ( + mb == 8 + ? raw[ly * stride + lx] == 0 + : (raw[ly * stride + lx / 8] & (128 >> (lx % 8))) == 0 + ) + elevations[destination] = float.NaN; + } + ); + } + } + stream.ThrowIfFailed(); + var info = Crop(image.Info, window, source); + var g = info.GeoTransform!; + return new( + new( + "remote-cog", + info.SrsName, + info.Width, + info.Height, + g.At(.5, .5), + new(g.A, g.D), + new(g.B, g.E), + noData, + elevations + ) + { + GeoTransform = g, + }, + stream.RequestCount, + stream.TransferredBytes, + image.Info.Width + ); + } + + private static Tiff Open(HttpRangeStream stream) + { + var header = new byte[4]; + if (stream.Read(header, 0, 4) != 4) + throw new InvalidDataException("Truncated TIFF header."); + stream.Position = 0; + var magic = header[0] == 73 ? header[2] | header[3] << 8 : header[2] << 8 | header[3]; + if (magic == 43) + throw new InvalidDataException( + "BigTIFF is not supported by this decoder; use a classic TIFF COG." + ); + var firstOffset = + header[0] == 73 + ? BinaryPrimitives.ReadUInt32LittleEndian(ReadBytes(stream, 4, 4)) + : BinaryPrimitives.ReadUInt32BigEndian(ReadBytes(stream, 4, 4)); + PreflightDirectory(stream, firstOffset); + stream.Position = 0; + TiffWarningSuppression.EnsureInstalled(); + var tiff = Tiff.ClientOpen("remote COG", "r", stream, new TiffStream()); + stream.ThrowIfFailed(); + return tiff + ?? throw new InvalidDataException("The ranged resource is not a supported TIFF."); + } + + private static List Directories( + Tiff tiff, + string source, + HttpRangeStream stream + ) + { + var first = GeoTiffReader.ReadImageInfo(tiff, source, stream.Length); + var output = new List(); + var seen = new HashSet(); + var pending = new Queue(); + void AddCurrent(bool subIfd) + { + stream.ThrowIfFailed(); + var offset = tiff.CurrentDirOffset(); + if (!seen.Add(offset)) + throw new InvalidDataException("Cyclic or duplicate TIFF image directory."); + if (seen.Count > 64) + throw new InvalidDataException( + "Remote TIFF exceeds its 64-directory metadata budget." + ); + var flags = Value(tiff, TiffTag.SUBFILETYPE, 0); + var width = Value(tiff, TiffTag.IMAGEWIDTH, 0); + var height = Value(tiff, TiffTag.IMAGELENGTH, 0); + if (width <= 0 || height <= 0) + throw new InvalidDataException("Invalid TIFF directory dimensions."); + if (output.Count == 0 && (flags & 4) != 0) + throw new InvalidDataException( + "The primary TIFF directory is a mask, not an image." + ); + if (output.Count > 0 && (flags & 5) == 0 && !subIfd) + throw new InvalidDataException("Multi-page TIFFs are not a COG overview pyramid."); + var sub = tiff.GetField(TiffTag.SUBIFD); + if (sub is not null) + { + var offsets = + sub[^1].TolongArray() + ?? throw new InvalidDataException("Invalid TIFF SubIFD offsets."); + if (offsets.Length > 64 || pending.Count + offsets.Length > 64) + throw new InvalidDataException("Too many TIFF SubIFDs."); + foreach (var child in offsets) + pending.Enqueue(child); + } + GeoReferencedRasterInfo info; + if ( + tiff.GetField(TiffTag.GEOTIFF_MODELPIXELSCALETAG) is not null + || tiff.GetField(TiffTag.GEOTIFF_MODELTRANSFORMATIONTAG) is not null + ) + info = GeoTiffReader.ReadImageInfo(tiff, source, stream.Length, first.SrsName); + else + { + if ( + first.Width % width != 0 + || first.Height % height != 0 + || first.Width / width != first.Height / height + ) + return; // Skip ambiguous unreferenced rounded overview grids; retain exact levels. + var factor = (double)first.Width / width; + var g = first.GeoTransform!; + var scaled = new RasterGeoTransform( + g.A * factor, + g.B * factor, + g.C, + g.D * factor, + g.E * factor, + g.F + ); + info = new( + source, + first.SrsName, + width, + height, + scaled.Bounds(width, height), + stream.Length + ) + { + GeoTransform = scaled, + }; + } + if (info.SrsName != first.SrsName || width > first.Width || height > first.Height) + throw new InvalidDataException( + "TIFF overview CRS/dimensions disagree with the primary image." + ); + var expected = first.GeoTransform!; + var actual = info.GeoTransform!; + var a = expected.Bounds(first.Width, first.Height); + var b = actual.Bounds(width, height); + var tolerance = Math.Max(1, Math.Max(a.MaxX - a.MinX, a.MaxY - a.MinY)) * 1e-8; + if ( + Math.Abs(a.MinX - b.MinX) > tolerance + || Math.Abs(a.MinY - b.MinY) > tolerance + || Math.Abs(a.MaxX - b.MaxX) > tolerance + || Math.Abs(a.MaxY - b.MaxY) > tolerance + ) + throw new InvalidDataException("TIFF overview and base footprints disagree."); + _ = TileDimensions(tiff); + output.Add(new(offset, info, flags)); + } + AddCurrent(false); + var next = PreflightDirectory(stream, tiff.CurrentDirOffset()); + while (next != 0) + { + SetDirectory(tiff, next, stream); + AddCurrent(false); + next = PreflightDirectory(stream, next); + } + stream.ThrowIfFailed(); + while (pending.Count > 0) + { + SetDirectory(tiff, pending.Dequeue(), stream); + AddCurrent(true); + } + return output; + } + + private static (ImageDirectory Image, Window Window) Select( + List directories, + SpatialContext2D context + ) + { + var choices = directories + .Where(d => (d.Flags & 4) == 0) + .OrderByDescending(d => d.Info.Width) + .Select(d => (Image: d, Window: WindowFor(d.Info, context))) + .ToList(); + var result = choices.FirstOrDefault(c => c.Window.Width <= 1024 && c.Window.Height <= 1024); + if (result.Image is null) + result = choices[^1]; + if (result.Window.Width <= 0 || result.Window.Height <= 0) + throw new InvalidDataException( + "Remote raster does not overlap the shared Spatial Context." + ); + if ((long)result.Window.Width * result.Window.Height > 4 * 1024 * 1024) + throw new InvalidDataException( + "Remote raster window exceeds four million pixels; narrow the area or use a source with overviews." + ); + return result; + } + + private static Window WindowFor(GeoReferencedRasterInfo info, SpatialContext2D context) + { + var bounds = Lod2AtomFeedReader.TransformBounds( + new(context.ResolvedSrs, context.PlacementBoundingBox), + info.SrsName + ); + var g = info.GeoTransform!; + var p = new[] + { + g.Pixel(bounds.MinX, bounds.MinY), + g.Pixel(bounds.MaxX, bounds.MinY), + g.Pixel(bounds.MaxX, bounds.MaxY), + g.Pixel(bounds.MinX, bounds.MaxY), + }; + var x = (int)Math.Clamp(Math.Floor(p.Min(c => c.X)), 0, info.Width); + var y = (int)Math.Clamp(Math.Floor(p.Min(c => c.Y)), 0, info.Height); + var right = (int)Math.Clamp(Math.Ceiling(p.Max(c => c.X)), 0, info.Width); + var bottom = (int)Math.Clamp(Math.Ceiling(p.Max(c => c.Y)), 0, info.Height); + return new(x, y, right - x, bottom - y); + } + + private static GeoReferencedRasterInfo Crop( + GeoReferencedRasterInfo info, + Window window, + string path + ) + { + var g = info.GeoTransform!; + var origin = g.At(window.X, window.Y); + var crop = new RasterGeoTransform(g.A, g.B, origin.X, g.D, g.E, origin.Y); + return new( + path, + info.SrsName, + window.Width, + window.Height, + crop.Bounds(window.Width, window.Height), + info.FileSizeBytes + ) + { + GeoTransform = crop, + }; + } + + private static (int Width, int Height) TileDimensions(Tiff tiff) + { + if (!tiff.IsTiled()) + throw new InvalidDataException( + "Remote raster is not tiled; provide a COG rather than a striped TIFF." + ); + if (Value(tiff, TiffTag.ORIENTATION, 1) != 1) + throw new InvalidDataException("Remote TIFF requires top-left orientation."); + var width = Value(tiff, TiffTag.TILEWIDTH, 0); + var height = Value(tiff, TiffTag.TILELENGTH, 0); + var bytes = tiff.TileSize(); + if ( + width <= 0 + || height <= 0 + || (long)width * height > 1024 * 1024 + || bytes <= 0 + || bytes > 32 * 1024 * 1024 + ) + throw new InvalidDataException("Remote raster tile exceeds the decoded tile budget."); + return (width, height); + } + + private static IEnumerable<(int X, int Y)> Tiles(Window w, int tw, int th) + { + var count = 0; + for (var y = w.Y / th * th; y < w.Y + w.Height; y += th) + for (var x = w.X / tw * tw; x < w.X + w.Width; x += tw) + { + if (++count > 256) + throw new InvalidDataException( + "Remote window exceeds 256 tiles; narrow the area or use overviews." + ); + yield return (x, y); + } + } + + private static void CopyPixels( + Window w, + int tw, + int th, + int x, + int y, + Action copy + ) + { + for (var row = Math.Max(y, w.Y); row < Math.Min(y + th, w.Y + w.Height); row++) + for (var col = Math.Max(x, w.X); col < Math.Min(x + tw, w.X + w.Width); col++) + copy(col - x, row - y, (row - w.Y) * w.Width + col - w.X); + } + + private static byte[] ReadTile(Tiff tiff, int x, int y, HttpRangeStream stream) + { + var buffer = new byte[tiff.TileSize()]; + var read = tiff.ReadEncodedTile(tiff.ComputeTile(x, y, 0, 0), buffer, 0, -1); + stream.ThrowIfFailed(); + if (read != buffer.Length) + throw new InvalidDataException("Truncated decoded TIFF tile."); + return buffer; + } + + // Bound TIFF field allocations before handing a directory to the decoder. + private static long PreflightDirectory(HttpRangeStream stream, long offset) + { + var little = ReadBytes(stream, 0, 2)[0] == 73; + if (offset < 8 || offset > stream.Length - 2) + throw new InvalidDataException("Invalid TIFF directory offset."); + var countBytes = ReadBytes(stream, offset, 2); + var count = little + ? BinaryPrimitives.ReadUInt16LittleEndian(countBytes) + : BinaryPrimitives.ReadUInt16BigEndian(countBytes); + if (count > 1024) + throw new InvalidDataException("Remote TIFF directory exceeds 1024 tags."); + var entries = ReadBytes(stream, offset + 2, checked(count * 12 + 4)); + long total = 0; + for (var i = 0; i < count; i++) + { + var entry = entries.AsSpan(i * 12, 12); + var type = little + ? BinaryPrimitives.ReadUInt16LittleEndian(entry.Slice(2, 2)) + : BinaryPrimitives.ReadUInt16BigEndian(entry.Slice(2, 2)); + var n = little + ? BinaryPrimitives.ReadUInt32LittleEndian(entry.Slice(4, 4)) + : BinaryPrimitives.ReadUInt32BigEndian(entry.Slice(4, 4)); + var size = type switch + { + 1 or 2 or 6 or 7 => 1, + 3 or 8 => 2, + 4 or 9 or 11 or 13 => 4, + 5 or 10 or 12 => 8, + _ => throw new InvalidDataException("Unsupported TIFF field type."), + }; + var bytes = (long)n * size; + total += bytes; + if (bytes > 8 * 1024 * 1024 || total > 16 * 1024 * 1024) + throw new InvalidDataException( + "Remote TIFF directory exceeds its field allocation budget." + ); + if (bytes > 4) + { + var location = little + ? BinaryPrimitives.ReadUInt32LittleEndian(entry.Slice(8, 4)) + : BinaryPrimitives.ReadUInt32BigEndian(entry.Slice(8, 4)); + if (location > stream.Length - bytes) + throw new InvalidDataException("TIFF field points outside the remote file."); + } + } + var tail = entries.AsSpan(count * 12, 4); + return little + ? BinaryPrimitives.ReadUInt32LittleEndian(tail) + : BinaryPrimitives.ReadUInt32BigEndian(tail); + } + + private static byte[] ReadBytes(HttpRangeStream stream, long offset, int length) + { + if (offset < 0 || offset > stream.Length - length) + throw new InvalidDataException("Truncated TIFF directory."); + var saved = stream.Position; + try + { + stream.Position = offset; + var bytes = new byte[length]; + if (stream.Read(bytes, 0, length) != length) + throw new InvalidDataException("Truncated TIFF directory."); + return bytes; + } + finally + { + stream.Position = saved; + } + } + + private static int Value(Tiff tiff, TiffTag tag, int fallback) => + tiff.GetField(tag)?[0].ToInt() ?? fallback; + + private static void SetDirectory(Tiff tiff, long offset, HttpRangeStream stream) + { + PreflightDirectory(stream, offset); + if (!tiff.SetSubDirectory(offset)) + { + stream.ThrowIfFailed(); + throw new InvalidDataException("Cannot read TIFF image directory."); + } + stream.ThrowIfFailed(); + } + + private static double? NoData(Tiff tiff) => TerrainRasterReader.ReadNoDataValue(tiff); + + private static double? ReadBaseNoData( + Tiff tiff, + ImageDirectory first, + ImageDirectory selected, + HttpRangeStream stream + ) + { + SetDirectory(tiff, first.Offset, stream); + var value = NoData(tiff); + SetDirectory(tiff, selected.Offset, stream); + return value; + } + + private static bool IsNoData(float value, double noData) => + double.IsNaN(noData) ? float.IsNaN(value) : value == (float)noData; +} diff --git a/RhinoSpatial.Core/FeatureCollectionPager.cs b/RhinoSpatial.Core/FeatureCollectionPager.cs new file mode 100644 index 0000000..8105345 --- /dev/null +++ b/RhinoSpatial.Core/FeatureCollectionPager.cs @@ -0,0 +1,210 @@ +using System.Globalization; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using System.Xml.Linq; + +namespace RhinoSpatial.Core; + +public record PagedFeatureResponse(string Text, string StatusNote, int PageCount); +public record FeaturePageContent(string Text, Uri Url); + +/// Keeps source geometries intact while assembling bounded JSON or GML pages. +public static class FeatureCollectionPager +{ + public const int PageSize = 1000; + public const int MaximumFeatures = 100000; + public const int MaximumPages = 100; + + public static Task LoadAsync(string firstUrl, + Func> fetch, int maxFeatures, + Func? offsetUrl = null, CancellationToken cancellationToken = default) => + LoadHttpAsync(firstUrl, async (url, token) => new FeaturePageContent(await fetch(url, token).ConfigureAwait(false), new Uri(url)), + maxFeatures, offsetUrl, cancellationToken); + + public static async Task LoadHttpAsync(string firstUrl, + Func> fetch, int maxFeatures, + Func? offsetUrl = null, CancellationToken cancellationToken = default) + { + if (maxFeatures < 0) throw new ArgumentOutOfRangeException(nameof(maxFeatures)); + using var timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + timeout.CancelAfter(TimeSpan.FromMinutes(2)); + var cap = maxFeatures > 0 ? Math.Min(maxFeatures, MaximumFeatures) : MaximumFeatures; + var visited = new HashSet(StringComparer.Ordinal); + var pageHashes = new HashSet(StringComparer.Ordinal); + var ids = new HashSet(StringComparer.Ordinal); + var records = new List(); + Page? first = null; + var url = firstUrl; + var pages = 0; + var rawCount = 0; + long bytes = 0; + long? matchedTotal = null; + var note = ""; + string? firstJsonSrs = null; + var anonymous = false; + try + { + while (true) + { + timeout.Token.ThrowIfCancellationRequested(); + if (++pages > MaximumPages || !visited.Add(url)) + throw new InvalidDataException("Feature pagination repeated a URL or exceeded 100 pages. No partial result was returned; reduce the area or set Max Features."); + var response = await fetch(url, timeout.Token).ConfigureAwait(false); + var body = response.Text; + url = response.Url.AbsoluteUri; + bytes += Encoding.UTF8.GetByteCount(body); + if (bytes > 128L * 1024 * 1024) + throw new InvalidDataException("Feature responses exceeded 128 MiB. Reduce the area or set Max Features."); + var page = Page.Read(body); + first ??= page; + if (matchedTotal.HasValue && page.Total.HasValue && matchedTotal != page.Total) + throw new InvalidDataException("The provider changed its matching feature total during pagination; retry the selection."); + matchedTotal ??= page.Total; + if (page.IsJson != first.IsJson) + throw new InvalidDataException("The provider changed feature encoding between pages."); + var fingerprint = Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(string.Join("\n", page.Records)))); + if (page.Records.Count > 0 && !pageHashes.Add(fingerprint)) + throw new InvalidDataException("The provider repeated a feature page instead of advancing. No partial result was returned."); + if (page.IsJson) + { + var srs = SpatialReferenceTransform.NormalizeSrs(GeoJsonReader.TryReadSourceSrs(body) ?? "EPSG:4326"); + if (firstJsonSrs is not null && firstJsonSrs != srs) + throw new InvalidDataException("The provider changed GeoJSON coordinate systems between pages."); + firstJsonSrs ??= srs; + } + var before = records.Count; + for (var i = 0; i < page.Records.Count && records.Count < cap; i++) + { + if (page.Keys[i].Length == 0) anonymous = true; + if (page.Keys[i].Length == 0 || ids.Add(page.Keys[i])) records.Add(page.Records[i]); + } + rawCount += page.Records.Count; + if (matchedTotal.HasValue && records.Count > matchedTotal.Value) + throw new InvalidDataException("The provider returned more feature records than its advertised matching total; retry the selection."); + var more = page.Next is not null || matchedTotal > rawCount || + (offsetUrl is not null && matchedTotal is null && page.Records.Count >= PageSize); + if (records.Count >= cap) + { + if (maxFeatures <= 0 || maxFeatures > MaximumFeatures) + { + if (more || page.Records.Count > records.Count - before) + throw new InvalidDataException("Feature selection reached the 100,000-feature safety limit. Reduce the area or set Max Features."); + } + note = maxFeatures > 0 && maxFeatures <= MaximumFeatures + ? $"Stopped at Max Features ({maxFeatures}); further matching features may exist." : ""; + break; + } + if (page.Records.Count == 0) + { + if (page.Next is not null || matchedTotal > rawCount) + throw new InvalidDataException("The provider returned an empty page before completing its advertised feature result."); + break; + } + if (records.Count == before) + throw new InvalidDataException("Feature pagination made no progress; all returned IDs were duplicates."); + if (!more) + { + if (matchedTotal > records.Count) + throw new InvalidDataException("The provider ended paging before returning its advertised unique feature total."); + if (matchedTotal is null) note = "Provider supplied no matching total; paging ended at its last advertised page."; + break; + } + if (page.Next is not null) + { + var next = new Uri(new Uri(url), page.Next); + if (next.Scheme is not ("http" or "https") || !string.IsNullOrEmpty(next.UserInfo)) + throw new InvalidDataException("Feature pagination requires an HTTP(S) next link without embedded credentials."); + url = next.AbsoluteUri; + } + else if (offsetUrl is not null) + url = offsetUrl(rawCount, Math.Min(PageSize, cap - records.Count)); + else + throw new InvalidDataException("The provider advertised more features but supplied no supported paging mechanism. No partial result was returned."); + } + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { throw new TimeoutException("Feature pagination timed out after two minutes. Reduce the selected area or set Max Features."); } + var identityNote = anonymous ? " Features without stable IDs are retained; identity-based deduplication is unavailable for those records." : ""; + return new PagedFeatureResponse(first!.Merge(records), $"Read {records.Count} {(anonymous ? "" : "unique ")}feature(s) in {pages} page(s). {note}{identityNote}".Trim(), pages); + } + + private sealed record Page(bool IsJson, string Original, List Records, List Keys, string? Next, long? Total) + { + public static Page Read(string body) + { + var records = new List(); + var keys = new List(); + if (body.TrimStart().StartsWith("{", StringComparison.Ordinal)) + { + using var document = JsonDocument.Parse(body); + var root = document.RootElement; + if (!root.TryGetProperty("features", out var features) || features.ValueKind != JsonValueKind.Array) + throw new InvalidDataException("Expected a GeoJSON FeatureCollection, not service metadata or an error document."); + foreach (var feature in features.EnumerateArray()) + { + var text = JsonSerializer.Serialize(feature); + records.Add(text); + keys.Add(feature.TryGetProperty("id", out var id) && id.ValueKind is JsonValueKind.String or JsonValueKind.Number + ? "id:" + id.ValueKind + ":" + id.GetRawText() : ""); + } + string? next = null; + if (root.TryGetProperty("links", out var links) && links.ValueKind == JsonValueKind.Array) + foreach (var link in links.EnumerateArray()) + if (link.TryGetProperty("rel", out var rel) && rel.GetString() == "next" && link.TryGetProperty("href", out var href)) + next = href.GetString(); + long? total = null; + if ((root.TryGetProperty("numberMatched", out var count) || root.TryGetProperty("totalFeatures", out count)) && count.TryGetInt64Safe(out var n) && n >= 0) total = n; + return new Page(true, body, records, keys, next, total); + } + var xml = XDocument.Parse(body); + var element = xml.Root ?? throw new InvalidDataException("Empty feature response."); + if (element.Name.LocalName != "FeatureCollection") + throw new InvalidDataException("Expected a WFS FeatureCollection: " + element.Value[..Math.Min(250, element.Value.Length)]); + foreach (var member in element.Elements().Where(e => e.Name.LocalName is "member" or "featureMember" or "featureMembers")) + foreach (var feature in member.Elements()) + { + var wrapper = new XElement(XName.Get("member", "http://www.opengis.net/wfs/2.0"), new XElement(feature)); + // Preserve inherited coordinate-system declarations and namespace bindings. + foreach (var attribute in element.Attributes().Where(a => a.IsNamespaceDeclaration || a.Name.LocalName is "srsName" or "srsDimension")) + if (wrapper.Attribute(attribute.Name) is null) wrapper.Add(new XAttribute(attribute)); + var text = wrapper.ToString(SaveOptions.DisableFormatting); + records.Add(text); + var id = feature.Attributes().FirstOrDefault(a => a.Name.LocalName is "id" or "fid")?.Value; + keys.Add(string.IsNullOrEmpty(id) ? "" : "id:" + id); + } + var totalText = (string?)element.Attribute("numberMatched"); + return new Page(false, body, records, keys, (string?)element.Attribute("next"), + long.TryParse(totalText, NumberStyles.Integer, CultureInfo.InvariantCulture, out var totalValue) && totalValue >= 0 ? totalValue : null); + } + + public string Merge(List records) + { + if (IsJson) + { + using var document = JsonDocument.Parse(Original); + using var stream = new MemoryStream(); + using (var writer = new Utf8JsonWriter(stream)) + { + writer.WriteStartObject(); + foreach (var property in document.RootElement.EnumerateObject()) + if (property.Name is not ("features" or "links" or "numberReturned" or "numberMatched" or "totalFeatures")) property.WriteTo(writer); + writer.WritePropertyName("features"); writer.WriteStartArray(); + foreach (var record in records) { using var value = JsonDocument.Parse(record); value.RootElement.WriteTo(writer); } + writer.WriteEndArray(); writer.WriteNumber("numberReturned", records.Count); writer.WriteEndObject(); + } + return Encoding.UTF8.GetString(stream.ToArray()); + } + var xml = XDocument.Parse(Original); + var root = xml.Root!; + root.Elements().Where(e => e.Name.LocalName is "member" or "featureMember" or "featureMembers").Remove(); + root.Attribute("next")?.Remove(); root.Attribute("previous")?.Remove(); + root.SetAttributeValue("numberReturned", records.Count); + foreach (var record in records) root.Add(XElement.Parse(record)); + return xml.ToString(SaveOptions.DisableFormatting); + } + } + + private static bool TryGetInt64Safe(this JsonElement element, out long value) + { value = 0; return element.ValueKind == JsonValueKind.Number && element.TryGetInt64(out value); } +} diff --git a/RhinoSpatial.Core/GeoJsonReader.cs b/RhinoSpatial.Core/GeoJsonReader.cs index a091dcc..12d3e68 100644 --- a/RhinoSpatial.Core/GeoJsonReader.cs +++ b/RhinoSpatial.Core/GeoJsonReader.cs @@ -52,6 +52,9 @@ public static List ReadFeatures(string geoJson, string sourceLayerNa using var document = JsonDocument.Parse(geoJson); var root = document.RootElement; + if (root.TryGetProperty("type", out var rootType) && rootType.GetString() == "Feature") + return new List { ReadFeature(root, sourceLayerName) }; + if (!root.TryGetProperty("features", out var featuresElement) || featuresElement.ValueKind != JsonValueKind.Array) { throw new InvalidOperationException("The GeoJSON response does not contain a valid features array."); diff --git a/RhinoSpatial.Core/GeoPackageReader.cs b/RhinoSpatial.Core/GeoPackageReader.cs new file mode 100644 index 0000000..a132145 --- /dev/null +++ b/RhinoSpatial.Core/GeoPackageReader.cs @@ -0,0 +1,673 @@ +using System.Buffers.Binary; +using System.Globalization; +using Microsoft.Data.Sqlite; +using NetTopologySuite.Geometries; +using NetTopologySuite.IO; +using NtsGeometry = NetTopologySuite.Geometries.Geometry; + +namespace RhinoSpatial.Core; + +public record GeoPackageLayer( + string Name, + string Title, + string GeometryColumn, + int SrsId, + string SourceSrs, + BoundingBox2D? Bounds, + string PrimaryKey +) +{ + public string GeometryType { get; init; } = string.Empty; +} + +public record GeoPackageReadResult( + List Features, + int ScannedCount, + bool UsedSpatialIndex, + bool ReachedFeatureLimit +); + +/// Read-only standard GeoPackage feature layers. No SQL supplied by the user is executed. +public static class GeoPackageReader +{ + private static readonly object ProviderLock = new(); + private const int MaxRows = 500000, + MaxFeatures = 100000, + MaxGeometryBytes = 16 * 1024 * 1024; + + private static void InitializeProvider() + { + lock (ProviderLock) + { + try + { + _ = SQLitePCL.raw.sqlite3_libversion_number(); + } + catch (Exception ex) when (ex.Message.Contains("SetProvider", StringComparison.Ordinal)) + { + if (OperatingSystem.IsWindows()) + SQLitePCL.raw.SetProvider(new SQLitePCL.SQLite3Provider_winsqlite3()); + else + SQLitePCL.raw.SetProvider(new SQLitePCL.SQLite3Provider_sqlite3()); + } + } + } + + private static SqliteConnection Open(string path) + { + if (!File.Exists(path)) + throw new FileNotFoundException("GeoPackage was not found.", path); + InitializeProvider(); + var connection = new SqliteConnection( + new SqliteConnectionStringBuilder + { + DataSource = Path.GetFullPath(path), + Mode = SqliteOpenMode.ReadOnly, + Cache = SqliteCacheMode.Private, + Pooling = false, + }.ToString() + ); + try + { + connection.Open(); + SQLitePCL.raw.sqlite3_limit( + connection.Handle, + SQLitePCL.raw.SQLITE_LIMIT_LENGTH, + 16 * 1024 * 1024 + ); + SQLitePCL.raw.sqlite3_limit( + connection.Handle, + SQLitePCL.raw.SQLITE_LIMIT_SQL_LENGTH, + 1024 * 1024 + ); + using var command = connection.CreateCommand(); + command.CommandText = + "PRAGMA query_only=ON; PRAGMA trusted_schema=OFF; PRAGMA application_id;"; + var id = Convert.ToInt64(command.ExecuteScalar(), CultureInfo.InvariantCulture); + if (id != 0x47504B47 && id != 0x47503130) + throw new InvalidDataException( + "This SQLite file does not declare a GeoPackage application ID." + ); + return connection; + } + catch + { + connection.Dispose(); + throw; + } + } + + public static List ReadLayers(string path, CancellationToken token = default) + { + token.ThrowIfCancellationRequested(); + using var connection = Open(path); + using var deadline = CancellationTokenSource.CreateLinkedTokenSource(token); + deadline.CancelAfter(TimeSpan.FromSeconds(30)); + using var interrupt = deadline.Token.Register(() => + SQLitePCL.raw.sqlite3_interrupt(connection.Handle) + ); + using var transaction = connection.BeginTransaction(deferred: true); + return Layers(connection, transaction, deadline.Token); + } + + private static List Layers( + SqliteConnection connection, + SqliteTransaction transaction, + CancellationToken token + ) + { + foreach ( + var table in new[] { "gpkg_contents", "gpkg_geometry_columns", "gpkg_spatial_ref_sys" } + ) + RequireOrdinaryTable(connection, transaction, table); + using var command = connection.CreateCommand(); + command.Transaction = transaction; + command.CommandText = """ + SELECT substr(c.table_name,1,257),substr(coalesce(c.identifier,c.table_name),1,1025),substr(g.column_name,1,257),g.srs_id, + substr(s.organization,1,32),s.organization_coordsys_id,c.min_x,c.min_y,c.max_x,c.max_y,c.srs_id,substr(g.geometry_type_name,1,32) + FROM gpkg_contents c LEFT JOIN gpkg_geometry_columns g ON g.table_name=c.table_name + LEFT JOIN gpkg_spatial_ref_sys s ON s.srs_id=g.srs_id WHERE c.data_type='features' LIMIT 129 + """; + var layers = new List(); + using (var reader = command.ExecuteReader()) + while (reader.Read()) + { + token.ThrowIfCancellationRequested(); + if (layers.Count >= 128) + throw new InvalidDataException( + "GeoPackage exceeds the 128 feature-layer metadata limit." + ); + if ( + reader.IsDBNull(2) + || reader.IsDBNull(3) + || reader.IsDBNull(10) + || reader.GetInt32(3) != reader.GetInt32(10) + ) + throw new InvalidDataException( + "GeoPackage feature-layer catalogs disagree or are incomplete." + ); + var name = reader.GetString(0); + var geometry = reader.GetString(2); + if ( + name.Length > 256 + || geometry.Length > 256 + || name.Length == 0 + || geometry.Length == 0 + ) + throw new InvalidDataException( + "GeoPackage has an invalid or oversized table/geometry name." + ); + var srs = + !reader.IsDBNull(4) + && reader.GetString(4).Equals("EPSG", StringComparison.OrdinalIgnoreCase) + && !reader.IsDBNull(5) + && reader.GetInt32(5) > 0 + ? $"EPSG:{reader.GetInt32(5)}" + : string.Empty; + BoundingBox2D? bounds = null; + if ( + !reader.IsDBNull(6) + && !reader.IsDBNull(7) + && !reader.IsDBNull(8) + && !reader.IsDBNull(9) + ) + { + bounds = new( + reader.GetDouble(6), + reader.GetDouble(7), + reader.GetDouble(8), + reader.GetDouble(9) + ); + if (!ValidBounds(bounds)) + throw new InvalidDataException("GeoPackage catalog extent is invalid."); + } + layers.Add( + new( + name, + reader.GetString(1), + geometry, + reader.GetInt32(3), + srs, + bounds, + string.Empty + ) + { + GeometryType = reader.IsDBNull(11) + ? string.Empty + : reader.GetString(11).ToUpperInvariant(), + } + ); + } + for (var i = 0; i < layers.Count; i++) + { + var layer = layers[i]; + RequireOrdinaryTable(connection, transaction, layer.Name); + var columns = Columns(connection, transaction, layer.Name); + if (!columns.Any(c => c.Name == layer.GeometryColumn)) + throw new InvalidDataException("GeoPackage geometry column is missing."); + var keys = columns.Where(c => c.Primary).ToList(); + if ( + keys.Count != 1 + || !keys[0].Type.Equals("INTEGER", StringComparison.OrdinalIgnoreCase) + ) + throw new InvalidDataException( + "GeoPackage feature table requires one INTEGER primary key." + ); + layers[i] = layer with { PrimaryKey = keys[0].Name }; + } + return layers; + } + + public static GeoPackageReadResult ReadFeatures( + string path, + string layerName, + SpatialContext2D context, + int maxFeatures = 0, + CancellationToken token = default + ) + { + if (maxFeatures < 0 || maxFeatures > MaxFeatures) + throw new ArgumentOutOfRangeException( + nameof(maxFeatures), + "GeoPackage supports at most 100000 output features per layer." + ); + token.ThrowIfCancellationRequested(); + using var connection = Open(path); + using var deadline = CancellationTokenSource.CreateLinkedTokenSource(token); + deadline.CancelAfter(TimeSpan.FromMinutes(2)); + using var interrupt = deadline.Token.Register(() => + SQLitePCL.raw.sqlite3_interrupt(connection.Handle) + ); + using var transaction = connection.BeginTransaction(deferred: true); + var layers = Layers(connection, transaction, deadline.Token); + var layer = + string.IsNullOrWhiteSpace(layerName) && layers.Count == 1 + ? layers[0] + : layers.SingleOrDefault(l => l.Name.Equals(layerName, StringComparison.Ordinal)); + if (layer is null) + throw new InvalidDataException( + "Select an exact feature-layer name from List WFS Layers; multi-layer GeoPackages require an explicit selection." + ); + if (string.IsNullOrEmpty(layer.SourceSrs)) + throw new InvalidDataException( + "GeoPackage layer does not declare a supported EPSG CRS; no CRS was guessed." + ); + if ( + !new[] + { + "GEOMETRY", + "POINT", + "LINESTRING", + "POLYGON", + "MULTIPOINT", + "MULTILINESTRING", + "MULTIPOLYGON", + "GEOMETRYCOLLECTION", + }.Contains(layer.GeometryType) + ) + throw new InvalidDataException( + "Unsupported GeoPackage feature-layer geometry type or extension." + ); + var bounds = Lod2AtomFeedReader.TransformBounds( + new(context.ResolvedSrs, context.RequestBoundingBox), + layer.SourceSrs + ); + var columns = Columns(connection, transaction, layer.Name) + .Where(c => c.Name != layer.GeometryColumn) + .ToList(); + var rtree = "rtree_" + layer.Name + "_" + layer.GeometryColumn; + var indexed = HasIndex(connection, transaction, layer, rtree); + using var command = connection.CreateCommand(); + command.Transaction = transaction; + var geom = "f." + Quote(layer.GeometryColumn); + command.CommandText = + $"SELECT f.{Quote(layer.PrimaryKey)},length({geom}),CASE WHEN length({geom})<={MaxGeometryBytes} THEN {geom} END" + + string.Concat( + columns.Select(c => + $",CASE WHEN typeof(f.{Quote(c.Name)})='blob' THEN '[BLOB '||length(f.{Quote(c.Name)})||' bytes]' ELSE substr(CAST(f.{Quote(c.Name)} AS TEXT),1,16384) END" + ) + ) + + $" FROM {Quote(layer.Name)} f" + + ( + indexed + ? $" JOIN {Quote(rtree)} r ON r.id=f.{Quote(layer.PrimaryKey)} WHERE r.minx<=@maxx AND r.maxx>=@minx AND r.miny<=@maxy AND r.maxy>=@miny" + : "" + ) + + $" ORDER BY f.{Quote(layer.PrimaryKey)} LIMIT {MaxRows + 1}"; + if (indexed) + { + command.Parameters.AddWithValue("@minx", bounds.MinX); + command.Parameters.AddWithValue("@maxx", bounds.MaxX); + command.Parameters.AddWithValue("@miny", bounds.MinY); + command.Parameters.AddWithValue("@maxy", bounds.MaxY); + } + var output = new List(); + var scanned = 0; + long bytes = 0, + points = 0, + attributeCharacters = 0; + var reached = false; + using var reader = command.ExecuteReader(); + while (reader.Read()) + { + deadline.Token.ThrowIfCancellationRequested(); + if (++scanned > MaxRows) + throw new InvalidDataException( + "GeoPackage exceeded the 500000-row query budget. Add a standard RTree index or narrow the layer." + ); + if (reader.IsDBNull(1)) + continue; + var length = reader.GetInt64(1); + bytes += length; + if (length > MaxGeometryBytes || bytes > 128L * 1024 * 1024) + throw new InvalidDataException("GeoPackage exceeded its geometry byte budget."); + if (reader.GetValue(2) is not byte[] blob) + throw new InvalidDataException( + "GeoPackage geometry is not a binary geometry value." + ); + var geometry = Decode(blob, layer.SrsId, ref points); + if ( + layer.GeometryType != "GEOMETRY" + && !layer.GeometryType.Equals( + geometry.GeometryType, + StringComparison.OrdinalIgnoreCase + ) + ) + throw new InvalidDataException( + "GeoPackage geometry type disagrees with its declared feature-layer type." + ); + if (geometry.IsEmpty) + continue; + var e = geometry.EnvelopeInternal; + if ( + e.MinX > bounds.MaxX + || e.MaxX < bounds.MinX + || e.MinY > bounds.MaxY + || e.MaxY < bounds.MinY + ) + continue; + var converted = new WfsGeometry { Type = geometry.GeometryType }; + Append(geometry, converted, layer.SourceSrs, context.ResolvedSrs); + var attributes = new Dictionary(StringComparer.Ordinal); + for (var i = 0; i < columns.Count; i++) + attributes[columns[i].Name] = reader.IsDBNull(3 + i) + ? null + : reader.GetString(3 + i); + attributeCharacters += attributes.Values.Sum(v => (long)(v?.Length ?? 0)); + if (attributeCharacters > 8 * 1024 * 1024) + throw new InvalidDataException( + "GeoPackage exceeded its attribute text budget; narrow the area or select fewer fields upstream." + ); + var feature = new WfsFeature( + reader.GetValue(0).ToString()!, + layer.Name, + converted, + attributes + ); + if ( + SpatialFeatureFilter + .FilterToBounds(new[] { feature }, context.RequestBoundingBox) + .Features.Count == 0 + ) + continue; + if (output.Count >= MaxFeatures) + throw new InvalidDataException( + "GeoPackage exceeded its 100000-feature output budget; narrow the area." + ); + output.Add(feature); + if (maxFeatures > 0 && output.Count >= maxFeatures) + { + reached = true; + break; + } + } + return new(output, scanned, indexed, reached); + } + + private static NtsGeometry Decode(byte[] blob, int srs, ref long points) + { + if ( + blob.Length < 8 + || blob[0] != 71 + || blob[1] != 80 + || blob[2] != 0 + || (blob[3] & 0xE0) != 0 + ) + throw new InvalidDataException( + "Unsupported or malformed GeoPackage binary geometry header." + ); + var little = (blob[3] & 1) != 0; + var code = (blob[3] >> 1) & 7; + var empty = (blob[3] & 16) != 0; + var envelopeSize = code switch + { + 0 => 0, + 1 => 32, + 2 or 3 => 48, + 4 => 64, + _ => throw new InvalidDataException("Invalid GeoPackage envelope code."), + }; + var header = 8 + envelopeSize; + if (blob.Length < header || ReadInt(blob.AsSpan(4, 4), little) != srs || empty && code != 0) + throw new InvalidDataException( + "GeoPackage geometry header CRS/envelope does not match its layer." + ); + var wkb = blob.AsSpan(header); + var offset = 0; + ValidateWkb(wkb, ref offset, ref points, 0); + if (offset != wkb.Length) + throw new InvalidDataException("Trailing bytes in GeoPackage WKB geometry."); + var geometry = new WKBReader { RepairRings = false }.Read(wkb.ToArray()); + if (geometry.IsEmpty != empty) + throw new InvalidDataException( + "GeoPackage empty-geometry flag disagrees with its WKB." + ); + return geometry; + } + + // Validate counts before NTS allocates coordinate sequences from untrusted WKB. + private static int ValidateWkb( + ReadOnlySpan data, + ref int offset, + ref long points, + int depth + ) + { + if (depth > 32 || offset > data.Length - 5 || data[offset] > 1) + throw new InvalidDataException("Invalid or excessively nested GeoPackage WKB."); + var little = data[offset++] != 0; + var type = ReadInt(data.Slice(offset, 4), little); + offset += 4; + var kind = type % 1000; + var dimensions = 2 + (type / 1000 is 1 or 3 ? 1 : 0) + (type / 1000 is 2 or 3 ? 1 : 0); + if (type < 1 || type / 1000 > 3 || kind is < 1 or > 7) + throw new InvalidDataException( + "Only standard GeoPackage point, line, polygon and collection WKB geometries are supported." + ); + if (kind == 1) + Coordinates(data, ref offset, 1, dimensions, little, ref points, true); + else + { + var count = CountValue(data, ref offset, little); + if (kind == 2) + { + if (count == 1) + throw new InvalidDataException( + "A WKB line requires zero or at least two positions." + ); + Coordinates(data, ref offset, count, dimensions, little, ref points, false); + } + else if (kind == 3) + { + if (count > 100000) + throw new InvalidDataException("Excessive polygon ring count."); + for (var i = 0; i < count; i++) + { + var n = CountValue(data, ref offset, little); + if (n < 4) + throw new InvalidDataException( + "Polygon ring has fewer than four positions." + ); + var start = offset; + Coordinates(data, ref offset, n, dimensions, little, ref points, false); + if ( + ReadDouble(data.Slice(start, 8), little) + != ReadDouble(data.Slice(offset - dimensions * 8, 8), little) + || ReadDouble(data.Slice(start + 8, 8), little) + != ReadDouble(data.Slice(offset - dimensions * 8 + 8, 8), little) + ) + throw new InvalidDataException( + "Unclosed WKB polygon ring; no geometry was repaired." + ); + } + } + else + { + if (count > 100000) + throw new InvalidDataException("Excessive geometry collection count."); + for (var i = 0; i < count; i++) + { + var child = ValidateWkb(data, ref offset, ref points, depth + 1); + if (kind is >= 4 and <= 6 && child != kind - 3) + throw new InvalidDataException("Invalid multi-geometry child type."); + } + } + } + return kind; + } + + private static int CountValue(ReadOnlySpan data, ref int offset, bool little) + { + if (offset > data.Length - 4) + throw new InvalidDataException("Truncated WKB count."); + var n = ReadInt(data.Slice(offset, 4), little); + offset += 4; + if (n < 0) + throw new InvalidDataException("Negative WKB count."); + return n; + } + + private static void Coordinates( + ReadOnlySpan data, + ref int offset, + int count, + int dimensions, + bool little, + ref long points, + bool emptyPoint + ) + { + if (count > (data.Length - offset) / (8 * dimensions) || (points += count) > 4000000) + throw new InvalidDataException("Truncated WKB or excessive coordinate count."); + for (var i = 0; i < count; i++) + { + var x = ReadDouble(data.Slice(offset, 8), little); + var y = ReadDouble(data.Slice(offset + 8, 8), little); + if ( + (!double.IsFinite(x) || !double.IsFinite(y)) + && !(emptyPoint && double.IsNaN(x) && double.IsNaN(y)) + ) + throw new InvalidDataException("WKB contains non-finite XY coordinates."); + offset += 8 * dimensions; + } + } + + private static int ReadInt(ReadOnlySpan value, bool little) => + little + ? BinaryPrimitives.ReadInt32LittleEndian(value) + : BinaryPrimitives.ReadInt32BigEndian(value); + + private static double ReadDouble(ReadOnlySpan value, bool little) => + BitConverter.Int64BitsToDouble( + little + ? BinaryPrimitives.ReadInt64LittleEndian(value) + : BinaryPrimitives.ReadInt64BigEndian(value) + ); + + private static void Append( + NtsGeometry geometry, + WfsGeometry output, + string source, + string target + ) + { + List Convert(NetTopologySuite.Geometries.Coordinate[] coordinates) => + coordinates + .Select(c => + SpatialReferenceTransform.TryTransformXY( + source, + target, + c.X, + c.Y, + out var x, + out var y + ) + && double.IsFinite(x) + && double.IsFinite(y) + ? new Coordinate2D(x, y) + : throw new InvalidDataException( + $"Cannot transform GeoPackage coordinates from '{source}' to '{target}'." + ) + ) + .ToList(); + switch (geometry) + { + case Polygon polygon: + output.OuterRings.Add(new(Convert(polygon.ExteriorRing.Coordinates))); + for (var i = 0; i < polygon.NumInteriorRings; i++) + output.OuterRings.Add(new(Convert(polygon.GetInteriorRingN(i).Coordinates))); + break; + case NetTopologySuite.Geometries.LineString line: + output.LineStrings.Add(new(Convert(line.Coordinates))); + break; + case Point point: + output.Points.AddRange(Convert(point.Coordinates)); + break; + case GeometryCollection collection: + for (var i = 0; i < collection.NumGeometries; i++) + Append(collection.GetGeometryN(i), output, source, target); + break; + default: + throw new InvalidDataException("Unsupported GeoPackage geometry type."); + } + } + + private record Column(string Name, string Type, bool Primary); + + private static List Columns( + SqliteConnection connection, + SqliteTransaction transaction, + string table + ) + { + using var command = connection.CreateCommand(); + command.Transaction = transaction; + command.CommandText = $"PRAGMA table_info({Quote(table)})"; + using var reader = command.ExecuteReader(); + var columns = new List(); + while (reader.Read()) + { + if (columns.Count >= 256) + throw new InvalidDataException("GeoPackage exceeds its 256-column limit."); + columns.Add(new(reader.GetString(1), reader.GetString(2), reader.GetInt32(5) > 0)); + } + return columns; + } + + private static bool HasIndex( + SqliteConnection connection, + SqliteTransaction transaction, + GeoPackageLayer layer, + string rtree + ) + { + using var command = connection.CreateCommand(); + command.Transaction = transaction; + command.CommandText = + "SELECT count(*) FROM sqlite_master WHERE type='table' AND name=@name AND lower(sql) LIKE 'create virtual table%using rtree%';"; + command.Parameters.AddWithValue("@name", rtree); + if (Convert.ToInt32(command.ExecuteScalar(), CultureInfo.InvariantCulture) != 1) + return false; + command.CommandText = + "SELECT count(*) FROM sqlite_master WHERE type='table' AND name='gpkg_extensions'"; + command.Parameters.Clear(); + if (Convert.ToInt32(command.ExecuteScalar(), CultureInfo.InvariantCulture) != 1) + return false; + RequireOrdinaryTable(connection, transaction, "gpkg_extensions"); + command.CommandText = "SELECT sqlite_compileoption_used('ENABLE_RTREE')"; + if (Convert.ToInt32(command.ExecuteScalar(), CultureInfo.InvariantCulture) == 0) + return false; + command.CommandText = + "SELECT count(*) FROM gpkg_extensions WHERE table_name=@table AND column_name=@column AND extension_name='gpkg_rtree_index'"; + command.Parameters.AddWithValue("@table", layer.Name); + command.Parameters.AddWithValue("@column", layer.GeometryColumn); + return Convert.ToInt32(command.ExecuteScalar(), CultureInfo.InvariantCulture) > 0; + } + + private static void RequireOrdinaryTable( + SqliteConnection connection, + SqliteTransaction transaction, + string table + ) + { + using var command = connection.CreateCommand(); + command.Transaction = transaction; + command.CommandText = + "SELECT count(*) FROM sqlite_master WHERE type='table' AND name=@name AND lower(sql) NOT LIKE 'create virtual table%'"; + command.Parameters.AddWithValue("@name", table); + if (Convert.ToInt32(command.ExecuteScalar(), CultureInfo.InvariantCulture) != 1) + throw new InvalidDataException( + "GeoPackage catalog/feature table is missing or is not an ordinary SQLite table." + ); + } + + private static string Quote(string value) => "\"" + value.Replace("\"", "\"\"") + "\""; + + private static bool ValidBounds(BoundingBox2D b) => + double.IsFinite(b.MinX) + && double.IsFinite(b.MinY) + && double.IsFinite(b.MaxX) + && double.IsFinite(b.MaxY) + && b.MinX <= b.MaxX + && b.MinY <= b.MaxY; +} diff --git a/RhinoSpatial.Core/GeoTiffInfoCache.cs b/RhinoSpatial.Core/GeoTiffInfoCache.cs index 06505b3..e280a25 100644 --- a/RhinoSpatial.Core/GeoTiffInfoCache.cs +++ b/RhinoSpatial.Core/GeoTiffInfoCache.cs @@ -6,7 +6,7 @@ namespace RhinoSpatial.Core { public static class GeoTiffInfoCache { - private readonly record struct CacheKey(long FileSizeBytes, long LastWriteUtcTicks); + private readonly record struct CacheKey(long FileSizeBytes, long LastWriteUtcTicks, string Sidecars); private readonly record struct CacheEntry(CacheKey Key, GeoReferencedRasterInfo RasterInfo); @@ -15,7 +15,7 @@ public static class GeoTiffInfoCache public static GeoReferencedRasterInfo GetOrRead(string filePath) { var fileInfo = new FileInfo(filePath); - var key = new CacheKey(fileInfo.Length, fileInfo.LastWriteTimeUtc.Ticks); + var key = new CacheKey(fileInfo.Length, fileInfo.LastWriteTimeUtc.Ticks, RasterSidecars.Signature(filePath)); if (EntriesByPath.TryGetValue(filePath, out var cachedEntry) && cachedEntry.Key.Equals(key)) { diff --git a/RhinoSpatial.Core/GeoTiffReader.cs b/RhinoSpatial.Core/GeoTiffReader.cs index 6528584..d0fb725 100644 --- a/RhinoSpatial.Core/GeoTiffReader.cs +++ b/RhinoSpatial.Core/GeoTiffReader.cs @@ -10,98 +10,60 @@ public static class GeoTiffReader private const ushort GeographicTypeGeoKey = 2048; private const ushort ProjectedCSTypeGeoKey = 3072; - public static GeoReferencedRasterInfo ReadImageInfo(string filePath) - { - TiffWarningSuppression.EnsureInstalled(); - using var tiff = Tiff.Open(filePath, "r"); - if (tiff is null) - { - throw new InvalidOperationException($"Failed to open GeoTIFF: {filePath}"); - } - - var width = tiff.GetField(TiffTag.IMAGEWIDTH)?[0].ToInt() ?? 0; - var height = tiff.GetField(TiffTag.IMAGELENGTH)?[0].ToInt() ?? 0; - if (width <= 0 || height <= 0) - { - throw new InvalidOperationException("The GeoTIFF image dimensions could not be read."); - } + public static GeoReferencedRasterInfo ReadImageInfo(string filePath) => ReadImageInfo(filePath,null); - var srsName = ReadSrsName(tiff); - if (string.IsNullOrWhiteSpace(srsName)) + public static GeoReferencedRasterInfo ReadImageInfo(string filePath,string? declaredFallbackSrs) + { + if (new[] { ".png", ".jpg", ".jpeg" }.Contains(Path.GetExtension(filePath), StringComparer.OrdinalIgnoreCase)) { - throw new InvalidOperationException("The GeoTIFF does not contain a readable EPSG code yet. The current RhinoSpatial GeoTIFF loader supports GeoTIFF files with embedded EPSG georeferencing."); + using var stream=File.OpenRead(filePath); + var info=StbImageNext.ImageInfo.FromStream(stream); + if(info.Width<=0 || info.Height<=0)throw new InvalidDataException("Invalid raster image dimensions."); + var sidecar=RasterSidecars.Read(filePath); + return new GeoReferencedRasterInfo(filePath,sidecar.Srs,info.Width,info.Height,sidecar.Transform.Bounds(info.Width,info.Height),new FileInfo(filePath).Length) { GeoTransform=sidecar.Transform }; } - - var boundingBox = ReadBoundingBox(tiff, width, height); - var fileSizeBytes = new FileInfo(filePath).Length; - return new GeoReferencedRasterInfo( - filePath, - srsName, - width, - height, - boundingBox, - fileSizeBytes); + TiffWarningSuppression.EnsureInstalled(); + using var tiff=Tiff.Open(filePath,"r"); + if(tiff is null) throw new InvalidOperationException("Failed to open raster: "+filePath); + return ReadImageInfo(tiff,filePath,new FileInfo(filePath).Length,declaredFallbackSrs,true); } - private static BoundingBox2D ReadBoundingBox(Tiff tiff, int width, int height) + internal static GeoReferencedRasterInfo ReadImageInfo(Tiff tiff,string name,long size,string? declaredFallbackSrs=null,bool sidecars=false) { - var pixelScale = TryReadDoubleArray(tiff, TiffTag.GEOTIFF_MODELPIXELSCALETAG); - var tiePoints = TryReadDoubleArray(tiff, TiffTag.GEOTIFF_MODELTIEPOINTTAG); + var width=tiff.GetField(TiffTag.IMAGEWIDTH)?[0].ToInt()??0; + var height=tiff.GetField(TiffTag.IMAGELENGTH)?[0].ToInt()??0; + if(width<=0 || height<=0) throw new InvalidDataException("Invalid raster dimensions."); + var srs=ReadSrsName(tiff); + var transform=ReadTransform(tiff); + transform ??= (sidecars?RasterSidecars.ReadTransform(name):throw new InvalidDataException("Remote TIFF requires embedded affine georeferencing.")); + if(string.IsNullOrWhiteSpace(srs))srs=string.IsNullOrWhiteSpace(declaredFallbackSrs)?(sidecars?RasterSidecars.ReadSrs(name):throw new InvalidDataException("Remote TIFF requires an embedded CRS.")):declaredFallbackSrs; + transform.Validate(); + return new GeoReferencedRasterInfo(name,srs,width,height,transform.Bounds(width,height),size) { GeoTransform=transform }; + } - if (pixelScale is { Length: >= 2 } && tiePoints is { Length: >= 6 }) + private static RasterGeoTransform? ReadTransform(Tiff tiff) + { + RasterGeoTransform? g=null; + var matrix=TryReadDoubleArray(tiff,TiffTag.GEOTIFF_MODELTRANSFORMATIONTAG); + if(matrix is { Length: >=16 }) { - var scaleX = Math.Abs(pixelScale[0]); - var scaleY = Math.Abs(pixelScale[1]); - if (scaleX <= 0.0 || scaleY <= 0.0) - { - throw new InvalidOperationException("The GeoTIFF pixel scale is invalid."); - } - - var rasterX = tiePoints[0]; - var rasterY = tiePoints[1]; - var modelX = tiePoints[3]; - var modelY = tiePoints[4]; - - var minX = modelX - rasterX * scaleX; - var maxY = modelY + rasterY * scaleY; - var maxX = minX + width * scaleX; - var minY = maxY - height * scaleY; - return new BoundingBox2D(minX, minY, maxX, maxY); + if(matrix.Any(v=>!double.IsFinite(v)) || matrix[12]!=0 || matrix[13]!=0 || matrix[15]!=1) + throw new InvalidDataException("Unsupported GeoTIFF projective transformation."); + g=new(matrix[0],matrix[1],matrix[3],matrix[4],matrix[5],matrix[7]); } - - var transform = TryReadDoubleArray(tiff, TiffTag.GEOTIFF_MODELTRANSFORMATIONTAG); - if (transform is { Length: >= 16 }) + else { - var scaleX = transform[0]; - var shearX = transform[1]; - var shearY = transform[4]; - var scaleY = transform[5]; - var translateX = transform[3]; - var translateY = transform[7]; - - if (Math.Abs(shearX) > 1e-9 || Math.Abs(shearY) > 1e-9) - { - throw new InvalidOperationException("The GeoTIFF uses rotated or sheared georeferencing, which the current RhinoSpatial GeoTIFF loader does not support yet."); - } - - if (Math.Abs(scaleX) <= 1e-9 || Math.Abs(scaleY) <= 1e-9) - { - throw new InvalidOperationException("The GeoTIFF transformation matrix is invalid."); - } - - var minX = translateX; - var maxY = translateY; - var maxX = minX + width * scaleX; - var minY = maxY + height * scaleY; - - return new BoundingBox2D( - Math.Min(minX, maxX), - Math.Min(minY, maxY), - Math.Max(minX, maxX), - Math.Max(minY, maxY)); + var scale=TryReadDoubleArray(tiff,TiffTag.GEOTIFF_MODELPIXELSCALETAG); + var tie=TryReadDoubleArray(tiff,TiffTag.GEOTIFF_MODELTIEPOINTTAG); + if(scale is {Length:>=2} && tie is {Length:>=6}) g=new(scale[0],0,tie[3]-tie[0]*scale[0],0,-scale[1],tie[4]+tie[1]*scale[1]); } - - throw new InvalidOperationException("The GeoTIFF does not expose supported georeferencing tags. The current RhinoSpatial GeoTIFF loader supports north-up GeoTIFF files with ModelTiepoint/ModelPixelScale or simple affine transform tags."); + if(g is null) return null; + var keys=TryReadUShortArray(tiff,TiffTag.GEOTIFF_GEOKEYDIRECTORYTAG); + if(keys is not null) + for(var i=4;i+3= geoKeys.Length) - { - break; - } - - var keyId = geoKeys[entryOffset]; - var tagLocation = geoKeys[entryOffset + 1]; - var valueOffset = geoKeys[entryOffset + 3]; - - if (tagLocation != 0) - { - continue; - } - - if (keyId == ProjectedCSTypeGeoKey || keyId == GeographicTypeGeoKey) - { - var normalizedSrs = $"EPSG:{valueOffset}"; - return normalizedSrs; - } - } - + foreach(var requested in new[]{ProjectedCSTypeGeoKey,GeographicTypeGeoKey}) + for(var offset=4;offset+30 && geoKeys[offset+3]!=32767) + return "EPSG:"+geoKeys[offset+3]; return string.Empty; } diff --git a/RhinoSpatial.Core/HttpRangeStream.cs b/RhinoSpatial.Core/HttpRangeStream.cs new file mode 100644 index 0000000..996481f --- /dev/null +++ b/RhinoSpatial.Core/HttpRangeStream.cs @@ -0,0 +1,263 @@ +using System.Net; +using System.Net.Http.Headers; + +namespace RhinoSpatial.Core; + +/// A bounded, representation-pinned seekable view over an HTTP byte-range resource. +public sealed class HttpRangeStream : Stream +{ + private const int BlockSize = 65536; + private readonly HttpClient _http; + private readonly string _url; + private readonly long _budget; + private readonly int _requestBudget; + private readonly CancellationTokenSource _timeout; + private readonly CancellationToken _token; + private readonly Dictionary _blocks = new(); + private readonly Queue _order = new(); + private long _length, + _position; + private string? _etag; + private DateTimeOffset? _modified; + private bool _disposed; + public long TransferredBytes { get; private set; } + public int RequestCount { get; private set; } + public string Validator => _etag ?? _modified?.ToString("R") ?? "none"; + public Exception? Failure { get; private set; } + + private HttpRangeStream( + HttpClient http, + string url, + long budget, + int requests, + TimeSpan timeout, + CancellationToken token + ) + { + if ( + !Uri.TryCreate(url, UriKind.Absolute, out var uri) + || uri.Scheme is not ("http" or "https") + || uri.UserInfo.Length > 0 + ) + throw new ArgumentException( + "Remote rasters require an HTTP(S) URL without embedded credentials." + ); + _http = http; + _url = url; + _budget = budget; + _requestBudget = requests; + _timeout = CancellationTokenSource.CreateLinkedTokenSource(token); + _timeout.CancelAfter(timeout); + _token = _timeout.Token; + } + + public static async Task OpenAsync( + HttpClient http, + string url, + CancellationToken token = default, + long maximumBytes = 64L * 1024 * 1024, + int maximumRequests = 512 + ) + { + var result = new HttpRangeStream( + http, + url, + maximumBytes, + maximumRequests, + TimeSpan.FromMinutes(2), + token + ); + try + { + await result.BlockAsync(0).ConfigureAwait(false); + return result; + } + catch + { + result.Dispose(); + throw; + } + } + + private async Task BlockAsync(long start) + { + _token.ThrowIfCancellationRequested(); + if (_blocks.TryGetValue(start, out var cached)) + return cached; + if (++RequestCount > _requestBudget) + throw new InvalidDataException( + "Remote raster exceeded its range-request budget; reduce the area." + ); + if (start > 0 && _etag is null && _modified is null) + throw new InvalidDataException( + "The range server supplies no strong ETag or Last-Modified validator; a consistent raster snapshot cannot be verified." + ); + var end = _length == 0 ? BlockSize - 1 : Math.Min(_length - 1, start + BlockSize - 1); + using var request = new HttpRequestMessage(HttpMethod.Get, _url); + request.Headers.Range = new RangeHeaderValue(start, end); + request.Headers.AcceptEncoding.ParseAdd("identity"); + if (_etag is not null) + request.Headers.IfRange = new RangeConditionHeaderValue( + new EntityTagHeaderValue(_etag) + ); + else if (_modified.HasValue) + request.Headers.IfRange = new RangeConditionHeaderValue(_modified.Value); + using var response = await _http + .SendAsync(request, HttpCompletionOption.ResponseHeadersRead, _token) + .ConfigureAwait(false); + if (response.StatusCode != HttpStatusCode.PartialContent) + throw new InvalidDataException( + $"Remote raster requires HTTP 206 byte ranges (received {(int)response.StatusCode}); no whole-file fallback was used." + ); + if ( + response.Content.Headers.ContentEncoding.Any(e => + !e.Equals("identity", StringComparison.OrdinalIgnoreCase) + ) + ) + throw new InvalidDataException( + "Encoded HTTP ranges cannot be used as TIFF byte offsets." + ); + var range = response.Content.Headers.ContentRange; + if ( + range is null + || range.Unit != "bytes" + || !range.HasLength + || !range.HasRange + || range.From != start + || range.To < start + || range.To > end + || range.Length <= 0 + || range.Length > 1024L * 1024 * 1024 * 1024 + || range.To != Math.Min(end, range.Length.GetValueOrDefault() - 1) + ) + throw new InvalidDataException("The server returned an inconsistent Content-Range."); + var strong = response.Headers.ETag is { IsWeak: false } tag ? tag.Tag : null; + var modified = response.Content.Headers.LastModified; + if (_length == 0) + { + _length = range.Length.GetValueOrDefault(); + _etag = strong; + _modified = modified; + } + else if ( + range.Length != _length + || (_etag is not null && strong != _etag) + || (_etag is null && _modified.HasValue && modified != _modified) + ) + throw new InvalidDataException( + "The remote raster changed during reading; retry to obtain one consistent snapshot." + ); + var expected = checked((int)(range.To!.Value - start + 1)); + if (TransferredBytes + expected > _budget) + throw new InvalidDataException( + "Remote raster exceeded its byte budget; reduce the area or use a coarser source." + ); + if ( + response.Content.Headers.ContentLength.HasValue + && response.Content.Headers.ContentLength != expected + ) + throw new InvalidDataException("Truncated or oversized HTTP range."); + await using var body = await response + .Content.ReadAsStreamAsync(_token) + .ConfigureAwait(false); + var data = new byte[expected]; + var read = 0; + while (read < expected) + { + var n = await body.ReadAsync(data.AsMemory(read, expected - read), _token) + .ConfigureAwait(false); + if (n == 0) + throw new InvalidDataException("Truncated HTTP byte range."); + read += n; + TransferredBytes += n; + } + if (await body.ReadAsync(new byte[1], _token).ConfigureAwait(false) != 0) + throw new InvalidDataException("HTTP range exceeded its declared size."); + while (_blocks.Count >= 256) + _blocks.Remove(_order.Dequeue()); + _blocks[start] = data; + _order.Enqueue(start); + return data; + } + + public override int Read(byte[] buffer, int offset, int count) + { + if (_disposed) + throw new ObjectDisposedException(nameof(HttpRangeStream)); + try + { + _token.ThrowIfCancellationRequested(); + if (offset < 0 || count < 0 || offset > buffer.Length - count) + throw new ArgumentOutOfRangeException(nameof(count)); + var total = (int)Math.Min(count, Math.Max(0, _length - _position)); + var copied = 0; + while (copied < total) + { + var start = _position / BlockSize * BlockSize; + var data = BlockAsync(start).GetAwaiter().GetResult(); + var index = (int)(_position - start); + var n = Math.Min(total - copied, data.Length - index); + if (n <= 0) + throw new EndOfStreamException("Invalid TIFF range offset."); + Buffer.BlockCopy(data, index, buffer, offset + copied, n); + copied += n; + _position += n; + } + return copied; + } + catch (Exception ex) + { + Failure = ex; + throw; + } + } + + public void ThrowIfFailed() + { + if (Failure is not null) + System.Runtime.ExceptionServices.ExceptionDispatchInfo.Capture(Failure).Throw(); + _token.ThrowIfCancellationRequested(); + } + + public override long Seek(long offset, SeekOrigin origin) + { + var position = origin switch + { + SeekOrigin.Begin => offset, + SeekOrigin.Current => checked(_position + offset), + SeekOrigin.End => checked(_length + offset), + _ => throw new ArgumentOutOfRangeException(nameof(origin)), + }; + if (position < 0) + throw new IOException("Negative TIFF seek."); + return _position = position; + } + + public override bool CanRead => !_disposed; + public override bool CanSeek => !_disposed; + public override bool CanWrite => false; + public override long Length => _length; + public override long Position + { + get => _position; + set => Seek(value, SeekOrigin.Begin); + } + + public override void Flush() { } + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override void Write(byte[] buffer, int offset, int count) => + throw new NotSupportedException(); + + protected override void Dispose(bool disposing) + { + if (disposing && !_disposed) + { + _disposed = true; + _timeout.Dispose(); + _blocks.Clear(); + } + base.Dispose(disposing); + } +} diff --git a/RhinoSpatial.Core/Lod2AtomClient.cs b/RhinoSpatial.Core/Lod2AtomClient.cs index e8bd3c4..186ae4b 100644 --- a/RhinoSpatial.Core/Lod2AtomClient.cs +++ b/RhinoSpatial.Core/Lod2AtomClient.cs @@ -12,7 +12,7 @@ namespace RhinoSpatial.Core; public record Lod2AtomSelection(IReadOnlyList Tiles, int FeedCount, int DownloadCount); public record Lod2AtomCachedTile(Lod2AtomLink Tile, string FilePath, bool UsedCache); -public sealed class Lod2AtomClient +public class AtomDownloadClient { private static readonly HttpClient SharedHttpClient = new() { Timeout = Timeout.InfiniteTimeSpan }; private static readonly ConcurrentDictionary FeedCache = new(StringComparer.Ordinal); @@ -23,10 +23,15 @@ public sealed class Lod2AtomClient private const long MaximumDownloadBytes = 512L * 1024 * 1024; private const long MaximumExpandedBytes = 1024L * 1024 * 1024; - public Lod2AtomClient(HttpClient? httpClient = null, string? cacheDirectory = null) + private readonly AtomDownloadKind _kind; + private readonly SourceDownloadCache _downloads; + + public AtomDownloadClient(HttpClient? httpClient = null, string? cacheDirectory = null, AtomDownloadKind kind = AtomDownloadKind.Buildings) { _http = httpClient ?? SharedHttpClient; - _cacheDirectory = cacheDirectory ?? DefaultCacheDirectory; + _kind = kind; + _cacheDirectory = cacheDirectory ?? (kind == AtomDownloadKind.Buildings ? DefaultCacheDirectory : Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "RhinoSpatial", "TerrainAtomCache")); + _downloads = new SourceDownloadCache(kind, _http, _cacheDirectory); } public async Task TrySelectAsync(string source, SpatialContext2D context, CancellationToken cancellationToken = default) @@ -67,7 +72,7 @@ bool Intersects(AtomSpatialExtent extent) cancellationToken.ThrowIfCancellationRequested(); if (++feedCount > 64 || current.Depth > 8) throw new InvalidOperationException("ATOM feed traversal limit reached. Select a more specific dataset feed."); - var feed = Lod2AtomFeedReader.Read(current.Document, current.Url, current.Extent); + var feed = Lod2AtomFeedReader.Read(current.Document, current.Url, current.Extent, _kind); downloadCount += feed.Downloads.Count; if (downloadCount > 50000) throw new InvalidOperationException("The ATOM catalogue exceeds the supported download-entry limit."); @@ -93,7 +98,7 @@ bool Intersects(AtomSpatialExtent extent) } } if (missing.Count > 0) - throw new InvalidOperationException($"The ATOM feed does not provide sufficient per-tile spatial metadata to select downloads safely. Example entries: {string.Join(", ", missing)}. No building archives were downloaded. Use a spatially indexed feed or local tiles."); + throw new InvalidOperationException($"The ATOM feed does not provide sufficient per-tile spatial metadata to select downloads safely. Example entries: {string.Join(", ", missing)}. No data archives were downloaded. Use a spatially indexed feed or local tiles."); if (tiles.Count > 32) throw new InvalidOperationException($"The area selects {tiles.Count} ATOM tiles; the limit is 32 per solution. Select a smaller study area."); return new Lod2AtomSelection(tiles.Values.OrderBy(tile => tile.Url.AbsoluteUri, StringComparer.Ordinal).ToList(), feedCount, downloadCount); @@ -168,129 +173,41 @@ private async Task> DownloadCoreAsync(Lod2Atom try { var prefix = $"ATOM {result.Count + 1}/{selection.Tiles.Count}"; - var cachedTile = await DownloadTileAsync(tile, note => progress?.Invoke($"{prefix}: {note}"), cancellationToken).ConfigureAwait(false); + var cachedTile = await _downloads.DownloadAsync(tile, note => progress?.Invoke($"{prefix}: {note}"), cancellationToken).ConfigureAwait(false); result.Add(cachedTile); progress?.Invoke($"{prefix}: {(cachedTile.UsedCache ? "reused cache" : "download complete")}"); } catch (Exception ex) when (ex is not OperationCanceledException) { - throw new InvalidOperationException($"Could not load ATOM tile '{tile.Title}': {ex.Message} Completed {result.Count}/{selection.Tiles.Count} tile(s); successful downloads remain cached. No partial building result was returned.", ex); + throw new InvalidOperationException($"Could not load ATOM tile '{tile.Title}': {ex.Message} Completed {result.Count}/{selection.Tiles.Count} tile(s); successful downloads remain cached. No partial source result was returned.", ex); } } return result; } - private sealed record CacheStamp(long Length, string Hash, DateTime DownloadedUtc); - - private async Task DownloadTileAsync(Lod2AtomLink tile, Action progress, CancellationToken cancellationToken) - { - Directory.CreateDirectory(_cacheDirectory); - var key = Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(tile.Url.AbsoluteUri + "\n" + tile.Revision))); - var target = Path.Combine(_cacheDirectory, key + ".zip"); - using var timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); - timeout.CancelAfter(TimeSpan.FromMinutes(3)); - var temporary = target + "." + Guid.NewGuid().ToString("N") + ".partial"; - try - { - // A filesystem lock also serializes writers from separate Rhino processes. - await using var cacheLock = await AcquireLockAsync(target + ".lock", timeout.Token).ConfigureAwait(false); - if (await IsValidCacheAsync(target, tile.Revision, timeout.Token).ConfigureAwait(false)) - return new Lod2AtomCachedTile(tile, target, true); - using var response = await _http.GetAsync(tile.Url, HttpCompletionOption.ResponseHeadersRead, timeout.Token).ConfigureAwait(false); - response.EnsureSuccessStatusCode(); - var expectedLength = response.Content.Headers.ContentLength; - if (expectedLength > MaximumDownloadBytes) - throw new InvalidOperationException("Archive exceeds the 512 MiB download limit."); - await using (var input = await response.Content.ReadAsStreamAsync(timeout.Token).ConfigureAwait(false)) - await using (var output = new FileStream(temporary, FileMode.CreateNew, FileAccess.Write, FileShare.None, 81920, useAsync: true)) - { - var buffer = new byte[81920]; - long count = 0; - long lastProgressBytes = 0; - int read; - while ((read = await input.ReadAsync(buffer, timeout.Token).ConfigureAwait(false)) != 0) - { - count += read; - if (count > MaximumDownloadBytes) throw new InvalidOperationException("Archive exceeds the 512 MiB download limit."); - await output.WriteAsync(buffer.AsMemory(0, read), timeout.Token).ConfigureAwait(false); - if (count - lastProgressBytes >= 1024 * 1024) - { - progress($"downloaded {count / (1024 * 1024)} MiB" + (expectedLength.HasValue ? $" / {expectedLength.Value / (1024 * 1024)} MiB" : "")); - lastProgressBytes = count; - } - } - if (expectedLength.HasValue && count != expectedLength.Value) - throw new InvalidDataException("The archive download was incomplete."); - } - await ValidateArchiveAsync(temporary, timeout.Token).ConfigureAwait(false); - await using var validated = File.OpenRead(temporary); - var stamp = new CacheStamp(validated.Length, Convert.ToHexString(await SHA256.HashDataAsync(validated, timeout.Token).ConfigureAwait(false)), DateTime.UtcNow); - validated.Close(); - File.Move(temporary, target, overwrite: true); - await File.WriteAllTextAsync(temporary, JsonSerializer.Serialize(stamp), timeout.Token).ConfigureAwait(false); - File.Move(temporary, target + ".json", overwrite: true); - return new Lod2AtomCachedTile(tile, target, false); - } - catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) - { - throw new TimeoutException("The ATOM archive download timed out after three minutes."); - } - finally - { - if (File.Exists(temporary)) File.Delete(temporary); - } - } - - private static async Task AcquireLockAsync(string path, CancellationToken cancellationToken) - { - while (true) - { - cancellationToken.ThrowIfCancellationRequested(); - try { return new FileStream(path, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None); } - catch (IOException) { await Task.Delay(150, cancellationToken).ConfigureAwait(false); } - } - } + public Task DownloadFileAsync(Lod2AtomLink tile, CancellationToken cancellationToken = default) => + _downloads.DownloadAsync(tile, cancellationToken: cancellationToken); - private static async Task IsValidCacheAsync(string path, string revision, CancellationToken cancellationToken) + public async Task TryReadReferenceAsync(string source, CancellationToken cancellationToken = default) { - try - { - if (!File.Exists(path) || !File.Exists(path + ".json")) return false; - var stamp = JsonSerializer.Deserialize(await File.ReadAllTextAsync(path + ".json", cancellationToken).ConfigureAwait(false)); - if (stamp is null || new FileInfo(path).Length != stamp.Length || - (string.IsNullOrWhiteSpace(revision) && DateTime.UtcNow - stamp.DownloadedUtc > TimeSpan.FromDays(1))) return false; - await using var stream = File.OpenRead(path); - return Convert.ToHexString(await SHA256.HashDataAsync(stream, cancellationToken).ConfigureAwait(false)) == stamp.Hash; - } - catch (Exception ex) when (ex is IOException or JsonException or InvalidDataException) { return false; } + var feed = await LoadFeedAsync(new Uri(source), true, cancellationToken).ConfigureAwait(false); + if (feed is null) return null; + var root = feed.Value.Document.Root!; + var extents = root.Elements(Lod2AtomFeedReader.Atom + "entry").Prepend(root) + .Select(Lod2AtomFeedReader.ReadExtent).Where(e => e is not null).Cast().ToList(); + if (extents.Count == 0) throw new InvalidDataException("The ATOM feed has no readable coverage extent; use its service-level feed or select an area manually."); + var boxes = extents.Select(e => Lod2AtomFeedReader.TransformBounds(e, "EPSG:4326")).ToList(); + var wgs = new BoundingBox2D(boxes.Min(b => b.MinX), boxes.Min(b => b.MinY), boxes.Max(b => b.MaxX), boxes.Max(b => b.MaxY)); + var codes = root.Elements(Lod2AtomFeedReader.Atom + "entry").Prepend(root) + .Select(Lod2AtomFeedReader.ReadCategorySrs).Where(s => s is not null).Distinct().ToList(); + var srs = codes.Count == 1 ? codes[0]! : "EPSG:4326"; + var native = Lod2AtomFeedReader.TransformBounds(new AtomSpatialExtent("EPSG:4326", wgs), srs); + return new ReferenceSourceMetadata("ATOM", srs, native, wgs, extents.Count); } +} - private static async Task ValidateArchiveAsync(string path, CancellationToken cancellationToken) - { - using var archive = ZipFile.OpenRead(path); - if (archive.Entries.Count > 10000) throw new InvalidDataException("ZIP contains too many entries."); - long expandedBytes = 0; - var buildingFiles = 0; - foreach (var entry in archive.Entries) - { - if (entry.Length > 256L * 1024 * 1024 || (expandedBytes += entry.Length) > MaximumExpandedBytes) - throw new InvalidDataException("ZIP exceeds the supported expanded-size limit (256 MiB per file, 1 GiB total)."); - var extension = Path.GetExtension(entry.FullName).ToLowerInvariant(); - if (extension is not (".gml" or ".citygml" or ".xml")) continue; - await using var stream = entry.Open(); - using var reader = XmlReader.Create(stream, new XmlReaderSettings - { - Async = true, DtdProcessing = DtdProcessing.Prohibit, XmlResolver = null, MaxCharactersInDocument = 256L * 1024 * 1024 - }); - await reader.MoveToContentAsync().ConfigureAwait(false); - if (reader.LocalName == "CityModel" && reader.NamespaceURI.StartsWith("http://www.opengis.net/citygml", StringComparison.Ordinal)) - { - buildingFiles++; - // Verify XML and decompression through EOF before publishing a completed cache file. - while (await reader.ReadAsync().ConfigureAwait(false)) cancellationToken.ThrowIfCancellationRequested(); - } - } - if (buildingFiles == 0) - throw new InvalidDataException("Unsupported or invalid archive: expected a ZIP containing CityGML building data. Other archive formats and remote CityJSON downloads are not supported."); - } +// Keep the original building client API available for existing callers. +public sealed class Lod2AtomClient : AtomDownloadClient +{ + public Lod2AtomClient(HttpClient? httpClient = null, string? cacheDirectory = null) : base(httpClient, cacheDirectory) { } } diff --git a/RhinoSpatial.Core/Lod2AtomFeedReader.cs b/RhinoSpatial.Core/Lod2AtomFeedReader.cs index 126aa87..0335c73 100644 --- a/RhinoSpatial.Core/Lod2AtomFeedReader.cs +++ b/RhinoSpatial.Core/Lod2AtomFeedReader.cs @@ -6,7 +6,7 @@ namespace RhinoSpatial.Core; public record AtomSpatialExtent(string SrsName, BoundingBox2D Bounds); -public record Lod2AtomLink(Uri Url, string Title, string Revision, AtomSpatialExtent? Extent); +public record Lod2AtomLink(Uri Url, string Title, string Revision, AtomSpatialExtent? Extent, string NativeSrs = ""); public record Lod2AtomFeed(IReadOnlyList Feeds, IReadOnlyList Downloads); /// Reads download metadata only; never opens building archives to locate tiles. @@ -15,7 +15,7 @@ public static class Lod2AtomFeedReader public static readonly XNamespace Atom = "http://www.w3.org/2005/Atom"; private static readonly XNamespace GeoRss = "http://www.georss.org/georss"; - public static Lod2AtomFeed Read(XDocument document, Uri source, AtomSpatialExtent? inheritedExtent = null) + public static Lod2AtomFeed Read(XDocument document, Uri source, AtomSpatialExtent? inheritedExtent = null, AtomDownloadKind kind = AtomDownloadKind.Buildings) { var root = document.Root; if (root?.Name != Atom + "feed") @@ -38,16 +38,16 @@ public static Lod2AtomFeed Read(XDocument document, Uri source, AtomSpatialExten foreach (var link in links) { var contentType = ((string?)link.Attribute("type") ?? "").Split(';')[0].Trim(); - if (contentType.Equals("text/csv", StringComparison.OrdinalIgnoreCase) || - contentType.StartsWith("image/", StringComparison.OrdinalIgnoreCase)) - throw new InvalidOperationException($"This ATOM download advertises '{contentType}', not CityGML building data. Load LoD2 Buildings requires a CityGML download feed. Terrain ATOM feeds are not supported; use local terrain files or a WCS source with Load Terrain."); + if (kind == AtomDownloadKind.Buildings && (contentType.Equals("text/csv", StringComparison.OrdinalIgnoreCase) || + contentType.StartsWith("image/", StringComparison.OrdinalIgnoreCase))) + throw new InvalidOperationException($"This ATOM download advertises '{contentType}', not CityGML building data. Load LoD2 Buildings requires a CityGML download feed. Use the terrain download feed with Load Terrain instead."); var url = ResolveUrl(link, source); var tileExtent = ReadExtent(link); if (tileExtent is null && links.Count == 1) tileExtent = extent; if (tileExtent is null && gridSize.HasValue && srs is not null) tileExtent = ReadDeclaredGridTile(url, srs, gridSize.Value, coverage); - downloads.Add(CreateLink(link, entry, url, tileExtent)); + downloads.Add(CreateLink(link, entry, url, tileExtent) with { NativeSrs = srs ?? "" }); } } @@ -83,7 +83,7 @@ private static bool IsDownloadLink(XElement link) var type = ((string?)link.Attribute("type") ?? "").ToLowerInvariant(); var href = ((string?)link.Attribute("href") ?? "").Split('?', '#')[0]; return (relation == "enclosure" && !type.Contains("atom+xml")) || type.Contains("zip") || type.Contains("gmz") || type.Contains("7z") || - new[] { ".zip", ".gmz", ".7z", ".gz", ".tar", ".rar" }.Any(extension => href.EndsWith(extension, StringComparison.OrdinalIgnoreCase)); + new[] { ".zip", ".gmz", ".7z", ".gz", ".tar", ".rar", ".cityjson", ".jsonl", ".json", ".geojson", ".gml", ".citygml", ".asc", ".xyz", ".csv", ".tif", ".tiff" }.Any(extension => href.EndsWith(extension, StringComparison.OrdinalIgnoreCase)); } public static Uri ResolveUrl(XElement link, Uri source) @@ -100,7 +100,7 @@ public static Uri ResolveUrl(XElement link, Uri source) return new UriBuilder(uri) { Fragment = "" }.Uri; } - private static string? ReadCategorySrs(XElement owner) + public static string? ReadCategorySrs(XElement owner) { var codes = owner.Elements(Atom + "category") .Select(category => Regex.Match((string?)category.Attribute("term") ?? "", @"EPSG(?::|/)(?:[^:/]*[:/])?(\d+)$", RegexOptions.IgnoreCase)) @@ -135,7 +135,7 @@ public static Uri ResolveUrl(XElement link, Uri source) var bounds = new BoundingBox2D(x, y, x + size, y + size); var extent = new AtomSpatialExtent(srs, bounds); var coverageBounds = TransformBounds(coverage, srs); - if (!Intersects(bounds, coverageBounds)) + if (!Intersects(bounds, new BoundingBox2D(coverageBounds.MinX - size, coverageBounds.MinY - size, coverageBounds.MaxX + size, coverageBounds.MaxY + size))) throw new InvalidOperationException($"The declared ATOM grid and coverage disagree for '{url.LocalPath}'. Tile placement cannot be determined safely."); return extent; } diff --git a/RhinoSpatial.Core/Models.cs b/RhinoSpatial.Core/Models.cs index 87ce868..7062234 100644 --- a/RhinoSpatial.Core/Models.cs +++ b/RhinoSpatial.Core/Models.cs @@ -125,7 +125,7 @@ public record TerrainRasterData( Coordinate2D OffsetVectorY, double? NoDataValue, float[] Elevations - ); + ) { public RasterGeoTransform? GeoTransform { get; init; } } public record WcsCoverageResult( string RequestUrl, @@ -142,7 +142,7 @@ public record GeoReferencedRasterInfo( int Height, BoundingBox2D BoundingBox, long FileSizeBytes - ); + ) { public RasterGeoTransform? GeoTransform { get; init; } } public record OsmAreaFeature( long Id, diff --git a/RhinoSpatial.Core/OgcApiFeaturesClient.cs b/RhinoSpatial.Core/OgcApiFeaturesClient.cs index 928ae8d..c2f0c92 100644 --- a/RhinoSpatial.Core/OgcApiFeaturesClient.cs +++ b/RhinoSpatial.Core/OgcApiFeaturesClient.cs @@ -1,138 +1,186 @@ -using System; -using System.Collections.Generic; -using System.Net.Http; +using System.Collections.Concurrent; +using System.Globalization; using System.Net.Http.Headers; -using System.Text; -using System.Threading.Tasks; +using System.Text.Json; -namespace RhinoSpatial.Core +namespace RhinoSpatial.Core; + +public record OgcFeatureCollection(string Id, string Title, string ItemsUrl, BoundingBox2D? Wgs84Bounds); +public record OgcFeatureResult(List Features, string StatusNote); + +public sealed class OgcApiFeaturesClient { - public class OgcApiFeaturesClient + private static readonly HttpClient SharedHttpClient = new() { Timeout = TimeSpan.FromSeconds(30) }; + private static readonly ConcurrentDictionary? Collections)> DiscoveryCache = new(); + private readonly HttpClient _http; + + static OgcApiFeaturesClient() { - private static readonly HttpClient SharedHttpClient = new() - { - Timeout = TimeSpan.FromSeconds(30) - }; + SharedHttpClient.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("RhinoSpatial", "1.0")); + SharedHttpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); + SharedHttpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/geo+json")); + } - static OgcApiFeaturesClient() - { - SharedHttpClient.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("RhinoSpatial", "1.0")); - SharedHttpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/geo+json")); - SharedHttpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); - SharedHttpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("*/*")); - } + public OgcApiFeaturesClient() : this(null) { } + public OgcApiFeaturesClient(HttpClient? httpClient) => _http = httpClient ?? SharedHttpClient; - public async Task> LoadFeaturesAsync( - string sourceUrl, - string sourceLayerName, - BoundingBox2D? wgs84BoundingBox, - int maxFeatures) + public async Task?> TryDiscoverAsync(string sourceUrl, CancellationToken cancellationToken = default) + { + if (!Uri.TryCreate(sourceUrl, UriKind.Absolute, out var source) || source.Scheme is not ("http" or "https")) return null; + if (System.Text.RegularExpressions.Regex.IsMatch(sourceUrl, @"[?&]service=(?:wfs|wms|wcs)(?:&|$)", System.Text.RegularExpressions.RegexOptions.IgnoreCase)) return null; + if (ReferenceEquals(_http, SharedHttpClient) && DiscoveryCache.TryGetValue(sourceUrl, out var cached) && cached.Expires > DateTime.UtcNow) + return cached.Collections; + if (source.AbsolutePath.TrimEnd('/').EndsWith("/items", StringComparison.OrdinalIgnoreCase)) + return new[] { new OgcFeatureCollection(CollectionName(source), CollectionName(source), sourceUrl, null) }; + using var timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + timeout.CancelAfter(TimeSpan.FromSeconds(45)); + var queue = new Queue(); queue.Enqueue(sourceUrl); + var visited = new HashSet(StringComparer.Ordinal); + var collections = new Dictionary(StringComparer.Ordinal); + var recognized = false; + while (queue.TryDequeue(out var url)) { - var requestUrl = BuildItemsRequestUrl(sourceUrl, wgs84BoundingBox, maxFeatures); - var response = await SharedHttpClient.GetAsync(requestUrl).ConfigureAwait(false); - var body = await response.Content.ReadAsStringAsync().ConfigureAwait(false); - - if (!response.IsSuccessStatusCode) + if (!visited.Add(url)) continue; + if (visited.Count > 32) throw new InvalidDataException("OGC API discovery exceeded 32 metadata pages."); + (string Text, Uri Url, string MediaType) response; + try { response = await SourceHttp.ReadAsync(_http, url, timeout.Token).ConfigureAwait(false); } + catch (HttpRequestException) when (!recognized && !LooksLikeOgcApiFeaturesUrl(sourceUrl)) { return Store(null); } + if (!response.Text.TrimStart().StartsWith("{", StringComparison.Ordinal)) { - throw new InvalidOperationException( - $"The OGC API Features endpoint returned {(int)response.StatusCode} {response.ReasonPhrase}. URL: {requestUrl}"); + if (recognized) throw new InvalidDataException("An OGC API metadata link did not return JSON."); + return Store(null); } - - return GeoJsonReader.ReadFeatures(body, sourceLayerName); - } - - public static bool LooksLikeOgcApiFeaturesUrl(string? sourceUrl) - { - if (string.IsNullOrWhiteSpace(sourceUrl) || - !Uri.TryCreate(sourceUrl.Trim(), UriKind.Absolute, out var uri)) + using var document = JsonDocument.Parse(response.Text); + var root = document.RootElement; + if (root.TryGetProperty("type", out var type) && type.GetString() == "FeatureCollection") { - return false; + if (!LooksLikeOgcApiFeaturesUrl(sourceUrl)) return Store(null); + return Store(new[] { new OgcFeatureCollection(CollectionName(source), CollectionName(source), sourceUrl, null) }); } - - var path = uri.AbsolutePath; - return path.Contains("/collections/", StringComparison.OrdinalIgnoreCase) || - path.Contains("/items", StringComparison.OrdinalIgnoreCase); - } - - private static string BuildItemsRequestUrl( - string sourceUrl, - BoundingBox2D? wgs84BoundingBox, - int maxFeatures) - { - var builder = new UriBuilder(sourceUrl.Trim()); - if (!builder.Path.Contains("/items", StringComparison.OrdinalIgnoreCase) && - builder.Path.Contains("/collections/", StringComparison.OrdinalIgnoreCase)) + if (root.TryGetProperty("collections", out var entries) && entries.ValueKind == JsonValueKind.Array) { - builder.Path = builder.Path.TrimEnd('/') + "/items"; + recognized = true; + foreach (var collection in entries.EnumerateArray()) AddCollection(collection, response.Url, false); + var next = Link(root, "next", response.Url); + if (next is not null) queue.Enqueue(next); } - - var query = ParseQuery(builder.Query); - query["f"] = "json"; - if (wgs84BoundingBox is not null) + else if (root.TryGetProperty("id", out _) && + (Link(root, "items", response.Url) is not null || LooksLikeOgcApiFeaturesUrl(sourceUrl))) { - query["bbox"] = string.Join( - ",", - FormatNumber(wgs84BoundingBox.MinX), - FormatNumber(wgs84BoundingBox.MinY), - FormatNumber(wgs84BoundingBox.MaxX), - FormatNumber(wgs84BoundingBox.MaxY)); + recognized = true; AddCollection(root, response.Url, true); } + else if (Link(root, "data", response.Url) is { } data) + { recognized = true; queue.Enqueue(data); } + else if (!recognized) return Store(null); + if (collections.Count > 1000) throw new InvalidDataException("OGC API discovery exceeded 1,000 collections; use a specific collection URL."); + } + return Store(recognized ? collections.Values.ToList() : null); - if (maxFeatures > 0) + IReadOnlyList? Store(IReadOnlyList? value) + { + if (ReferenceEquals(_http, SharedHttpClient)) { - query["limit"] = maxFeatures.ToString(System.Globalization.CultureInfo.InvariantCulture); + if (DiscoveryCache.Count > 128) DiscoveryCache.Clear(); + DiscoveryCache[sourceUrl] = (DateTime.UtcNow.AddMinutes(5), value); } - - builder.Query = BuildQuery(query); - return builder.Uri.ToString(); + return value; } - private static Dictionary ParseQuery(string query) + void AddCollection(JsonElement value, Uri baseUrl, bool isSingle) { - var values = new Dictionary(StringComparer.OrdinalIgnoreCase); - var trimmed = query.TrimStart('?'); - if (string.IsNullOrWhiteSpace(trimmed)) + if (!value.TryGetProperty("id", out var idElement) || idElement.ValueKind != JsonValueKind.String) return; + if (value.TryGetProperty("itemType", out var itemType) && itemType.GetString() != "feature") return; + var id = idElement.GetString()!; + var self = Link(value, "self", baseUrl); + var items = Link(value, "items", baseUrl); + if (items is null) { - return values; + var builder = new UriBuilder(self ?? baseUrl.AbsoluteUri); + if (self is null && !isSingle) builder.Path = builder.Path.TrimEnd('/') + "/" + Uri.EscapeDataString(id); + builder.Path = builder.Path.TrimEnd('/') + "/items"; + items = builder.Uri.AbsoluteUri; } - - foreach (var part in trimmed.Split('&', StringSplitOptions.RemoveEmptyEntries)) + BoundingBox2D? bounds = null; + if (value.TryGetProperty("extent", out var extent) && extent.TryGetProperty("spatial", out var spatial) && + (!spatial.TryGetProperty("crs", out var crs) || crs.GetString()?.Contains("CRS84", StringComparison.OrdinalIgnoreCase) == true) && + spatial.TryGetProperty("bbox", out var boxes) && boxes.ValueKind == JsonValueKind.Array) { - var separatorIndex = part.IndexOf('=', StringComparison.Ordinal); - if (separatorIndex < 0) + foreach (var box in boxes.EnumerateArray()) { - values[Uri.UnescapeDataString(part)] = string.Empty; - continue; + if (box.ValueKind != JsonValueKind.Array || box.GetArrayLength() is not (4 or 6)) continue; + if (box.EnumerateArray().Any(v => v.ValueKind != JsonValueKind.Number)) continue; + var n = box.GetArrayLength() / 2; + var b = new BoundingBox2D(box[0].GetDouble(), box[1].GetDouble(), box[n].GetDouble(), box[n + 1].GetDouble()); + if (!double.IsFinite(b.MinX) || !double.IsFinite(b.MaxX) || !double.IsFinite(b.MinY) || !double.IsFinite(b.MaxY) || + b.MinX < -180 || b.MinX > 180 || b.MaxX < -180 || b.MaxX > 180 || b.MinY < -90 || b.MaxY > 90 || b.MinY > b.MaxY) continue; + // A wrapped collection extent needs a conservative world-width map view. + if (b.MinX > b.MaxX) b = new BoundingBox2D(-180, b.MinY, 180, b.MaxY); + bounds = bounds is null ? b : new BoundingBox2D(Math.Min(bounds.MinX, b.MinX), Math.Min(bounds.MinY, b.MinY), Math.Max(bounds.MaxX, b.MaxX), Math.Max(bounds.MaxY, b.MaxY)); } - - values[Uri.UnescapeDataString(part[..separatorIndex])] = - Uri.UnescapeDataString(part[(separatorIndex + 1)..]); } - - return values; + collections[id] = new OgcFeatureCollection(id, value.TryGetProperty("title", out var title) ? title.GetString() ?? id : id, items, bounds); } + } - private static string BuildQuery(IReadOnlyDictionary values) - { - var builder = new StringBuilder(); - foreach (var entry in values) - { - if (builder.Length > 0) - { - builder.Append('&'); - } + public async Task> LoadFeaturesAsync(string sourceUrl, string sourceLayerName, BoundingBox2D? wgs84BoundingBox, int maxFeatures) => + (await LoadFeaturesWithStatusAsync(sourceUrl, sourceLayerName, wgs84BoundingBox, maxFeatures).ConfigureAwait(false)).Features; - builder.Append(Uri.EscapeDataString(entry.Key)); - builder.Append('='); - builder.Append(Uri.EscapeDataString(entry.Value)); - } + public async Task LoadFeaturesWithStatusAsync(string sourceUrl, string sourceLayerName, + BoundingBox2D? wgs84BoundingBox, int maxFeatures, CancellationToken cancellationToken = default) + { + var collections = await TryDiscoverAsync(sourceUrl, cancellationToken).ConfigureAwait(false) + ?? throw new InvalidDataException("The URL is not an OGC API Features source."); + var collection = collections.FirstOrDefault(c => c.Id == sourceLayerName) ?? + (collections.Count == 1 ? collections[0] : throw new InvalidOperationException("Select an OGC API collection using List WFS Layers and the Layer input.")); + var url = BuildItemsRequestUrl(collection.ItemsUrl, wgs84BoundingBox, maxFeatures); + var result = await FeatureCollectionPager.LoadHttpAsync(url, + async (pageUrl, token) => { var page = await SourceHttp.ReadAsync(_http, pageUrl, token).ConfigureAwait(false); return new FeaturePageContent(page.Text, page.Url); }, + maxFeatures, cancellationToken: cancellationToken).ConfigureAwait(false); + var returnedSrs = GeoJsonReader.TryReadSourceSrs(result.Text); + if (returnedSrs is not null && SpatialReferenceTransform.NormalizeSrs(returnedSrs) != "EPSG:4326") + throw new InvalidDataException("The OGC API returned a non-default GeoJSON CRS despite a longitude/latitude request."); + return new OgcFeatureResult(GeoJsonReader.ReadFeatures(result.Text, collection.Id), result.StatusNote); + } - return builder.ToString(); - } + public static bool LooksLikeOgcApiFeaturesUrl(string? sourceUrl) => + Uri.TryCreate(sourceUrl, UriKind.Absolute, out var uri) && + (uri.AbsolutePath.Contains("/collections/", StringComparison.OrdinalIgnoreCase) || + uri.AbsolutePath.TrimEnd('/').EndsWith("/collections", StringComparison.OrdinalIgnoreCase) || + uri.AbsolutePath.Contains("/items", StringComparison.OrdinalIgnoreCase)); + + private static string CollectionName(Uri uri) + { + var parts = uri.AbsolutePath.Split('/', StringSplitOptions.RemoveEmptyEntries); + var index = Array.FindIndex(parts, p => p == "collections"); + return index >= 0 && index + 1 < parts.Length ? Uri.UnescapeDataString(parts[index + 1]) : "features"; + } - private static string FormatNumber(double value) + private static string? Link(JsonElement owner, string relation, Uri source) + { + if (!owner.TryGetProperty("links", out var links) || links.ValueKind != JsonValueKind.Array) return null; + var matches = links.EnumerateArray().Where(l => l.TryGetProperty("rel", out var rel) && rel.GetString() == relation && l.TryGetProperty("href", out _)) + .OrderByDescending(l => l.TryGetProperty("type", out var type) && type.GetString()?.Contains("json", StringComparison.OrdinalIgnoreCase) == true); + foreach (var link in matches) { - return value.ToString("G17", System.Globalization.CultureInfo.InvariantCulture); + var result = new Uri(source, link.GetProperty("href").GetString()!); + if (result.Scheme is not ("http" or "https") || !string.IsNullOrEmpty(result.UserInfo)) throw new InvalidDataException("Unsupported OGC API link URL."); + return result.AbsoluteUri; } + return null; + } + + private static string BuildItemsRequestUrl(string sourceUrl, BoundingBox2D? bounds, int maxFeatures) + { + var builder = new UriBuilder(sourceUrl); + var query = builder.Query.TrimStart('?').Split('&', StringSplitOptions.RemoveEmptyEntries) + .Select(p => p.Split('=', 2)).ToDictionary(p => Uri.UnescapeDataString(p[0]), p => p.Length == 2 ? Uri.UnescapeDataString(p[1]) : "", StringComparer.OrdinalIgnoreCase); + var pageSize = query.TryGetValue("limit", out var limit) && int.TryParse(limit, out var n) && n > 0 ? Math.Min(n, FeatureCollectionPager.PageSize) : FeatureCollectionPager.PageSize; + query["f"] = "json"; + query["limit"] = (maxFeatures > 0 ? Math.Min(maxFeatures, pageSize) : pageSize).ToString(CultureInfo.InvariantCulture); + // This client requests GeoJSON in its default longitude/latitude CRS. + query.Remove("crs"); query.Remove("bbox-crs"); + if (bounds is not null) query["bbox"] = string.Join(",", new[] { bounds.MinX, bounds.MinY, bounds.MaxX, bounds.MaxY }.Select(x => x.ToString("G17", CultureInfo.InvariantCulture))); + builder.Query = string.Join("&", query.Select(p => Uri.EscapeDataString(p.Key) + "=" + Uri.EscapeDataString(p.Value))); + return builder.Uri.AbsoluteUri; } } diff --git a/RhinoSpatial.Core/RasterGeoTransform.cs b/RhinoSpatial.Core/RasterGeoTransform.cs new file mode 100644 index 0000000..76566f8 --- /dev/null +++ b/RhinoSpatial.Core/RasterGeoTransform.cs @@ -0,0 +1,67 @@ +using System.Globalization; +using System.Security.Cryptography; + +namespace RhinoSpatial.Core; + +/// Maps pixel corners to source coordinates; sample centers are at (col+.5,row+.5). +public record RasterGeoTransform(double A, double B, double C, double D, double E, double F) +{ + public Coordinate2D At(double column, double row) => new(A * column + B * row + C, D * column + E * row + F); + public void Validate() + { + if (new[] { A, B, C, D, E, F }.Any(v => !double.IsFinite(v)) || !double.IsFinite(A * E - B * D) || A * E - B * D == 0 || Math.Abs(A * E - B * D) <= 1e-12 * (Math.Abs(A * E) + Math.Abs(B * D))) + throw new InvalidDataException("Raster georeferencing must be finite and invertible."); + } + public Coordinate2D Pixel(double x, double y) + { var det = A * E - B * D; return new((E * (x - C) - B * (y - F)) / det, (-D * (x - C) + A * (y - F)) / det); } + public BoundingBox2D Bounds(int width, int height) + { + var p = new[] { At(0, 0), At(width, 0), At(width, height), At(0, height) }; + return new(p.Min(p => p.X), p.Min(p => p.Y), p.Max(p => p.X), p.Max(p => p.Y)); + } + public static RasterGeoTransform FromWorldFile(string path) + { + var values = File.ReadAllLines(path).Where(l => !string.IsNullOrWhiteSpace(l)).Select(l => double.Parse(l, NumberStyles.Float, CultureInfo.InvariantCulture)).ToArray(); + if (values.Length != 6) throw new InvalidDataException("World file must contain six numeric lines."); + var g = new RasterGeoTransform(values[0], values[2], values[4] - (values[0] + values[2]) / 2, values[1], values[3], values[5] - (values[1] + values[3]) / 2); + g.Validate(); return g; + } +} + +public static class RasterSidecars +{ + public static IEnumerable Files(string raster) + { + var directory = Path.GetDirectoryName(Path.GetFullPath(raster))!; + var ext = Path.GetExtension(raster).TrimStart('.'); + var stem = Path.GetFileNameWithoutExtension(raster); + var names = ext.Length == 0 ? new[] { stem + ".wld", stem + ".prj" } : new[] { stem + "." + ext[0] + ext[^1] + "w", stem + "." + ext + "w", stem + ".wld", stem + ".prj" }; + var files = Directory.EnumerateFiles(directory, stem + ".*", new EnumerationOptions { MatchCasing = MatchCasing.CaseInsensitive }).ToList(); + foreach (var name in names) + { + var file = files.FirstOrDefault(f => Path.GetFileName(f).Equals(name, StringComparison.OrdinalIgnoreCase)); + if (file is not null) yield return file; + } + } + public static string Signature(string raster) => string.Join("|", Files(raster).Select(p => + { + if (new FileInfo(p).Length > 65536) return p + ":" + new FileInfo(p).Length + ":" + File.GetLastWriteTimeUtc(p).Ticks; + return p + ":" + Convert.ToHexString(SHA256.HashData(File.ReadAllBytes(p))); + })); + public static RasterGeoTransform ReadTransform(string raster) + { + var world = Files(raster).FirstOrDefault(p => !p.EndsWith(".prj", StringComparison.OrdinalIgnoreCase)); + if (world is null) throw new InvalidDataException("Raster requires readable embedded georeferencing or a world file."); + if (new FileInfo(world).Length > 65536) throw new InvalidDataException("World file exceeds 64 KiB."); + return RasterGeoTransform.FromWorldFile(world); + } + public static string ReadSrs(string raster) + { + var prj = Files(raster).FirstOrDefault(p => p.EndsWith(".prj", StringComparison.OrdinalIgnoreCase)); + if (prj is null) throw new InvalidDataException("Raster requires an embedded CRS or a resolvable .prj; no context CRS was assumed."); + if (new FileInfo(prj).Length > 65536) throw new InvalidDataException("Raster .prj exceeds 64 KiB."); + var srs = ShapefileFeatureReader.NormalizeKnownProjection(File.ReadAllText(prj)); + return string.IsNullOrWhiteSpace(srs) ? throw new InvalidDataException("Raster .prj CRS is unsupported.") : srs; + } + public static (RasterGeoTransform Transform, string Srs) Read(string raster) => (ReadTransform(raster), ReadSrs(raster)); +} diff --git a/RhinoSpatial.Core/RasterImageCodec.cs b/RhinoSpatial.Core/RasterImageCodec.cs new file mode 100644 index 0000000..5161cc4 --- /dev/null +++ b/RhinoSpatial.Core/RasterImageCodec.cs @@ -0,0 +1,37 @@ +using StbImageNext; +using System.Buffers.Binary; +using System.IO.Compression; +using System.Text; + +namespace RhinoSpatial.Core; + +public static class RasterImageCodec +{ + public static byte[] DecodeRgba(byte[] bytes, int width, int height) + { + using var input = new MemoryStream(bytes); + var info = ImageInfo.FromStream(input); + if (info.Width != width || info.Height != height || (long)width * height > 4194304) throw new InvalidDataException("Tile dimensions disagree with its declared matrix."); + return ImageResult.FromMemory(bytes, ColorComponents.RedGreenBlueAlpha).Data; + } + public static void WritePng(string path, int width, int height, byte[] rgba) + { + if (width <= 0 || height <= 0 || (long)width * height * 4 != rgba.Length) throw new InvalidDataException("Invalid RGBA image dimensions."); + using var output = File.Create(path); + output.Write(new byte[] { 137, 80, 78, 71, 13, 10, 26, 10 }); + var header = new byte[13]; BinaryPrimitives.WriteInt32BigEndian(header.AsSpan(0, 4), width); BinaryPrimitives.WriteInt32BigEndian(header.AsSpan(4, 4), height); header[8] = 8; header[9] = 6; + Chunk(output, "IHDR", header); + using var compressed = new MemoryStream(); + using (var zlib = new ZLibStream(compressed, CompressionLevel.Fastest, true)) + for (var row = 0; row < height; row++) { zlib.WriteByte(0); zlib.Write(rgba, row * width * 4, width * 4); } + Chunk(output, "IDAT", compressed.ToArray()); Chunk(output, "IEND", Array.Empty()); + } + private static void Chunk(Stream stream, string name, byte[] bytes) + { + Span length = stackalloc byte[4]; BinaryPrimitives.WriteInt32BigEndian(length, bytes.Length); stream.Write(length); + var type = Encoding.ASCII.GetBytes(name); stream.Write(type); stream.Write(bytes); + uint crc = 0xffffffff; + foreach (var b in type.Concat(bytes)) { crc ^= b; for (var bit = 0; bit < 8; bit++) crc = (crc >> 1) ^ ((crc & 1) != 0 ? 0xedb88320u : 0); } + BinaryPrimitives.WriteUInt32BigEndian(length, ~crc); stream.Write(length); + } +} diff --git a/RhinoSpatial.Core/RasterImageMeshBuilder.cs b/RhinoSpatial.Core/RasterImageMeshBuilder.cs new file mode 100644 index 0000000..8409973 --- /dev/null +++ b/RhinoSpatial.Core/RasterImageMeshBuilder.cs @@ -0,0 +1,70 @@ +namespace RhinoSpatial.Core; + +public record RasterImageVertex(double X, double Y, double U, double V); +public record RasterImageMeshData(List Vertices, List<(int A, int B, int C)> Faces) { public double CoverageRatio { get; init; } } + +public static class RasterImageMeshBuilder +{ + public static RasterImageMeshData Build(GeoReferencedRasterInfo info, SpatialContext2D context, int divisions = 16) + { + if (divisions < 1 || divisions > 128) throw new ArgumentOutOfRangeException(nameof(divisions)); + var g = info.GeoTransform ?? new RasterGeoTransform((info.BoundingBox.MaxX - info.BoundingBox.MinX) / info.Width, 0, info.BoundingBox.MinX, 0, -(info.BoundingBox.MaxY - info.BoundingBox.MinY) / info.Height, info.BoundingBox.MaxY); + g.Validate(); + var selected=Lod2AtomFeedReader.TransformBounds(new(context.ResolvedSrs,context.PlacementBoundingBox),info.SrsName); + var pixelCorners=new[]{g.Pixel(selected.MinX,selected.MinY),g.Pixel(selected.MaxX,selected.MinY),g.Pixel(selected.MaxX,selected.MaxY),g.Pixel(selected.MinX,selected.MaxY)}; + var u0=Math.Clamp(pixelCorners.Min(p=>p.X)/info.Width,0,1);var u1=Math.Clamp(pixelCorners.Max(p=>p.X)/info.Width,0,1); + var r0=Math.Clamp(pixelCorners.Min(p=>p.Y)/info.Height,0,1);var r1=Math.Clamp(pixelCorners.Max(p=>p.Y)/info.Height,0,1); + if(u1<=u0 || r1<=r0)return new(new(),new()); + var grid = new RasterImageVertex[(divisions + 1) * (divisions + 1)]; + for (var r = 0; r <= divisions; r++) for (var c = 0; c <= divisions; c++) + { + var uv = new Coordinate2D(u0+(u1-u0)*c/divisions,r0+(r1-r0)*r/divisions); var p = g.At(uv.X * info.Width, uv.Y * info.Height); + if (!SpatialReferenceTransform.TryTransformXY(info.SrsName, context.ResolvedSrs, p.X, p.Y, out var x, out var y) || !double.IsFinite(x) || !double.IsFinite(y)) + throw new InvalidDataException($"Cannot transform raster from '{info.SrsName}' into '{context.ResolvedSrs}'."); + grid[r * (divisions + 1) + c] = new(x, y, uv.X, 1 - uv.Y); + } + var vertices = new List(); var faces = new List<(int, int, int)>(); + var b = context.PlacementBoundingBox; double keptArea = 0; + for (var r = 0; r < divisions; r++) for (var c = 0; c < divisions; c++) + { + var i = r * (divisions + 1) + c; + // Triangles preserve an unambiguous UV interpolation when projection is nonlinear. + foreach (var triangle in new[] { new[] { grid[i], grid[i + 1], grid[i + divisions + 2] }, new[] { grid[i], grid[i + divisions + 2], grid[i + divisions + 1] } }) + { + var polygon = triangle.ToList(); + polygon = Clip(polygon, p => p.X - b.MinX); + polygon = Clip(polygon, p => b.MaxX - p.X); + polygon = Clip(polygon, p => p.Y - b.MinY); + polygon = Clip(polygon, p => b.MaxY - p.Y); + if (polygon.Count < 3) continue; + var start = vertices.Count; + vertices.AddRange(polygon.Select(p => p with { X = p.X - (context.UseAbsoluteCoordinates ? 0 : context.PlacementOrigin.X), Y = p.Y - (context.UseAbsoluteCoordinates ? 0 : context.PlacementOrigin.Y) })); + for (var n = 1; n < polygon.Count - 1; n++) + { + var a = polygon[0]; var d = polygon[n]; var e = polygon[n + 1]; + var cross = Cross(a, d, e); + if (Math.Abs(cross) > 1e-18) { keptArea += Math.Abs((d.U-a.U)*(e.V-a.V)-(d.V-a.V)*(e.U-a.U))/2; faces.Add(cross > 0 ? (start, start + n, start + n + 1) : (start, start + n + 1, start + n)); } + } + } + } + return new(vertices, faces) { CoverageRatio = Math.Clamp(keptArea,0,1) }; + } + private static double Cross(RasterImageVertex a, RasterImageVertex b, RasterImageVertex c) => (b.X - a.X) * (c.Y - a.Y) - (b.Y - a.Y) * (c.X - a.X); + private static List Clip(List input, Func distance) + { + var output = new List(); if (input.Count == 0) return output; + var previous = input[^1]; var a = distance(previous); + foreach (var point in input) + { + var b = distance(point); + if ((a >= 0) != (b >= 0)) + { + var t = a / (a - b); + output.Add(new(previous.X + t * (point.X - previous.X), previous.Y + t * (point.Y - previous.Y), previous.U + t * (point.U - previous.U), previous.V + t * (point.V - previous.V))); + } + if (b >= 0) output.Add(point); + previous = point; a = b; + } + return output; + } +} diff --git a/RhinoSpatial.Core/ReferenceSourceMetadataReader.cs b/RhinoSpatial.Core/ReferenceSourceMetadataReader.cs index 4f19cfd..b52f26e 100644 --- a/RhinoSpatial.Core/ReferenceSourceMetadataReader.cs +++ b/RhinoSpatial.Core/ReferenceSourceMetadataReader.cs @@ -19,8 +19,8 @@ public static class ReferenceSourceMetadataReader private const int MaximumDirectoryFileCount = 256; private static readonly string[] SupportedFileExtensions = { - ".tif", ".tiff", ".shp", ".geojson", ".json", ".cityjson", - ".gml", ".xml", ".asc", ".xyz", ".csv", ".zip" + ".tif", ".tiff", ".shp", ".geojson", ".json", ".cityjson", ".jsonl", ".png", ".jpg", ".jpeg", + ".gml", ".xml", ".asc", ".xyz", ".csv", ".zip", ".gpkg" }; public static bool IsLocalSource(string? source) @@ -118,9 +118,11 @@ private static ReferenceSourceMetadata ReadFile( var extension = Path.GetExtension(filePath).ToLowerInvariant(); return extension switch { - ".tif" or ".tiff" => ReadGeoTiff(filePath), + ".tif" or ".tiff" or ".png" or ".jpg" or ".jpeg" => ReadGeoTiff(filePath), + ".gpkg" => ReadGeoPackage(filePath), ".shp" => ReadShapefile(filePath, fallbackSrs), ".geojson" => ReadJson(File.ReadAllText(filePath), "GeoJSON", fallbackSrs), + ".jsonl" => ReadCityJsonSequence(filePath, fallbackSrs), ".json" or ".cityjson" => ReadJson(File.ReadAllText(filePath), "JSON", fallbackSrs), ".gml" or ".xml" => ReadGml(filePath, fallbackSrs), ".asc" or ".xyz" or ".csv" => ReadTerrainText(filePath, fallbackSrs), @@ -129,12 +131,27 @@ private static ReferenceSourceMetadata ReadFile( }; } + private static ReferenceSourceMetadata ReadCityJsonSequence(string filePath, string fallbackSrs) + { + using var stream = File.OpenRead(filePath); + var metadata = CityJsonReader.ReadSourceMetadata(stream); + return Create("CityJSONSeq", string.IsNullOrWhiteSpace(metadata.SrsName) ? fallbackSrs : metadata.SrsName, metadata.BoundingBox, 1); + } + private static ReferenceSourceMetadata ReadGeoTiff(string filePath) { var raster = GeoTiffInfoCache.GetOrRead(filePath); return Create("GeoTIFF", raster.SrsName, raster.BoundingBox, 1); } + private static ReferenceSourceMetadata ReadGeoPackage(string filePath) + { + var layers=GeoPackageReader.ReadLayers(filePath); + if(layers.Any(l=>string.IsNullOrWhiteSpace(l.SourceSrs)||l.Bounds is null||!TryTransformBoundsToWgs84(l.SourceSrs,l.Bounds,out _))) + throw new InvalidDataException("GeoPackage Reference Source requires transformable EPSG CRS and catalog extents for every feature layer; geometry is not scanned to infer missing metadata."); + return Combine(layers.Select(l=>Create("GeoPackage",l.SourceSrs,l.Bounds,1)),"GeoPackage"); + } + private static ReferenceSourceMetadata ReadShapefile( string filePath, string fallbackSrs) @@ -218,6 +235,7 @@ private static ReferenceSourceMetadata ReadZip( extension.Equals(".xml", StringComparison.OrdinalIgnoreCase) || extension.Equals(".json", StringComparison.OrdinalIgnoreCase) || extension.Equals(".cityjson", StringComparison.OrdinalIgnoreCase) || + extension.Equals(".jsonl", StringComparison.OrdinalIgnoreCase) || extension.Equals(".geojson", StringComparison.OrdinalIgnoreCase); }) .Take(MaximumDirectoryFileCount) @@ -243,6 +261,8 @@ private static ReferenceSourceMetadata ReadZip( continue; } + if(extension.Equals(".jsonl",StringComparison.OrdinalIgnoreCase)) + { var m=CityJsonReader.ReadSourceMetadata(stream);metadata.Add(Create("CityJSONSeq",string.IsNullOrWhiteSpace(m.SrsName)?fallbackSrs:m.SrsName,m.BoundingBox,1));continue; } using var reader = new StreamReader(stream); metadata.Add(ReadJson(reader.ReadToEnd(), "JSON", fallbackSrs)); } diff --git a/RhinoSpatial.Core/RemoteReferenceMetadataReader.cs b/RhinoSpatial.Core/RemoteReferenceMetadataReader.cs new file mode 100644 index 0000000..9c76ed3 --- /dev/null +++ b/RhinoSpatial.Core/RemoteReferenceMetadataReader.cs @@ -0,0 +1,67 @@ +using System.Collections.Concurrent; + +namespace RhinoSpatial.Core; + +/// Uses discovery metadata endpoints and rejects known data-file paths. +public sealed class RemoteReferenceMetadataReader +{ + private static readonly ConcurrentDictionary Cache = new(); + private readonly AtomDownloadClient _atom; + private readonly OgcApiFeaturesClient _ogc; + private readonly bool _cache; + private readonly WmtsClient _wmts; + private readonly CogReader _cog; + public RemoteReferenceMetadataReader(HttpClient? http = null) + { _atom = new AtomDownloadClient(http); _ogc = new OgcApiFeaturesClient(http); _cache = http is null; _wmts=new WmtsClient(http); _cog=new CogReader(http); } + + public async Task TryReadAsync(string source, string? layer = null, CancellationToken cancellationToken = default) + { + if (!Uri.TryCreate(source, UriKind.Absolute, out var uri) || uri.Scheme is not ("http" or "https")) return null; + if (System.Text.RegularExpressions.Regex.IsMatch(source, @"[?&]service=(wfs|wms|wcs)(?:&|$)", System.Text.RegularExpressions.RegexOptions.IgnoreCase)) return null; + if(CogReader.LooksLike(source)) + { + var cogKey="COG\n"+source; + if(_cache&&Cache.TryGetValue(cogKey,out var prior)&&prior.Expires>DateTime.UtcNow)return prior.Value; + var info=await _cog.ReadMetadataAsync(source,cancellationToken).ConfigureAwait(false); + var metadata=new ReferenceSourceMetadata("Remote COG",info.SrsName,info.BoundingBox, + ReferenceSourceMetadataReader.TryTransformBoundsToWgs84(info.SrsName,info.BoundingBox,out var wgs)?wgs:null,1); + if(_cache){if(Cache.Count>128)Cache.Clear();Cache[cogKey]=(DateTime.UtcNow.AddMinutes(5),metadata);} + return metadata; + } + if (new[] { ".zip", ".geojson", ".cityjson", ".jsonl", ".png", ".jpg", ".jpeg", ".tif", ".tiff", ".asc", ".xyz", ".csv", ".gml", ".citygml" } + .Contains(Path.GetExtension(uri.AbsolutePath), StringComparer.OrdinalIgnoreCase)) + throw new InvalidDataException("Use the provider's service/feed metadata URL as Reference Source, or a supported local file. Remote data files are not downloaded to orient the map."); + if(WmtsClient.LooksLike(source)) + { + var wmts=await _wmts.TryLoadCapabilitiesAsync(source,cancellationToken).ConfigureAwait(false); + if(wmts is null) throw new InvalidDataException("WMTS reference metadata was not found."); + var selected=string.IsNullOrWhiteSpace(layer)?wmts.Layers:wmts.Layers.Where(l=>l.Id==layer).ToList(); + if(selected.Count==0)throw new InvalidDataException("WMTS reference layer was not found."); + var extents=selected.Select(l=>l.Wgs84Bounds).Where(b=>b is not null).Cast().ToList(); + var bounds=extents.Count==0?null:new BoundingBox2D(extents.Min(b=>b.MinX),extents.Min(b=>b.MinY),extents.Max(b=>b.MaxX),extents.Max(b=>b.MaxY)); + return new ReferenceSourceMetadata("WMTS","EPSG:4326",bounds,bounds,selected.Count); + } + var key = source + "\n" + layer; + if (_cache && Cache.TryGetValue(key, out var cached) && cached.Expires > DateTime.UtcNow) return cached.Value; + // The parent collection holds metadata; /items would download feature data. + var metadataUrl = source; + if (uri.AbsolutePath.TrimEnd('/').EndsWith("/items", StringComparison.OrdinalIgnoreCase)) + { + var parent = new UriBuilder(uri); parent.Path = parent.Path.TrimEnd('/')[..^6]; parent.Query = "f=json"; + metadataUrl = parent.Uri.AbsoluteUri; + } + var collections = await _ogc.TryDiscoverAsync(metadataUrl, cancellationToken).ConfigureAwait(false); + ReferenceSourceMetadata? result; + if (collections is not null) + { + var selected = string.IsNullOrWhiteSpace(layer) ? collections : collections.Where(c => c.Id == layer).ToList(); + if (selected.Count == 0) throw new InvalidDataException("The selected OGC API reference collection was not found."); + var bounds = selected.Select(c => c.Wgs84Bounds).Where(b => b is not null).Cast().ToList(); + var union = bounds.Count == 0 ? null : new BoundingBox2D(bounds.Min(b => b.MinX), bounds.Min(b => b.MinY), bounds.Max(b => b.MaxX), bounds.Max(b => b.MaxY)); + result = new ReferenceSourceMetadata("OGC API Features", "EPSG:4326", union, union, selected.Count); + } + else result = await _atom.TryReadReferenceAsync(source, cancellationToken).ConfigureAwait(false); + if (_cache) { if (Cache.Count > 128) Cache.Clear(); Cache[key] = (DateTime.UtcNow.AddMinutes(5), result); } + return result; + } +} diff --git a/RhinoSpatial.Core/RhinoSpatial.Core.csproj b/RhinoSpatial.Core/RhinoSpatial.Core.csproj index 0abf7ad..4c38aad 100644 --- a/RhinoSpatial.Core/RhinoSpatial.Core.csproj +++ b/RhinoSpatial.Core/RhinoSpatial.Core.csproj @@ -9,6 +9,10 @@ + + + + diff --git a/RhinoSpatial.Core/ShapefileFeatureReader.cs b/RhinoSpatial.Core/ShapefileFeatureReader.cs index c0e407a..5021a0b 100644 --- a/RhinoSpatial.Core/ShapefileFeatureReader.cs +++ b/RhinoSpatial.Core/ShapefileFeatureReader.cs @@ -133,20 +133,20 @@ private static string ResolveSourceSrs(string shapefilePath, string fallbackSrs) var projectionText = File.ReadAllText(prjPath); var normalized = NormalizeKnownProjection(projectionText); - return string.IsNullOrWhiteSpace(normalized) ? fallbackSrs : normalized; + return string.IsNullOrWhiteSpace(normalized) + ? throw new InvalidOperationException("The Shapefile .prj contains an unsupported CRS; no context-CRS assumption was used.") : normalized; } - private static string NormalizeKnownProjection(string projectionText) + public static string NormalizeKnownProjection(string projectionText) { var text = projectionText ?? string.Empty; - var authorityMatch = Regex.Match( - text, - "AUTHORITY\\[\\s*\"EPSG\"\\s*,\\s*\"(?\\d+)\"\\s*\\]", - RegexOptions.IgnoreCase); - if (authorityMatch.Success) + try { - return $"EPSG:{authorityMatch.Groups["epsg"].Value}"; + var coordinateSystem = new ProjNet.CoordinateSystems.CoordinateSystemFactory().CreateFromWkt(text); + if (coordinateSystem.Authority?.Equals("EPSG", StringComparison.OrdinalIgnoreCase) == true && coordinateSystem.AuthorityCode > 0) + return "EPSG:" + coordinateSystem.AuthorityCode.ToString(CultureInfo.InvariantCulture); } + catch { /* Unsupported WKT dialects may still have a known, explicit CRS name below. */ } if (text.Contains("ETRS_1989_UTM_Zone_32N", StringComparison.OrdinalIgnoreCase) || text.Contains("ETRS89 / UTM zone 32N", StringComparison.OrdinalIgnoreCase)) @@ -166,8 +166,9 @@ private static string NormalizeKnownProjection(string projectionText) return "EPSG:3857"; } - if (text.Contains("WGS_1984", StringComparison.OrdinalIgnoreCase) || - text.Contains("WGS 84", StringComparison.OrdinalIgnoreCase)) + if ((text.TrimStart().StartsWith("GEOGCS[", StringComparison.OrdinalIgnoreCase) || + text.TrimStart().StartsWith("GEOGCRS[", StringComparison.OrdinalIgnoreCase)) && + (text.Contains("WGS_1984", StringComparison.OrdinalIgnoreCase) || text.Contains("WGS 84", StringComparison.OrdinalIgnoreCase))) { return "EPSG:4326"; } diff --git a/RhinoSpatial.Core/SourceDownloadCache.cs b/RhinoSpatial.Core/SourceDownloadCache.cs new file mode 100644 index 0000000..32796a3 --- /dev/null +++ b/RhinoSpatial.Core/SourceDownloadCache.cs @@ -0,0 +1,269 @@ +using System.Globalization; +using System.IO.Compression; +using System.Net.Http; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using System.Xml; + +namespace RhinoSpatial.Core; + +public enum AtomDownloadKind { Buildings, Terrain, Vector } + +internal sealed class LimitedReadStream : Stream +{ + private readonly Stream inner; + private readonly long maximum; + public LimitedReadStream(Stream stream, long limit) { inner = stream; maximum = limit; } + private long _read; + private int Count(int count) { _read += count; if (_read > maximum) throw new InvalidDataException("ZIP expanded beyond its declared entry size."); return count; } + public override int Read(byte[] buffer, int offset, int count) => Count(inner.Read(buffer, offset, count)); + public override int Read(Span buffer) => Count(inner.Read(buffer)); + public override async ValueTask ReadAsync(Memory buffer, CancellationToken token = default) => Count(await inner.ReadAsync(buffer, token).ConfigureAwait(false)); + public override Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken token) => ReadAsync(buffer.AsMemory(offset, count), token).AsTask(); + public override bool CanRead => true; + public override bool CanSeek => false; + public override bool CanWrite => false; + public override long Length => throw new NotSupportedException(); + public override long Position { get => _read; set => throw new NotSupportedException(); } + public override void Flush() => throw new NotSupportedException(); + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + public override void SetLength(long value) => throw new NotSupportedException(); + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + protected override void Dispose(bool disposing) { if (disposing) inner.Dispose(); base.Dispose(disposing); } +} + +/// Validated, atomic payload cache shared by direct downloads and ATOM services. +public sealed class SourceDownloadCache +{ + private static readonly HttpClient SharedHttp = new() { Timeout = Timeout.InfiniteTimeSpan }; + private static readonly string[] Extensions = { ".zip", ".geojson", ".cityjson", ".jsonl", ".gml", ".tif", ".asc", ".csv" }; + private readonly HttpClient _http; + private readonly string _directory; + private readonly AtomDownloadKind _kind; + private sealed record Stamp(long Length, string Hash, DateTime DownloadedUtc); + public SourceDownloadCache(AtomDownloadKind kind, HttpClient? http = null, string? directory = null) + { + _kind = kind; _http = http ?? SharedHttp; + _directory = directory ?? Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), + "RhinoSpatial", kind == AtomDownloadKind.Buildings ? "Lod2AtomCache" : kind == AtomDownloadKind.Terrain ? "TerrainAtomCache" : "VectorDownloadCache"); + } + + public async Task DownloadAsync(Lod2AtomLink tile, Action? progress = null, CancellationToken cancellationToken = default) + { + Directory.CreateDirectory(_directory); + var key = Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(tile.Url.AbsoluteUri + "\n" + tile.Revision))); + var stem = Path.Combine(_directory, key); + var temporary = stem + "." + Guid.NewGuid().ToString("N") + ".partial"; + using var timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + timeout.CancelAfter(TimeSpan.FromMinutes(3)); + try + { + await using var cacheLock = await LockAsync(stem + ".lock", timeout.Token).ConfigureAwait(false); + foreach (var suffix in Extensions) + if (await ValidCacheAsync(stem + suffix, tile.Revision, timeout.Token).ConfigureAwait(false)) + return new Lod2AtomCachedTile(tile, stem + suffix, true); + using var response = await _http.GetAsync(tile.Url, HttpCompletionOption.ResponseHeadersRead, timeout.Token).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + var maximum = _kind == AtomDownloadKind.Vector ? 128L * 1024 * 1024 : 512L * 1024 * 1024; + var expected = response.Content.Headers.ContentLength; + if (expected > maximum) throw new InvalidDataException("Download exceeds the supported payload size limit."); + await using (var input = await response.Content.ReadAsStreamAsync(timeout.Token).ConfigureAwait(false)) + await using (var output = new FileStream(temporary, FileMode.CreateNew, FileAccess.Write, FileShare.None, 81920, true)) + { + var buffer = new byte[81920]; long count = 0; long previous = 0; int read; + while ((read = await input.ReadAsync(buffer, timeout.Token).ConfigureAwait(false)) > 0) + { + count += read; + if (count > maximum) throw new InvalidDataException("Download exceeds the supported payload size limit."); + await output.WriteAsync(buffer.AsMemory(0, read), timeout.Token).ConfigureAwait(false); + if (count - previous > 1024 * 1024) { progress?.Invoke($"downloaded {count / (1024 * 1024)} MiB"); previous = count; } + } + if (expected.HasValue && count != expected) throw new InvalidDataException("The source download was incomplete."); + } + var extension = await InspectPayloadAsync(temporary, _kind, timeout.Token).ConfigureAwait(false); + var target = stem + extension; + Stamp stamp; + await using (var input = File.OpenRead(temporary)) + stamp = new Stamp(input.Length, Convert.ToHexString(await SHA256.HashDataAsync(input, timeout.Token).ConfigureAwait(false)), DateTime.UtcNow); + File.Move(temporary, target, true); + await File.WriteAllTextAsync(temporary, JsonSerializer.Serialize(stamp), timeout.Token).ConfigureAwait(false); + File.Move(temporary, target + ".json", true); + return new Lod2AtomCachedTile(tile, target, false); + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { throw new TimeoutException("The source download timed out after three minutes."); } + finally { if (File.Exists(temporary)) File.Delete(temporary); } + } + + private static async Task LockAsync(string path, CancellationToken token) + { + while (true) + { + token.ThrowIfCancellationRequested(); + try { return new FileStream(path, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None); } + catch (IOException) { await Task.Delay(150, token).ConfigureAwait(false); } + } + } + private static async Task ValidCacheAsync(string path, string revision, CancellationToken token) + { + try + { + if (!File.Exists(path) || !File.Exists(path + ".json")) return false; + var stamp = JsonSerializer.Deserialize(await File.ReadAllTextAsync(path + ".json", token).ConfigureAwait(false)); + if (stamp is null || new FileInfo(path).Length != stamp.Length || + (string.IsNullOrWhiteSpace(revision) && DateTime.UtcNow - stamp.DownloadedUtc > TimeSpan.FromDays(1))) return false; + await using var input = File.OpenRead(path); + return Convert.ToHexString(await SHA256.HashDataAsync(input, token).ConfigureAwait(false)) == stamp.Hash; + } + catch (Exception ex) when (ex is IOException or JsonException) { return false; } + } + + private static async Task InspectPayloadAsync(string path, AtomDownloadKind kind, CancellationToken token) + { + var header = new byte[512]; int length; + await using (var input = File.OpenRead(path)) length = await input.ReadAsync(header, token).ConfigureAwait(false); + if (length >= 4 && header[0] == 'P' && header[1] == 'K') + { + using var archive = ZipFile.OpenRead(path); + ValidateEntries(archive); + var supported = 0; + var names = archive.Entries.Select(e => e.FullName).ToHashSet(StringComparer.OrdinalIgnoreCase); + foreach (var entry in archive.Entries) + { + token.ThrowIfCancellationRequested(); + var extension = Path.GetExtension(entry.FullName).ToLowerInvariant(); + if (kind == AtomDownloadKind.Vector && extension == ".shp") + { + foreach (var companion in new[] { ".shx", ".dbf", ".prj" }) + if (!names.Contains(Path.ChangeExtension(entry.FullName, companion))) throw new InvalidDataException("Shapefile ZIP is missing a matching " + companion + " companion file."); + supported++; continue; + } + var candidate = kind switch + { + AtomDownloadKind.Buildings => extension is ".gml" or ".citygml" or ".xml" or ".json" or ".cityjson" or ".jsonl" or ".geojson", + AtomDownloadKind.Terrain => extension is ".tif" or ".tiff" or ".asc" or ".xyz" or ".csv", + _ => extension is ".json" or ".geojson" + }; + if (!candidate) continue; + await using var stream = new LimitedReadStream(entry.Open(), entry.Length); + if (kind == AtomDownloadKind.Buildings && extension is ".gml" or ".citygml" or ".xml") + { if (await IsBuildingXmlAsync(stream, token).ConfigureAwait(false)) supported++; } + else if (extension is ".json" or ".geojson" or ".cityjson" or ".jsonl") + { if (await IsSupportedJsonAsync(stream, kind, token).ConfigureAwait(false)) supported++; } + else + { + // Read through EOF to verify decompression before promoting a completed cache entry. + var buffer = new byte[81920]; while (await stream.ReadAsync(buffer, token).ConfigureAwait(false) > 0) { } + supported++; + } + } + if (supported == 0) throw new InvalidDataException($"Archive contains no supported {kind} data. Expected a ZIP with CityGML/CityJSON buildings, terrain grids, or vector files for the selected loader."); + return ".zip"; + } + if (new FileInfo(path).Length > 256L * 1024 * 1024) throw new InvalidDataException("Direct data file exceeds 256 MiB; use a smaller dataset."); + var text = Encoding.UTF8.GetString(header, 0, length).TrimStart('\uFEFF', ' ', '\r', '\n', '\t'); + if (text.StartsWith("{", StringComparison.Ordinal)) + { + await using var input = File.OpenRead(path); + if (await IsSupportedJsonAsync(input, kind, token).ConfigureAwait(false)) + return kind == AtomDownloadKind.Buildings ? ".cityjson" : ".geojson"; + } + if (kind == AtomDownloadKind.Buildings && text.StartsWith("<", StringComparison.Ordinal)) + { + await using var input = File.OpenRead(path); + if (await IsBuildingXmlAsync(input, token).ConfigureAwait(false)) return ".gml"; + } + if (kind == AtomDownloadKind.Terrain) + { + if (length >= 4 && ((header[0] == 'I' && header[1] == 'I') || (header[0] == 'M' && header[1] == 'M'))) + { GeoTiffReader.ReadImageInfo(path); return ".tif"; } + if (text.StartsWith("ncols", StringComparison.OrdinalIgnoreCase)) return ".asc"; + foreach (var line in text.Split('\n')) + { + var parts = line.Split(new[] { ' ', '\t', ',', ';' }, StringSplitOptions.RemoveEmptyEntries); + if (parts.Length >= 3 && parts.Take(3).All(p => double.TryParse(p, NumberStyles.Float, CultureInfo.InvariantCulture, out _))) return ".csv"; + } + } + throw new InvalidDataException($"Unsupported {kind} download; its content does not match a supported data file or ZIP archive."); + } + + private static async Task IsSupportedJsonAsync(Stream input, AtomDownloadKind kind, CancellationToken token) + { + if (kind == AtomDownloadKind.Buildings) return CityJsonReader.Validate(input, requireCrs: true, cancellationToken: token); + using var document = await JsonDocument.ParseAsync(input, cancellationToken: token).ConfigureAwait(false); + var root = document.RootElement; + if (!root.TryGetProperty("type", out var type) || type.ValueKind != JsonValueKind.String) return false; + return kind == AtomDownloadKind.Vector && type.GetString() is "FeatureCollection" or "Feature"; + } + private static async Task IsBuildingXmlAsync(Stream input, CancellationToken token) + { + using var reader = XmlReader.Create(input, new XmlReaderSettings { Async = true, DtdProcessing = DtdProcessing.Prohibit, XmlResolver = null, MaxCharactersInDocument = 256L * 1024 * 1024 }); + await reader.MoveToContentAsync().ConfigureAwait(false); + if (reader.LocalName != "CityModel" || !reader.NamespaceURI.StartsWith("http://www.opengis.net/citygml", StringComparison.Ordinal)) return false; + while (await reader.ReadAsync().ConfigureAwait(false)) token.ThrowIfCancellationRequested(); + return true; + } + internal static void ValidateEntries(ZipArchive archive) + { + if (archive.Entries.Count > 10000) throw new InvalidDataException("ZIP contains too many entries."); + long size = 0; var names = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var entry in archive.Entries) + { + var name = entry.FullName.Replace('\\', '/'); + if (name.StartsWith('/') || name.Contains(':') || name.Split('/').Any(p => p is ".." or ".") || + ((entry.ExternalAttributes >> 16) & 0xF000) == 0xA000 || !names.Add(name)) + throw new InvalidDataException("ZIP contains an unsafe, symbolic-link or duplicate entry path."); + if (entry.Length > 256L * 1024 * 1024 || (size += entry.Length) > 1024L * 1024 * 1024) + throw new InvalidDataException("ZIP exceeds the expanded-size limit (256 MiB per file, 1 GiB total)."); + } + } +} + +public sealed class SourceFilesLease : IDisposable +{ + private readonly string? _temporary; + public IReadOnlyList Files { get; } + private SourceFilesLease(IReadOnlyList files, string? temporary) { Files = files; _temporary = temporary; } + public static SourceFilesLease Open(string path, CancellationToken cancellationToken = default) + { + if (!Path.GetExtension(path).Equals(".zip", StringComparison.OrdinalIgnoreCase)) return new SourceFilesLease(new[] { path }, null); + var directory = Path.Combine(Path.GetTempPath(), "rhinospatial-source-" + Guid.NewGuid().ToString("N")); + try + { + using var archive = ZipFile.OpenRead(path); SourceDownloadCache.ValidateEntries(archive); + Directory.CreateDirectory(directory); + foreach (var entry in archive.Entries) + { + if (string.IsNullOrEmpty(entry.Name)) continue; + var target = Path.Combine(directory, entry.FullName.Replace('\\', '/')); + Directory.CreateDirectory(Path.GetDirectoryName(target)!); + cancellationToken.ThrowIfCancellationRequested(); + using var input = entry.Open(); using var output = File.Create(target); + var buffer = new byte[81920]; long count = 0; int read; + while ((read = input.Read(buffer)) > 0) + { + cancellationToken.ThrowIfCancellationRequested(); count += read; + if (count > entry.Length || count > 256L * 1024 * 1024) throw new InvalidDataException("ZIP expanded beyond its declared entry size."); + output.Write(buffer, 0, read); + } + if (count != entry.Length) throw new InvalidDataException("ZIP entry is truncated."); + } + var extracted = Directory.GetFiles(directory, "*", SearchOption.AllDirectories); + foreach (var shape in extracted.Where(f => Path.GetExtension(f).Equals(".shp", StringComparison.OrdinalIgnoreCase))) + foreach (var extension in new[] { ".dbf", ".shx", ".prj", ".cpg" }) + { + var expected = Path.ChangeExtension(shape, extension); + var actual = extracted.FirstOrDefault(f => f.Equals(expected, StringComparison.OrdinalIgnoreCase)); + if (actual is not null && actual != expected) + { + var intermediate = actual + ".normalize-" + Guid.NewGuid().ToString("N"); + File.Move(actual, intermediate); File.Move(intermediate, expected); + } + } + return new SourceFilesLease(Directory.GetFiles(directory, "*", SearchOption.AllDirectories), directory); + } + catch { if (Directory.Exists(directory)) Directory.Delete(directory, true); throw; } + } + public void Dispose() { if (_temporary is not null && Directory.Exists(_temporary)) Directory.Delete(_temporary, true); } +} diff --git a/RhinoSpatial.Core/SourceHttp.cs b/RhinoSpatial.Core/SourceHttp.cs new file mode 100644 index 0000000..703ddbe --- /dev/null +++ b/RhinoSpatial.Core/SourceHttp.cs @@ -0,0 +1,39 @@ +using System.Net.Http; +using System.Text; + +namespace RhinoSpatial.Core; + +internal static class SourceHttp +{ + public static async Task<(string Text, Uri Url, string MediaType)> ReadAsync(HttpClient http, string url, + CancellationToken cancellationToken = default, int maximumBytes = 16 * 1024 * 1024) + { + using var timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + timeout.CancelAfter(TimeSpan.FromSeconds(30)); + using var response = await http.GetAsync(url, HttpCompletionOption.ResponseHeadersRead, timeout.Token).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + if (response.Content.Headers.ContentLength > maximumBytes) + throw new InvalidDataException("Source response exceeds the supported size limit."); + await using var input = await response.Content.ReadAsStreamAsync(timeout.Token).ConfigureAwait(false); + using var output = new MemoryStream(); + var buffer = new byte[81920]; + int count; + while ((count = await input.ReadAsync(buffer, timeout.Token).ConfigureAwait(false)) > 0) + { + if (output.Length + count > maximumBytes) throw new InvalidDataException("Source response exceeds the supported size limit."); + output.Write(buffer, 0, count); + } + var data = output.ToArray(); + var charset = response.Content.Headers.ContentType?.CharSet?.Trim('"'); + if (string.IsNullOrWhiteSpace(charset)) + { + var declaration = Encoding.ASCII.GetString(data, 0, Math.Min(data.Length, 256)); + var match = System.Text.RegularExpressions.Regex.Match(declaration, "^\\s*<\\?xml[^>]*encoding=['\"]([^'\"]+)['\"]", System.Text.RegularExpressions.RegexOptions.IgnoreCase); + if (match.Success) charset = match.Groups[1].Value; + } + var encoding = string.IsNullOrWhiteSpace(charset) ? Encoding.UTF8 : Encoding.GetEncoding(charset); + using var decoded = new StreamReader(new MemoryStream(data), encoding, true); + return (await decoded.ReadToEndAsync().ConfigureAwait(false), response.RequestMessage?.RequestUri ?? new Uri(url), + response.Content.Headers.ContentType?.MediaType ?? ""); + } +} diff --git a/RhinoSpatial.Core/SpatialReferenceTransform.cs b/RhinoSpatial.Core/SpatialReferenceTransform.cs index 2494901..1e0e437 100644 --- a/RhinoSpatial.Core/SpatialReferenceTransform.cs +++ b/RhinoSpatial.Core/SpatialReferenceTransform.cs @@ -82,6 +82,11 @@ public static bool TryTransformXY(string sourceSrs, string targetSrs, double x, private static CoordinateSystem? CreateCoordinateSystem(string normalizedSrs) { + if(normalizedSrs.StartsWith("EPSG:",StringComparison.Ordinal)&&int.TryParse(normalizedSrs.AsSpan(5),out var epsg)) + { + if(epsg is >=32601 and <=32660)return ProjectedCoordinateSystem.WGS84_UTM(epsg-32600,true); + if(epsg is >=32701 and <=32760)return ProjectedCoordinateSystem.WGS84_UTM(epsg-32700,false); + } return normalizedSrs switch { "EPSG:4326" => GeographicCoordinateSystem.WGS84, diff --git a/RhinoSpatial.Core/TerrainAtomLoader.cs b/RhinoSpatial.Core/TerrainAtomLoader.cs new file mode 100644 index 0000000..fa17284 --- /dev/null +++ b/RhinoSpatial.Core/TerrainAtomLoader.cs @@ -0,0 +1,133 @@ +namespace RhinoSpatial.Core; + +public record TerrainAtomResult(TerrainRasterData Raster, BoundingBox2D Bounds, string Status); + +public sealed class TerrainAtomLoader +{ + private readonly AtomDownloadClient _client; + public TerrainAtomLoader(HttpClient? http = null, string? cacheDirectory = null) => _client = new AtomDownloadClient(http, cacheDirectory, AtomDownloadKind.Terrain); + + public async Task TryLoadAsync(string source, SpatialContext2D context, + Action? progress = null, CancellationToken cancellationToken = default) + { + var selection = await _client.TrySelectAsync(source, context, cancellationToken).ConfigureAwait(false); + if (selection is null) return null; + if (selection.Tiles.Count == 0) throw new InvalidOperationException("No terrain ATOM tiles intersect the selected Spatial Context."); + var downloads = await _client.DownloadAsync(selection, progress, cancellationToken).ConfigureAwait(false); + var rasters = new List(); + long samples = 0; + foreach (var download in downloads) + { + cancellationToken.ThrowIfCancellationRequested(); + using var files = SourceFilesLease.Open(download.FilePath, cancellationToken); + foreach (var file in files.Files) + { + var extension = Path.GetExtension(file).ToLowerInvariant(); + if (extension is not (".tif" or ".tiff" or ".asc" or ".csv" or ".xyz")) continue; + progress?.Invoke("Reading terrain " + Path.GetFileName(download.Tile.Url.LocalPath)); + var srs = download.Tile.NativeSrs; + TerrainRasterData raster; + if (extension is ".tif" or ".tiff") + { + var metadata = GeoTiffReader.ReadImageInfo(file); + srs = metadata.SrsName; + raster = TerrainRasterReader.ReadRaster(file, Path.GetFileName(file), srs); + } + else + { + if (string.IsNullOrWhiteSpace(srs)) + throw new InvalidDataException("Terrain text downloads require an unambiguous source CRS in the ATOM category metadata."); + var window = Lod2AtomFeedReader.TransformBounds(new AtomSpatialExtent(context.ResolvedSrs, context.RequestBoundingBox), srs); + raster = TerrainTextGridReader.ReadWindow(file, Path.GetFileName(file), srs, window, cancellationToken).Raster; + } + var bounds = Lod2AtomFeedReader.TransformBounds(new AtomSpatialExtent(context.ResolvedSrs, context.RequestBoundingBox), srs); + raster = TerrainRasterMosaic.Crop(raster, bounds); + samples += (long)raster.Width * raster.Height; + if (samples > 8388608) throw new InvalidOperationException("Terrain ATOM windows exceed eight million source samples; select a smaller area."); + rasters.Add(raster); + } + } + if (rasters.Count == 0) throw new InvalidDataException("Selected terrain archives contained no readable terrain grids."); + var mosaic = TerrainRasterMosaic.Merge(rasters, cancellationToken); + var finalBounds = TerrainRasterMosaic.Bounds(mosaic); + return new TerrainAtomResult(mosaic, finalBounds, + $"ATOM terrain: {downloads.Count} selected tile(s), {downloads.Count(t => t.UsedCache)} cached, {mosaic.Width}x{mosaic.Height} samples, {mosaic.SrsName}. Tiles: {string.Join(", ", downloads.Select(t => Path.GetFileName(t.Tile.Url.LocalPath)))}."); + } +} + +public static class TerrainRasterMosaic +{ + private const int MaximumCells = 4194304; + private static void Validate(TerrainRasterData raster) + { + if (raster.Width < 2 || raster.Height < 2 || raster.OffsetVectorX.X <= 0 || raster.OffsetVectorY.Y >= 0 || + Math.Abs(raster.OffsetVectorX.Y) > 1e-9 || Math.Abs(raster.OffsetVectorY.X) > 1e-9) + throw new InvalidDataException("Terrain ATOM mosaics require regular north-up grids with at least 2x2 samples."); + } + public static BoundingBox2D Bounds(TerrainRasterData raster) => new(raster.Origin.X, + raster.Origin.Y + (raster.Height - 1) * raster.OffsetVectorY.Y, + raster.Origin.X + (raster.Width - 1) * raster.OffsetVectorX.X, raster.Origin.Y); + + public static TerrainRasterData Crop(TerrainRasterData raster, BoundingBox2D window) + { + Validate(raster); + var x0 = Math.Clamp((int)Math.Floor((window.MinX - raster.Origin.X) / raster.OffsetVectorX.X), 0, raster.Width - 1); + var x1 = Math.Clamp((int)Math.Ceiling((window.MaxX - raster.Origin.X) / raster.OffsetVectorX.X), 0, raster.Width - 1); + var y0 = Math.Clamp((int)Math.Floor((raster.Origin.Y - window.MaxY) / -raster.OffsetVectorY.Y), 0, raster.Height - 1); + var y1 = Math.Clamp((int)Math.Ceiling((raster.Origin.Y - window.MinY) / -raster.OffsetVectorY.Y), 0, raster.Height - 1); + var width = x1 - x0 + 1; var height = y1 - y0 + 1; + if (width < 2 || height < 2 || (long)width * height > MaximumCells) + throw new InvalidDataException("Terrain window has fewer than 2x2 or more than four million samples; adjust the selected area."); + var samples = new float[width * height]; + for (var row = 0; row < height; row++) Array.Copy(raster.Elevations, (row + y0) * raster.Width + x0, samples, row * width, width); + var geo = raster.GeoTransform; + if (geo is not null) { var corner=geo.At(x0,y0); geo=geo with {C=corner.X,F=corner.Y}; } + return raster with { GeoTransform=geo, Width = width, Height = height, Origin = new Coordinate2D(raster.Origin.X + x0 * raster.OffsetVectorX.X, raster.Origin.Y + y0 * raster.OffsetVectorY.Y), Elevations = samples }; + } + + public static TerrainRasterData Merge(IReadOnlyList rasters, CancellationToken cancellationToken = default) + { + if (rasters.Count == 0) throw new ArgumentException("No terrain rasters."); + var first = rasters[0]; + foreach (var raster in rasters) + { + Validate(raster); + if (SpatialReferenceTransform.NormalizeSrs(raster.SrsName) != SpatialReferenceTransform.NormalizeSrs(first.SrsName) || + Math.Abs(raster.OffsetVectorX.X - first.OffsetVectorX.X) > 1e-6 || Math.Abs(raster.OffsetVectorY.Y - first.OffsetVectorY.Y) > 1e-6) + throw new InvalidDataException("Terrain tiles have different coordinate systems or grid spacing. They cannot be mosaicked without resampling."); + } + var boxes = rasters.Select(Bounds).ToList(); + var minX = boxes.Min(b => b.MinX); var maxY = boxes.Max(b => b.MaxY); + var dx = first.OffsetVectorX.X; var dy = -first.OffsetVectorY.Y; + int Index(double value) + { + var rounded = Math.Round(value); + if (!double.IsFinite(value) || Math.Abs(value - rounded) > 1e-5 || rounded < 0 || rounded > int.MaxValue) + throw new InvalidDataException("Terrain tile origins are not aligned to one regular grid."); + return (int)rounded; + } + var width = checked(Index((boxes.Max(b => b.MaxX) - minX) / dx) + 1); + var height = checked(Index((maxY - boxes.Min(b => b.MinY)) / dy) + 1); + if ((long)width * height > MaximumCells) throw new InvalidDataException("Terrain mosaic exceeds four million cells; select a smaller area."); + var values = Enumerable.Repeat(float.NaN, width * height).ToArray(); + foreach (var raster in rasters) + { + var x0 = Index((raster.Origin.X - minX) / dx); var y0 = Index((maxY - raster.Origin.Y) / dy); + for (var row = 0; row < raster.Height; row++) + { + cancellationToken.ThrowIfCancellationRequested(); + for (var column = 0; column < raster.Width; column++) + { + var value = raster.Elevations[row * raster.Width + column]; + if (TerrainMeshBuilder.IsNoData(raster, value)) continue; + var index = (row + y0) * width + column + x0; + if (float.IsFinite(values[index]) && Math.Abs(values[index] - value) > 0.01f) + throw new InvalidDataException("Overlapping terrain tiles disagree on elevation; no artificial seam was created."); + values[index] = value; + } + } + } + return first with { CoverageId = "ATOM terrain mosaic", Width = width, Height = height, + Origin = new Coordinate2D(minX, maxY), NoDataValue = double.NaN, Elevations = values }; + } +} diff --git a/RhinoSpatial.Core/TerrainMeshBuilder.cs b/RhinoSpatial.Core/TerrainMeshBuilder.cs index 0480136..0d71c1f 100644 --- a/RhinoSpatial.Core/TerrainMeshBuilder.cs +++ b/RhinoSpatial.Core/TerrainMeshBuilder.cs @@ -49,8 +49,9 @@ public static TerrainMeshData Build( continue; } - var sourceX = sampleBounds.MinX + xs[col] * stepX; - var sourceY = sampleBounds.MaxY - ys[row] * stepY; + var sourcePoint = raster.GeoTransform?.At(xs[col]+0.5,ys[row]+0.5); + var sourceX = sourcePoint?.X ?? sampleBounds.MinX + xs[col] * stepX; + var sourceY = sourcePoint?.Y ?? sampleBounds.MaxY - ys[row] * stepY; if (!SpatialReferenceTransform.TryTransformXY( raster.SrsName, context.ResolvedSrs, sourceX, sourceY, out var x, out var y) || !double.IsFinite(x) || !double.IsFinite(y)) diff --git a/RhinoSpatial.Core/TerrainRasterReader.cs b/RhinoSpatial.Core/TerrainRasterReader.cs index ecd17d6..3e75c7e 100644 --- a/RhinoSpatial.Core/TerrainRasterReader.cs +++ b/RhinoSpatial.Core/TerrainRasterReader.cs @@ -38,21 +38,24 @@ public static TerrainRasterData ReadRaster(string filePath, string coverageId, s ReadScanlineRaster(tiff, width, height, bitsPerSample, sampleFormat, elevations); } - var origin = new Coordinate2D(0, 0); - var offsetX = new Coordinate2D(1, 0); - var offsetY = new Coordinate2D(0, -1); + GeoReferencedRasterInfo? geo=null; + try { geo=GeoTiffReader.ReadImageInfo(filePath,srsName); } + catch (InvalidDataException) when (tiff.GetField(TiffTag.GEOTIFF_MODELPIXELSCALETAG) is null && tiff.GetField(TiffTag.GEOTIFF_MODELTRANSFORMATIONTAG) is null) { } + var origin = geo?.GeoTransform?.At(0.5,0.5) ?? new Coordinate2D(0, 0); + var offsetX = geo?.GeoTransform is { } gx ? new Coordinate2D(gx.A,gx.D) : new Coordinate2D(1,0); + var offsetY = geo?.GeoTransform is { } gy ? new Coordinate2D(gy.B,gy.E) : new Coordinate2D(0,-1); var noData = ReadNoDataValue(tiff); return new TerrainRasterData( coverageId, - srsName, + geo?.SrsName ?? srsName, width, height, origin, offsetX, offsetY, noData, - elevations); + elevations) { GeoTransform=geo?.GeoTransform }; } private static void ReadScanlineRaster( @@ -120,7 +123,7 @@ private static void ReadTiledRaster( } } - private static float ReadSample(byte[] buffer, int sampleIndex, int bitsPerSample, SampleFormat sampleFormat) + internal static float ReadSample(byte[] buffer, int sampleIndex, int bitsPerSample, SampleFormat sampleFormat) { if (bitsPerSample == 64 && sampleFormat == SampleFormat.IEEEFP) { @@ -158,7 +161,7 @@ private static float ReadSample(byte[] buffer, int sampleIndex, int bitsPerSampl throw new InvalidOperationException($"Unsupported terrain raster sample format: {bitsPerSample}-bit {sampleFormat}."); } - private static double? ReadNoDataValue(Tiff tiff) + internal static double? ReadNoDataValue(Tiff tiff) { var noDataField = tiff.GetField(TiffTag.GDAL_NODATA); if (noDataField is null || noDataField.Length == 0) @@ -166,13 +169,14 @@ private static float ReadSample(byte[] buffer, int sampleIndex, int bitsPerSampl return null; } - var text = noDataField[0].ToString(); + // Unknown ASCII TIFF tags expose a count followed by the value. + var text = noDataField[^1].ToString().TrimEnd('\0'); if (double.TryParse(text, System.Globalization.NumberStyles.Float, System.Globalization.CultureInfo.InvariantCulture, out var value)) { return value; } - return null; + throw new InvalidDataException("Invalid GeoTIFF NoData value."); } } } diff --git a/RhinoSpatial.Core/TerrainTextGridReader.cs b/RhinoSpatial.Core/TerrainTextGridReader.cs index 18c708a..b112549 100644 --- a/RhinoSpatial.Core/TerrainTextGridReader.cs +++ b/RhinoSpatial.Core/TerrainTextGridReader.cs @@ -222,11 +222,21 @@ private static TerrainTextGridSourceMetadata ReadXyzGridMetadata( "XYZ/CSV terrain grid"); } - private static TerrainTextGridReadResult ReadXyzGrid(string filePath, string coverageId, string sourceSrs) + public static TerrainTextGridReadResult ReadWindow(string filePath, string coverageId, string sourceSrs, + BoundingBox2D bounds, System.Threading.CancellationToken cancellationToken = default) + { + if (Path.GetExtension(filePath).Equals(".asc", StringComparison.OrdinalIgnoreCase)) + return ReadRaster(filePath, coverageId, sourceSrs); + return ReadXyzGrid(filePath, coverageId, sourceSrs, bounds, cancellationToken); + } + + private static TerrainTextGridReadResult ReadXyzGrid(string filePath, string coverageId, string sourceSrs, + BoundingBox2D? bounds = null, System.Threading.CancellationToken cancellationToken = default) { var samples = new List<(double X, double Y, float Z)>(); foreach (var line in File.ReadLines(filePath)) { + cancellationToken.ThrowIfCancellationRequested(); var parts = SplitFields(line); if (parts.Length < 3 || !double.TryParse(parts[0], NumberStyles.Float, CultureInfo.InvariantCulture, out var x) || @@ -237,6 +247,8 @@ private static TerrainTextGridReadResult ReadXyzGrid(string filePath, string cov } RequireFinitePosition(x, y); + if (bounds is not null && (x < bounds.MinX || x > bounds.MaxX || y < bounds.MinY || y > bounds.MaxY)) continue; + if (bounds is not null && samples.Count >= 4194304) throw new InvalidOperationException("Terrain window exceeds four million samples; select a smaller area."); samples.Add((x, y, z)); } diff --git a/RhinoSpatial.Core/VectorSourceResolver.cs b/RhinoSpatial.Core/VectorSourceResolver.cs index e83c38d..81a6f73 100644 --- a/RhinoSpatial.Core/VectorSourceResolver.cs +++ b/RhinoSpatial.Core/VectorSourceResolver.cs @@ -10,7 +10,9 @@ public enum VectorSourceKind Wfs, OgcApiFeatures, LocalShapefile, - LocalGeoJson + LocalGeoJson, + RemoteDownload, + LocalGeoPackage } public record VectorSourceInfo( @@ -34,6 +36,9 @@ public static VectorSourceInfo Resolve(string? source) return new VectorSourceInfo(string.Empty, VectorSourceKind.Unsupported, string.Empty); } + if (IsLocalFile(trimmed, ".gpkg")) + return new VectorSourceInfo(trimmed,VectorSourceKind.LocalGeoPackage,string.Empty); + if (IsLocalFile(trimmed, ".shp")) { return new VectorSourceInfo( @@ -58,6 +63,10 @@ public static VectorSourceInfo Resolve(string? source) ResolveOgcApiFeaturesLayerName(trimmed)); } + if (IsHttpUrl(trimmed) && new[] { ".geojson", ".json", ".zip" }.Contains(Path.GetExtension(new Uri(trimmed).AbsolutePath), StringComparer.OrdinalIgnoreCase) && + !System.Text.RegularExpressions.Regex.IsMatch(trimmed, @"[?&]service=wfs(?:&|$)", System.Text.RegularExpressions.RegexOptions.IgnoreCase)) + return new VectorSourceInfo(trimmed, VectorSourceKind.RemoteDownload, Path.GetFileNameWithoutExtension(new Uri(trimmed).AbsolutePath)); + if (IsHttpUrl(trimmed)) { return new VectorSourceInfo(trimmed, VectorSourceKind.Wfs, string.Empty); @@ -95,6 +104,8 @@ private static string ResolveOgcApiFeaturesLayerName(string sourceUrl) return Uri.UnescapeDataString(pathParts[collectionIndex + 1]); } + if (pathParts.LastOrDefault()?.Equals("collections", StringComparison.OrdinalIgnoreCase) == true) return string.Empty; + return string.IsNullOrWhiteSpace(pathParts.LastOrDefault()) ? "OGC API Features" : Uri.UnescapeDataString(pathParts.Last()); diff --git a/RhinoSpatial.Core/WfsClient.cs b/RhinoSpatial.Core/WfsClient.cs index 529bd03..324ce6b 100644 --- a/RhinoSpatial.Core/WfsClient.cs +++ b/RhinoSpatial.Core/WfsClient.cs @@ -5,6 +5,7 @@ using System.Net.Http.Headers; using System.Text; using System.Threading.Tasks; +using System.Threading; namespace RhinoSpatial.Core { @@ -31,6 +32,7 @@ public class WfsClient "MAXFEATURES", "COUNT", "OUTPUTFORMAT", + "STARTINDEX", "BBOX" }; @@ -40,17 +42,25 @@ static WfsClient() SharedHttpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("*/*")); } + private readonly HttpClient _http; + public WfsClient() : this(null) { } + public WfsClient(HttpClient? httpClient) => _http = httpClient ?? SharedHttpClient; + private sealed record FeatureResponseCacheEntry(WfsFeatureResponse Response, DateTimeOffset StoredAtUtc); - public async Task> LoadFeaturesAsync(WfsRequestOptions options) + public Task> LoadFeaturesAsync(WfsRequestOptions options) => LoadFeaturesAsync(options, CancellationToken.None); + + public async Task> LoadFeaturesAsync(WfsRequestOptions options, CancellationToken cancellationToken) { - var result = await LoadFeaturesWithStatusAsync(options); + var result = await LoadFeaturesWithStatusAsync(options, cancellationToken); return result.Features; } - public async Task LoadFeaturesWithStatusAsync(WfsRequestOptions options) + public Task LoadFeaturesWithStatusAsync(WfsRequestOptions options) => LoadFeaturesWithStatusAsync(options, CancellationToken.None); + + public async Task LoadFeaturesWithStatusAsync(WfsRequestOptions options, CancellationToken cancellationToken) { - var featureResponse = await LoadFeatureResponseAsync(options); + var featureResponse = await LoadFeatureResponseAsync(options, cancellationToken); if (TryReadFeatures(featureResponse.ResponseText, featureResponse.AppliedOptions.TypeName, out var features)) { @@ -60,35 +70,40 @@ public async Task LoadFeaturesWithStatusAsync(WfsRequestOp throw CreateUnsupportedFeatureResponseException(featureResponse.ResponseText); } - public async Task LoadFeatureResponseAsync(WfsRequestOptions options) + public Task LoadFeatureResponseAsync(WfsRequestOptions options) => LoadFeatureResponseAsync(options, CancellationToken.None); + + public async Task LoadFeatureResponseAsync(WfsRequestOptions options, CancellationToken cancellationToken) { - options = await PrepareRequestOptionsAsync(options); + options = await PrepareRequestOptionsAsync(options).WaitAsync(cancellationToken); var cacheKey = BuildFeatureResponseCacheKey(options); - if (TryGetCachedFeatureResponse(cacheKey, allowStale: false, out var cachedResponse)) + if (ReferenceEquals(_http, SharedHttpClient) && TryGetCachedFeatureResponse(cacheKey, allowStale: false, out var cachedResponse)) { return cachedResponse; } - var loadTask = FeatureResponseTaskCache.GetOrAdd(cacheKey, _ => LoadFeatureResponseUncachedAsync(options)); + var useSharedTask = !cancellationToken.CanBeCanceled && ReferenceEquals(_http, SharedHttpClient); + var loadTask = useSharedTask + ? FeatureResponseTaskCache.GetOrAdd(cacheKey, _ => LoadFeatureResponseUncachedAsync(options)) + : LoadFeatureResponseUncachedAsync(options, cancellationToken); try { var response = await loadTask; - FeatureResponseCache[cacheKey] = new FeatureResponseCacheEntry(response, DateTimeOffset.UtcNow); + if (ReferenceEquals(_http, SharedHttpClient)) FeatureResponseCache[cacheKey] = new FeatureResponseCacheEntry(response, DateTimeOffset.UtcNow); return response; } - catch (Exception ex) when (IsTransientFailure(ex) && + catch (Exception ex) when (!cancellationToken.IsCancellationRequested && ReferenceEquals(_http, SharedHttpClient) && IsTransientFailure(ex) && TryGetCachedFeatureResponse(cacheKey, allowStale: true, out var staleCachedResponse)) { return staleCachedResponse with { - StatusNote = "Using cached WFS result because the live WFS service was temporarily unavailable." + StatusNote = staleCachedResponse.StatusNote + " Using cached WFS result because the live WFS service was temporarily unavailable." }; } finally { - if (loadTask.IsCompleted) + if (useSharedTask && loadTask.IsCompleted) { FeatureResponseTaskCache.TryRemove(cacheKey, out _); } @@ -107,6 +122,7 @@ public async Task LoadCapabilitiesAsync(string baseUrl) var cacheKey = string.IsNullOrWhiteSpace(requestedVersion) ? normalizedBaseUrl : $"{normalizedBaseUrl}|VERSION={requestedVersion}"; + if (!ReferenceEquals(_http, SharedHttpClient)) return await LoadCapabilitiesUncachedAsync(normalizedBaseUrl, requestedVersion); var loadTask = CapabilitiesCache.GetOrAdd(cacheKey, _ => LoadCapabilitiesUncachedAsync(normalizedBaseUrl, requestedVersion)); try @@ -115,7 +131,7 @@ public async Task LoadCapabilitiesAsync(string baseUrl) } catch { - CapabilitiesCache.TryRemove(normalizedBaseUrl, out _); + CapabilitiesCache.TryRemove(cacheKey, out _); throw; } } @@ -168,6 +184,8 @@ public static string BuildGetFeatureRequestUrl(WfsRequestOptions options) builder.Append(Uri.EscapeDataString(options.OutputFormat)); } + if (options.StartIndex > 0) builder.Append("&STARTINDEX=" + options.StartIndex); + if (options.BoundingBox is not null) { builder.Append("&BBOX="); @@ -249,7 +267,7 @@ private async Task LoadCapabilitiesUncachedAsync(string nor throw lastException ?? new InvalidOperationException("The WFS capabilities request sequence did not produce a capabilities response."); } - private async Task LoadFeatureResponseUncachedAsync(WfsRequestOptions options) + private async Task LoadFeatureResponseUncachedAsync(WfsRequestOptions options, CancellationToken cancellationToken = default) { Exception? lastRequestException = null; @@ -259,12 +277,23 @@ private async Task LoadFeatureResponseUncachedAsync(WfsReque try { - var response = await GetStringAsync(requestUrl); + var pageOptions = CloneOptions(candidateOptions); + var supportsOffset = pageOptions.Version.StartsWith("2.", StringComparison.Ordinal); + if (supportsOffset) pageOptions.MaxFeatures = options.MaxFeatures > 0 + ? Math.Min(options.MaxFeatures, FeatureCollectionPager.PageSize) : FeatureCollectionPager.PageSize; + var paged = await FeatureCollectionPager.LoadHttpAsync(BuildGetFeatureRequestUrl(pageOptions), + async (url, token) => { var page = await SourceHttp.ReadAsync(_http, url, token).ConfigureAwait(false); return new FeaturePageContent(page.Text, page.Url); }, + options.MaxFeatures, + supportsOffset ? (offset, count) => { + var next = CloneOptions(pageOptions); next.StartIndex = candidateOptions.StartIndex + offset; next.MaxFeatures = count; + return BuildGetFeatureRequestUrl(next); + } : null, cancellationToken).ConfigureAwait(false); + var response = paged.Text; var statusNote = lastRequestException is null ? string.Empty : $"Retried WFS request with output format '{candidateOptions.OutputFormat}' after the provider rejected an earlier request."; - return new WfsFeatureResponse(response, candidateOptions, statusNote); + return new WfsFeatureResponse(response, candidateOptions, (statusNote + " " + paged.StatusNote).Trim()); } catch (Exception ex) when (ShouldTryNextRequestCandidate(ex)) { @@ -330,6 +359,7 @@ private static void AppendIfMissing(List sequence, WfsRequest GetFeatureBaseUrl = source.GetFeatureBaseUrl, TypeName = source.TypeName, MaxFeatures = source.MaxFeatures, + StartIndex = source.StartIndex, Version = version, SrsName = source.SrsName, OutputFormat = outputFormat, @@ -345,6 +375,7 @@ private static WfsRequestOptions CloneOptions(WfsRequestOptions options) GetFeatureBaseUrl = options.GetFeatureBaseUrl, TypeName = options.TypeName, MaxFeatures = options.MaxFeatures, + StartIndex = options.StartIndex, Version = options.Version, SrsName = options.SrsName, OutputFormat = options.OutputFormat, @@ -429,6 +460,8 @@ private static string BuildFeatureResponseCacheKey(WfsRequestOptions options) builder.Append('|'); builder.Append(options.MaxFeatures); builder.Append('|'); + builder.Append(options.StartIndex); + builder.Append('|'); builder.Append(options.Version); builder.Append('|'); builder.Append(options.SrsName); @@ -548,7 +581,7 @@ private async Task GetStringAsync(string requestUrl) try { using var request = new HttpRequestMessage(HttpMethod.Get, requestUrl); - using var response = await SharedHttpClient.SendAsync(request); + using var response = await _http.SendAsync(request); var responseText = await response.Content.ReadAsStringAsync(); if (!response.IsSuccessStatusCode) diff --git a/RhinoSpatial.Core/WfsRequestOptions.cs b/RhinoSpatial.Core/WfsRequestOptions.cs index 30cbf7c..601cc5a 100644 --- a/RhinoSpatial.Core/WfsRequestOptions.cs +++ b/RhinoSpatial.Core/WfsRequestOptions.cs @@ -10,6 +10,8 @@ public class WfsRequestOptions public int MaxFeatures { get; set; } + public int StartIndex { get; set; } + public string Version { get; set; } = "1.1.0"; public string SrsName { get; set; } = string.Empty; diff --git a/RhinoSpatial.Core/WmtsClient.cs b/RhinoSpatial.Core/WmtsClient.cs new file mode 100644 index 0000000..c4bf8fa --- /dev/null +++ b/RhinoSpatial.Core/WmtsClient.cs @@ -0,0 +1,202 @@ +using System.Collections.Concurrent; +using System.Globalization; +using System.Security.Cryptography; +using System.Text; +using System.Xml; +using System.Xml.Linq; + +namespace RhinoSpatial.Core; + +public record WmtsMatrix(string Id, double Resolution, Coordinate2D TopLeft, int TileWidth, int TileHeight, int Width, int Height); +public record WmtsMatrixSet(string Id, string Srs, List Matrices); +public record WmtsLimits(int MinRow, int MaxRow, int MinColumn, int MaxColumn); +public record WmtsLink(string Set, Dictionary Limits); +public record WmtsLayer(string Id, string Title, string Style, List Formats, List Links, Dictionary Dimensions, Dictionary Templates, BoundingBox2D? Wgs84Bounds); +public record WmtsCapabilities(List Layers, Dictionary Sets, string TileUrl); +public record WmtsTile(int Row, int Column, string Url); +public record WmtsPlan(WmtsLayer Layer, WmtsMatrixSet Set, WmtsMatrix Matrix, List Tiles, int MinRow, int MinColumn, int Width, int Height, RasterGeoTransform Transform, bool CoverageClipped); +public record WmtsImageResult(string LocalFilePath, GeoReferencedRasterInfo Raster, WmtsPlan Plan, bool UsedCache, string Status); + +public sealed class WmtsClient +{ + private static readonly HttpClient SharedHttp = new() { Timeout = Timeout.InfiniteTimeSpan }; + private static readonly ConcurrentDictionary Cache = new(); + private static readonly ConcurrentDictionary Gates = new(); + private readonly HttpClient _http; private readonly string _directory; + public WmtsClient(HttpClient? http = null, string? cacheDirectory = null) + { _http = http ?? SharedHttp; _directory = cacheDirectory ?? Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "RhinoSpatial", "WmtsCache"); } + public static bool LooksLike(string source) => source.Contains("wmts", StringComparison.OrdinalIgnoreCase); + public async Task TryLoadCapabilitiesAsync(string source, CancellationToken token = default) + { + if (System.Text.RegularExpressions.Regex.IsMatch(source, @"[?&]service=(?:wms|wfs|wcs)(?:&|$)", System.Text.RegularExpressions.RegexOptions.IgnoreCase)) return null; + if (ReferenceEquals(_http, SharedHttp) && Cache.TryGetValue(source, out var cached) && cached.Expires > DateTime.UtcNow) return cached.Value; + var uri = new Uri(source); var query = Query(uri.Query); + var url = source; + if (!uri.AbsolutePath.EndsWith(".xml", StringComparison.OrdinalIgnoreCase)) + url = WithQuery(source, new() { { "SERVICE", "WMTS" }, { "REQUEST", "GetCapabilities" }, { "VERSION", "1.0.0" } }); + WmtsCapabilities? result; + try + { + var response = await SourceHttp.ReadAsync(_http, url, token).ConfigureAwait(false); + using var reader = XmlReader.Create(new StringReader(response.Text), new XmlReaderSettings { DtdProcessing = DtdProcessing.Prohibit, XmlResolver = null }); + var doc = XDocument.Load(reader); + if (doc.Root?.Name.NamespaceName != "http://www.opengis.net/wmts/1.0") + { if (LooksLike(source)) throw new InvalidDataException("The source did not return WMTS capabilities."); result = null; } + else result = ReadCapabilities(doc, response.Url, query); + } + catch (Exception ex) when (!LooksLike(source) && ex is HttpRequestException or XmlException) { result = null; } + if (ReferenceEquals(_http, SharedHttp)) { if (Cache.Count > 128) Cache.Clear(); Cache[source] = (DateTime.UtcNow.AddMinutes(5), result); } + return result; + } + public static WmtsCapabilities ReadCapabilities(XDocument doc, Uri responseUrl, Dictionary? overrides = null) + { + XNamespace w = "http://www.opengis.net/wmts/1.0", o = "http://www.opengis.net/ows/1.1", x = "http://www.w3.org/1999/xlink"; + var contents = doc.Root?.Element(w + "Contents") ?? throw new InvalidDataException("WMTS capabilities has no Contents."); + var sets = new Dictionary(StringComparer.Ordinal); + foreach (var set in contents.Elements(w + "TileMatrixSet")) + { + var id = Text(set, o + "Identifier"); var declared = Text(set, o + "SupportedCRS"); + var srs = declared.Contains("CRS84", StringComparison.OrdinalIgnoreCase) || declared.Contains("CRS:84", StringComparison.OrdinalIgnoreCase) ? "EPSG:4326" : SpatialReferenceTransform.NormalizeSrs(declared); + if (!SpatialReferenceTransform.TryTransformXY(srs, "EPSG:4326", 0, 0, out _, out _)) continue; + var swap = OgcUrlUtilities.UsesLatitudeLongitudeAxisOrder(declared); + var degrees = srs is "EPSG:4326" or "EPSG:4258" or "EPSG:4283" or "EPSG:7844" or "EPSG:7423"; + var matrices = new List(); + foreach (var m in set.Elements(w + "TileMatrix")) + { + var top = Numbers(Text(m, w + "TopLeftCorner")); if (top.Length != 2) throw new InvalidDataException("Invalid WMTS top-left coordinate."); + var resolution = Double(Text(m, w + "ScaleDenominator")) * 0.00028 / (degrees ? 111319.49079327358 : 1); + var tw = Positive(Text(m, w + "TileWidth")); var th = Positive(Text(m, w + "TileHeight")); + if (resolution <= 0 || !double.IsFinite(resolution) || tw > 2048 || th > 2048) throw new InvalidDataException("Unsupported WMTS matrix resolution/tile size."); + matrices.Add(new(Text(m, o + "Identifier"), resolution, new(swap ? top[1] : top[0], swap ? top[0] : top[1]), tw, th, Positive(Text(m, w + "MatrixWidth")), Positive(Text(m, w + "MatrixHeight")))); + if (matrices.Count > 128) throw new InvalidDataException("WMTS exceeds 128 matrices per set."); + } + sets.Add(id, new(id, srs, matrices)); if (sets.Count > 128) throw new InvalidDataException("WMTS exceeds 128 matrix sets."); + } + var layers = new List(); + foreach (var l in contents.Elements(w + "Layer")) + { + var links = new List(); + foreach (var link in l.Elements(w + "TileMatrixSetLink")) + { + var limits = new Dictionary(StringComparer.Ordinal); + foreach (var limit in link.Element(w + "TileMatrixSetLimits")?.Elements(w + "TileMatrixLimits") ?? Enumerable.Empty()) + { + var minR = Nonnegative(Text(limit, w + "MinTileRow")); var maxR = Nonnegative(Text(limit, w + "MaxTileRow")); var minC = Nonnegative(Text(limit, w + "MinTileCol")); var maxC = Nonnegative(Text(limit, w + "MaxTileCol")); + if (minR > maxR || minC > maxC) throw new InvalidDataException("Invalid WMTS tile limits."); + limits.Add(Text(limit, w + "TileMatrix"), new(minR, maxR, minC, maxC)); + } + links.Add(new(Text(link, w + "TileMatrixSet"), limits)); + } + var dimensions = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var d in l.Elements(w + "Dimension")) + { + var name = Text(d, o + "Identifier"); var values = d.Elements(w + "Value").Select(e => e.Value).ToList(); + var value = overrides is not null && overrides.TryGetValue(name, out var supplied) ? supplied : (string?)d.Element(w + "Default") ?? (values.Count == 1 ? values[0] : ""); + dimensions.Add(name, value); + } + var templates = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var resource in l.Elements(w + "ResourceURL").Where(e => (string?)e.Attribute("resourceType") == "tile")) + templates[(string?)resource.Attribute("format") ?? ""] = Resolve(responseUrl, (string?)resource.Attribute("template") ?? "").Replace("%7B", "{", StringComparison.OrdinalIgnoreCase).Replace("%7D", "}", StringComparison.OrdinalIgnoreCase); + var bbox = l.Element(o + "WGS84BoundingBox"); BoundingBox2D? bounds = null; + if (bbox is not null) { var lo = Numbers(Text(bbox, o + "LowerCorner")); var hi = Numbers(Text(bbox, o + "UpperCorner")); if (lo.Length == 2 && hi.Length == 2) bounds = new(lo[0], lo[1], hi[0], hi[1]); } + var style = l.Elements(w + "Style").FirstOrDefault(e => (string?)e.Attribute("isDefault") is "true" or "1") ?? l.Element(w + "Style"); + layers.Add(new(Text(l, o + "Identifier"), (string?)l.Element(o + "Title") ?? Text(l, o + "Identifier"), style is null ? "" : Text(style, o + "Identifier"), l.Elements(w + "Format").Select(e => e.Value).ToList(), links, dimensions, templates, bounds)); + if (layers.Count > 2000) throw new InvalidDataException("WMTS exceeds 2000 layers."); + } + var operation = doc.Descendants(o + "Operation").FirstOrDefault(e => (string?)e.Attribute("name") == "GetTile"); + var get = operation?.Descendants(o + "Get").FirstOrDefault(e => !e.Descendants(o + "Value").Any() || e.Descendants(o + "Value").Any(v => v.Value == "KVP")); + var endpoint = (string?)get?.Attribute(x + "href"); + return new(layers, sets, string.IsNullOrWhiteSpace(endpoint) ? "" : OgcUrlUtilities.PreferSecureSameHostOperationUrl(responseUrl.AbsoluteUri, Resolve(responseUrl, endpoint))); + } + public static WmtsPlan Plan(WmtsCapabilities capabilities, string? layerName, SpatialContext2D context, string format = "image/png") + { + var layer = string.IsNullOrWhiteSpace(layerName) ? capabilities.Layers.FirstOrDefault() : capabilities.Layers.FirstOrDefault(l => l.Id == layerName); + if (layer is null) throw new InvalidDataException("The requested WMTS layer was not found."); + if (layer.Dimensions.Values.Any(string.IsNullOrWhiteSpace)) throw new InvalidDataException("WMTS dimension has no unambiguous default; set it in the source URL query."); + if (layer.Wgs84Bounds is { } coverage && coverage.MinX <= coverage.MaxX && coverage.MinY <= coverage.MaxY) + { + var requested = Lod2AtomFeedReader.TransformBounds(new(context.ResolvedSrs, context.PlacementBoundingBox), "EPSG:4326"); + if (requested.MinX > coverage.MaxX || requested.MaxX < coverage.MinX || requested.MinY > coverage.MaxY || requested.MaxY < coverage.MinY) + throw new InvalidDataException("WMTS layer declares no coverage for the selected Spatial Context."); + } + var selectedFormat = layer.Formats.FirstOrDefault(f => f == format && f is "image/png" or "image/jpeg") ?? layer.Formats.FirstOrDefault(f => f is "image/png" or "image/jpeg") ?? throw new InvalidDataException("WMTS layer has no supported PNG/JPEG format."); + foreach (var link in layer.Links.OrderByDescending(l => capabilities.Sets.TryGetValue(l.Set, out var s) && s.Srs == context.ResolvedSrs)) + { + if (!capabilities.Sets.TryGetValue(link.Set, out var set)) continue; + var bounds = Lod2AtomFeedReader.TransformBounds(new(context.ResolvedSrs, context.PlacementBoundingBox), set.Srs); + var desired = Math.Max(bounds.MaxX - bounds.MinX, bounds.MaxY - bounds.MinY) / 1024; + if (desired <= 0) throw new InvalidDataException("WMTS requires a nonempty selected area."); + foreach (var m in set.Matrices.OrderBy(m => Math.Abs(Math.Log(m.Resolution / desired)))) + { + if (link.Limits.Count > 0 && !link.Limits.ContainsKey(m.Id)) continue; + var sx = m.Resolution * m.TileWidth; var sy = m.Resolution * m.TileHeight; + var c0 = Math.Floor((bounds.MinX - m.TopLeft.X) / sx); var c1 = Math.Ceiling((bounds.MaxX - m.TopLeft.X) / sx) - 1; + var r0 = Math.Floor((m.TopLeft.Y - bounds.MaxY) / sy); var r1 = Math.Ceiling((m.TopLeft.Y - bounds.MinY) / sy) - 1; + var limits = link.Limits.GetValueOrDefault(m.Id) ?? new(0, m.Height - 1, 0, m.Width - 1); + var loC = Math.Max(Math.Max(c0, 0), limits.MinColumn); var hiC = Math.Min(Math.Min(c1, m.Width - 1), limits.MaxColumn); + var loR = Math.Max(Math.Max(r0, 0), limits.MinRow); var hiR = Math.Min(Math.Min(r1, m.Height - 1), limits.MaxRow); + if (hiC < loC || hiR < loR) continue; + var minC = (int)loC; var maxC = (int)hiC; var minR = (int)loR; var maxR = (int)hiR; + var count = (long)(maxC - minC + 1) * (maxR - minR + 1); var width = (long)(maxC - minC + 1) * m.TileWidth; var height = (long)(maxR - minR + 1) * m.TileHeight; + if (count > 64 || width * height > 4194304 || width > 8192 || height > 8192) continue; + var tiles = new List(); + for (var r = minR; r <= maxR; r++) for (var c = minC; c <= maxC; c++) + tiles.Add(new(r, c, TileUrl(capabilities, layer, set, m, r, c, selectedFormat))); + return new(layer, set, m, tiles, minR, minC, (int)width, (int)height, new(m.Resolution, 0, m.TopLeft.X + minC * sx, 0, -m.Resolution, m.TopLeft.Y - minR * sy), c0 < minC || c1 > maxC || r0 < minR || r1 > maxR); + } + } + throw new InvalidDataException("No supported WMTS matrix covers this area within 64 tiles/four million pixels; reduce the area or select another layer."); + } + public static string TileUrl(WmtsCapabilities caps, WmtsLayer layer, WmtsMatrixSet set, WmtsMatrix matrix, int row, int column, string format) + { + var values = new Dictionary(layer.Dimensions, StringComparer.OrdinalIgnoreCase) { { "Layer", layer.Id }, { "Style", layer.Style }, { "TileMatrixSet", set.Id }, { "TileMatrix", matrix.Id }, { "TileRow", row.ToString(CultureInfo.InvariantCulture) }, { "TileCol", column.ToString(CultureInfo.InvariantCulture) }, { "Format", format } }; + if (layer.Templates.TryGetValue(format, out var template)) + { + foreach (var v in values) template = template.Replace("{" + v.Key + "}", Uri.EscapeDataString(v.Value), StringComparison.OrdinalIgnoreCase); + if (template.Contains('{')) throw new InvalidDataException("WMTS tile template has an unresolved dimension."); + return template; + } + if (string.IsNullOrWhiteSpace(caps.TileUrl)) throw new InvalidDataException("WMTS advertises neither a usable REST tile template nor KVP GetTile endpoint."); + values["SERVICE"] = "WMTS"; values["REQUEST"] = "GetTile"; values["VERSION"] = "1.0.0"; + return WithQuery(caps.TileUrl, values); + } + public async Task DownloadAsync(WmtsPlan plan, CancellationToken token = default) + { + using var timeout = CancellationTokenSource.CreateLinkedTokenSource(token); timeout.CancelAfter(TimeSpan.FromMinutes(2)); + var key = Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(System.Text.Json.JsonSerializer.Serialize(new { plan.Transform, plan.Width, plan.Height, plan.Set.Srs, Urls = plan.Tiles.Select(t => t.Url) })))); + Directory.CreateDirectory(_directory); var target = Path.Combine(_directory, key + ".png"); var gate = Gates.GetOrAdd(target, _ => new(1, 1)); + await gate.WaitAsync(timeout.Token).ConfigureAwait(false); + try + { + bool cached = false; + if (File.Exists(target) && DateTime.UtcNow - File.GetLastWriteTimeUtc(target) < TimeSpan.FromDays(1)) + try { using var cachedStream = File.OpenRead(target); var info = StbImageNext.ImageInfo.FromStream(cachedStream); cached = info.Width == plan.Width && info.Height == plan.Height && File.Exists(target + ".sha256") && File.ReadAllText(target + ".sha256") == Convert.ToHexString(SHA256.HashData(File.ReadAllBytes(target))); } catch { } + if (!cached) + { + var rgba = new byte[checked(plan.Width * plan.Height * 4)]; long total = 0; + foreach (var tile in plan.Tiles) + { + using var response = await _http.GetAsync(tile.Url, HttpCompletionOption.ResponseHeadersRead, timeout.Token).ConfigureAwait(false); response.EnsureSuccessStatusCode(); + await using var input = await response.Content.ReadAsStreamAsync(timeout.Token).ConfigureAwait(false); using var bytes = new MemoryStream(); var buffer = new byte[65536]; int n; + while ((n = await input.ReadAsync(buffer, timeout.Token).ConfigureAwait(false)) > 0) + { total += n; if (bytes.Length + n > 4 * 1024 * 1024 || total > 64L * 1024 * 1024) throw new InvalidDataException("WMTS transfer exceeds tile/batch byte limits."); bytes.Write(buffer, 0, n); } + var pixels = RasterImageCodec.DecodeRgba(bytes.ToArray(), plan.Matrix.TileWidth, plan.Matrix.TileHeight); + for (var row = 0; row < plan.Matrix.TileHeight; row++) Buffer.BlockCopy(pixels, row * plan.Matrix.TileWidth * 4, rgba, ((tile.Row - plan.MinRow) * plan.Matrix.TileHeight * plan.Width + (tile.Column - plan.MinColumn) * plan.Matrix.TileWidth + row * plan.Width) * 4, plan.Matrix.TileWidth * 4); + } + timeout.Token.ThrowIfCancellationRequested(); var temporary = target + "." + Guid.NewGuid().ToString("N") + ".tmp"; + try { RasterImageCodec.WritePng(temporary, plan.Width, plan.Height, rgba); timeout.Token.ThrowIfCancellationRequested(); var checksum = Convert.ToHexString(SHA256.HashData(File.ReadAllBytes(temporary))); File.Move(temporary, target, true); File.WriteAllText(target + ".sha256", checksum); } finally { if (File.Exists(temporary)) File.Delete(temporary); } + } + var raster = new GeoReferencedRasterInfo(target, plan.Set.Srs, plan.Width, plan.Height, plan.Transform.Bounds(plan.Width, plan.Height), new FileInfo(target).Length) { GeoTransform = plan.Transform }; + return new(target, raster, plan, cached, $"{(cached ? "Using cached" : "Loaded")} WMTS '{plan.Layer.Id}', matrix set '{plan.Set.Id}', matrix '{plan.Matrix.Id}', {plan.Set.Srs}, resolution {plan.Matrix.Resolution.ToString("G6", CultureInfo.InvariantCulture)}, {plan.Tiles.Count} tiles, {plan.Width}x{plan.Height}." + (plan.CoverageClipped ? " Declared matrix coverage clips the selected area." : "")); + } + finally { gate.Release(); } + } + private static string Text(XElement e, XName name) => (string?)e.Element(name) ?? throw new InvalidDataException("Missing WMTS " + name.LocalName); + private static double Double(string s) { var d = double.Parse(s, NumberStyles.Float, CultureInfo.InvariantCulture); if (!double.IsFinite(d)) throw new InvalidDataException("Nonfinite WMTS coordinate."); return d; } + private static double[] Numbers(string s) => s.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries).Select(Double).ToArray(); + private static int Positive(string s) { var n = Nonnegative(s); return n > 0 ? n : throw new InvalidDataException("WMTS size must be positive."); } + private static int Nonnegative(string s) { var n = int.Parse(s, CultureInfo.InvariantCulture); return n >= 0 ? n : throw new InvalidDataException("WMTS limit must be nonnegative."); } + private static string Resolve(Uri basis, string href) { var uri = new Uri(basis, href); if (uri.Scheme is not ("http" or "https") || uri.UserInfo.Length > 0) throw new InvalidDataException("WMTS links require HTTP(S) without embedded credentials."); return uri.AbsoluteUri; } + private static Dictionary Query(string query) => query.TrimStart('?').Split('&', StringSplitOptions.RemoveEmptyEntries).Select(p => p.Split('=', 2)).GroupBy(p => Uri.UnescapeDataString(p[0]), StringComparer.OrdinalIgnoreCase).ToDictionary(g => g.Key, g => g.Last().Length > 1 ? Uri.UnescapeDataString(g.Last()[1]) : "", StringComparer.OrdinalIgnoreCase); + private static string WithQuery(string url, Dictionary values) { var uri = new UriBuilder(url); var query = Query(uri.Query); foreach (var pair in values) query[pair.Key] = pair.Value; uri.Query = string.Join("&", query.Select(p => Uri.EscapeDataString(p.Key) + "=" + Uri.EscapeDataString(p.Value))); return uri.Uri.AbsoluteUri; } +} diff --git a/RhinoSpatial/LoadGeoTiffComponent.cs b/RhinoSpatial/LoadGeoTiffComponent.cs index 81af3c0..b3037d4 100644 --- a/RhinoSpatial/LoadGeoTiffComponent.cs +++ b/RhinoSpatial/LoadGeoTiffComponent.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Globalization; using System.IO; +using System.Linq; using System.Threading.Tasks; using GH_IO.Serialization; using Grasshopper.Kernel; @@ -42,11 +43,12 @@ private sealed class RequestData public string FilePath { get; init; } = string.Empty; public SpatialContext2D SpatialContext { get; init; } = null!; public string RequestSignature { get; init; } = string.Empty; + public System.Threading.CancellationToken CancellationToken { get; init; } } public LoadGeoTiffComponent() : base("Load GeoTIFF", "Load GeoTIFF", - "Load a georeferenced GeoTIFF raster and align it to the shared RhinoSpatial spatial context.", + "Load local georeferenced imagery or a remote COG window and align it to the shared RhinoSpatial spatial context.", "RhinoSpatial", "Sources") { } @@ -55,7 +57,7 @@ public LoadGeoTiffComponent() protected override void RegisterInputParams(GH_InputParamManager pManager) { - pManager.AddTextParameter("GeoTIFF File", "GeoTIFF", "Path to a georeferenced GeoTIFF raster file.", GH_ParamAccess.item); + pManager.AddTextParameter("GeoTIFF File", "GeoTIFF", "Local GeoTIFF/world-file image path or HTTP(S) COG TIFF URL.", GH_ParamAccess.item); pManager.AddTextParameter("Spatial Context", "Spatial Context", "Shared RhinoSpatial spatial context from the Spatial Context component.", GH_ParamAccess.item); } @@ -63,7 +65,7 @@ protected override void RegisterOutputParams(GH_OutputParamManager pManager) { pManager.AddMeshParameter("Image Mesh", "Image Mesh", "Aligned local mesh for the GeoTIFF image.", GH_ParamAccess.item); pManager.AddGenericParameter("Material", "Material", "Material with the GeoTIFF already attached.", GH_ParamAccess.item); - pManager.AddTextParameter("Image File", "Image File", "Local GeoTIFF file used for the aligned raster source.", GH_ParamAccess.item); + pManager.AddTextParameter("Image File", "Image File", "Local source image or selected remote COG PNG window used by the material.", GH_ParamAccess.item); pManager.AddTextParameter("Status", "Status", "Status or warning information from the GeoTIFF loader.", GH_ParamAccess.item); } @@ -129,7 +131,7 @@ private bool TryGetRequestData(IGH_DataAccess dataAccess, out RequestData reques return false; } - if (!File.Exists(filePath)) + if (!File.Exists(filePath.Trim()) && !CogReader.LooksLike(filePath.Trim())) { AddRuntimeMessage(GH_RuntimeMessageLevel.Error, $"GeoTIFF file not found: {filePath}"); return false; @@ -145,7 +147,8 @@ private bool TryGetRequestData(IGH_DataAccess dataAccess, out RequestData reques { FilePath = filePath.Trim(), SpatialContext = spatialContext, - RequestSignature = BuildRequestSignature(filePath.Trim(), spatialContext) + RequestSignature = BuildRequestSignature(filePath.Trim(), spatialContext), + CancellationToken = CancelToken }; return true; @@ -153,7 +156,9 @@ private bool TryGetRequestData(IGH_DataAccess dataAccess, out RequestData reques private SolveResults Compute(RequestData requestData) { - var rasterInfo = GeoTiffInfoCache.GetOrRead(requestData.FilePath); + var remote=CogReader.LooksLike(requestData.FilePath) + ?new CogReader().ReadImageAsync(requestData.FilePath,requestData.SpatialContext,requestData.CancellationToken).GetAwaiter().GetResult():null; + var rasterInfo = remote?.Image??GeoTiffInfoCache.GetOrRead(requestData.FilePath); if (!TryCreateImageMesh( rasterInfo, @@ -165,8 +170,8 @@ private SolveResults Compute(RequestData requestData) { return new SolveResults { - Status = $"The GeoTIFF could not be aligned from '{rasterInfo.SrsName}' into the current Spatial Context SRS '{requestData.SpatialContext.ResolvedSrs}'.", - MessageLevel = GH_RuntimeMessageLevel.Error + Status = "The raster footprint does not overlap the selected Spatial Context.", + MessageLevel = GH_RuntimeMessageLevel.Warning }; } @@ -185,6 +190,12 @@ private SolveResults Compute(RequestData requestData) }; } + if(remote is not null)return new SolveResults + { + ImageFilePath=rasterInfo.LocalFilePath,ImageMesh=imageMesh, + Status=$"Loaded remote COG window {rasterInfo.Width}×{rasterInfo.Height} ({rasterInfo.SrsName}) from {remote.OverviewWidth}-pixel overview, {remote.RangeRequests} range request(s), {FormatFileSize(remote.TransferredBytes)} transferred. Aligned to the shared Spatial Context.", + MessageLevel=GH_RuntimeMessageLevel.Remark + }; var largeRasterNote = BuildLargeRasterNote(rasterInfo); if (clippedToSpatialContext) { @@ -272,112 +283,20 @@ private void ApplySolveResults(IGH_DataAccess dataAccess, SolveResults result) dataAccess.SetData(3, result.Status); } - private static bool TryCreateImageMesh( - GeoReferencedRasterInfo rasterInfo, - SpatialContext2D spatialContext, - out Mesh imageMesh, - out BoundingBox2D transformedBoundingBox, - out bool clippedToSpatialContext, - out double overlapRatio) + internal static bool TryCreateImageMesh( + GeoReferencedRasterInfo rasterInfo, SpatialContext2D spatialContext, + out Mesh imageMesh, out BoundingBox2D transformedBoundingBox, + out bool clippedToSpatialContext, out double overlapRatio) { - imageMesh = new Mesh(); - transformedBoundingBox = new BoundingBox2D(0.0, 0.0, 0.0, 0.0); - clippedToSpatialContext = false; - overlapRatio = 0.0; - - var corners = new[] - { - new Coordinate2D(rasterInfo.BoundingBox.MinX, rasterInfo.BoundingBox.MinY), - new Coordinate2D(rasterInfo.BoundingBox.MaxX, rasterInfo.BoundingBox.MinY), - new Coordinate2D(rasterInfo.BoundingBox.MaxX, rasterInfo.BoundingBox.MaxY), - new Coordinate2D(rasterInfo.BoundingBox.MinX, rasterInfo.BoundingBox.MaxY) - }; - - var transformedCorners = new Point3d[4]; - var offsetX = spatialContext.UseAbsoluteCoordinates ? 0.0 : spatialContext.PlacementOrigin.X; - var offsetY = spatialContext.UseAbsoluteCoordinates ? 0.0 : spatialContext.PlacementOrigin.Y; - - for (var index = 0; index < corners.Length; index++) - { - var sourceCorner = corners[index]; - if (!SpatialReferenceTransform.TryTransformXY( - rasterInfo.SrsName, - spatialContext.ResolvedSrs, - sourceCorner.X, - sourceCorner.Y, - out var x, - out var y)) - { - return false; - } - - transformedCorners[index] = new Point3d(x - offsetX, y - offsetY, 0.0); - } - - transformedBoundingBox = BoundingBox2DFromPoints(transformedCorners); - - var placedSpatialContextBoundingBox = RhinoSpatialContextTools.CreatePlacedBoundingBox( - spatialContext.PlacementBoundingBox, - spatialContext.PlacementOrigin, - spatialContext.UseAbsoluteCoordinates); - - var fullArea = RhinoSpatialContextTools.CalculateBoundingBoxArea(transformedBoundingBox); - if (fullArea > 0.0 && - RhinoSpatialContextTools.TryIntersectBoundingBoxes( - transformedBoundingBox, - placedSpatialContextBoundingBox, - out var intersectionBoundingBox)) - { - overlapRatio = RhinoSpatialContextTools.CalculateBoundingBoxArea(intersectionBoundingBox) / fullArea; - if (overlapRatio > 0.0 && overlapRatio < 0.999) - { - clippedToSpatialContext = true; - - var minU = (float)((intersectionBoundingBox.MinX - transformedBoundingBox.MinX) / (transformedBoundingBox.MaxX - transformedBoundingBox.MinX)); - var maxU = (float)((intersectionBoundingBox.MaxX - transformedBoundingBox.MinX) / (transformedBoundingBox.MaxX - transformedBoundingBox.MinX)); - var minV = (float)((intersectionBoundingBox.MinY - transformedBoundingBox.MinY) / (transformedBoundingBox.MaxY - transformedBoundingBox.MinY)); - var maxV = (float)((intersectionBoundingBox.MaxY - transformedBoundingBox.MinY) / (transformedBoundingBox.MaxY - transformedBoundingBox.MinY)); - - imageMesh = RhinoSpatialContextTools.CreateTexturedBoundingBoxMesh( - intersectionBoundingBox, - new Coordinate2D(0.0, 0.0), - true, - minU, - minV, - maxU, - maxV); - - return true; - } - } - - imageMesh = RhinoSpatialContextTools.CreateTexturedBoundingBoxMesh( - transformedBoundingBox, - new Coordinate2D(0.0, 0.0), - true, - 0.0f, - 0.0f, - 1.0f, - 1.0f); - return true; - } - - private static BoundingBox2D BoundingBox2DFromPoints(IEnumerable points) - { - var minX = double.PositiveInfinity; - var minY = double.PositiveInfinity; - var maxX = double.NegativeInfinity; - var maxY = double.NegativeInfinity; - - foreach (var point in points) - { - minX = Math.Min(minX, point.X); - minY = Math.Min(minY, point.Y); - maxX = Math.Max(maxX, point.X); - maxY = Math.Max(maxY, point.Y); - } - - return new BoundingBox2D(minX, minY, maxX, maxY); + var data=RasterImageMeshBuilder.Build(rasterInfo,spatialContext); + imageMesh=new Mesh(); + foreach(var p in data.Vertices) {imageMesh.Vertices.Add(p.X,p.Y,0);imageMesh.TextureCoordinates.Add((float)p.U,(float)p.V);} + foreach(var f in data.Faces) imageMesh.Faces.AddFace(f.A,f.B,f.C); + imageMesh.Normals.ComputeNormals();imageMesh.Compact(); + transformedBoundingBox=data.Vertices.Count==0 ? new BoundingBox2D(0,0,0,0) : + new BoundingBox2D(data.Vertices.Min(p=>p.X),data.Vertices.Min(p=>p.Y),data.Vertices.Max(p=>p.X),data.Vertices.Max(p=>p.Y)); + clippedToSpatialContext=data.CoverageRatio<0.999; overlapRatio=data.CoverageRatio; + return data.Faces.Count>0; } private static string BuildLargeRasterNote(GeoReferencedRasterInfo rasterInfo) @@ -498,11 +417,12 @@ public override void BakeGeometry(RhinoDoc doc, ObjectAttributes att, List private static string BuildRequestSignature(string filePath, SpatialContext2D spatialContext) { + if(CogReader.LooksLike(filePath))return string.Empty; // Revalidate the remote representation on a new solve. var fileInfo = new FileInfo(filePath); return string.Create( CultureInfo.InvariantCulture, - $"{fileInfo.FullName}|{fileInfo.Length}|{fileInfo.LastWriteTimeUtc.Ticks}|" + + $"{fileInfo.FullName}|{fileInfo.Length}|{fileInfo.LastWriteTimeUtc.Ticks}|{RasterSidecars.Signature(filePath)}|" + $"{spatialContext.ResolvedSrs}|{spatialContext.UseAbsoluteCoordinates}|" + $"{FormatBoundingBox(spatialContext.PlacementBoundingBox)}|" + $"{FormatCoordinate(spatialContext.PlacementOrigin)}"); diff --git a/RhinoSpatial/LoadTerrainComponent.cs b/RhinoSpatial/LoadTerrainComponent.cs index ea62d5f..5c1c55c 100644 --- a/RhinoSpatial/LoadTerrainComponent.cs +++ b/RhinoSpatial/LoadTerrainComponent.cs @@ -17,6 +17,7 @@ public class LoadTerrainComponent : GH_TaskCapableComponent ComputeForSourceAsync( SpatialContext2D spatialContext, CancellationToken cancellationToken) { + if(CogReader.LooksLike(source.BaseUrl)) + { + var remote=await new CogReader().ReadTerrainAsync(source.BaseUrl,spatialContext,cancellationToken).ConfigureAwait(false); + var raster=remote.Raster;var bounds=raster.GeoTransform!.Bounds(raster.Width,raster.Height); + var mesh=BuildTerrainMesh(raster,bounds,spatialContext,out var baseline); + SpatialTerrainCache.Store(spatialContext,raster.SrsName,bounds,raster,baseline); + return new SolveResults { TerrainMeshes=new List{mesh}, + Status=$"Loaded remote COG DEM window {raster.Width}×{raster.Height} ({raster.SrsName}), {remote.RangeRequests} range request(s), {remote.TransferredBytes:N0} bytes transferred. "+BuildTerrainAlignmentNote(spatialContext.UseAbsoluteCoordinates)+BuildMissingElevationNote(raster), + MessageLevel=GH_RuntimeMessageLevel.Remark }; + } + if (source.Kind == TerrainSourceKind.GlobalSkadiTiles) { return await ComputeGlobalTerrainFallbackAsync(source, spatialContext, cancellationToken); @@ -160,6 +173,29 @@ private async Task ComputeForSourceAsync( return ComputeLocalTextGridTerrain(source, spatialContext); } + if (!System.Text.RegularExpressions.Regex.IsMatch(source.BaseUrl, @"[?&]service=wcs(?:&|$)", System.Text.RegularExpressions.RegexOptions.IgnoreCase)) + { + var progressActive = 1; + TerrainAtomResult? atom; + try + { + atom = await _atom.TryLoadAsync(source.BaseUrl, spatialContext, progress => + Eto.Forms.Application.Instance.AsyncInvoke(() => + { + if (System.Threading.Volatile.Read(ref progressActive) != 0 && !cancellationToken.IsCancellationRequested && OnPingDocument() is not null) + { Message = progress; OnDisplayExpired(false); } + }), cancellationToken).ConfigureAwait(false); + } + finally { System.Threading.Volatile.Write(ref progressActive, 0); } + if (atom is not null) + { + var terrain = BuildTerrainMesh(atom.Raster, atom.Bounds, spatialContext, out var baseline); + SpatialTerrainCache.Store(spatialContext, atom.Raster.SrsName, atom.Bounds, atom.Raster, baseline); + return new SolveResults { TerrainMeshes = new List { terrain }, + Status = atom.Status + " " + BuildTerrainAlignmentNote(spatialContext.UseAbsoluteCoordinates) + BuildMissingElevationNote(atom.Raster), MessageLevel = GH_RuntimeMessageLevel.Remark }; + } + } + try { var capabilities = await _wcsClient.LoadCapabilitiesAsync(source.BaseUrl); diff --git a/RhinoSpatial/Lod2LoadComponent.cs b/RhinoSpatial/Lod2LoadComponent.cs index 7e139d6..f218e3b 100644 --- a/RhinoSpatial/Lod2LoadComponent.cs +++ b/RhinoSpatial/Lod2LoadComponent.cs @@ -241,6 +241,15 @@ private SolveResults Compute(RequestData requestData, CancellationToken cancella return ComputeLocalCityGmlSource(requestData); } + if (new[] { ".zip", ".gmz", ".cityjson", ".json", ".jsonl", ".gml", ".citygml", ".xml" } + .Contains(Path.GetExtension(new Uri(requestData.Source).AbsolutePath), StringComparer.OrdinalIgnoreCase) && + !System.Text.RegularExpressions.Regex.IsMatch(requestData.Source, @"[?&]service=wfs(?:&|$)", System.Text.RegularExpressions.RegexOptions.IgnoreCase)) + { + var tile = new Lod2AtomLink(new Uri(requestData.Source), "Building download", "", null); + var downloaded = _atomClient.DownloadFileAsync(tile, cancellationToken).GetAwaiter().GetResult(); + return ComputeDownloadedBuildings(requestData, new[] { downloaded }); + } + // Explicit WFS URLs retain their existing request path without an extra discovery GET. if (!System.Text.RegularExpressions.Regex.IsMatch(requestData.Source, @"[?&]service=wfs(?:&|$)", System.Text.RegularExpressions.RegexOptions.IgnoreCase)) { @@ -350,15 +359,7 @@ private SolveResults ComputeAtomSource(RequestData requestData, Lod2AtomSelectio Volatile.Write(ref progressActive, 0); } cancellationToken.ThrowIfCancellationRequested(); - var localRequest = new RequestData - { - Source = downloaded[0].FilePath, - SourceKind = Lod2SourceKind.Zip, - LocalSourcePaths = downloaded.Select(tile => tile.FilePath).ToList(), - SpatialContext = requestData.SpatialContext, - LayerName = requestData.LayerName - }; - var result = ComputeLocalCityGmlSource(localRequest); + var result = ComputeDownloadedBuildings(requestData, downloaded); var cachedCount = downloaded.Count(tile => tile.UsedCache); return new SolveResults { @@ -369,6 +370,47 @@ private SolveResults ComputeAtomSource(RequestData requestData, Lod2AtomSelectio }; } + private SolveResults ComputeDownloadedBuildings(RequestData requestData, IReadOnlyList downloads) + { + var sources = new Dictionary<(bool Json, Lod2SourceKind Kind), List>(); + foreach (var download in downloads) + { + var kind = Path.GetExtension(download.FilePath).Equals(".zip", StringComparison.OrdinalIgnoreCase) ? Lod2SourceKind.Zip : Lod2SourceKind.File; + if (EnumerateLocalSourceFiles(download.FilePath, kind).Any(file => + { using var stream = file.OpenRead(); return Lod2GmlReader.IsCityGmlDocument(stream); })) Add(false); + if (EnumerateLocalCityJsonSourceFiles(download.FilePath, kind).Any(file => IsCityJsonSourceFile(file))) Add(true); + void Add(bool json) + { + if (!sources.TryGetValue((json, kind), out var files)) sources[(json, kind)] = files = new List(); + files.Add(download.FilePath); + } + } + var groups = new List(); + foreach (var group in sources) + { + var local = new RequestData { Source = group.Value[0], SourceKind = group.Key.Kind, + LocalSourcePaths = group.Value, SpatialContext = requestData.SpatialContext, LayerName = requestData.LayerName }; + var result = group.Key.Json ? ComputeLocalCityJsonSource(local) : ComputeLocalCityGmlSource(local); + if (result.MessageLevel == GH_RuntimeMessageLevel.Error) throw new InvalidDataException(result.Status); + groups.Add(result); + } + if (groups.Count == 0) throw new InvalidDataException("Downloaded source contained no CityGML or CityJSON building documents."); + if (groups.Count == 1) return groups[0]; + var tree = new GH_Structure(); + for (var group = 0; group < groups.Count; group++) + for (var branch = 0; branch < groups[group].BrepTree.PathCount; branch++) + tree.AppendRange(groups[group].BrepTree.Branches[branch], + new GH_Path(new[] { group }.Concat(groups[group].BrepTree.Paths[branch].Indices).ToArray())); + return new SolveResults { BrepTree = tree, BuildingCount = groups.Sum(g => g.BuildingCount), + Status = string.Join(" ", groups.Select(g => g.Status)), MessageLevel = GH_RuntimeMessageLevel.Remark }; + } + + private static bool IsCityJsonSourceFile(LocalCityGmlSourceFile file) + { + using var stream = file.OpenRead(); + return CityJsonReader.IsCityJson(stream); + } + private SolveResults ComputeLocalCityGmlSource(RequestData requestData) { var timings = new LocalCityGmlTimings(); @@ -516,7 +558,9 @@ private SolveResults ComputeSafe(RequestData requestData, CancellationToken canc private SolveResults ComputeLocalCityJsonSource(RequestData requestData) { - var sourceFiles = EnumerateLocalCityJsonSourceFiles(requestData.Source, requestData.SourceKind).ToList(); + var sourceFiles = (requestData.LocalSourcePaths ?? new[] { requestData.Source }) + .SelectMany(source => EnumerateLocalCityJsonSourceFiles(source, requestData.SourceKind)) + .Where(file => requestData.LocalSourcePaths is null || IsCityJsonSourceFile(file)).ToList(); if (sourceFiles.Count == 0) { return new SolveResults @@ -535,10 +579,10 @@ private SolveResults ComputeLocalCityJsonSource(RequestData requestData) CityJsonSourceMetadata metadata; try { - var jsonText = sourceFile.ReadText(); - metadata = CityJsonReader.ReadSourceMetadata(jsonText); + using var sourceStream = sourceFile.OpenRead(); + metadata = CityJsonReader.ReadSourceMetadata(sourceStream); } - catch + catch when (requestData.LocalSourcePaths is null) { if (failedFileNames.Count < 5) { @@ -548,6 +592,9 @@ private SolveResults ComputeLocalCityJsonSource(RequestData requestData) continue; } + if (requestData.LocalSourcePaths is not null && (string.IsNullOrWhiteSpace(metadata.SrsName) || + !SpatialReferenceTransform.TryTransformXY(metadata.SrsName, requestData.SpatialContext.ResolvedSrs, 0, 0, out _, out _))) + throw new InvalidDataException("Downloaded CityJSON has missing or unsupported source CRS metadata."); var candidateSrs = ResolveLocalCityGmlSrs(metadata.SrsName, requestData.SpatialContext); if (metadata.BoundingBox is not null && @@ -613,15 +660,15 @@ private SolveResults ComputeLocalCityJsonSource(RequestData requestData) { try { - var jsonText = candidate.File.ReadText(); + using var cityStream = candidate.File.OpenRead(); var displayName = string.IsNullOrWhiteSpace(requestData.LayerName) ? candidate.File.DisplayName : requestData.LayerName; - returnedBuildings.AddRange(CityJsonReader.ReadBuildings(jsonText, displayName, requestBoundingBox)); + returnedBuildings.AddRange(CityJsonReader.ReadBuildings(cityStream, displayName, requestBoundingBox)); parsedFileCount++; totalByteCount += candidate.File.ByteLength > 0 ? candidate.File.ByteLength - : Encoding.UTF8.GetByteCount(jsonText); + : 0; } catch { @@ -1113,7 +1160,8 @@ private static bool IsCityJsonFileName(string fileName) { var extension = Path.GetExtension(fileName); return extension.Equals(".json", StringComparison.OrdinalIgnoreCase) || - extension.Equals(".cityjson", StringComparison.OrdinalIgnoreCase); + extension.Equals(".cityjson", StringComparison.OrdinalIgnoreCase) || + extension.Equals(".jsonl", StringComparison.OrdinalIgnoreCase); } private static bool LocalSourceContainsCityJson(string source, Lod2SourceKind sourceKind) diff --git a/RhinoSpatial/SpatialContextComponent.cs b/RhinoSpatial/SpatialContextComponent.cs index 9d283cf..568668d 100644 --- a/RhinoSpatial/SpatialContextComponent.cs +++ b/RhinoSpatial/SpatialContextComponent.cs @@ -20,6 +20,7 @@ public class SpatialContextComponent : GH_TaskCapableComponent target, GH_RuntimeMessageLevel.Warning); } + try + { + var remote = _remoteReference.TryReadAsync(requestData.BaseUrl, + RhinoSpatialInputParser.ParseLayerName(requestData.LayerSelection)).GetAwaiter().GetResult(); + if (remote is not null) + { + var supported = NormalizeSupportedMapSrs(remote.SrsName); + return string.IsNullOrEmpty(supported) + ? ("EPSG:3857", "Reference CRS is not supported by the map helper; using EPSG:3857.", GH_RuntimeMessageLevel.Warning) + : (supported, string.Empty, null); + } + } + catch (Exception ex) + { return ("EPSG:3857", "Could not read reference metadata: " + ex.Message, GH_RuntimeMessageLevel.Warning); } + if (ReferenceSourceMetadataReader.IsLocalSource(requestData.BaseUrl)) { try @@ -746,6 +762,15 @@ private async Task OpenMapAsync(string? baseUrl, string? layerSelection, string? return (null, normalizedRequestedSrs); } + try + { + var remote = await _remoteReference.TryReadAsync(baseUrl, RhinoSpatialInputParser.ParseLayerName(layerSelection ?? string.Empty)); + if (remote is not null) + return (remote.Wgs84BoundingBox, string.IsNullOrWhiteSpace(normalizedRequestedSrs) + ? NormalizeSupportedMapSrs(remote.SrsName) : normalizedRequestedSrs); + } + catch { return (null, normalizedRequestedSrs); } + if (ReferenceSourceMetadataReader.IsLocalSource(baseUrl)) { try diff --git a/RhinoSpatial/SpatialTerrainCache.cs b/RhinoSpatial/SpatialTerrainCache.cs index 4398ddd..c451d46 100644 --- a/RhinoSpatial/SpatialTerrainCache.cs +++ b/RhinoSpatial/SpatialTerrainCache.cs @@ -22,7 +22,7 @@ public static void Store( double elevationBase) { var key = RhinoSpatialContextTools.CreateSpatialContextKey(spatialContext); - EntriesByContext[key] = new Entry(terrainSrs, boundingBox, raster, elevationBase); + EntriesByContext[key] = new Entry(raster.GeoTransform is null ? terrainSrs : raster.SrsName, raster.GeoTransform?.Bounds(raster.Width,raster.Height) ?? boundingBox, raster, elevationBase); } public static bool TrySamplePlacedElevation( @@ -82,6 +82,12 @@ private static bool TrySampleRaster(Entry entry, double x, double y, out double var gridX = (x - entry.BoundingBox.MinX) / spanX * (raster.Width - 1); var gridY = (entry.BoundingBox.MaxY - y) / spanY * (raster.Height - 1); + if (raster.GeoTransform is { } geo) + { + var pixel=geo.Pixel(x,y); gridX=pixel.X-0.5; gridY=pixel.Y-0.5; + if(gridX<0 || gridY<0 || gridX>raster.Width-1 || gridY>raster.Height-1) return false; + } + var x0 = Math.Clamp((int)Math.Floor(gridX), 0, raster.Width - 1); var y0 = Math.Clamp((int)Math.Floor(gridY), 0, raster.Height - 1); var x1 = Math.Min(x0 + 1, raster.Width - 1); diff --git a/RhinoSpatial/WfsListLayersComponent.cs b/RhinoSpatial/WfsListLayersComponent.cs index 6e6b97a..818faf0 100644 --- a/RhinoSpatial/WfsListLayersComponent.cs +++ b/RhinoSpatial/WfsListLayersComponent.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Threading.Tasks; using Grasshopper.Kernel; using RhinoSpatial.Core; @@ -26,7 +27,7 @@ private class RequestData public WfsListLayersComponent() : base("List WFS Layers", "WFS Layers", - "List available layers from a WFS GetCapabilities response.", + "List WFS layers, OGC API Features collections, or local GeoPackage feature layers.", "RhinoSpatial", "Layers") { } @@ -35,7 +36,7 @@ public WfsListLayersComponent() protected override void RegisterInputParams(GH_InputParamManager pManager) { - pManager.AddTextParameter("WFS Service URL", "WFS URL", "Base URL of the WFS service.", GH_ParamAccess.item); + pManager.AddTextParameter("WFS Service URL", "WFS URL", "WFS/OGC API service URL or local GeoPackage path.", GH_ParamAccess.item); } protected override void RegisterOutputParams(GH_OutputParamManager pManager) @@ -117,7 +118,15 @@ private bool TryGetRequestData(IGH_DataAccess dataAccess, out RequestData reques private SolveResults Compute(RequestData requestData) { - var layers = _wfsClient.LoadLayersAsync(requestData.BaseUrl).GetAwaiter().GetResult(); + if (VectorSourceResolver.Resolve(requestData.BaseUrl).Kind == VectorSourceKind.LocalGeoPackage) + { + var localLayers = GeoPackageReader.ReadLayers(requestData.BaseUrl,CancelToken); + return new SolveResults { Layers = localLayers.Select(l => new WfsLayerInfo(l.Name,l.Title,l.SourceSrs,new List(), + l.Bounds is not null && ReferenceSourceMetadataReader.TryTransformBoundsToWgs84(l.SourceSrs,l.Bounds,out var wgs)?wgs:null)).ToList() }; + } + var collections = new OgcApiFeaturesClient().TryDiscoverAsync(requestData.BaseUrl).GetAwaiter().GetResult(); + var layers = collections is null ? _wfsClient.LoadLayersAsync(requestData.BaseUrl).GetAwaiter().GetResult() + : System.Linq.Enumerable.ToList(System.Linq.Enumerable.Select(collections, c => new WfsLayerInfo(c.Id, c.Title, "EPSG:4326", new List(), c.Wgs84Bounds))); return new SolveResults { diff --git a/RhinoSpatial/WfsLoadComponent.cs b/RhinoSpatial/WfsLoadComponent.cs index dfa38d3..aaae2a0 100644 --- a/RhinoSpatial/WfsLoadComponent.cs +++ b/RhinoSpatial/WfsLoadComponent.cs @@ -15,6 +15,7 @@ public class WfsLoadComponent : GH_TaskCapableComponent RequestedLayerNames { get; init; } = new(); public int MaxFeatures { get; init; } @@ -63,10 +66,11 @@ public WfsLoadComponent() protected override void RegisterInputParams(GH_InputParamManager pManager) { - pManager.AddTextParameter("WFS Service URL", "WFS URL", "Base URL of the WFS service, OGC API Features collection/items URL, or path to a local Shapefile (.shp) or GeoJSON (.geojson/.json). If left empty, RhinoSpatial will try to inherit it from the connected Layer input.", GH_ParamAccess.item); - pManager.AddTextParameter("Layer", "Layer", "One or more layer names or layer entries. Use List Item to choose one layer, or merge explicit selections if you want to load several layers. Optional for local Shapefile, local GeoJSON, and OGC API Features sources.", GH_ParamAccess.list); - pManager.AddIntegerParameter("Max Features", "Max Features", "Maximum number of features to request. Use 0 to request all available features.", GH_ParamAccess.item, 0); + pManager.AddTextParameter("WFS Service URL", "WFS URL", "WFS service, OGC API Features root/collection/items URL, direct GeoJSON or vector ZIP URL, or local Shapefile/GeoJSON/GeoPackage path. If left empty, RhinoSpatial will try to inherit it from the connected Layer input.", GH_ParamAccess.item); + pManager.AddTextParameter("Layer", "Layer", "One or more layer names or layer entries. Use List Item to choose one layer, or merge explicit selections if you want to load several layers. Optional for files and a single OGC API collection; choose a collection explicitly when an API root exposes several.", GH_ParamAccess.list); + pManager.AddIntegerParameter("Max Features", "Max Features", "Per-layer feature cap. Use 0 to follow available pages within the selected area and safety limits; read Status for completeness diagnostics.", GH_ParamAccess.item, 0); pManager.AddTextParameter("Spatial Context", "Spatial Context", "Shared RhinoSpatial spatial context. This is required so WFS, WMS, LoD2, terrain, GeoTIFF, and OSM outputs stay aligned.", GH_ParamAccess.item); + pManager[1].Optional = true; } protected override void RegisterOutputParams(GH_OutputParamManager pManager) @@ -128,10 +132,7 @@ private bool TryGetRequestData(IGH_DataAccess dataAccess, out RequestData reques dataAccess.GetData(0, ref baseUrl); - if (!dataAccess.GetDataList(1, layerSelections)) - { - return false; - } + dataAccess.GetDataList(1, layerSelections); dataAccess.GetData(2, ref maxFeatures); dataAccess.GetData(3, ref spatialContextText); @@ -155,7 +156,7 @@ private bool TryGetRequestData(IGH_DataAccess dataAccess, out RequestData reques var sourceInfo = VectorSourceResolver.Resolve(baseUrl); if (!sourceInfo.IsSupported) { - AddRuntimeMessage(GH_RuntimeMessageLevel.Error, $"WFS URL must be a WFS service URL, an OGC API Features collection/items URL, a local .shp file path, or a local .geojson/.json file path. Source was not found or is unsupported: {baseUrl}"); + AddRuntimeMessage(GH_RuntimeMessageLevel.Error, $"WFS URL must be a WFS service URL, an OGC API Features URL, a supported vector download URL, or a local Shapefile/GeoJSON/GeoPackage path. Source was not found or is unsupported: {baseUrl}"); return false; } @@ -166,14 +167,14 @@ private bool TryGetRequestData(IGH_DataAccess dataAccess, out RequestData reques .Distinct(StringComparer.OrdinalIgnoreCase) .ToList(); - if (requestedLayerNames.Count == 0 && sourceInfo.CanUseDefaultLayerName) + if (requestedLayerNames.Count == 0 && sourceInfo.CanUseDefaultLayerName && !string.IsNullOrWhiteSpace(sourceInfo.DefaultLayerName)) { requestedLayerNames.Add(sourceInfo.DefaultLayerName); } - if (requestedLayerNames.Count == 0) + if (maxFeatures < 0) { - AddRuntimeMessage(GH_RuntimeMessageLevel.Error, "At least one valid layer selection is required."); + AddRuntimeMessage(GH_RuntimeMessageLevel.Error, "Max Features must be zero or a positive number."); return false; } @@ -200,6 +201,7 @@ private bool TryGetRequestData(IGH_DataAccess dataAccess, out RequestData reques RequestedLayerNames = requestedLayerNames, MaxFeatures = maxFeatures, SpatialContext = spatialContext, + CancellationToken = CancelToken, SourceKind = sourceInfo.Kind }; @@ -210,15 +212,17 @@ private SolveResults Compute(RequestData requestData) { return requestData.SourceKind switch { + VectorSourceKind.LocalGeoPackage => ComputeLocalGeoPackage(requestData), VectorSourceKind.LocalShapefile => ComputeLocalShapefile(requestData), VectorSourceKind.LocalGeoJson => ComputeLocalGeoJson(requestData), - VectorSourceKind.OgcApiFeatures => ComputeOgcApiFeatures(requestData), - _ => ComputeWfs(requestData) + VectorSourceKind.RemoteDownload => ComputeRemoteVector(requestData), + _ => ComputeRemoteService(requestData) }; } private SolveResults ComputeWfs(RequestData requestData) { + if (requestData.RequestedLayerNames.Count == 0) throw new InvalidOperationException("Select a WFS layer with List WFS Layers."); var resolvedSrsName = requestData.SpatialContext.ResolvedSrs; var features = new List(); var statusNotes = new List(); @@ -234,7 +238,7 @@ private SolveResults ComputeWfs(RequestData requestData) BoundingBox = requestData.SpatialContext.RequestBoundingBox }; - var layerResult = _wfsClient.LoadFeaturesWithStatusAsync(layerRequestOptions).GetAwaiter().GetResult(); + var layerResult = _wfsClient.LoadFeaturesWithStatusAsync(layerRequestOptions, requestData.CancellationToken).GetAwaiter().GetResult(); var filterResult = SpatialFeatureFilter.FilterToBounds( layerResult.Features, requestData.SpatialContext.RequestBoundingBox); @@ -265,42 +269,161 @@ private SolveResults ComputeWfs(RequestData requestData) maxFeaturesText, resolvedSrsText, statusNotes), - ResolveMessageLevel(features.Count, statusNotes.Count > 0), - statusNotes.Count > 0); + ResolveMessageLevel(features.Count, statusNotes.Any(note => note.Contains("Using cached WFS", StringComparison.Ordinal) || note.StartsWith("Filtered ", StringComparison.Ordinal))), + statusNotes.Any(note => note.Contains("Using cached WFS", StringComparison.Ordinal))); } - private SolveResults ComputeOgcApiFeatures(RequestData requestData) + private SolveResults ComputeRemoteService(RequestData requestData) { - var layerName = requestData.RequestedLayerNames[0]; - var sourceFeatures = _ogcApiFeaturesClient.LoadFeaturesAsync( - requestData.BaseUrl, - layerName, - requestData.SpatialContext.Wgs84BoundingBox, - requestData.MaxFeatures) - .GetAwaiter() - .GetResult(); - var projectedFeatures = WfsFeatureGeometryTransformer.TransformFeatures( - sourceFeatures, - "EPSG:4326", - requestData.SpatialContext.ResolvedSrs); - var filterResult = SpatialFeatureFilter.FilterToBounds( - projectedFeatures, - requestData.SpatialContext.RequestBoundingBox); - var features = filterResult.Features; - var outputData = BuildVectorOutput(requestData, features); - var maxFeaturesText = FormatMaxFeatures(requestData.MaxFeatures); - var coordinateText = FormatCoordinateMode(requestData.SpatialContext); - var filterNote = BuildOversizedFilterNote(filterResult, "feature"); - var filterSuffix = FormatOptionalStatusNote(filterNote); - var status = features.Count == 0 - ? $"No features were found for OGC API Features source '{Path.GetFileName(requestData.BaseUrl)}' inside the current Spatial Context.{filterSuffix}" - : $"Loaded {features.Count} feature(s) and {outputData.GeometryItemCount} geometry item(s) from OGC API Features source '{layerName}' with {maxFeaturesText} features, assumed GeoJSON CRS EPSG:4326, {coordinateText}{filterSuffix}"; + var collections = _ogcApiFeaturesClient.TryDiscoverAsync(requestData.BaseUrl, requestData.CancellationToken).GetAwaiter().GetResult(); + if (collections is null) + { + if (requestData.RequestedLayerNames.Count == 0) return ComputeRemoteVector(requestData); + return ComputeWfs(requestData); + } + if (collections.Count == 0) throw new InvalidDataException("This OGC API exposes no feature collections."); + var names = requestData.RequestedLayerNames; + if (names.Count == 0 || (collections.Count == 1 && !names.Contains(collections[0].Id))) + names = collections.Count == 1 ? new List { collections[0].Id } + : throw new InvalidOperationException("This OGC API has several collections. Use List WFS Layers and select the collection(s) to load."); + if (names.Any(name => !collections.Any(c => c.Id == name))) + throw new InvalidOperationException("An OGC API collection selection was not found. Refresh List WFS Layers."); + var resolved = CopyRequest(requestData, requestData.BaseUrl, VectorSourceKind.OgcApiFeatures, names); + var features = new List(); + var notes = new List(); + foreach (var name in names) + { + var result = _ogcApiFeaturesClient.LoadFeaturesWithStatusAsync(requestData.BaseUrl, name, + requestData.SpatialContext.Wgs84BoundingBox, requestData.MaxFeatures, requestData.CancellationToken).GetAwaiter().GetResult(); + var projected = WfsFeatureGeometryTransformer.TransformFeatures(result.Features, "EPSG:4326", requestData.SpatialContext.ResolvedSrs); + var filtered = SpatialFeatureFilter.FilterToBounds(projected, requestData.SpatialContext.RequestBoundingBox); + features.AddRange(filtered.Features); + notes.Add(name + ": " + result.StatusNote); + if (filtered.OversizedEnclosingFeatureCount > 0) notes.Add(BuildOversizedFilterNote(filtered, "feature")); + } + var output = BuildVectorOutput(resolved, features); + return BuildSolveResults(output, features.Count, + $"Loaded {features.Count} OGC API feature(s), {output.GeometryItemCount} geometry item(s), {FormatCoordinateMode(requestData.SpatialContext)}. {string.Join(" ", notes)}", + features.Count == 0 ? GH_RuntimeMessageLevel.Warning : GH_RuntimeMessageLevel.Remark); + } + private static RequestData CopyRequest(RequestData source, string path, VectorSourceKind kind, List layers) => new() + { + BaseUrl = path, SourceKind = kind, RequestedLayerNames = layers, + MaxFeatures = source.MaxFeatures, SpatialContext = source.SpatialContext, CancellationToken = source.CancellationToken + }; + + private SolveResults ComputeRemoteVector(RequestData requestData) + { + var tile = new Lod2AtomLink(new Uri(requestData.BaseUrl), "Vector download", "", null); + var downloaded = _downloads.DownloadAsync(tile, cancellationToken: requestData.CancellationToken).GetAwaiter().GetResult(); + using var files = SourceFilesLease.Open(downloaded.FilePath, requestData.CancellationToken); + var candidates = files.Files.Where(path => Path.GetExtension(path).Equals(".shp", StringComparison.OrdinalIgnoreCase) || + Path.GetExtension(path).Equals(".geojson", StringComparison.OrdinalIgnoreCase) || Path.GetExtension(path).Equals(".json", StringComparison.OrdinalIgnoreCase)).ToList(); + var features = new List(); + var names = new List(); + var notes = new List(); + var failed = false; + foreach (var file in candidates) + { + requestData.CancellationToken.ThrowIfCancellationRequested(); + var name = candidates.Count == 1 && requestData.RequestedLayerNames.Count == 1 + ? requestData.RequestedLayerNames[0] : Path.GetFileNameWithoutExtension(file); + if (requestData.RequestedLayerNames.Count > 0 && candidates.Count > 1 && !requestData.RequestedLayerNames.Contains(name)) continue; + if (Path.GetExtension(file).Equals(".shp", StringComparison.OrdinalIgnoreCase)) + { + var metadata = ShapefileFeatureReader.ReadSourceMetadata(file); + if (string.IsNullOrWhiteSpace(metadata.SourceSrs)) throw new InvalidDataException("Downloaded Shapefile must have a resolvable source CRS in its .prj file."); + names.Add(name); + var read = ShapefileFeatureReader.ReadFeatures(file, name, requestData.SpatialContext, requestData.MaxFeatures); + features.AddRange(read.Features); + failed |= read.FailedFeatureCount > 0; + notes.Add($"{name}: scanned {read.SourceFeatureCount}, outside {read.SkippedOutsideContextCount}, failed {read.FailedFeatureCount}."); + } + else + { + var json = File.ReadAllText(file); + using var document = System.Text.Json.JsonDocument.Parse(json); + if (!document.RootElement.TryGetProperty("type", out var type) || type.GetString() is not ("Feature" or "FeatureCollection")) continue; + names.Add(name); + var projected = WfsFeatureGeometryTransformer.TransformFeatures(GeoJsonReader.ReadFeatures(json, name), + GeoJsonReader.TryReadSourceSrs(json) ?? "EPSG:4326", requestData.SpatialContext.ResolvedSrs); + var filtered = SpatialFeatureFilter.FilterToBounds(projected, requestData.SpatialContext.RequestBoundingBox); + features.AddRange(requestData.MaxFeatures > 0 ? filtered.Features.Take(requestData.MaxFeatures) : filtered.Features); + if (filtered.OversizedEnclosingFeatureCount > 0) notes.Add(BuildOversizedFilterNote(filtered, "feature")); + } + } + if (names.Count == 0) throw new InvalidDataException("The vector download contains no matching Shapefile or GeoJSON layer."); + var resolved = CopyRequest(requestData, downloaded.FilePath, VectorSourceKind.RemoteDownload, names); + var output = BuildVectorOutput(resolved, features); + return BuildSolveResults(output, features.Count, + $"Loaded {features.Count} feature(s) from a validated vector download ({(downloaded.UsedCache ? "cached" : "downloaded")}), {FormatCoordinateMode(requestData.SpatialContext)}. {string.Join(" ", notes)}", + features.Count == 0 || failed ? GH_RuntimeMessageLevel.Warning : GH_RuntimeMessageLevel.Remark); + } + + private SolveResults ComputeLocalGeoPackage(RequestData requestData) + { + using var deadline = System.Threading.CancellationTokenSource.CreateLinkedTokenSource( + requestData.CancellationToken + ); + deadline.CancelAfter(TimeSpan.FromMinutes(2)); + var names = requestData.RequestedLayerNames; + var layers = GeoPackageReader.ReadLayers(requestData.BaseUrl, deadline.Token); + if (names.Count == 0) + { + if (layers.Count != 1) + throw new InvalidOperationException( + "Select a GeoPackage feature layer using List WFS Layers." + ); + names = new List { layers[0].Name }; + } + var resolved = CopyRequest( + requestData, + requestData.BaseUrl, + VectorSourceKind.LocalGeoPackage, + names + ); + var features = new List(); + var notes = new List(); + long pointCount = 0, + attributeCharacters = 0; + foreach (var name in names) + { + var read = GeoPackageReader.ReadFeatures( + requestData.BaseUrl, + name, + requestData.SpatialContext, + requestData.MaxFeatures, + deadline.Token + ); + pointCount += read.Features.Sum(f => + (long)f.Geometry.Points.Count + + f.Geometry.OuterRings.Sum(r => (long)r.Points.Count) + + f.Geometry.LineStrings.Sum(l => (long)l.Points.Count) + ); + attributeCharacters += read.Features.Sum(f => + f.Attributes.Values.Sum(v => (long)(v?.Length ?? 0)) + ); + if ( + features.Count + read.Features.Count > 100000 + || pointCount > 4000000 + || attributeCharacters > 8 * 1024 * 1024 + ) + throw new InvalidOperationException( + "GeoPackage selection exceeded its combined feature/coordinate/attribute budget; narrow the area or select fewer layers." + ); + features.AddRange(read.Features); + notes.Add( + $"{name}: {read.Features.Count} feature(s), {read.ScannedCount} row(s), {(read.UsedSpatialIndex ? "RTree index" : "bounded scan")}{(read.ReachedFeatureLimit ? ", reached requested feature limit" : "")}." + ); + } + var output = BuildVectorOutput(resolved, features); return BuildSolveResults( - outputData, + output, features.Count, - status, - ResolveMessageLevel(features.Count, filterResult.OversizedEnclosingFeatureCount > 0)); + $"Loaded read-only GeoPackage: {string.Join(" ", notes)} {FormatCoordinateMode(requestData.SpatialContext)}", + features.Count == 0 ? GH_RuntimeMessageLevel.Warning : GH_RuntimeMessageLevel.Remark + ); } private SolveResults ComputeLocalShapefile(RequestData requestData) diff --git a/RhinoSpatial/WmsListLayersComponent.cs b/RhinoSpatial/WmsListLayersComponent.cs index ba2b32c..970e3ca 100644 --- a/RhinoSpatial/WmsListLayersComponent.cs +++ b/RhinoSpatial/WmsListLayersComponent.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Threading.Tasks; using Grasshopper.Kernel; using RhinoSpatial.Core; @@ -26,7 +27,7 @@ private class RequestData public WmsListLayersComponent() : base("List WMS Layers", "WMS Layers", - "List available layers from a WMS GetCapabilities response.", + "List WMS layers or WMTS imagery layers from capabilities.", "RhinoSpatial", "Layers") { } @@ -35,7 +36,7 @@ public WmsListLayersComponent() protected override void RegisterInputParams(GH_InputParamManager pManager) { - pManager.AddTextParameter("WMS Service URL", "WMS URL", "Base URL of the WMS service.", GH_ParamAccess.item); + pManager.AddTextParameter("WMS Service URL", "WMS URL", "WMS service URL or WMTS capabilities URL.", GH_ParamAccess.item); } protected override void RegisterOutputParams(GH_OutputParamManager pManager) @@ -115,7 +116,9 @@ private bool TryGetRequestData(IGH_DataAccess dataAccess, out RequestData reques private SolveResults Compute(RequestData requestData) { - var layers = _wmsClient.LoadLayersAsync(requestData.BaseUrl).GetAwaiter().GetResult(); + var wmts=new WmtsClient().TryLoadCapabilitiesAsync(requestData.BaseUrl,CancelToken).GetAwaiter().GetResult(); + var layers=wmts is null ? _wmsClient.LoadLayersAsync(requestData.BaseUrl).GetAwaiter().GetResult() + : wmts.Layers.Select(l=>new WmsLayerInfo(l.Id,l.Title,l.Links.Where(link=>wmts.Sets.ContainsKey(link.Set)).Select(link=>wmts.Sets[link.Set].Srs).ToList(),l.Wgs84Bounds)).ToList(); return new SolveResults { diff --git a/RhinoSpatial/WmsLoadComponent.cs b/RhinoSpatial/WmsLoadComponent.cs index a0ee1e9..4c70d11 100644 --- a/RhinoSpatial/WmsLoadComponent.cs +++ b/RhinoSpatial/WmsLoadComponent.cs @@ -19,6 +19,7 @@ namespace RhinoSpatial public class WmsLoadComponent : GH_TaskCapableComponent { private readonly WmsClient _wmsClient = new(); + private readonly WmtsClient _wmtsClient = new(); private Mesh? _previewMesh; private DisplayMaterial? _previewMaterial; private string? _previewImageFilePath; @@ -51,8 +52,8 @@ public WmsLoadComponent() protected override void RegisterInputParams(GH_InputParamManager pManager) { - pManager.AddTextParameter("WMS Service URL", "WMS URL", "Base URL of the WMS service. Leave empty to use the built-in fallback map/imagery source sequence.", GH_ParamAccess.item); - pManager.AddTextParameter("Layer", "Layer", "Optional WMS layer name to request. Leave empty to let RhinoSpatial choose a usable requestable layer automatically.", GH_ParamAccess.item, string.Empty); + pManager.AddTextParameter("WMS Service URL", "WMS URL", "WMS service URL or WMTS capabilities URL. Leave empty to use the built-in fallback map/imagery source sequence.", GH_ParamAccess.item); + pManager.AddTextParameter("Layer", "Layer", "Optional WMS or WMTS layer name to request. Leave empty to let RhinoSpatial choose a usable requestable layer automatically.", GH_ParamAccess.item, string.Empty); pManager.AddTextParameter("Spatial Context", "Spatial Context", "Shared RhinoSpatial spatial context from the Spatial Context component. No WFS inputs are required for the WMS workflow.", GH_ParamAccess.item); pManager.AddTextParameter("Format", "Format", "Requested image format.", GH_ParamAccess.item, "image/png"); @@ -329,6 +330,16 @@ private SolveResults Compute(RequestData requestData) private SolveResults ComputeForSource(ResolvedImagerySource source, SpatialContext2D spatialContext, string format) { + var wmts=_wmtsClient.TryLoadCapabilitiesAsync(source.BaseUrl,CancelToken).GetAwaiter().GetResult(); + if(wmts is not null) + { + var plan=WmtsClient.Plan(wmts,source.PreferredLayerName,spatialContext,format); + var image=_wmtsClient.DownloadAsync(plan,CancelToken).GetAwaiter().GetResult(); + if(!LoadGeoTiffComponent.TryCreateImageMesh(image.Raster,spatialContext,out var mesh,out _,out _,out _)) + throw new InvalidOperationException("WMTS imagery does not overlap the selected Spatial Context."); + return new SolveResults { ImageFilePath=image.LocalFilePath, ImageMesh=mesh, + GetMapUrl=string.Join(System.Environment.NewLine,plan.Tiles.Select(t=>t.Url)),Status=image.Status,MessageLevel=GH_RuntimeMessageLevel.Remark }; + } var capabilities = _wmsClient.LoadCapabilitiesAsync(source.BaseUrl).GetAwaiter().GetResult(); var resolvedLayer = ResolveLayerName(capabilities, source.PreferredLayerName); if (string.IsNullOrWhiteSpace(resolvedLayer.LayerName)) diff --git a/THIRD-PARTY-NOTICES.md b/THIRD-PARTY-NOTICES.md index d528557..eb264a5 100644 --- a/THIRD-PARTY-NOTICES.md +++ b/THIRD-PARTY-NOTICES.md @@ -57,6 +57,33 @@ RhinoSpatial uses NetTopologySuite.IO.Esri.Shapefile to read local `.shp` vector RhinoSpatial uses SharpGLTF.Core to decode glTF/GLB tile content for the optional Google 3D Tiles reference viewer. +### StbImageNext + +- Package: `StbImageNext` +- Version: `1.0.0` +- Project URL: [StbImageNext](https://github.com/StbSharp/StbImageNext) +- Package metadata: [NuGet 1.0.0](https://www.nuget.org/packages/StbImageNext/1.0.0) +- Declared license: Unlicense OR MIT +- License texts: [Unlicense](https://licenses.nuget.org/Unlicense) or [MIT](https://licenses.nuget.org/MIT) +- Package authors: StbImageSharpTeam; based on the stb image libraries + +RhinoSpatial uses the managed .NET Standard assembly for bounded PNG/JPEG tile +decoding and local image-header metadata. It adds no native binaries or further +package dependencies. Encoded WMTS mosaics preserve their source pixel colors +and alpha; Google 3D Tiles are unaffected. The dependency is included in package +inspection for this unreleased batch. + +### Microsoft.Data.Sqlite and SQLitePCLRaw + +- `Microsoft.Data.Sqlite.Core` 8.0.32: [EF Core project](https://github.com/dotnet/efcore), MIT; © Microsoft Corporation; license copyright .NET Foundation and Contributors. +- `SQLitePCLRaw.core` 2.1.13, `SQLitePCLRaw.provider.sqlite3` 2.1.11 and `SQLitePCLRaw.provider.winsqlite3` 2.1.11: [SQLitePCLRaw project](https://github.com/ericsink/SQLitePCL.raw), Apache-2.0; Copyright 2014–2024 SourceGear, LLC. + +These managed assemblies provide read-only local GeoPackage access. They use the +host's SQLite library (`sqlite3` on macOS/Linux, `winsqlite3` on Windows 10+). +RhinoSpatial does not bundle another native SQLite binary. Existing initialized +providers are retained. The full license texts are reproduced below; this +unreleased dependency change is included in package inspection and platform CI. + ### ProjNET - Package: `ProjNET` @@ -92,3 +119,451 @@ RhinoSpatial uses Mapzen Skadi terrain tiles as the built-in global terrain fall - RhinoSpatial itself remains licensed under MIT. - These third-party libraries are distributed as separate assemblies in the release package. - Original copyrights, notices, and licenses for those libraries continue to apply to those libraries. + +## SQLite dependency license texts + +### Microsoft.Data.Sqlite.Core — MIT + +```text +The MIT License (MIT) + +Copyright (c) .NET Foundation and Contributors + +All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +``` + +### SQLitePCLRaw — Apache License 2.0 + +Copyright 2014–2024 SourceGear, LLC. + +```text +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` + +### SQLitePCLRaw upstream NOTICE + +This upstream notice also covers optional configurations of SQLitePCLRaw. + +```text +---------------------------------------------------------------- +Copyright on SQLitePCL.raw +---------------------------------------------------------------- + +Version prior to 2.0 were labeled with the copyright owned by +Zumero. In 2.0, this changed to SourceGear. There is no legal +distinction, as Zumero is simply a dba name for SourceGear. + +And in either case, the open source license remains the same, +Apache v2. + +---------------------------------------------------------------- +License for SQLite +---------------------------------------------------------------- + +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** + + +---------------------------------------------------------------- +License for MS Open Tech +---------------------------------------------------------------- + +// Copyright © Microsoft Open Technologies, Inc. +// All Rights Reserved +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy +// of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// +// See the Apache 2 License for the specific language governing permissions and +// limitations under the License. + + +---------------------------------------------------------------- +License for SQLCipher +---------------------------------------------------------------- + +Copyright (c) 2008, ZETETIC LLC +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the ZETETIC LLC nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY ZETETIC LLC ''AS IS'' AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL ZETETIC LLC BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +---------------------------------------------------------------- +License for OpenSSL libcrypto +---------------------------------------------------------------- + + LICENSE ISSUES + ============== + + The OpenSSL toolkit stays under a dual license, i.e. both the conditions of + the OpenSSL License and the original SSLeay license apply to the toolkit. + See below for the actual license texts. Actually both licenses are BSD-style + Open Source licenses. In case of any license issues related to OpenSSL + please contact openssl-core@openssl.org. + + OpenSSL License + --------------- + +/* ==================================================================== + * Copyright (c) 1998-2011 The OpenSSL Project. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * 3. All advertising materials mentioning features or use of this + * software must display the following acknowledgment: + * "This product includes software developed by the OpenSSL Project + * for use in the OpenSSL Toolkit. (http://www.openssl.org/)" + * + * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to + * endorse or promote products derived from this software without + * prior written permission. For written permission, please contact + * openssl-core@openssl.org. + * + * 5. Products derived from this software may not be called "OpenSSL" + * nor may "OpenSSL" appear in their names without prior written + * permission of the OpenSSL Project. + * + * 6. Redistributions of any form whatsoever must retain the following + * acknowledgment: + * "This product includes software developed by the OpenSSL Project + * for use in the OpenSSL Toolkit (http://www.openssl.org/)" + * + * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY + * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR + * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + * ==================================================================== + * + * This product includes cryptographic software written by Eric Young + * (eay@cryptsoft.com). This product includes software written by Tim + * Hudson (tjh@cryptsoft.com). + * + */ + + Original SSLeay License + ----------------------- + +/* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com) + * All rights reserved. + * + * This package is an SSL implementation written + * by Eric Young (eay@cryptsoft.com). + * The implementation was written so as to conform with Netscapes SSL. + * + * This library is free for commercial and non-commercial use as long as + * the following conditions are aheared to. The following conditions + * apply to all code found in this distribution, be it the RC4, RSA, + * lhash, DES, etc., code; not just the SSL code. The SSL documentation + * included with this distribution is covered by the same copyright terms + * except that the holder is Tim Hudson (tjh@cryptsoft.com). + * + * Copyright remains Eric Young's, and as such any Copyright notices in + * the code are not to be removed. + * If this package is used in a product, Eric Young should be given attribution + * as the author of the parts of the library used. + * This can be in the form of a textual message at program startup or + * in documentation (online or textual) provided with the package. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * "This product includes cryptographic software written by + * Eric Young (eay@cryptsoft.com)" + * The word 'cryptographic' can be left out if the rouines from the library + * being used are not cryptographic related :-). + * 4. If you include any Windows specific code (or a derivative thereof) from + * the apps directory (application code) you must include an acknowledgement: + * "This product includes software written by Tim Hudson (tjh@cryptsoft.com)" + * + * THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * The licence and distribution terms for any publically available version or + * derivative of this code cannot be changed. i.e. this code cannot simply be + * copied and put under another distribution licence + * [including the GNU Public Licence.] + */ +``` diff --git a/docs/COMPATIBILITY_IMPLEMENTATION_PLAN.md b/docs/COMPATIBILITY_IMPLEMENTATION_PLAN.md new file mode 100644 index 0000000..68cc040 --- /dev/null +++ b/docs/COMPATIBILITY_IMPLEMENTATION_PLAN.md @@ -0,0 +1,54 @@ +# Source compatibility implementation plan + +Started 2026-11-01 from 0.4.0 on `codex/source-compatibility`. +The [research audit](COMPATIBILITY_RESEARCH.md) records the full backlog. + +## Authorized scope + +1. **Feature completeness:** bounded OGC API and WFS pagination, explicit user + limits, duplicate/repeated-page protection and honest completeness diagnostics. +2. **Dataset discovery:** follow OGC API landing-page and collection links; allow + the existing layer list and vector loader to select collections without adding + component parameters. +3. **Direct downloads:** recognize remote GeoJSON and supported vector archives, + and extend building downloads to CityJSON. Validate formats, companion files, + coordinate metadata and archive paths before use. +4. **Terrain ATOM:** share feed selection and validated download infrastructure + with the building workflow; load intersecting terrain tiles, retain NoData and + one elevation baseline, and reject unsupported or ambiguous placement. +5. **Reference metadata:** use ATOM feed coverage/CRS to orient Spatial Context + without downloading geometry. Saved selections retain priority. + +## Validation and examples + +- Add deterministic HTTP fixtures for discovery, pagination, download failures, + cache reuse, limits and cancellation, plus terrain tile placement checks. +- Run the existing six suites and the new compatibility suite, a Release build, + JSON validation and a final diff review. +- Exercise small live provider requests where available. Record URLs, selected + layers/collections, bounds, CRS and actual results in the source manifest and + a copyable example guide; distinguish live results from fixture-only cases. +- Record remaining Rhino visual checks explicitly. Preserve component GUIDs, + input/output layouts, localization and elevation rules. No Google export, + extraction, persistent tile caching or key storage. +- Update component documentation and an Unreleased changelog. This task does not + publish another version or change the installed plugin automatically. + +## Progress + +- [x] Research recorded and implementation branch created. +- [x] Feature pagination and completeness checks. +- [x] OGC API discovery and existing component integration. +- [x] Direct vector/building downloads. +- [x] Terrain ATOM loading and shared placement. +- [x] ATOM reference metadata. +- [x] Saved example URLs, documentation and final validation. + +## Recorded results + +- Release solution build passed with zero warnings and errors; all seven suites passed, including 26 new compatibility checks. Component GUIDs and parameter names/order were compared against the 0.4.0 baseline; source manifest JSON and unique ids were checked. +- Live results and exact reproduction inputs are saved in [the example guide](../examples/COMPATIBILITY_WORKFLOWS.md): API-root discovery, five-page capped loading, single- and two-tile DGM1 meshes, direct GeoJSON/Shapefile ZIPs and a declared-CRS CityJSON source. +- The Delft-hosted older sample failed TLS on this machine; the cjio maintainer fixture provided a successful alternative. Certificate validation was retained. +- New Rhino/Grasshopper visual checks and Windows runtime checks remain on [the manual checklist](../examples/VALIDATION.md#unreleased-source-compatibility), including progress/cancellation, saved definitions and cross-layer XY/Z alignment. +- This first batch did not implement priorities 6–10. The subsequent [medium-priority plan](MEDIUM_COMPATIBILITY_PLAN.md) covers 6–8; COG and GeoPackage remain later work. +- Public release 0.4.0 and the installed plugin were not changed. diff --git a/docs/COMPATIBILITY_RESEARCH.md b/docs/COMPATIBILITY_RESEARCH.md new file mode 100644 index 0000000..2965deb --- /dev/null +++ b/docs/COMPATIBILITY_RESEARCH.md @@ -0,0 +1,42 @@ +# Compatibility research — 2026-11-01 + +Reviewed RhinoSpatial 0.4.0 at `2bd239ee409fdb06f2aa49aa81c2cffbcc1d2be5` against official provider and format documentation. This is a code/documentation audit, not a live reproduction of every provider failure. Priorities are recommendations; no runtime behavior was changed. + +Implementation of priorities 1–5 is tracked in the [implementation plan](COMPATIBILITY_IMPLEMENTATION_PLAN.md) and [example guide](../examples/COMPATIBILITY_WORKFLOWS.md). Priorities 6–8 are implemented in the [medium-priority follow-up](MEDIUM_COMPATIBILITY_PLAN.md), with [saved examples](../examples/MEDIUM_COMPATIBILITY_WORKFLOWS.md). Priorities 9–10 are implemented in the [low-priority follow-up](LOW_COMPATIBILITY_PLAN.md), with [COG/GeoPackage how-to examples](../examples/LOW_COMPATIBILITY_WORKFLOWS.md). The evidence below describes the audited 0.4.0 baseline, before those changes. + +## Highest-value improvements + +| Priority | Gap | Current evidence | Proposed behavior and acceptance check | +| --- | --- | --- | --- | +| 1 | Bounded feature pagination and completeness diagnostics | `RhinoSpatial.Core/OgcApiFeaturesClient.cs:25` performs one request, then parses features without following collection links. `WfsClient.cs:157` sends feature limits but has no paging loop. | Follow OGC API next links and supported WFS paging until the user cap or end of results. Detect repeated pages, deduplicate IDs, preserve bounds, and report truncation. Test a server whose page limit is smaller than the requested feature count. Existing MaxFeatures remains a deliberate safety limit. | +| 2 | Terrain ATOM downloads | `docs/COMPONENT_REFERENCE.md:275` explicitly excludes terrain ATOM. `LoadTerrainComponent.cs` routes WCS and local files. Berlin officially publishes DGM1 through ATOM. | Reuse bounded feed discovery/download infrastructure with terrain-specific archive validation, tile placement and mosaicking. Verify adjacent DGM tiles, CRS, seams, overlapping edges and NoData. Do not reuse the building filename convention without validating terrain metadata. | +| 3 | OGC API landing-page and collection discovery | `OgcApiFeaturesClient.cs:44` recognizes paths containing /collections/ or /items. `VectorSourceResolver.cs:61` classifies other HTTP URLs as WFS. | Accept the API landing page, follow its data/collection links and expose collection titles/extents. Test a root URL with two collections and relative links; avoid guessing URL paths. | +| 4 | ATOM metadata as Spatial Context Reference Source | `docs/COMPONENT_REFERENCE.md:281` explicitly says this is unsupported. | Use feed extent/CRS metadata to orient the map without downloading geometry. Test missing and ambiguous metadata, geographic axis order and preservation of saved-selection priority. | +| 5 | Direct data-download URLs and packaged vector files | `VectorSourceResolver.cs:37–63` supports local SHP/GeoJSON; a remote .geojson URL falls through to WFS. `Lod2AtomClient.cs:294` explicitly excludes remote CityJSON. | Detect response type/content before service routing; safely download bounded GeoJSON and appropriate archives, including Shapefile companion files. Extend building downloads to validated CityJSON separately. Test redirects, misleading extensions, archive limits and missing CRS. | +| 6 | CityJSON LoD selection and sequence support | `CityJsonReader.cs:24` parses a single JSON document. Lines 44–51 traverse all geometry entries without selecting their lod. There is no CityJSONFeature sequence or GeometryInstance template handling. | First select one suitable LoD per object, rather than combining all supplied LoDs. Then add .city.jsonl streaming and template resolution where building datasets require them. Multi-LoD duplication is a code-derived risk, not a reproduced visual bug. Test a two-LoD building, a sequence header with shared transform, and a template instance. | + +## Useful follow-on work + +| Priority | Gap | Current evidence | Proposed behavior and acceptance check | +| --- | --- | --- | --- | +| 7 | Broader raster georeferencing | `GeoTiffReader.cs:32` requires an embedded EPSG code, and line 84 explicitly rejects rotated/sheared transforms. No world-file reader was found. | Add supported affine transforms and sidecar georeferencing (.tfw/.jgw/.pgw with a resolvable CRS). Test rotated imagery and pixel-center conventions. A world file alone does not identify the CRS; fail clearly instead of assuming one. | +| 8 | WMTS imagery | Existing imagery clients implement WMS; no WMTS or tile-matrix-set client was found in the reviewed C# code. | Support an additional delivery mechanism within imagery: read capabilities, choose a suitable tile matrix/resolution, request only intersecting tiles and align the mosaic. Test a projected tile matrix, not just Web Mercator. This is not a request to add caching to Google 3D Tiles. | +| 9 | Remote Cloud Optimized GeoTIFF access | GeoTIFF readers open local paths with Tiff.Open; no HTTP range reader was found. | Read appropriate overviews and spatial windows from a remote COG using bounded byte-range requests. A compatible local COG may already work as an ordinary TIFF; the missing capability is efficient remote subset access. Test transferred byte budgets and a server without range support. | +| 10 | GeoPackage vector input | VectorSourceKind and resolver contain WFS, OGC API Features, local Shapefile and local GeoJSON, but no .gpkg route or reader. | Add read-only feature-layer discovery and bounded spatial reads from a GeoPackage. Start with vector features, not every optional GeoPackage extension. Test multiple layers, CRS metadata and polygon holes. This is a larger optional format extension; defer until a real project supplies a useful test dataset. | + +## Official references + +- [OGC API Features Core: landing pages, collections and paging](https://docs.ogc.org/is/17-069r3/17-069r3.html) +- [Berlin DGM1 official ATOM service metadata](https://gdi.berlin.de/geonetwork/srv/api/records/fa02f9e1-a0df-3be1-b0aa-bc624c0c7ff5) +- [Berlin DGM1 provider documentation](https://gdi.berlin.de/data/dgm1/docs/dgm1.pdf) +- [CityJSON specification: LoD and geometry templates](https://www.cityjson.org/specs/) +- [CityJSON sequences and CityJSONFeature](https://www.cityjson.org/cityjsonseq/) +- [OGC GeoTIFF georeferencing standard](https://docs.ogc.org/is/19-008r4/19-008r4.html) +- [Esri world-file documentation](https://pro.arcgis.com/en/pro-app/3.6/help/data/imagery/world-files-for-raster-datasets.htm) +- [OGC WMTS standard](https://www.ogc.org/standards/wmts/) +- [OGC Cloud Optimized GeoTIFF standard](https://docs.ogc.org/is/21-026/21-026.html) +- [OGC GeoPackage specification](https://www.geopackage.org/spec/) + +## Suggested order + +Start with pagination/completeness and terrain ATOM. Add landing-page discovery and ATOM reference metadata next. Keep these inside the current components where practical, preserve their GUIDs and ordinary connections, and verify with bounded provider requests plus deterministic fixtures. Choose subsequent format work from actual user datasets, rather than treating this list as a mandatory expansion roadmap. diff --git a/docs/COMPONENT_REFERENCE.md b/docs/COMPONENT_REFERENCE.md index ff082ba..30140bc 100644 --- a/docs/COMPONENT_REFERENCE.md +++ b/docs/COMPONENT_REFERENCE.md @@ -8,6 +8,8 @@ RhinoSpatial is organized around one rule: Every loader consumes the same `Spatial Context`, so maps, vector data, terrain, buildings, rasters, OSM context, and visual reference layers can share one Rhino and Grasshopper study space. +> The source-compatibility additions described here are **Unreleased**. The public 0.4.0 package and installed plugin are unchanged. See the [example recipes](../examples/COMPATIBILITY_WORKFLOWS.md) for tested inputs. + ## Component groups The Grasshopper tab is organized as: @@ -63,7 +65,8 @@ The component returns one `Spatial Context` output for all aligned source and vi Supported reference metadata includes: - WFS, WMS, and WCS service URLs -- GeoTIFF +- OGC API Features landing/collection metadata, ATOM feed coverage and WMTS capabilities (Unreleased) +- GeoTIFF and supported world-file TIFF/PNG/JPEG imagery - Shapefile - GeoJSON - CityJSON @@ -93,7 +96,7 @@ Terrain, LoD2 buildings, and localized Google 3D Tiles participate in one shared ### List WFS Layers -Connect a WFS service URL to inspect the layers advertised by its `GetCapabilities` response. Each output entry combines the service layer name and title. Use `List Item` to select one layer, or explicitly merge several entries for a multi-layer WFS request. +Connect a WFS service URL to inspect the layers advertised by its `GetCapabilities` response. OGC API Features roots and collection metadata URLs are also accepted; the component follows advertised data/items links and returns collection ids and titles. Each output entry combines the service layer name and title. Use `List Item` to select one layer, or explicitly merge several entries for a multi-layer WFS request. Public interface: @@ -102,7 +105,7 @@ Public interface: ### List WMS Layers -Connect a WMS service URL to inspect its requestable map layers. Use `List Item` to choose the layer needed by `Load WMS`. +Connect a WMS service URL or WMTS capabilities URL to inspect its imagery layers. Use `List Item` to choose the layer needed by `Load WMS`. Public interface: @@ -118,19 +121,32 @@ Use `Load WFS` for parcels, planning layers, boundaries, roads, building footpri The public component name is retained for Grasshopper compatibility, but the accepted source category is broader than WFS: - WFS service URLs -- OGC API Features collection or items URLs +- OGC API Features landing-page, collection-list, collection or items URLs +- direct GeoJSON URLs and ZIPs containing GeoJSON or complete Shapefile companion sets - local Shapefiles (`.shp`) - local GeoJSON (`.geojson` or GeoJSON `.json`) Public inputs: - `WFS URL` — service URL, OGC API Features URL, or local vector-file path -- `Layer` — one or more WFS layer entries; optional for local files and OGC API Features +- `Layer` — one or more layer entries; optional for a single OGC API collection or sole GeoPackage feature layer and ordinary single files; choose explicitly for multiple collections/GeoPackage layers - `Max Features` — request limit; the default `0` requests all available features within the bounded query - `Spatial Context` — the shared selected area and placement context The `Geometry` output is a tree grouped by layer and feature. +### Unreleased: local GeoPackage features + +Connect a local `.gpkg` path to List WFS Layers and Load WFS. Catalog names identify +actual feature tables; select explicitly for multiple layers, or leave Layer +empty for a sole feature layer. Read-only queries use a standard advertised RTree +when available, otherwise a bounded scan. Declared EPSG organization codes govern +the CRS; polygon exterior/hole rings remain feature-grouped closed curves, with +points/lines/multi-geometries/collections and string attributes in the existing XY +vector contract. Z/M are not a building/elevation import. Raster tables, editing, +SQL, curved/extended geometry and automatic remote GeoPackage downloads are not +supported. See [limits and saved examples](../examples/LOW_COMPATIBILITY_WORKFLOWS.md). + ### Typical WFS workflow 1. Connect the service URL to `List WFS Layers`. @@ -138,9 +154,13 @@ The `Geometry` output is a tree grouped by layer and feature. 3. Define the area with `Spatial Context`. 4. Connect the service URL, chosen layer, and shared context to `Load WFS`. -For local vector files, connect the file path directly to `WFS URL`. `Layer` can remain empty and is used only as a label when provided. OGC API Features URLs can also be connected directly; RhinoSpatial requests GeoJSON features for the context bounding box. +For Shapefile/GeoJSON, connect the file path directly to `WFS URL`. `Layer` can remain empty and is used only as a label when provided. OGC API Features URLs can also be connected directly; RhinoSpatial requests GeoJSON features for the context bounding box. + +`Max Features` is a deliberate per-layer cap. `0` follows available pages within the bounded query. OGC API next links and WFS next links are supported; WFS 2 can also advance `STARTINDEX` by the actual returned count. Duplicate ids are removed. Identical records without stable ids remain separate, with a deduplication caveat; declared GeoJSON CRS changes between pages are rejected. Repeated pages, failed required pages and known incomplete responses fail explicitly instead of returning partial selections. Without a matching total, the status states that completeness cannot be independently verified. Legacy WFS without next links cannot be assumed pageable. + +Pagination has safety limits of 100 pages, 100,000 unique features, 128 MiB total text and two minutes; a single response is limited to 16 MiB. Hitting a safety limit while more data is advertised returns an error; hitting the user's cap is reported as intentional. These bounds apply to service paging, not a promise that every server implements filtering correctly. -`Max Features` limits the request when a provider supports it. Use `0` to request all features available within the bounded query, but keep broad or dense study areas conservative. +Direct vector downloads are limited to 128 MiB compressed, 256 MiB per expanded file and 1 GiB expanded per archive. ZIPs require `.shp`, `.shx`, `.dbf` and a resolvable `.prj` for each imported Shapefile. Unsafe paths, symlinks, duplicate paths and incomplete transfers are rejected. Validated payloads persist under `RhinoSpatial/VectorDownloadCache`; revisionless URLs refresh after one day. Source JSON is inspected by content, and GeoJSON may contain a FeatureCollection or a single Feature. Unknown declared CRS metadata fails rather than assuming the context. ### Outputs and parsing @@ -165,7 +185,9 @@ Use `Load WMS` for orthophotos, aerial imagery, map layers, and raster overlays 3. Define the area with `Spatial Context`. 4. Connect the WMS URL, layer, format, and context to `Load WMS`. -One component requests one WMS layer. Use several components when multiple image layers should remain explicit. +WMTS is also accepted through these existing components. Use a WMTS capabilities URL with `List WMS Layers` and `Load WMS`; the selected layer uses advertised matrix sets, styles, dimensions and PNG/JPEG formats. `GetMap URL` returns the tile request URLs in WMTS mode. Runtime status reports the selected matrix, resolution, native CRS and tile count. + +One component requests one WMS/WMTS layer. Use several components when multiple image layers should remain explicit. Public inputs: @@ -187,11 +209,13 @@ When a layer does not support the context's primary SRS but advertises another c WMS 1.3 geographic requests use the authority axis order required by the standard. -## Load GeoTIFF: local georeferenced rasters +WMTS supports KVP GetTile and advertised REST templates. It chooses a resolution near a 1024-pixel study view within 64 tiles and 4,194,304 mosaic pixels (8192 per axis). Limits are 4 MiB per tile, 64 MiB per batch and two minutes per mosaic. Matrix limits and geographic axis order are respected; unsupported CRS, ambiguous dimensions and failed required tiles return explicit errors. Dimension defaults are used; an explicit source-URL query value can select a dimension without a new component parameter. Completed PNG mosaics persist under `RhinoSpatial/WmtsCache`, expire after one day, and are checked against SHA-256 before reuse. Delete the cache while idle to reclaim space. No partial mosaic is published after a required tile failure. + +## Load GeoTIFF: georeferenced rasters Use `Load GeoTIFF` for a local georeferenced image such as an orthophoto, map export, satellite image, or other raster that should align with the selected study area. -Connect a `.tif` or `.tiff` path and the shared `Spatial Context`. +Connect a `.tif`/`.tiff`, `.png`, `.jpg` or `.jpeg` path and the shared `Spatial Context`. PNG/JPEG and ordinary TIFF need a matching world file (`.pgw`, `.jgw`, `.tfw`, long-extension variants or `.wld`) and a resolvable `.prj` unless usable geospatial metadata is embedded. Public inputs: @@ -207,6 +231,20 @@ Outputs include: The file must contain readable geospatial metadata. If RhinoSpatial cannot determine the coordinate system or the raster does not overlap the selected context, it returns a clear warning rather than guessing a placement. +Embedded TIFF georeferencing takes priority. The reader supports finite invertible affine transforms, including rotation, shear and reflection. World-file coefficients refer to pixel centers; GeoTIFF PixelIsArea/PixelIsPoint conventions are honored. Reference extents use all four footprint corners. A bounded mesh grid projects the actual footprint through the shared context, clips it with interpolated UVs, and localizes it without stretching the image over its bounding rectangle. Sidecar changes invalidate cached metadata and component results. A world file without a resolvable CRS fails explicitly. + +### Unreleased: remote COG imagery + +The same GeoTIFF input accepts an HTTP(S) `.tif/.tiff` URL, including URL query +parameters. The client validates byte ranges and a stable representation, selects +usable overviews/intersecting tiles, preserves masks/NoData and writes only the +selected PNG window. Status reports resolution, request count and transferred +bytes. The existing mesh/material/image-path outputs retain affine placement, +UV clipping and localization. Remote inputs revalidate on a new solve. Reference +Source reads headers only; saved selections retain priority. BigTIFF, striped +files, unsupported masks/codecs and range refusal return explicit errors. See +[the complete budgets and recipes](../examples/LOW_COMPATIBILITY_WORKFLOWS.md). + ## Load Terrain: elevation and ground surfaces Use `Load Terrain` to create ground geometry aligned with the same study area as imagery, vectors, and buildings. @@ -214,14 +252,16 @@ Use `Load Terrain` to create ground geometry aligned with the same study area as Supported sources include: - WCS services +- ATOM feeds containing supported GeoTIFF, ASC or regular XYZ/CSV terrain tiles - local GeoTIFF DEM (`.tif` or `.tiff`) +- **unreleased:** remote single-band COG DEM via HTTP byte ranges - Esri ASCII Grid (`.asc`) - regular XYZ or CSV point grids with x, y, z columns - the built-in quick global land-elevation fallback Public inputs: -- `Terrain URL` — WCS URL or local DEM/grid path; leave empty for the quick global fallback +- `Terrain URL` — WCS URL, terrain ATOM feed, remote COG DEM (development) or local DEM/grid path; leave empty for the quick global fallback - `Coverage` — optional WCS coverage id or EPSG override for local text grids - `Spatial Context` — the shared selected area and placement context @@ -229,12 +269,16 @@ Outputs are the aligned `Terrain` mesh list and a `Status` message. `Coverage` identifies a WCS coverage where required. For local text grids it can also provide an EPSG override such as `EPSG:25832`; otherwise RhinoSpatial uses the current context SRS. +GeoTIFF terrain vertices use the same affine pixel centers as imagery, and cached elevation sampling inverts that transform. Rotated local DEMs are supported; terrain ATOM mosaics still require compatible north-up grids and reject other grids explicitly. + Terrain coordinates are transformed from the source SRS into the shared context before localization, including when the source uses a different projected CRS. Unsupported transformations and non-overlapping WCS coverage return explicit errors. Missing elevation samples leave holes in the surface, including during downsampling; RhinoSpatial does not replace them with zero-height terrain. A raster without connected usable cells returns an error. Esri ASCII Grid samples are placed at cell centers, while reference metadata describes the full cell footprint. Matching `xllcorner`/`yllcorner` and `xllcenter`/`yllcenter` headers are supported. `NODATA_value` is optional and defaults to `-9999`. Cell values may wrap across text lines; their total must match `ncols × nrows`. Invalid headers, truncated data, and extra cell values are rejected. These conventions follow the [Esri ASCII raster format](https://help.arcgis.com/en/arcgisdesktop/10.0/help/009t/009t0000000z000000.htm). XYZ/CSV terrain requires finite, evenly spaced X and Y coordinates. Sparse regular grids retain missing cells as NoData; irregularly spaced samples return an error instead of being stretched onto an artificial grid. +For ATOM, connect the feed directly and leave `Coverage` empty. Text grids require a declared native feed CRS; GeoTIFFs use embedded metadata. Tiles are selected from supported spatial metadata or explicitly declared UTM grids. Compatible north-up grids are cropped to the study area and merged before mesh construction, so adjacent samples and the elevation baseline remain consistent. Incompatible spacing/CRS/alignment and conflicting overlap heights fail explicitly. The mosaic is limited to 4,194,304 cells and eight million aggregate cropped source samples. See the [Berlin DGM1 example](../examples/COMPATIBILITY_WORKFLOWS.md#berlin-dgm1-terrain-from-atom). Downloads use the shared ATOM limits below and persist under `RhinoSpatial/TerrainAtomCache`. + The built-in fallback is intended for small-area orientation and short request times. It fails quickly with a useful status when the area is too large, no usable elevation samples are available, or the request cannot complete promptly. Project work should prefer official or project-specific terrain. The WCS path supports ordinary single-part GeoTIFF responses and multipart GeoTIFF responses. Where capabilities omit a WGS84 extent, RhinoSpatial can use `DescribeCoverage` metadata to orient the context. @@ -246,7 +290,8 @@ Use `Load LoD2 Buildings` for official building massing, roof forms, and 3D urba One `LoD2 Source` input accepts: - a LoD2 WFS service URL -- an ATOM download-service URL with selectable CityGML ZIP tiles +- an ATOM download-service URL with selectable CityGML/CityJSON tiles +- a direct supported building-file or ZIP/GMZ download URL - local CityGML/GML/XML - local CityJSON - a folder containing supported building files @@ -256,7 +301,7 @@ For WFS sources, select a building layer when needed. For ATOM and local data, ` Public inputs: -- `LoD2 Source` — WFS or ATOM URL, supported local file, folder, or ZIP path +- `LoD2 Source` — WFS/ATOM URL, direct supported download URL, local file, folder, or ZIP path - `Layer` — optional WFS layer name or local output label - `Spatial Context` — the shared selected area and placement context @@ -268,17 +313,23 @@ Connect the service URL and a `Spatial Context`; no tile naming, manual download Tile selection supports GeoRSS Simple boxes/polygons and GeoRSS GML envelopes, including native projected bounds. GeoRSS Simple coordinates are latitude/longitude independently of a dataset's CRS category. GML uses its declared CRS and authority axis order; an omitted CRS defaults to WGS84 latitude/longitude. Relative links and `xml:base` are resolved against the response URL. -Some feeds, including Berlin's, publish a single dataset extent and many download links. These require additional tile information. The supported grid convention is a declared square grid size, an EPSG:25832 or EPSG:25833 category, an enclosing geographic extent, and filenames ending in `_EEE_NNNN` with southwest UTM coordinates in kilometres. Berlin explicitly links a `1x1 Km` tile index. This rule depends on metadata and the supported naming convention, not the service hostname. Arbitrary numeric filenames or whole-dataset bounds are never sufficient on their own. A feed with missing, ambiguous or unsupported spatial metadata returns a limitation before downloading building archives. Start from the service feed so its enclosing extent remains available. +Some feeds, including Berlin's, publish a single dataset extent and many download links. These require additional tile information. The supported grid convention is a declared square grid size, an EPSG:25832 or EPSG:25833 category, an enclosing geographic extent, and filenames ending in `_EEE_NNNN` with southwest UTM coordinates in kilometres. Berlin buildings explicitly link a `1x1 Km` tile index; terrain uses a declared `2x2 Km` index. Coarse enclosing coverage allows a one-declared-tile margin for edge/buffer tiles; distant contradictory grids are rejected. This rule depends on metadata and the supported naming convention, not the service hostname. Arbitrary numeric filenames or whole-dataset bounds are never sufficient on their own. A feed with missing, ambiguous or unsupported spatial metadata returns a limitation before downloading building archives. Start from the service feed so its enclosing extent remains available. -ATOM currently imports ZIP/GMZ containers holding CityGML (`.gml`, `.citygml`, `.xml`). Other archive formats and remote CityJSON downloads return an explicit unsupported-format error. Local CityJSON/file/folder/ZIP workflows remain supported. Downloaded CityGML must declare a source CRS that can transform into the context; no guessed CRS or approximate bounding-box placement is used. Standard AdV names such as `urn:adv:crs:ETRS89_UTM33*DE_DHHN2016_NH` resolve to UTM XY; published heights are retained without a vertical-datum conversion. +ATOM and direct building download URLs support CityGML (`.gml`, `.citygml`, `.xml`) and CityJSON (`.json`, `.cityjson`), including ZIP/GMZ containers. Downloaded CityJSON/CityJSONSeq must declare the header `metadata.referenceSystem`. Standard CityJSONSeq (`.city.jsonl` / `.jsonl`) and affine row-major GeometryInstance templates are supported. Local CityJSON/file/folder/ZIP workflows remain supported. Downloaded CityGML must declare a source CRS that can transform into the context; no guessed CRS or approximate bounding-box placement is used. Standard AdV names such as `urn:adv:crs:ETRS89_UTM33*DE_DHHN2016_NH` resolve to UTM XY; published heights are retained without a vertical-datum conversion. -Use a building dataset, not a terrain feed: Berlin's `/data/a_lod2/atom/` supplies LoD2 buildings, while `/data/dgm1/atom/` supplies CSV terrain grids. Explicit CSV/raster download MIME types are rejected before grid selection with a source-format explanation. `Load Terrain` supports local grids and WCS, but does not yet read ATOM feeds. +Use a building dataset, not a terrain feed: Berlin's `/data/a_lod2/atom/` supplies LoD2 buildings, while `/data/dgm1/atom/` supplies CSV terrain grids. Explicit CSV/raster download MIME types are rejected before grid selection with a source-format explanation. Use the DGM1 feed directly with `Load Terrain`; its terrain-specific parser and mosaic path are separate from the building reader. -Completed ZIPs persist in the user's local application-data folder under `RhinoSpatial/Lod2AtomCache`. Repeated solutions reuse them, including after restarting Rhino. The cache key includes the download URL and feed entry's `updated` value. With no revision value, archives refresh after one day. Cached length and SHA-256 are checked before reuse. Downloads are written to temporary files, checked for complete transfer and readable CityGML, and promoted only on success. Failures can be retried; successfully completed tiles remain cached. Delete this cache directory while RhinoSpatial is idle to reclaim space or force fresh downloads. Feed metadata is refreshed after five minutes within the running process. +Completed ZIPs persist in the user's local application-data folder under `RhinoSpatial/Lod2AtomCache`. Repeated solutions reuse them, including after restarting Rhino. The cache key includes the download URL and feed entry's `updated` value. With no revision value, archives refresh after one day. Cached length and SHA-256 are checked before reuse. Downloads are written to temporary files, checked for complete transfer and supported readable payloads, and promoted only on success. Failures can be retried; successfully completed tiles remain cached. Delete this cache directory while RhinoSpatial is idle to reclaim space or force fresh downloads. Feed metadata is refreshed after five minutes within the running process. The component displays tile/download progress while working. `Status` reports selected tile filenames, downloaded and cached counts, cache location, parsed buildings, source CRS and conversion diagnostics. A failed required download or CityGML parse returns an error rather than presenting a partial area as complete. -Limits are 64 feeds, 8 subfeed levels, 50,000 download entries and 32 selected tiles per solution; 512 MiB compressed per archive, 256 MiB per expanded file and 1 GiB total expanded per archive. Feed requests time out after 30 seconds, discovery after two minutes, individual downloads after three minutes and the download batch after five minutes. Reduce the study area or use a more specific spatially indexed feed if a limit is reached. ATOM metadata is not yet supported by `Spatial Context → Reference Source`; define the area with the map or use an existing supported reference. +Limits are 64 feeds, 8 subfeed levels, 50,000 download entries and 32 selected tiles per solution; 512 MiB compressed per archive, 256 MiB per expanded file and 1 GiB total expanded per archive. Feed requests time out after 30 seconds, discovery after two minutes, individual downloads after three minutes and the download batch after five minutes. Reduce the study area or use a more specific spatially indexed feed if a limit is reached. ATOM feed coverage and native CRS are supported by `Spatial Context → Reference Source`. Use a service feed containing coverage; a dataset feed lacking coverage reports that limitation. Reference processing reads feed metadata without downloading archives. Saved map selections retain priority. + +### CityJSON detail, templates and sequences + +For each CityObject, import all geometry parts at one selected numeric LoD: the highest at or below 2.2, or the lowest higher value if none is available below that threshold. Geometry without any LoD retains legacy behavior; mixing declared and unspecified levels fails because a complete representation cannot be chosen safely. Hierarchy-wide LoD selection and material/texture import are not added. + +CityJSONSeq records are streamed with independent vertex indices and shared header transform, CRS and templates. GeometryInstance vertices use the declared matrix followed by the transformed reference-point anchor; template vertices are not quantized a second time. Invalid indices, duplicate building ids, malformed later records and unsupported selected primitives fail explicitly. Limits are 64 × 1024² characters per record, 512 × 1024² stream characters, 100,000 returned building objects and four million selected source surface points. Local/ZIP/direct downloads use the same reader. Reference metadata can stop at a declared header extent without constructing geometry. See the [medium-priority examples](../examples/MEDIUM_COMPATIBILITY_WORKFLOWS.md). ### Filtering and performance @@ -384,3 +435,13 @@ RhinoSpatial generally prefers: Fallback data should not be presented as equivalent to richer official sources. It should remain bounded and responsive; when a fallback cannot serve the selected context cleanly, it should fail with useful status text instead of leaving Grasshopper apparently stuck. For representative providers, known limitations, and regression notes, see the [Test Source Catalogue](TEST_SOURCES.md). For release checks, see the [Validation Checklist](../examples/VALIDATION.md). + +### Unreleased reference metadata additions + +Local GeoPackage Reference Source uses all feature-layer catalog extents and +transformable declared EPSG metadata without querying geometry. Missing extents +fail rather than trigger a scan. Remote COG Reference Source reads geospatial TIFF +headers through a tighter range budget without decoding/importing pixels. These +follow the existing rule that saved selections take priority. Explicit WGS84 UTM +north/south EPSG:32601–32660 / 32701–32760 are supported transformations; source +heights are not vertically reinterpreted. diff --git a/docs/LOW_COMPATIBILITY_PLAN.md b/docs/LOW_COMPATIBILITY_PLAN.md new file mode 100644 index 0000000..4cf3090 --- /dev/null +++ b/docs/LOW_COMPATIBILITY_PLAN.md @@ -0,0 +1,84 @@ +# Low-priority compatibility: how-to and implementation plan + +Started 2026-11-02 from `f20afe4` on `codex/low-source-compatibility`. +This batch implements research priorities 9–10 and builds on draft PR #7. +Public/installed 0.4.0 is unchanged. No merge, installation or release is planned. + +## Intended user workflow + +1. Select one study area with the existing `Spatial Context`. +2. For remote COG imagery, connect an HTTP(S) TIFF URL to `Load GeoTIFF`. For a + single-band elevation COG, use `Load Terrain`. The client reads metadata and + relevant tiles/overviews with byte ranges and reports transferred bytes and + selected resolution. A server that ignores ranges returns a limitation; it + does not trigger a hidden whole-file download. +3. A remote TIFF used as `Reference Source` reads geospatial headers only, without + decoding a raster window. Saved selections retain priority. +4. For GeoPackage vectors, connect a local `.gpkg` path to `List WFS Layers`, + choose feature layers and use the same path/layers with `Load WFS`. A sole + feature layer may be loaded without Layer. Use a local path + as `Reference Source` for catalogue CRS/extents without feature loading. +5. Read source status for limits, CRS, empty coverage, unsupported extensions and + failures. Keep all aligned components on the same context. + +## Implementation order + +- **Remote raster transport:** bounded seekable HTTP range source, block reuse, + strict Content-Range checks, representation validators, byte/request/time + budgets and cancellation. No full-download fallback or Google changes. +- **COG subset readers:** use existing LibTiff decoding through custom streams; + discover usable tiled image/overview/mask directories, preserve exact affine + placement and NoData, read only intersecting tiles and write a bounded image + window or elevation raster. Keep unsupported arrangements explicit. +- **GeoPackage reader:** use Microsoft.Data.Sqlite.Core with the platform SQLite + provider (macOS/Linux sqlite3, Windows winsqlite3), read-only private connections + and bounded queries. Reuse NetTopologySuite WKB decoding, validate GeoPackage + headers/CRS/standard geometry, preserve polygon-hole rings, use advertised + RTree indexes or a bounded fallback scan, and retain existing vector grouping. + Do not implement raster tables, editing or optional geometry extensions. +- **Integration:** add source routing behind existing components and reference + readers without changing GUIDs or parameter layouts. Share context/CRS, + localization and terrain baseline behavior. + +## Validation and examples + +- Deterministic HTTP fixtures for range/validator/cancellation failures, byte + budgets, overview/window choice, masks, imagery and terrain placement. +- Temporary GeoPackages for multiple layers, quoted identifiers, RTree/fallback + filtering, CRS, holes, attributes, malformed/unsupported geometries, limits + and source immutability. +- Save a small declared-CRS GeoPackage and COG fixtures plus copyable public URLs, + bounds, commands and actual live results. Mark unavailable sources honestly. +- Run Release build, all eight existing suites and the new low-priority suite; + verify manifest JSON, component interfaces, diff, dependencies and package + contents. Update notices, reference/catalogue/roadmap/changelog and manual + Rhino/Windows checks. +- Commit/push and open a stacked draft PR targeting the medium-priority branch. + +## Progress + +- [x] Startup, scope and how-to plan recorded. +- [x] Bounded range transport and remote raster readers. +- [x] Read-only GeoPackage reader and source integration. +- [x] Saved examples and documentation. +- [x] Local validation and stacked draft PR workflow. Platform CI results are recorded on the PR; native Rhino validation remains manual. + +## Implementation results + +Release build: zero warnings/errors. All nine local suites passed, including 43 +new checks. Live Sentinel/Copernicus range windows and the GDAL public GeoPackage +passed; exact URLs/bounds/results are in the how-to. Three fixtures and a loopback +range server are saved. Component GUIDs/names/parameter order match the base. + +The SQLite decision uses Microsoft.Data.Sqlite.Core 8.0.32 with SQLitePCLRaw.core +2.1.13 and sqlite3/winsqlite3 providers 2.1.11. The preview package contains all +four managed assemblies, notices/license texts and no new native binaries or +local AI context. Existing release staging is untouched. Platform CI covers +macOS/Windows; installed Rhino assembly resolution/material/bake checks remain. + +An incidental bug found by the NoData fixtures is fixed for local DEMs too: +LibTiff's unknown ASCII tags return count + value, so the value must be parsed +instead of the count. Explicit WGS84 UTM north/south transformations were added +after the live Sentinel test exposed that missing CRS support. No source CRS or +vertical datum is guessed. Unreferenced rounded overview grids are skipped when +their mapping is ambiguous; BigTIFF remains an explicit decoder limitation. diff --git a/docs/MEDIUM_COMPATIBILITY_PLAN.md b/docs/MEDIUM_COMPATIBILITY_PLAN.md new file mode 100644 index 0000000..54f24f9 --- /dev/null +++ b/docs/MEDIUM_COMPATIBILITY_PLAN.md @@ -0,0 +1,36 @@ +# Medium-priority compatibility plan + +Started 2026-11-01 from `8aff1e8` on `codex/medium-source-compatibility`. +This batch builds on draft PR #6; its PR will target that branch until #6 merges. +Scope is research priorities 6–8. COG and GeoPackage remain later work. + +1. **CityJSON:** select one numeric LoD per object (prefer the highest supplied LoD at or below 2.2, otherwise the lowest higher LoD), retain all geometry parts at that LoD, resolve template instances with the specified matrix/anchor, and stream CityJSONSeq records using shared header transform/CRS/templates. Support local, ZIP and validated direct downloads without new component parameters. Fail explicitly on malformed indices/transforms or unsupported primitives instead of producing partial rings. +2. **Raster georeferencing:** share a finite, invertible affine transform for embedded TIFF and world-file imagery. Respect area/point raster conventions, prefer embedded spatial metadata, require a resolvable CRS for sidecars and include sidecars in cache invalidation. Preserve the true raster footprint/UV mapping during projection, clipping and localization. Use the same transform for terrain sample centers; reject unsupported terrain mosaic grids explicitly. +3. **WMTS:** integrate capabilities/layer discovery, advertised matrix sets, bounded tile selection and mosaic loading with existing WMS components and reference metadata. Respect CRS axis order, matrix limits, styles, formats and default dimensions. Support KVP and advertised REST tile templates. Keep unknown CRS or failed required tiles explicit; do not add a provider catalogue or affect Google tiles. + +## Validation and examples + +- Deterministic checks for mixed LoDs, templates, sequences, invalid geometry, affine rotation/shear, pixel-center placement, missing CRS, cache invalidation, WMTS projected matrices, limits, request URLs, cancellation and failures. +- Small live provider/file probes where available; save URLs, layer/matrix choices, CRS/bounds and observed outcomes in an example guide and source manifest. Keep fixtures and live evidence distinct. +- Release build, all existing suites plus the medium-priority suite, JSON/interface/diff checks, and GitHub validation. +- Update reference/catalogue/roadmap/changelog and manual Rhino/Windows checklist. Preserve component GUIDs and parameter layouts, shared context/localization/elevation and metadata-only reference intent. +- Commit, push and open a separate draft PR. No installation, merge, version bump or public publication is part of this batch. + +## Progress + +- [x] Scope, upstream dependency and implementation plan recorded. +- [x] CityJSON LoD selection, templates, sequences and source routing. +- [x] Affine/world-file imagery and terrain placement. +- [x] WMTS discovery, bounded imagery and reference integration. +- [x] Examples, documentation and validation. + +## Recorded results — 2026-11-02 + +- Implemented all three items behind existing components. Component GUIDs and input/output parameter names/order match the previous batch. +- All eight local suites passed, including 27 medium-priority checks. Release solution build passed with zero warnings/errors. Source manifest JSON, unique ids, fixture paths and diff checks passed. +- [Saved recipes](../examples/MEDIUM_COMPATIBILITY_WORKFLOWS.md) include the verified PDOK WMTS URL/layer/native bbox, a public multi-LoD parser fixture, and six bundled declared-CRS synthetic files. Model/sequence/template each parsed one building/six surfaces; the rotated image produced a Core mesh. +- Live PDOK: 19 layers, six supported sets, 36 JPEG tiles at 0.42 m/pixel, 1536 × 1536 mosaic, final clipped Core mesh 512 triangles. Cache reuse and mosaic continuity verified. Public cjio multi-LoD: ten buildings/348 surfaces at LoD 2.2; missing CRS is explicitly labeled parser-only. Existing declared-CRS Delft sample still parsed 160 buildings/5,563 surfaces at LoD1. +- Final raster review added source-window projection for large images, with a global-geographic-to-projected-context regression; UV coverage remains relative to the original image. +- Dependency: managed `StbImageNext` 1.0.0, no native/further package dependencies. Notices updated. An explicitly unreleased temporary package preview contained 17 files / 3,156,137 compressed bytes; the 62,976-byte decoder and dependency manifest were present, stale decoder and AI files absent. Public release staging was preserved. +- New interactive Rhino material/UV/bake/saved-definition and Windows checks remain in [the manual checklist](../examples/VALIDATION.md#unreleased-medium-priority-compatibility). Headless/Core probes are not proof of those native surfaces. +- COG and GeoPackage remain later work. No merge, version bump, public release or installed-plugin replacement occurred. diff --git a/docs/PROJECT_SCOPE.md b/docs/PROJECT_SCOPE.md index 27789e9..893a7c2 100644 --- a/docs/PROJECT_SCOPE.md +++ b/docs/PROJECT_SCOPE.md @@ -41,7 +41,7 @@ The complete planned core source set is present: | WMS | Provides maps, orthophotos, aerial imagery, and raster overlays | | LoD2 Buildings | Provides official 3D building massing and roof geometry | | Terrain | Provides ground and elevation context | -| GeoTIFF | Places local georeferenced raster data | +| GeoTIFF | Places local georeferenced rasters; development also supports remote COG windows | | OSM | Provides lightweight buildings, roads, water, green areas, and rail | | Google 3D Tiles | Provides optional photogrammetric visual reference context | @@ -128,9 +128,9 @@ Lightweight does not mean replacing mature TIFF, topology, CRS, Shapefile, or gl ### Vector data -`Load WFS` retains its public name for compatibility with existing Grasshopper definitions, but its source category is broader than web-only WFS. It also supports OGC API Features, local Shapefiles, and local GeoJSON. +`Load WFS` retains its public name for compatibility with existing Grasshopper definitions, but its source category is broader than web-only WFS. It also supports OGC API Features, local Shapefiles, and local GeoJSON. Unreleased development adds standard read-only GeoPackage feature layers with bounded spatial queries through the same components. -These sources remain general vector context. Shapefile and GeoJSON are not treated as LoD2 building imports. +These sources remain general vector context. Shapefile, GeoJSON and GeoPackage feature tables are not treated as LoD2 building imports. Their vector contract is XY curves/points; polygon holes remain boundary curves. ### LoD2 buildings diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 58046ec..08a8ac7 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -43,6 +43,16 @@ The current release already provides: These are the starting conditions for future work, not unfinished roadmap promises. +## Unreleased compatibility batch + +The [research backlog](COMPATIBILITY_RESEARCH.md) is recorded and priorities 1–5 +are implemented on the source-compatibility branch: bounded feature paging, API-root +collection discovery, direct vector/building downloads, terrain ATOM and reference +metadata. [The implementation plan](COMPATIBILITY_IMPLEMENTATION_PLAN.md) records +validation and remaining work; [example recipes](../examples/COMPATIBILITY_WORKFLOWS.md) +save tested URLs and bounds. Interactive Rhino/Windows checks remain before release. +The [medium-priority follow-up](MEDIUM_COMPATIBILITY_PLAN.md) implements priorities 6–8: CityJSON detail/sequence/templates, affine/world-file rasters and WMTS. Its [saved examples](../examples/MEDIUM_COMPATIBILITY_WORKFLOWS.md) include synthetic declared-CRS data and live PDOK imagery. Native Rhino/Windows validation remains. The [low-priority plan](LOW_COMPATIBILITY_PLAN.md) implements priorities 9–10: bounded remote COG windows and read-only local GeoPackage features, with [saved examples](../examples/LOW_COMPATIBILITY_WORKFLOWS.md), platform CI and explicit format limits. + ## Now: real-use stabilization The most useful next improvements should come from real project use rather than speculative expansion. @@ -123,9 +133,9 @@ Later work should still strengthen the existing source categories rather than wi Possible candidates include: -- GeoPackage as an additional local vector/project-data format after a deliberate SQLite and geometry-parser decision; +- additional GeoPackage extensions only when justified by representative projects; the unreleased standard vector reader is implemented; - stronger CityJSON coverage after more representative real sources are tested; -- Cloud Optimized GeoTIFF only after remote range requests, caching, and policy behavior are designed; +- additional remote COG arrangements/codecs such as BigTIFF after decoder support and allocation limits are verified; bounded classic-TIFF range access is implemented; - optional output refinements that clearly improve architectural site workflows. These are candidates, not commitments or release blockers. diff --git a/docs/TEST_SOURCES.md b/docs/TEST_SOURCES.md index d0af1e0..b70e079 100644 --- a/docs/TEST_SOURCES.md +++ b/docs/TEST_SOURCES.md @@ -44,6 +44,34 @@ Use these together with: `dotnet run --project RhinoSpatial.Sandbox.csproj -- lod2atom ` Add `download` to download/reuse selected archives and parse buildings through the existing CityGML reader. +## Unreleased source-compatibility validation (2026-11-01) + +The [implementation plan](COMPATIBILITY_IMPLEMENTATION_PLAN.md) records scope and +status; [example workflows](../examples/COMPATIBILITY_WORKFLOWS.md) save copyable +URLs, context bounds, layer choices and reproduction commands. The source manifest +includes verification dates and distinguishes development-only behavior. + +`dotnet run --project RhinoSpatial.Sandbox.csproj -c Release -- source-compatibility-check` +exercises deterministic HTTP fixtures for pagination, caps, totals, duplicate and +repeated pages, redirects, discovery, cancellation, validated downloads, unsafe or +incomplete archives, projected WKT authority, terrain mosaics and metadata-only +references. It is included in CI and makes no external requests. + +Live probes found 16 collections at the pygeoapi root and loaded five lakes across +five pages with an intentional cap. Berlin DGM1 yielded a 400 × 400 single-tile +raster and a 200 × 200 two-tile mosaic, both built into Core meshes. Direct Natural +Earth GeoJSON parsed 177 features, its Shapefile ZIP returned three sample features, +and the cjio Delft CityJSON fixture parsed 160 buildings. These are download/parser/ +Core-placement checks; new interactive Rhino/Grasshopper and Windows validation +remains pending. The older Delft-hosted CityJSON URL failed TLS in this batch; +historical local-file results below remain historical results. + +## Unreleased medium-priority validation (2026-11-02) + +[The plan](MEDIUM_COMPATIBILITY_PLAN.md) and [example guide](../examples/MEDIUM_COMPATIBILITY_WORKFLOWS.md) record priorities 6–8 and exact inputs. `medium-compatibility-check` tests LoD selection, sequence headers/records/cancellation, instance matrices, invalid geometry, world files/CRS/cache invalidation, embedded affine/PixelIsPoint GeoTIFFs, image UV clipping/reprojection, terrain centers, projected/geographic WMTS, matrix limits, tile ordering/alpha, corruption, failures and metadata-only references. The suite uses temporary fixtures and no network. + +Live PDOK capabilities exposed 19 layers/six supported sets; bounded native EPSG:28992 imagery loaded 36 JPEG tiles at 0.42 m/pixel, yielding a 1536 × 1536 PNG and a 512-triangle clipped Core mesh. The mosaic was visually inspected for continuity, not in Rhino. The cjio multi-LoD fixture parsed ten buildings/348 selected surfaces at LoD 2.2; it lacks CRS metadata, so it is parser-only evidence. Bundled declared-CRS synthetic models/sequence/template each parsed one building/six surfaces, and the rotated PNG generated 512 Core triangles. New Rhino/Windows checks remain pending. + ## Deterministic correctness regressions The source-reader checks generate temporary, non-private fixtures for ASCII corner/center origins, reference extents, optional NoData, wrapped/truncated/excess cell values, invalid headers, sparse and irregular XYZ grids, non-finite positions, GeoJSON null attributes, empty geometries, malformed points, and versioned EPSG names. They cover format behavior documented by [Esri](https://help.arcgis.com/en/arcgisdesktop/10.0/help/009t/009t0000000z000000.htm) and [GeoJSON RFC 7946](https://www.rfc-editor.org/rfc/rfc7946.html). @@ -73,7 +101,7 @@ Bounded comparison requests for the same Museum Island EPSG:25833 bounds also re - WFS: `https://gdi.berlin.de/services/wfs/alkis?SERVICE=WFS&REQUEST=GetCapabilities`, layer `alkis:gebaeudeflaechen` - WMS: `https://gdi.berlin.de/services/wms/alkis_gebaeude?SERVICE=WMS&REQUEST=GetCapabilities`, layer `gebaeude` -These confirm bounded source loading and compatible native coordinate metadata. They do **not** prove visual alignment, Brep quality, progress display or saved-definition compatibility in Rhino. Complete the ATOM entries in `examples/VALIDATION.md` before release. ATOM reference-map preloading remains deferred; use the map helper or another supported reference source. +These confirm bounded source loading and compatible native coordinate metadata. They do **not** prove visual alignment, Brep quality, progress display or saved-definition compatibility in Rhino. Complete the ATOM entries in `examples/VALIDATION.md` before release. The unreleased compatibility branch adds ATOM reference-map preloading from feed metadata; interactive validation of that addition remains pending. ## Current Tested Sources @@ -215,7 +243,7 @@ These are not release blockers, but they are the most sensible future source-for - **CityJSON** Basic local CityJSON building import is now supported through `Load LoD2 Buildings`. More real CityJSON sources should be added to this test catalogue before calling it mature. - **GeoPackage** - Still a good candidate for local vector/project data because it can hold layers, CRS metadata, and mixed geometry more robustly than loose shapefiles. It remains unimplemented for now because robust GeoPackage support needs a deliberate SQLite/geometry-BLOB parser choice and should not be slipped in as a fragile partial reader. + Implemented in the unreleased low-priority batch: local read-only standard feature tables, catalog metadata, RTree/bounded queries and standard WKB with hole curves. Raster tables, curved/extended geometry and editing remain outside scope. See the low-priority how-to and fixtures. - **Shapefile** Local Shapefile is now supported as a vector source through `Load WFS`, using the existing shared Spatial Context and geometry tree output. It is not treated as LoD2 building import; many municipal Shapefiles are 2D footprints or thematic vector layers rather than true LoD2 geometry. - **Local GeoJSON** @@ -223,4 +251,21 @@ These are not release blockers, but they are the most sensible future source-for - **OGC API Features** Basic GeoJSON OGC API Features item/collection URLs are now supported through the vector loader path. They are not treated as WFS internally; RhinoSpatial requests GeoJSON items with the Spatial Context WGS84 bbox and transforms them into the shared context. - **Cloud Optimized GeoTIFF** - Still unimplemented. COG support would be a remote-raster workflow, not just a local GeoTIFF variation, and should wait until range-request/cache behavior is designed. + Implemented in the unreleased low-priority batch using bounded HTTP ranges and selected classic-TIFF image/DEM windows. Stable representations, byte/time/allocation limits, usable overviews, masks, NoData and header-only references are tested; BigTIFF and unsupported layouts remain explicit limitations. + +## Unreleased low-priority validation (2026-11-02) + +[Plan](LOW_COMPATIBILITY_PLAN.md) · [how-to, URLs and exact bounds](../examples/LOW_COMPATIBILITY_WORKFLOWS.md). +`low-compatibility-check` contains 43 network-independent cases: range refusal, +invalid headers, truncation, mutation, validator/cache/budget/cancellation behavior, +COG metadata/windows/overviews/masks/NoData/alpha/placement, decoder allocation +limits, explicit WGS84 UTM transformations, and GeoPackage catalogs/quoted names/ +RTree/fallback/CRS/holes/attributes/endian/envelopes/collections/malformed geometry/ +limits/cancellation/source immutability. All eight prior suites also pass locally. +CI now runs the build and all nine suites on macOS and Windows. + +Live probes: Sentinel TCI 11×12 pixels / 2,686,976 range bytes; Copernicus DSM +4×4 samples / 3,080,192 bytes; GDAL public GeoPackage 10 features / 10 rings with +RTree. Results are parser/core placement evidence; Rhino visual/Windows-host +checks remain pending. External bytes are not committed; the saved fixtures are +synthetic and reproducible with `low-probe examples`. diff --git a/examples/COMPATIBILITY_WORKFLOWS.md b/examples/COMPATIBILITY_WORKFLOWS.md new file mode 100644 index 0000000..d2b50e4 --- /dev/null +++ b/examples/COMPATIBILITY_WORKFLOWS.md @@ -0,0 +1,140 @@ +# Source compatibility examples + +These additions are **Unreleased** and require a build from the source-compatibility +branch. The public 0.4.0 package has not been replaced. URLs and results below were +checked on 2026-11-01; public servers can change. + +Use the existing Grasshopper examples as starting points: `01-wms-wfs-basics.gh` +for vectors, `02-geotiff-terrain.gh` for terrain, and `03-lod2-buildings.gh` for +buildings. These recipes do not add new binary Grasshopper definitions. Keep one +shared `Spatial Context` for every layer in a study. The machine-readable inputs +are also saved in [sources.json](sources.json). + +## Discover collections from an API root + +Connect this URL to `List WFS Layers`, then select `lakes | Large Lakes`: + +```text +https://demo.pygeoapi.io/master/ +``` + +Connect the same URL and selected entry to `Load WFS`. Set `Max Features` to `5` +for a small check. Define a context in North America; the bounded probe used +EPSG:4326 bounds `-100,30,-50,60`. The root exposed 16 collections during testing. +For a root containing several collections, choose a layer explicitly; a sole +collection can be loaded with `Layer` empty. + +To exercise pagination, use this URL directly in `Load WFS`, with `Layer` empty: + +```text +https://demo.pygeoapi.io/master/collections/lakes/items?limit=1 +``` + +The bounded probe returned five unique features across five pages and reported +the deliberate `Max Features = 5` stop. `0` follows available pages subject to +the documented safety limits. A provider without a matching total receives an +explicit completeness caveat. + +An API root or collection metadata URL can also be used as `Reference Source`. +`Reference Layer = lakes` selects its extent. This only orients the map; draw a +smaller study area before loading. A saved selection still takes priority. + +## Berlin DGM1 terrain from ATOM + +Connect this URL to `Load Terrain → Terrain URL`; leave `Coverage` empty: + +```text +https://gdi.berlin.de/data/dgm1/atom/ +``` + +The same URL can be connected to `Spatial Context → Reference Source` to infer +EPSG:25833 and the feed coverage. No terrain archives are fetched for reference +metadata. Draw a small area near Museum Island, Berlin. + +Repeatable native bounds and observed results: + +| EPSG:25833 bounds (minX,minY,maxX,maxY) | Selected archives | Result | +| --- | --- | --- | +| `391300,5820100,391700,5820500` | `DGM1_390_5820.zip` | 400 × 400 samples; 160,000 mesh vertices and 159,201 quad faces; heights 30.75–36.85 m | +| `391900,5820100,392100,5820300` | `DGM1_390_5820.zip`, `DGM1_392_5820.zip` | 200 × 200 joined samples; 40,000 vertices and 39,601 quad faces; heights 31.17–42.21 m | + +These are sandbox/Core mesh results, not a completed visual Rhino alignment +check. The map helper selects rectangles interactively; the precise bounds above +can be reproduced with the sandbox commands below. Each source ZIP is about +16–17 MiB, even for a small window. Subsequent requests reuse validated downloads. +Berlin publishes a declared 2 km UTM grid; sample centers retain their `.5 m` +coordinates. Neighboring grids must agree in CRS, spacing and alignment. Missing +samples remain holes; conflicting overlaps fail explicitly. + +To compare buildings and terrain in Rhino, connect the **same** context to +`Load LoD2 Buildings` using: + +```text +https://gdi.berlin.de/data/a_lod2/atom/ +``` + +Leave its `Layer` empty. The building feed uses a different declared 1 km grid; +the terrain feed belongs in `Load Terrain`. + +## Direct GeoJSON and Shapefile ZIP downloads + +Connect either URL directly to `Load WFS → WFS URL`, with `Layer` empty: + +```text +https://raw.githubusercontent.com/nvkelso/natural-earth-vector/master/geojson/ne_110m_admin_0_countries.geojson +https://naturalearth.s3.amazonaws.com/110m_cultural/ne_110m_admin_0_countries.zip +``` + +Both contain EPSG:4326 country boundaries. The GeoJSON download parsed 177 +features; the ZIP passed validation and its Shapefile reader returned three +bounded sample features. Use these coarse world datasets for compatibility +checks. For a small site, use an appropriately detailed local dataset: the +existing filter may exclude country outlines that surround the whole site. + +ZIPs must contain complete `.shp`, `.shx`, `.dbf` and resolvable `.prj` sets. +Downloads are validated and cached; failed or unsafe archives are not promoted. +Remote data files are loader inputs. Download a file first if you want to use +its local metadata as a `Reference Source`. + +## Direct CityJSON buildings + +Connect this maintainer test dataset to `Load LoD2 Buildings → LoD2 Source`, +with `Layer` empty: + +```text +https://raw.githubusercontent.com/cityjson/cjio/master/tests/data/delft.json +``` + +The live download parsed 160 buildings. Its declared CRS is EPSG:7415 and its +native XY extent is `84616.468,447422.999,85140.839,447750.636`. It contains LoD1 +geometry, which the existing building reader accepts. It does not validate +multi-LoD selection, CityJSON sequences or geometry templates. + +For an accurately oriented study area, save a local copy and connect that path +to `Reference Source`; use a supported map SRS such as EPSG:3857, then select +the building extent in Delft. Load the original remote URL through the building +loader. EPSG:7415 uses the supported RD New horizontal transform; source Z +values are retained without vertical-datum conversion. + +The older [Delfshaven CityJSON example](https://3d.bk.tudelft.nl/opendata/cityjson/3dcities/v2.0/3-20-DELFSHAVEN.city.json) +failed TLS validation from this machine during this batch. Historical local-file +parser results remain in the catalogue; it is not a verified direct-download +example for this batch. Certificate checks were retained. + +## Reproduce the live probes + +Run from the repository root after a Release build: + +```sh +dotnet run --project RhinoSpatial.Sandbox.csproj -c Release --no-build -- compatibility-probe discover 'https://demo.pygeoapi.io/master/' +dotnet run --project RhinoSpatial.Sandbox.csproj -c Release --no-build -- ogcapi 'https://demo.pygeoapi.io/master/collections/lakes/items?limit=1' '-100,30,-50,60' 5 +dotnet run --project RhinoSpatial.Sandbox.csproj -c Release --no-build -- compatibility-probe reference 'https://gdi.berlin.de/data/dgm1/atom/' +dotnet run --project RhinoSpatial.Sandbox.csproj -c Release --no-build -- compatibility-probe terrain 'https://gdi.berlin.de/data/dgm1/atom/' EPSG:25833 391300,5820100,391700,5820500 +dotnet run --project RhinoSpatial.Sandbox.csproj -c Release --no-build -- compatibility-probe terrain 'https://gdi.berlin.de/data/dgm1/atom/' EPSG:25833 391900,5820100,392100,5820300 +dotnet run --project RhinoSpatial.Sandbox.csproj -c Release --no-build -- compatibility-probe vector 'https://raw.githubusercontent.com/nvkelso/natural-earth-vector/master/geojson/ne_110m_admin_0_countries.geojson' +dotnet run --project RhinoSpatial.Sandbox.csproj -c Release --no-build -- compatibility-probe vector 'https://naturalearth.s3.amazonaws.com/110m_cultural/ne_110m_admin_0_countries.zip' +dotnet run --project RhinoSpatial.Sandbox.csproj -c Release --no-build -- compatibility-probe buildings 'https://raw.githubusercontent.com/cityjson/cjio/master/tests/data/delft.json' +``` + +The deterministic `source-compatibility-check` command uses temporary fixtures +and makes no external requests. Live probes are deliberately separate from CI. diff --git a/examples/LOW_COMPATIBILITY_WORKFLOWS.md b/examples/LOW_COMPATIBILITY_WORKFLOWS.md new file mode 100644 index 0000000..da78207 --- /dev/null +++ b/examples/LOW_COMPATIBILITY_WORKFLOWS.md @@ -0,0 +1,144 @@ +# Low-priority compatibility: how to use and verify + +These workflows are **Unreleased**, on `codex/low-source-compatibility`, building +on the two earlier compatibility batches. The installed/public 0.4.0 is unchanged. +The implementation plan was recorded before coding in +[LOW_COMPATIBILITY_PLAN.md](../docs/LOW_COMPATIBILITY_PLAN.md). + +## Start with small, repeatable examples + +Keep one `Spatial Context` for all components. The [fixtures/low](fixtures/low) +files are synthetic EPSG:25833 examples near Berlin, not survey data or imagery. +Select `392000,5819968,392032,5820000` for the 32 × 32 m test area. +A fixture path can orient a fresh Reference Source; saved selections still win. + +1. Start the fixture server from the repository root: + + ```sh + python3 examples/serve_range.py + ``` + +2. Connect `http://127.0.0.1:8765/imagery-mask.cog.tif` to the existing + `Load GeoTIFF → GeoTIFF` input and connect the context. Expect a 32 × 32 pixel + colored window with a transparent mask hole. The image keeps its real affine + footprint and is clipped/localized with the shared context. +3. Connect `http://127.0.0.1:8765/elevation-mask.cog.tif` to `Load Terrain`. + Expect a sloping elevation grid with a missing corner sample and an internal + mask hole. Source samples, cell centers and the shared elevation baseline are + retained. The imagery and DEM cover the same grid. +4. Connect the local [local-layers.gpkg](fixtures/low/local-layers.gpkg) path to + `List WFS Layers → WFS Service URL`. Choose `buildings` or `survey "points"` + using List Item. Connect that selection and the same file path to `Load WFS`. + In the test area, `buildings` returns one feature and two closed boundary + curves: the exterior and courtyard. Survey points returns one point. The + geometry tree and per-feature attributes follow the normal vector contract. + All vector output is XY; Z/M are not a terrain/building-height import. +5. For a sole GeoPackage feature layer, Layer can be empty. Multiple layers need + explicit selections. A GeoPackage Reference Source uses catalog CRS/extents + for all feature layers, without reading feature geometry. Missing metadata + fails explicitly rather than scanning to invent an extent. + +`python -m http.server` does not consistently implement byte ranges; use the +included fixture server. Stop it with Ctrl-C. Existing definitions +`gh/01-wms-wfs-basics.gh` and `gh/02-geotiff-terrain.gh` can be reused by replacing +source inputs; component names, GUIDs and input/output order are unchanged. + +Recreate fixtures and run deterministic checks: + +```sh +dotnet run --project RhinoSpatial.Sandbox.csproj -c Release -- low-probe examples examples/fixtures/low +dotnet run --project RhinoSpatial.Sandbox.csproj -c Release -- low-compatibility-check +``` + +## Public URLs verified on 2026-11-02 + +Use EPSG:25833 bounds `391000,5819900,391100,5820000` for both remote rasters. +These are small Berlin test windows; image date/cloud conditions and DSM height +semantics remain properties of the source. + +**Sentinel imagery:** + +```text +https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/32/U/QD/2024/6/S2A_32UQD_20240609_0_L2A/TCI.tif +``` + +Connect to Load GeoTIFF. The live probe read an 11 × 12 pixel window from the +10980 × 10980 EPSG:32632 image using 41 range requests and 2,686,976 transferred +bytes, producing 556 clipped mesh triangles. Explicit WGS84 UTM EPSG:32601–32660 +and 32701–32760 transformations are supported; the source is not relabeled as +ETRS89. See the [provider catalogue](https://registry.opendata.aws/sentinel-2-l2a-cogs/) +and its data terms. + +**Copernicus surface elevation:** + +```text +https://copernicus-dem-30m.s3.amazonaws.com/Copernicus_DSM_COG_10_N52_00_E013_00_DEM/Copernicus_DSM_COG_10_N52_00_E013_00_DEM.tif +``` + +Connect to Load Terrain. The live probe read a 4 × 4 window, 16 finite samples, +from the 2400 × 3600 EPSG:4326 raster with 47 requests / 3,080,192 bytes. This +[dataset is a DSM](https://registry.opendata.aws/copernicus-dem/), including +vegetation/buildings; it is not a bare-earth DTM. Published heights are retained +without a vertical-datum conversion. Observe the provider's license. + +**Small public GeoPackage parser example:** + +```text +https://raw.githubusercontent.com/OSGeo/gdal/master/autotest/ogr/data/gpkg/poly_golden.gpkg +``` + +Download this 106,496-byte GDAL test file to a local `.gpkg`; connect the local +path, not the download URL. Select `poly`, EPSG:27700 bounds +`478315,4762880,481646,4765611`. The live probe returned 10 features and 10 rings +through its RTree. This is GDAL regression data with declared test coordinates, +not a real project-site recommendation. Native Rhino placement remains manual. + +Copyable sandbox probes: + +```sh +dotnet run --project RhinoSpatial.Sandbox.csproj -c Release -- low-probe cog '' EPSG:25833 391000,5819900,391100,5820000 +dotnet run --project RhinoSpatial.Sandbox.csproj -c Release -- low-probe dem '' EPSG:25833 391000,5819900,391100,5820000 +dotnet run --project RhinoSpatial.Sandbox.csproj -c Release -- low-probe gpkg /path/to/poly_golden.gpkg poly EPSG:27700 478315,4762880,481646,4765611 +``` + +## Deliberate limits + +COG reads require HTTP 206 with correct Content-Range and a stable strong ETag or +Last-Modified for reads spanning blocks. There is no whole-file fallback. Classic +tiled TIFF imagery/one-band DEMs, exact georeferencing, usable overviews, standard +internal 1/8-bit masks and NoData are supported through LibTiff codecs. BigTIFF, +striped rasters, unsupported codecs/orientation/mask layouts fail explicitly. +Ambiguous unreferenced rounded overview grids are skipped, keeping usable levels. +An image with NoData needs contiguous 8/16-bit gray/palette/RGB samples. + +Each operation is limited to two minutes, 64 MiB of transferred ranges, 512 range +requests, four million output pixels, 256 tiles per selected directory, 64 image +directories and bounded decoder allocations. Header-only references have a 4 MiB / +64-request limit. A 64 KiB range block can incidentally include nearby pixel bytes, +but references never decode pixels. A source's internal tile size controls how +many bytes are needed even for a small window. + +Only selected PNG windows persist under `RhinoSpatial/CogWindows` in the user's +local application-data folder. A solve revalidates the remote representation; +range blocks are reused in memory for that operation. Delete the window folder +while idle to reclaim space. No whole remote TIFF is stored there. + +GeoPackage support is local, read-only, standard feature tables and standard WKB +points/lines/polygons/multi-geometries/collections. Holes remain separate closed +curves with the feature's grouping. Optional curved/extended geometry, raster +tables, editing, SQL input and automatic remote GeoPackage downloads are outside +this batch. Declared EPSG organization codes determine the CRS, not a guessed +interpretation of a local SRS id. Catalog references require transformable extents. + +Queries use an advertised RTree when the platform supports it, otherwise a bounded +scan. Budgets include 128 catalog layers, 256 columns, 500,000 scanned rows per +layer, 16 MiB per geometry/SQLite value, 128 MiB geometry bytes per layer, four +million coordinates, 100,000 output features and eight million attribute +characters per component selection. Text values are capped at 16,384 characters; +BLOB attributes are labeled by byte count. `Max Features` remains a reported +per-layer cap. Limits/malformed geometry fail instead of publishing partial data. + +Microsoft.Data.Sqlite uses the host's SQLite: macOS/Linux `sqlite3`, Windows 10+ +`winsqlite3`. Existing initialized providers are retained. No new native library +is bundled. CI checks both platform paths; Rhino visual/Windows host validation +is still a release gate, listed in [VALIDATION.md](VALIDATION.md). diff --git a/examples/MEDIUM_COMPATIBILITY_WORKFLOWS.md b/examples/MEDIUM_COMPATIBILITY_WORKFLOWS.md new file mode 100644 index 0000000..9100335 --- /dev/null +++ b/examples/MEDIUM_COMPATIBILITY_WORKFLOWS.md @@ -0,0 +1,111 @@ +# Medium-priority compatibility examples + +These workflows are **Unreleased** and require the medium-source-compatibility +build, which also contains the previous compatibility batch. Public/installed +0.4.0 is unchanged. Checks below were recorded on 2026-11-02. + +Keep one `Spatial Context` for a study. Existing definitions `02-geotiff-terrain.gh` +and `03-lod2-buildings.gh` can be reused by replacing source inputs. Component +names, GUIDs and parameter layouts are retained; these recipes do not add new +binary Grasshopper definitions. + +## Small local examples with declared coordinates + +The [fixtures/medium](fixtures/medium) folder contains synthetic test data, not +survey data or real aerial imagery: + +| File | Connect to | Expected result | +| --- | --- | --- | +| [building-lods.city.json](fixtures/medium/building-lods.city.json) | `Load LoD2 Buildings → LoD2 Source` | One building, six surfaces, selected LoD 2.2; the alternate LoD1 shell is omitted | +| [building-sequence.city.jsonl](fixtures/medium/building-sequence.city.jsonl) | Same input | Same six surfaces, using the sequence header's transform and CRS | +| [building-template.city.json](fixtures/medium/building-template.city.json) | Same input | One six-surface building from a unit-cube template, scaled and translated from its native anchor | +| [rotated-raster.png](fixtures/medium/rotated-raster.png) | `Load GeoTIFF → GeoTIFF` | A colored raster with its true rotated/sheared footprint and UV mapping | + +Keep `rotated-raster.pgw` and `rotated-raster.prj` beside the PNG. All fixtures use +EPSG:25833 near Berlin; a shared study rectangle +`391296,5820096,391314,5820110` covers them. Use a fixture path as +`Spatial Context → Reference Source` to orient a fresh map, then select the area. +Leave the building `Layer` input empty. Saved map selections still take priority. +The synthetic building base is 32 m; its source Z values remain unchanged before +the normal shared-baseline placement. + +The PNG is 4 × 4 pixels. Its world-file center for pixel (0,0) is +`391300,5820106`; its column vector is `(2,0.5)` and row vector `(1,-2)`. +The image is clipped to the study area after projection, preserving texture +coordinates. A world file alone cannot identify a CRS: removing the `.prj` +must produce an explicit error, rather than assuming the context's CRS. + +Recreate and inspect these files from the repository root: + +```sh +dotnet run --project RhinoSpatial.Sandbox.csproj -c Release -- medium-probe examples examples/fixtures/medium +dotnet run --project RhinoSpatial.Sandbox.csproj -c Release --no-build -- medium-probe cityjson examples/fixtures/medium/building-lods.city.json +dotnet run --project RhinoSpatial.Sandbox.csproj -c Release --no-build -- medium-probe cityjson examples/fixtures/medium/building-sequence.city.jsonl +dotnet run --project RhinoSpatial.Sandbox.csproj -c Release --no-build -- medium-probe cityjson examples/fixtures/medium/building-template.city.json +dotnet run --project RhinoSpatial.Sandbox.csproj -c Release --no-build -- medium-probe raster examples/fixtures/medium/rotated-raster.png +``` + +## Public CityJSON multi-LoD parser example + +```text +https://raw.githubusercontent.com/cityjson/cjio/master/tests/data/multi_lod.json +``` + +The maintainer fixture parsed as 10 buildings and 348 selected surfaces, all at +LoD 2.2. It has **no declared CRS**, so this is a parser example. It does not prove +alignment, and the direct-download validator intentionally rejects it until +coordinate metadata is supplied. Use the bundled declared-CRS fixtures above +for an end-to-end loader example. The public Delft fixture from the previous +guide remains a declared-CRS download example. + +Selection is per CityObject: choose the highest numeric LoD at or below 2.2, or +the lowest higher LoD if none is available below that threshold. Retain all +geometry parts at the selected level. This does not add hierarchy-wide LoD +selection, material/texture import or a public LoD selector. + +The [official CityJSONSeq page](https://www.cityjson.org/cityjsonseq/) links real +sequence samples. Its Delft data host failed TLS from this machine during this +batch; those files are not advertised here as verified downloads. The bundled +sequence exercises shared transform/CRS and independent record vertex indices. + +## Live WMTS imagery from PDOK + +Connect this capabilities URL to `List WMS Layers`, select `2025_ortho25`, and +connect the same URL/layer to `Load WMS`: + +```text +https://service.pdok.nl/hwh/luchtfotorgb/wmts/v1_0?SERVICE=WMTS&REQUEST=GetCapabilities +``` + +The existing WMS component names remain for saved-definition compatibility. +WMTS metadata can also be used as `Reference Source`, with the layer id as +`Reference Layer`; no tiles are requested for this reference step. + +The bounded live check used EPSG:28992 bounds +`84600,447400,85200,447900` near Delft. It discovered 19 layers and six supported +matrix sets, selected native matrix set `EPSG:28992`, matrix `13`, resolution +0.42 m/pixel, and loaded 36 JPEG tiles into a 1536 × 1536 PNG mosaic. The clipped +Core mesh had 512 triangles. The saved mosaic was visually inspected for tile +continuity; this is not a completed Rhino material/alignment test. + +For the map helper, use EPSG:3857, orient near Delft and draw a small area. The +provider advertises compatible matrix sets; grids are read from capabilities. +The precise native check can be reproduced with: + +```sh +dotnet run --project RhinoSpatial.Sandbox.csproj -c Release --no-build -- medium-probe wmts 'https://service.pdok.nl/hwh/luchtfotorgb/wmts/v1_0?SERVICE=WMTS&REQUEST=GetCapabilities' 2025_ortho25 EPSG:28992 84600,447400,85200,447900 +``` + +Read the component's runtime status for matrix/resolution/tile counts. The +existing `GetMap URL` output contains the WMTS tile request URLs in this mode. +Default dimensions are used; a provider with an ambiguous dimension needs an +explicit value in the source URL query. Failed required tiles return an error +and do not publish a partial mosaic. + +## Validation limits + +The `medium-compatibility-check` suite uses deterministic fixtures, including +embedded affine GeoTIFFs, PixelIsPoint, projected/geographic WMTS, matrix limits, +tile order, cache corruption, alpha and cancellation. It makes no external +requests. Rhino preview, texture orientation, baking, saved definitions and +Windows runtime validation remain on [the manual checklist](VALIDATION.md). diff --git a/examples/README.md b/examples/README.md index 72f91f8..11d1bf2 100644 --- a/examples/README.md +++ b/examples/README.md @@ -54,6 +54,15 @@ Public example services can change or become temporarily unavailable. For produc If an older saved definition shows Grasshopper IO archive warnings after a component layout change, replace the affected RhinoSpatial component once and reconnect it. The definitions in this repository should represent the current component layout and open without those warnings. +## Compatibility workflows (Unreleased) + +The [source compatibility recipes](COMPATIBILITY_WORKFLOWS.md) save tested URLs, +collection choices, bounds and observed results for API-root discovery, paging, +Berlin DGM1 ATOM terrain, direct GeoJSON/Shapefile ZIPs and CityJSON downloads. +These require the development build; the public 0.4.0 package is unchanged. + +The [medium-priority examples](MEDIUM_COMPATIBILITY_WORKFLOWS.md) add declared-CRS CityJSON/sequence/template files, a rotated world-file raster, and a verified public WMTS URL with native test bounds. These also require the development build. + ## Source and testing references - [Curated example source list](sources.json) @@ -77,3 +86,5 @@ The set intentionally favors complete design studies over isolated technical pro The source manifest combines broad fallback references, international compatibility sources, and deeper German/Hessen test data. Hessen remains useful for LoD2, terrain, cadastral, building, and parcel workflows, while international services help expose provider and coordinate-system differences. Local CityGML folders and ZIP archives can vary greatly in structure and size. Validation should record whether RhinoSpatial skipped files by metadata bounds, filtered buildings before conversion, or had to inspect a large single tile. + +The [low-priority how-to](LOW_COMPATIBILITY_WORKFLOWS.md) adds remote COG imagery/elevation, local GeoPackage layers, a loopback range server, synthetic fixtures and verified public URLs. These require the unreleased development build. diff --git a/examples/VALIDATION.md b/examples/VALIDATION.md index ca185ed..9cbf531 100644 --- a/examples/VALIDATION.md +++ b/examples/VALIDATION.md @@ -145,3 +145,53 @@ It is intentionally small and practical. The point is to confirm that the core d - run `scripts/build-yak.sh` on a machine with Rhino's Yak CLI available - confirm the generated Yak package includes the same user-facing docs, including `INSTALL.md` - confirm ignored staging/package artifacts are not committed + + +## Unreleased source compatibility + +Use [COMPATIBILITY_WORKFLOWS.md](COMPATIBILITY_WORKFLOWS.md) and record the Rhino +version/platform and actual results. These checks are pending for this batch: + +- [ ] Existing saved definitions open without changed GUIDs or parameter-layout warnings. +- [ ] API-root layer listing, explicit multi-collection choice and optional single-collection Layer work in Grasshopper. +- [ ] Paging status distinguishes the user cap, unknown totals and an incomplete/failed page. +- [ ] Berlin DGM1 single- and two-tile windows load with responsive asynchronous progress; cancellation and retry work. +- [ ] DGM1 mesh, building roofs and WMS agree in XY and Z under one localized context, regardless of solve order. +- [ ] NoData holes stay open and no seam appears at the terrain tile boundary. +- [ ] ATOM/API references orient a fresh map without geometry downloads, while saved selections retain priority. +- [ ] Direct GeoJSON, Shapefile ZIP and CityJSON URLs load; invalid CRS/companions return clear messages. +- [ ] Repeat the relevant checks on Windows before a public release. + + +## Unreleased medium-priority compatibility + +Use [MEDIUM_COMPATIBILITY_WORKFLOWS.md](MEDIUM_COMPATIBILITY_WORKFLOWS.md). These native checks remain pending: + +- [ ] Open existing definitions with unchanged GUIDs and parameter layouts on macOS and Windows. +- [ ] Load each bundled CityJSON/sequence/template example: one six-surface building at the expected coordinates, with no alternate-LoD overlay. +- [ ] Load the rotated PNG with both sidecars, compare its footprint to matching vectors, clip the study area, and verify UV orientation, alpha, preview and baking. +- [ ] Edit a world file while keeping the raster unchanged; recompute and confirm both placement and reference extent update. +- [ ] Verify embedded rotated/PixelIsPoint DEM alignment and cached OSM/building height sampling in localized and absolute modes. +- [ ] List and load PDOK WMTS through existing WMS components, check the selected layer/matrix diagnostics and compare imagery to buildings/vectors in one context. +- [ ] Confirm WMTS Reference Source reads capabilities without tile downloads and saved selections retain priority. +- [ ] Cancel a tile batch and retry; confirm no partial image is presented and cached results remain usable. + +## Unreleased low-priority compatibility + +Local Release build and all nine sandbox suites pass, including 43 new cases. +The low-priority [plan](../docs/LOW_COMPATIBILITY_PLAN.md) and +[how-to](LOW_COMPATIBILITY_WORKFLOWS.md) record source URLs, test bounds, limits +and three reproducible fixtures. Live Sentinel/Copernicus range subsets and the +GDAL public GeoPackage were verified; these are not native Rhino screenshots. + +Before merge/release: + +- [ ] Open an existing definition in Rhino 8 on macOS and Windows; verify unchanged connections/GUIDs and responsive cancellation. +- [ ] Use the fixture range server: compare imagery/terrain mask holes, correct pixel locations, shared baseline and reload behavior in local/absolute placement. +- [ ] Test public Sentinel and Copernicus windows; inspect materials, clipping and bake from the selected PNG window. +- [ ] List/select both GeoPackage layers, verify courtyard and attribute/feature grouping, sole-layer behavior and multi-layer output branches. +- [ ] Use GeoPackage/COG Reference Source on a fresh map, then confirm saved selections retain priority and no geometry is imported. +- [ ] Test installed-package assembly resolution in Rhino on both platforms. Windows platform-provider sandbox CI is useful but does not replace Rhino host validation. + +No installed plugin, public version, release assets or Food4Rhino listing are +changed by this implementation batch. diff --git a/examples/fixtures/low/elevation-mask.cog.tif b/examples/fixtures/low/elevation-mask.cog.tif new file mode 100644 index 0000000..b9c8ed8 Binary files /dev/null and b/examples/fixtures/low/elevation-mask.cog.tif differ diff --git a/examples/fixtures/low/imagery-mask.cog.tif b/examples/fixtures/low/imagery-mask.cog.tif new file mode 100644 index 0000000..75a552c Binary files /dev/null and b/examples/fixtures/low/imagery-mask.cog.tif differ diff --git a/examples/fixtures/low/local-layers.gpkg b/examples/fixtures/low/local-layers.gpkg new file mode 100644 index 0000000..4f60079 Binary files /dev/null and b/examples/fixtures/low/local-layers.gpkg differ diff --git a/examples/fixtures/medium/building-lods.city.json b/examples/fixtures/medium/building-lods.city.json new file mode 100644 index 0000000..690b716 --- /dev/null +++ b/examples/fixtures/medium/building-lods.city.json @@ -0,0 +1,228 @@ +{ + "type": "CityJSON", + "version": "2.0", + "metadata": { + "referenceSystem": "https://www.opengis.net/def/crs/EPSG/0/25833", + "geographicalExtent": [ + 391300, + 5820100, + 32, + 391308, + 5820106, + 46 + ] + }, + "transform": { + "scale": [ + 1, + 1, + 1 + ], + "translate": [ + 391300, + 5820100, + 32 + ] + }, + "CityObjects": { + "example-building": { + "type": "Building", + "geometry": [ + { + "type": "Solid", + "lod": "1", + "boundaries": [ + [ + [ + [ + 0, + 3, + 2, + 1 + ] + ], + [ + [ + 0, + 1, + 5, + 4 + ] + ], + [ + [ + 1, + 2, + 6, + 5 + ] + ], + [ + [ + 2, + 3, + 7, + 6 + ] + ], + [ + [ + 3, + 0, + 4, + 7 + ] + ], + [ + [ + 4, + 5, + 6, + 7 + ] + ] + ] + ] + }, + { + "type": "Solid", + "lod": "2.2", + "boundaries": [ + [ + [ + [ + 8, + 11, + 10, + 9 + ] + ], + [ + [ + 8, + 9, + 13, + 12 + ] + ], + [ + [ + 9, + 10, + 14, + 13 + ] + ], + [ + [ + 10, + 11, + 15, + 14 + ] + ], + [ + [ + 11, + 8, + 12, + 15 + ] + ], + [ + [ + 12, + 13, + 14, + 15 + ] + ] + ] + ] + } + ] + } + }, + "vertices": [ + [ + 0, + 0, + 0 + ], + [ + 8, + 0, + 0 + ], + [ + 8, + 6, + 0 + ], + [ + 0, + 6, + 0 + ], + [ + 0, + 0, + 10 + ], + [ + 8, + 0, + 10 + ], + [ + 8, + 6, + 10 + ], + [ + 0, + 6, + 10 + ], + [ + 0, + 0, + 0 + ], + [ + 8, + 0, + 0 + ], + [ + 8, + 6, + 0 + ], + [ + 0, + 6, + 0 + ], + [ + 0, + 0, + 12 + ], + [ + 8, + 0, + 12 + ], + [ + 8, + 6, + 14 + ], + [ + 0, + 6, + 14 + ] + ] +} diff --git a/examples/fixtures/medium/building-sequence.city.jsonl b/examples/fixtures/medium/building-sequence.city.jsonl new file mode 100644 index 0000000..a0e6117 --- /dev/null +++ b/examples/fixtures/medium/building-sequence.city.jsonl @@ -0,0 +1,2 @@ +{"type":"CityJSON","version":"2.0","metadata":{"referenceSystem":"https://www.opengis.net/def/crs/EPSG/0/25833","geographicalExtent":[391300,5820100,32,391308,5820106,46]},"transform":{"scale":[1,1,1],"translate":[391300,5820100,32]},"CityObjects":{},"vertices":[]} +{"type":"CityJSONFeature","id":"example-building","CityObjects":{"example-building":{"type":"Building","geometry":[{"type":"Solid","lod":"1","boundaries":[[[[0,3,2,1]],[[0,1,5,4]],[[1,2,6,5]],[[2,3,7,6]],[[3,0,4,7]],[[4,5,6,7]]]]},{"type":"Solid","lod":"2.2","boundaries":[[[[8,11,10,9]],[[8,9,13,12]],[[9,10,14,13]],[[10,11,15,14]],[[11,8,12,15]],[[12,13,14,15]]]]}]}},"vertices":[[0,0,0],[8,0,0],[8,6,0],[0,6,0],[0,0,10],[8,0,10],[8,6,10],[0,6,10],[0,0,0],[8,0,0],[8,6,0],[0,6,0],[0,0,12],[8,0,12],[8,6,14],[0,6,14]]} diff --git a/examples/fixtures/medium/building-template.city.json b/examples/fixtures/medium/building-template.city.json new file mode 100644 index 0000000..273e3e7 --- /dev/null +++ b/examples/fixtures/medium/building-template.city.json @@ -0,0 +1,156 @@ +{ + "type": "CityJSON", + "version": "2.0", + "metadata": { + "referenceSystem": "https://www.opengis.net/def/crs/EPSG/0/25833", + "geographicalExtent": [ + 391300, + 5820100, + 32, + 391308, + 5820106, + 46 + ] + }, + "CityObjects": { + "template-building": { + "type": "Building", + "geometry": [ + { + "type": "GeometryInstance", + "template": 0, + "boundaries": [ + 0 + ], + "transformationMatrix": [ + 8, + 0, + 0, + 0, + 0, + 6, + 0, + 0, + 0, + 0, + 12, + 0, + 0, + 0, + 0, + 1 + ] + } + ] + } + }, + "vertices": [ + [ + 391300, + 5820100, + 32 + ] + ], + "geometry-templates": { + "templates": [ + { + "type": "Solid", + "lod": "2.2", + "boundaries": [ + [ + [ + [ + 0, + 3, + 2, + 1 + ] + ], + [ + [ + 0, + 1, + 5, + 4 + ] + ], + [ + [ + 1, + 2, + 6, + 5 + ] + ], + [ + [ + 2, + 3, + 7, + 6 + ] + ], + [ + [ + 3, + 0, + 4, + 7 + ] + ], + [ + [ + 4, + 5, + 6, + 7 + ] + ] + ] + ] + } + ], + "vertices-templates": [ + [ + 0, + 0, + 0 + ], + [ + 1, + 0, + 0 + ], + [ + 1, + 1, + 0 + ], + [ + 0, + 1, + 0 + ], + [ + 0, + 0, + 1 + ], + [ + 1, + 0, + 1 + ], + [ + 1, + 1, + 1 + ], + [ + 0, + 1, + 1 + ] + ] + } +} diff --git a/examples/fixtures/medium/rotated-raster.pgw b/examples/fixtures/medium/rotated-raster.pgw new file mode 100644 index 0000000..a25b947 --- /dev/null +++ b/examples/fixtures/medium/rotated-raster.pgw @@ -0,0 +1,6 @@ +2 +0.5 +1 +-2 +391300 +5820106 diff --git a/examples/fixtures/medium/rotated-raster.png b/examples/fixtures/medium/rotated-raster.png new file mode 100644 index 0000000..b64d69e Binary files /dev/null and b/examples/fixtures/medium/rotated-raster.png differ diff --git a/examples/fixtures/medium/rotated-raster.prj b/examples/fixtures/medium/rotated-raster.prj new file mode 100644 index 0000000..24ff765 --- /dev/null +++ b/examples/fixtures/medium/rotated-raster.prj @@ -0,0 +1 @@ +PROJCS["ETRS89 / UTM zone 33N",GEOGCS["ETRS89",DATUM["European_Terrestrial_Reference_System_1989",SPHEROID["GRS 1980",6378137,298.257222101]],PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433]],PROJECTION["Transverse_Mercator"],PARAMETER["latitude_of_origin",0],PARAMETER["central_meridian",15],PARAMETER["scale_factor",0.9996],PARAMETER["false_easting",500000],PARAMETER["false_northing",0],UNIT["metre",1],AUTHORITY["EPSG","25833"]] diff --git a/examples/serve_range.py b/examples/serve_range.py new file mode 100644 index 0000000..633f1e4 --- /dev/null +++ b/examples/serve_range.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +"""Serve the small low-priority fixtures on loopback with proper HTTP byte ranges.""" +import argparse +import hashlib +import re +from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument("--port", type=int, default=8765) +args = parser.parse_args() +root = (Path(__file__).parent / "fixtures" / "low").resolve() + + +class Handler(SimpleHTTPRequestHandler): + def __init__(self, *a, **kw): + super().__init__(*a, directory=str(root), **kw) + + def send_head(self): + path = Path(self.translate_path(self.path)).resolve() + if not path.is_relative_to(root) or not path.is_file(): + self.send_error(404) + return None + if path.stat().st_size > 16 * 1024 * 1024: + self.send_error(413, "This server is for small examples only") + return None + data = path.read_bytes() + size = len(data) + etag = '"' + hashlib.sha256(data).hexdigest() + '"' + requested = self.headers.get("Range") + if self.headers.get("If-Range") not in (None, etag): + requested = None + start, end = 0, size - 1 + if requested: + match = re.fullmatch(r"bytes=(\d+)-(\d*)", requested) + if not match: + self.send_error(416, "One explicit byte range is required") + return None + start = int(match[1]) + end = min(int(match[2]) if match[2] else size - 1, size - 1) + if start > end: + self.send_response(416) + self.send_header("Content-Range", f"bytes */{size}") + self.send_header("Content-Length", "0") + self.end_headers() + return None + self.send_response(206 if requested else 200) + self.send_header("Content-Type", self.guess_type(str(path))) + self.send_header("Accept-Ranges", "bytes") + self.send_header("ETag", etag) + self.send_header("Content-Length", str(end - start + 1)) + if requested: + self.send_header("Content-Range", f"bytes {start}-{end}/{size}") + self.end_headers() + import io + return io.BytesIO(data[start:end + 1]) + + +print(f"Serving synthetic fixtures at http://127.0.0.1:{args.port}/", flush=True) +try: + ThreadingHTTPServer(("127.0.0.1", args.port), Handler).serve_forever() +except KeyboardInterrupt: + pass diff --git a/examples/sources.json b/examples/sources.json index 51d6bff..1fc500e 100644 --- a/examples/sources.json +++ b/examples/sources.json @@ -230,8 +230,14 @@ "notes": [ "Download and unzip before using with Load WFS local Shapefile mode", "Tested with sandbox shapefile command", - "EPSG:4326, 177 country features" - ] + "EPSG:4326, 177 country features", + "Unreleased: connect the direct_download_url ZIP directly to Load WFS, Layer empty.", + "2026-11-01: validated 214976-byte direct payload, EPSG:4326 metadata and three bounded sample features." + ], + "last_verified": "2026-11-01", + "availability": "Unreleased source-compatibility branch; not the public 0.4.0 package", + "example_guide": "COMPATIBILITY_WORKFLOWS.md", + "direct_download_url": "https://naturalearth.s3.amazonaws.com/110m_cultural/ne_110m_admin_0_countries.zip" }, { "id": "shapefile-naturalearth-populated-places", @@ -296,8 +302,23 @@ "notes": [ "Tested with sandbox ogcapi command", "Use directly in the WFS URL input; RhinoSpatial treats it as OGC API Features, not WFS", - "Good regression source for bbox-filtered GeoJSON item endpoints" - ] + "Good regression source for bbox-filtered GeoJSON item endpoints", + "2026-11-01: limit=1 probe read 5 unique features across 5 pages, then explicitly stopped at Max Features 5." + ], + "last_verified": "2026-11-01", + "availability": "Unreleased source-compatibility branch; not the public 0.4.0 package", + "example_guide": "COMPATIBILITY_WORKFLOWS.md", + "pagination_url": "https://demo.pygeoapi.io/master/collections/lakes/items?limit=1", + "max_features": 5, + "context": { + "srs": "EPSG:4326", + "bbox": [ + -100, + 30, + -50, + 60 + ] + } }, { "id": "wfs-geoserver-demo-naturalearth", @@ -623,7 +644,8 @@ "notes": [ "Downloaded and tested with sandbox lod2file command", "Parsed 853 buildings and 15482 surfaces", - "Source SRS EPSG:28992 is read and transformed explicitly for shared Spatial Context alignment" + "Source SRS EPSG:28992 is read and transformed explicitly for shared Spatial Context alignment", + "2026-11-01 direct-download retry failed TLS verification on this machine; historical local parser results above are unchanged. Use cityjson-cjio-delft-download for the verified direct-download recipe." ] }, { @@ -857,6 +879,329 @@ "Use only for bounded contextual preview aligned to Spatial Context", "Not a substitute for official editable project data" ] + }, + { + "id": "terrain-berlin-dgm1-atom", + "type": "Terrain ATOM", + "purpose": "Official terrain feed discovery, downloads and adjacent-grid mosaic", + "url": "https://gdi.berlin.de/data/dgm1/atom/", + "notes": [ + "Connect to Load Terrain, Coverage empty; the same feed is supported as Spatial Context Reference Source without archive downloads.", + "Declared 2 km EPSG:25833 grid; source heights and half-meter sample centers retained.", + "400 x 400 single-tile Core raster/mesh and 200 x 200 two-tile mosaic verified live; Rhino visual alignment pending.", + "16-17 MiB ZIP downloads per tile; subsequent validated cache reuse." + ], + "last_verified": "2026-11-01", + "availability": "Unreleased source-compatibility branch; not the public 0.4.0 package", + "example_guide": "COMPATIBILITY_WORKFLOWS.md", + "context": { + "srs": "EPSG:25833", + "bbox": [ + 391300, + 5820100, + 391700, + 5820500 + ] + }, + "boundary_context": { + "srs": "EPSG:25833", + "bbox": [ + 391900, + 5820100, + 392100, + 5820300 + ] + } + }, + { + "id": "ogcapi-pygeoapi-root", + "type": "OGC API Features", + "purpose": "Landing-page discovery and existing layer-list workflow", + "url": "https://demo.pygeoapi.io/master/", + "notes": [ + "16 feature collections discovered live; select lakes | Large Lakes using List WFS Layers.", + "Use the same root and layer with Load WFS. Multi-collection roots require an explicit collection choice.", + "Root/collection metadata supported as Reference Source; select a small context before loading." + ], + "last_verified": "2026-11-01", + "availability": "Unreleased source-compatibility branch; not the public 0.4.0 package", + "example_guide": "COMPATIBILITY_WORKFLOWS.md", + "layer": "lakes", + "max_features": 5, + "context": { + "srs": "EPSG:4326", + "bbox": [ + -100, + 30, + -50, + 60 + ] + } + }, + { + "id": "geojson-naturalearth-countries-download", + "type": "GeoJSON download", + "purpose": "Direct remote vector-file loading", + "url": "https://raw.githubusercontent.com/nvkelso/natural-earth-vector/master/geojson/ne_110m_admin_0_countries.geojson", + "notes": [ + "Connect directly to Load WFS with Layer empty; EPSG:4326.", + "Validated 838726-byte payload and parsed 177 features live.", + "Coarse world polygons are compatibility data; existing oversized-outline filtering can exclude them in small site contexts." + ], + "last_verified": "2026-11-01", + "availability": "Unreleased source-compatibility branch; not the public 0.4.0 package", + "example_guide": "COMPATIBILITY_WORKFLOWS.md" + }, + { + "id": "cityjson-cjio-delft-download", + "type": "CityJSON download", + "purpose": "Direct building-download and declared Dutch CRS check", + "url": "https://raw.githubusercontent.com/cityjson/cjio/master/tests/data/delft.json", + "notes": [ + "Connect to Load LoD2 Buildings with Layer empty; live parser returned 160 buildings.", + "CityJSON 2.0 maintainer fixture containing LoD1 geometry; not a multi-LoD, sequence or template test.", + "Source EPSG:7415; RD New XY transform supported, source heights retained.", + "For Reference Source use a downloaded local copy and a supported map SRS such as EPSG:3857. Rhino visual checks pending." + ], + "last_verified": "2026-11-01", + "availability": "Unreleased source-compatibility branch; not the public 0.4.0 package", + "example_guide": "COMPATIBILITY_WORKFLOWS.md", + "native_extent": { + "srs": "EPSG:7415", + "bbox": [ + 84616.468, + 447422.999, + 85140.839, + 447750.636 + ] + } + }, + { + "id": "wmts-pdok-ortho25", + "type": "WMTS", + "purpose": "Projected native WMTS selection, JPEG decoding, mosaic and shared image placement", + "url": "https://service.pdok.nl/hwh/luchtfotorgb/wmts/v1_0?SERVICE=WMTS&REQUEST=GetCapabilities", + "layer": "2025_ortho25", + "matrix_set": "EPSG:28992", + "matrix": "13", + "context": { + "srs": "EPSG:28992", + "bbox": [ + 84600, + 447400, + 85200, + 447900 + ] + }, + "notes": [ + "Live: 19 layers and six supported matrix sets; 36 JPEG tiles at 0.42 m/pixel, 1536 x 1536 PNG mosaic, clipped Core mesh 512 triangles.", + "Use List WMS Layers / Load WMS with the same capabilities URL. Map-helper users can select Delft in EPSG:3857.", + "Mosaic continuity inspected; Rhino textures/alignment and Windows runtime pending." + ], + "last_verified": "2026-11-02", + "availability": "Unreleased medium-source-compatibility branch; not the public 0.4.0 package", + "example_guide": "MEDIUM_COMPATIBILITY_WORKFLOWS.md" + }, + { + "id": "cityjson-cjio-multi-lod", + "type": "CityJSON parser fixture", + "purpose": "Public multi-LoD selection reference", + "url": "https://raw.githubusercontent.com/cityjson/cjio/master/tests/data/multi_lod.json", + "notes": [ + "Parsed 10 buildings and 348 selected surfaces at LoD 2.2.", + "No declared CRS: parser-only evidence, not verified placement; direct downloader correctly requires CRS metadata." + ], + "last_verified": "2026-11-02", + "availability": "Unreleased medium-source-compatibility branch; not the public 0.4.0 package", + "example_guide": "MEDIUM_COMPATIBILITY_WORKFLOWS.md" + }, + { + "id": "cityjson-medium-local-examples", + "type": "CityJSON / CityJSONSeq", + "purpose": "Self-contained declared-CRS LoD, template and sequence examples", + "url": null, + "paths": [ + "fixtures/medium/building-lods.city.json", + "fixtures/medium/building-sequence.city.jsonl", + "fixtures/medium/building-template.city.json" + ], + "context": { + "srs": "EPSG:25833", + "bbox": [ + 391296, + 5820096, + 391314, + 5820110 + ] + }, + "notes": [ + "Synthetic Berlin-coordinate examples, not real project buildings. Each parses as one building / six surfaces at LoD 2.2.", + "Connect directly to Load LoD2 Buildings, Layer empty. A local path can also orient Reference Source." + ], + "last_verified": "2026-11-02", + "availability": "Unreleased medium-source-compatibility branch; not the public 0.4.0 package", + "example_guide": "MEDIUM_COMPATIBILITY_WORKFLOWS.md" + }, + { + "id": "raster-medium-affine-local", + "type": "World-file raster", + "purpose": "Rotated/sheared image footprint, pixel-center and UV example", + "url": null, + "path": "fixtures/medium/rotated-raster.png", + "sidecars": [ + "fixtures/medium/rotated-raster.pgw", + "fixtures/medium/rotated-raster.prj" + ], + "context": { + "srs": "EPSG:25833", + "bbox": [ + 391296, + 5820096, + 391314, + 5820110 + ] + }, + "notes": [ + "Synthetic 4 x 4 colored PNG; keep both sidecars alongside it.", + "Pixel (0,0) center 391300,5820106; column vector (2,0.5), row vector (1,-2).", + "Connect to existing Load GeoTIFF. Core mesh produced 512 triangles; native Rhino validation pending." + ], + "last_verified": "2026-11-02", + "availability": "Unreleased medium-source-compatibility branch; not the public 0.4.0 package", + "example_guide": "MEDIUM_COMPATIBILITY_WORKFLOWS.md" + }, + { + "id": "cog-low-sentinel-berlin", + "type": "Remote COG imagery", + "purpose": "Range subset imagery with declared WGS84 UTM CRS", + "url": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/32/U/QD/2024/6/S2A_32UQD_20240609_0_L2A/TCI.tif", + "context": { + "srs": "EPSG:25833", + "bbox": [ + 391000, + 5819900, + 391100, + 5820000 + ] + }, + "notes": [ + "Load GeoTIFF: verified 11x12 window, 41 requests, 2686976 bytes, 556 core mesh triangles; EPSG:32632 source.", + "Provider terms: https://registry.opendata.aws/sentinel-2-l2a-cogs/; image date/cloud conditions are source properties. Native Rhino validation pending." + ], + "last_verified": "2026-11-02", + "availability": "Unreleased low-source-compatibility branch; not the public 0.4.0 package", + "example_guide": "LOW_COMPATIBILITY_WORKFLOWS.md" + }, + { + "id": "cog-low-copernicus-berlin", + "type": "Remote COG DEM", + "purpose": "One-band range subset elevation with native geographic CRS", + "url": "https://copernicus-dem-30m.s3.amazonaws.com/Copernicus_DSM_COG_10_N52_00_E013_00_DEM/Copernicus_DSM_COG_10_N52_00_E013_00_DEM.tif", + "context": { + "srs": "EPSG:25833", + "bbox": [ + 391000, + 5819900, + 391100, + 5820000 + ] + }, + "notes": [ + "Load Terrain: verified 4x4 window, 16 finite elevations, 47 requests, 3080192 bytes. EPSG:4326 source.", + "DSM includes vegetation/buildings, not bare-earth terrain. Source heights retained; no vertical datum conversion. Provider terms: https://registry.opendata.aws/copernicus-dem/" + ], + "last_verified": "2026-11-02", + "availability": "Unreleased low-source-compatibility branch; not the public 0.4.0 package", + "example_guide": "LOW_COMPATIBILITY_WORKFLOWS.md" + }, + { + "id": "geopackage-low-gdal-public", + "type": "Local GeoPackage download example", + "purpose": "Public standard GeoPackage metadata/RTree parser regression", + "url": "https://raw.githubusercontent.com/OSGeo/gdal/master/autotest/ogr/data/gpkg/poly_golden.gpkg", + "context": { + "srs": "EPSG:27700", + "bbox": [ + 478315, + 4762880, + 481646, + 4765611 + ] + }, + "notes": [ + "Download to a local .gpkg path; List WFS Layers / Load WFS; select poly.", + "106496-byte GDAL test fixture; 10 features / 10 rings, RTree query verified. Declared regression coordinates, not a real project-site recommendation." + ], + "last_verified": "2026-11-02", + "availability": "Unreleased low-source-compatibility branch; not the public 0.4.0 package", + "example_guide": "LOW_COMPATIBILITY_WORKFLOWS.md" + }, + { + "id": "cog-low-local-imagery", + "type": "Synthetic COG imagery", + "purpose": "Repeatable mask/window/placement example", + "url": "http://127.0.0.1:8765/imagery-mask.cog.tif", + "path": "fixtures/low/imagery-mask.cog.tif", + "context": { + "srs": "EPSG:25833", + "bbox": [ + 392000, + 5819968, + 392032, + 5820000 + ] + }, + "notes": [ + "Start python3 examples/serve_range.py; Load GeoTIFF uses the HTTP URL. Synthetic RGB grid with an internal mask, not real imagery." + ], + "last_verified": "2026-11-02", + "availability": "Unreleased low-source-compatibility branch; not the public 0.4.0 package", + "example_guide": "LOW_COMPATIBILITY_WORKFLOWS.md" + }, + { + "id": "cog-low-local-dem", + "type": "Synthetic COG DEM", + "purpose": "Repeatable source elevations/NoData/mask example", + "url": "http://127.0.0.1:8765/elevation-mask.cog.tif", + "path": "fixtures/low/elevation-mask.cog.tif", + "context": { + "srs": "EPSG:25833", + "bbox": [ + 392000, + 5819968, + 392032, + 5820000 + ] + }, + "notes": [ + "Start fixture range server; connect URL to Load Terrain. Shared affine grid, missing corner sample and internal mask hole." + ], + "last_verified": "2026-11-02", + "availability": "Unreleased low-source-compatibility branch; not the public 0.4.0 package", + "example_guide": "LOW_COMPATIBILITY_WORKFLOWS.md" + }, + { + "id": "geopackage-low-local", + "type": "Synthetic GeoPackage", + "purpose": "Layer discovery, polygon courtyard, attributes, quoted point-layer name", + "url": null, + "path": "fixtures/low/local-layers.gpkg", + "context": { + "srs": "EPSG:25833", + "bbox": [ + 392000, + 5819968, + 392032, + 5820000 + ] + }, + "notes": [ + "List WFS Layers / Load WFS: buildings returns one feature/two closed rings via RTree; survey \"points\" returns one point via bounded scan.", + "Reference Source reads both layers' catalog extents without feature geometry. No source mutation; native Rhino validation pending." + ], + "last_verified": "2026-11-02", + "availability": "Unreleased low-source-compatibility branch; not the public 0.4.0 package", + "example_guide": "LOW_COMPATIBILITY_WORKFLOWS.md" } ] } diff --git a/sandbox/LowCompatibilityChecks.cs b/sandbox/LowCompatibilityChecks.cs new file mode 100644 index 0000000..d8cd181 --- /dev/null +++ b/sandbox/LowCompatibilityChecks.cs @@ -0,0 +1,993 @@ +using System.Buffers.Binary; +using System.Net; +using System.Net.Http.Headers; +using System.Security.Cryptography; +using RhinoSpatial.Core; + +namespace RhinoSpatial.Sandbox; + +internal static class LowCompatibilityChecks +{ + private static int _count; + + private static void Require(bool condition, string message) + { + if (!condition) + throw new Exception(message); + } + + private static void Near(double actual, double expected) => + Require(Math.Abs(actual - expected) < 1e-6, $"Expected {expected}, got {actual}."); + + private static async Task Check(string name, Func run) + { + await run(); + _count++; + Console.WriteLine("PASS " + name); + } + + private static Task Sync(Action action) + { + action(); + return Task.CompletedTask; + } + + private static async Task Fails(Func action, string text = "") + { + try + { + await action(); + } + catch (Exception ex) + when (ex + is InvalidDataException + or InvalidOperationException + or OperationCanceledException + or ArgumentException + or Microsoft.Data.Sqlite.SqliteException + ) + { + if (text.Length > 0) + Require( + ex.Message.Contains(text, StringComparison.OrdinalIgnoreCase), + "Unexpected error: " + ex.Message + ); + return; + } + throw new Exception("Invalid source was accepted."); + } + + internal sealed class RangeHandler(byte[] data, string mode = "") : HttpMessageHandler + { + public readonly List<(long Start, long End)> Ranges = new(); + public int Requests => Ranges.Count; + + protected override Task SendAsync( + HttpRequestMessage request, + CancellationToken token + ) + { + token.ThrowIfCancellationRequested(); + var range = + request.Headers.Range?.Ranges.Single() + ?? throw new Exception("Missing Range header."); + var start = range.From!.Value; + var end = Math.Min(range.To!.Value, data.LongLength - 1); + Ranges.Add((start, end)); + Require( + request.Headers.AcceptEncoding.Any(e => e.Value == "identity"), + "Encoded ranges requested." + ); + if (mode == "ignore") + return Task.FromResult( + new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new ByteArrayContent(Array.Empty()), + } + ); + if (start >= data.Length) + return Task.FromResult( + new HttpResponseMessage(HttpStatusCode.RequestedRangeNotSatisfiable) + ); + var payload = data.AsSpan((int)start, (int)(end - start + 1)).ToArray(); + if (mode == "truncate") + payload = payload[..^1]; + var response = new HttpResponseMessage(HttpStatusCode.PartialContent) + { + Content = new ByteArrayContent(payload), + }; + response.Content.Headers.ContentRange = new ContentRangeHeaderValue( + mode == "wrong" ? start + 1 : start, + end, + data.LongLength + ); + if (mode != "no-validator") + response.Headers.ETag = new EntityTagHeaderValue( + mode == "changed" && Requests > 1 ? "\"changed\"" : "\"fixture-v1\"" + ); + if (Requests > 1 && mode != "no-validator") + Require(request.Headers.IfRange is not null, "Missing representation pin."); + return Task.FromResult(response); + } + } + + public static async Task RunAsync() + { + _count = 0; + var directory = Path.Combine( + Path.GetTempPath(), + "rhinospatial-low-check-" + Guid.NewGuid().ToString("N") + ); + Directory.CreateDirectory(directory); + var context = Lod2AtomChecks.Context("EPSG:25833", new(392000, 5819968, 392032, 5820000)); + try + { + var raw = Enumerable.Range(0, 400000).Select(i => (byte)(i % 251)).ToArray(); + await Check( + "byte ranges seek, pin representation and reuse cached blocks", + async () => + { + var handler = new RangeHandler(raw); + using var http = new HttpClient(handler); + using var stream = await HttpRangeStream.OpenAsync( + http, + "https://fixture.test/data.tif" + ); + stream.Position = 140000; + var b = new byte[30]; + Require( + stream.Read(b, 0, b.Length) == 30 && b.SequenceEqual(raw[140000..140030]), + "Incorrect range bytes." + ); + stream.Position = 140002; + Require(stream.Read(b, 0, b.Length) == b.Length, "Short cached range read."); + Require( + handler.Requests == 2 && stream.TransferredBytes == 131072, + "Range block not reused." + ); + stream.Position = raw.Length - 4; + Require( + stream.Read(b, 0, 30) == 4 && stream.Read(b, 0, 30) == 0, + "EOF not bounded." + ); + } + ); + foreach (var mode in new[] { "ignore", "wrong", "truncate" }) + await Check( + "range server " + mode + " rejected without fallback", + async () => + { + using var http = new HttpClient(new RangeHandler(raw, mode)); + await Fails(async () => + { + using var s = await HttpRangeStream.OpenAsync( + http, + "https://fixture.test/a.tif" + ); + }); + } + ); + foreach (var mode in new[] { "changed", "no-validator" }) + await Check( + "range snapshot " + mode + " rejected", + async () => + { + using var http = new HttpClient(new RangeHandler(raw, mode)); + using var s = await HttpRangeStream.OpenAsync( + http, + "https://fixture.test/a.tif" + ); + await Fails(() => + Sync(() => + { + s.Position = 140000; + s.ReadByte(); + }) + ); + } + ); + await Check( + "range byte budget enforced", + async () => + { + using var http = new HttpClient(new RangeHandler(raw)); + using var s = await HttpRangeStream.OpenAsync( + http, + "https://fixture.test/a.tif", + maximumBytes: 65536 + ); + await Fails( + () => + Sync(() => + { + s.Position = 140000; + s.ReadByte(); + }), + "budget" + ); + } + ); + await Check( + "range request budget enforced", + async () => + { + using var http = new HttpClient(new RangeHandler(raw)); + using var s = await HttpRangeStream.OpenAsync( + http, + "https://fixture.test/a.tif", + maximumRequests: 1 + ); + await Fails( + () => + Sync(() => + { + s.Position = 140000; + s.ReadByte(); + }), + "budget" + ); + } + ); + await Check( + "range cancellation honored before IO", + async () => + { + using var http = new HttpClient(new RangeHandler(raw)); + await Fails(async () => + { + using var s = await HttpRangeStream.OpenAsync( + http, + "https://fixture.test/a.tif", + new CancellationToken(true) + ); + }); + } + ); + var imagery = LowCompatibilityFixtures.Cog(); + await Check( + "COG reference reads headers without tile ranges", + async () => + { + var handler = new RangeHandler(imagery); + using var http = new HttpClient(handler); + var metadata = await new RemoteReferenceMetadataReader(http).TryReadAsync( + "https://fixture.test/image.tif" + ); + Require( + metadata?.SrsName == "EPSG:25833" + && metadata.NativeBoundingBox + == new BoundingBox2D(392000, 5819872, 392128, 5820000), + "Wrong COG reference extent." + ); + Require( + handler.Ranges.All(r => r.End < 65536), + "Reference fetched tile ranges." + ); + } + ); + await Check( + "COG selected window pixels, affine, mask and local mesh", + async () => + { + using var http = new HttpClient(new RangeHandler(imagery)); + var result = await new CogReader(http).ReadImageAsync( + "https://fixture.test/image.tif", + context + ); + var image = result.Image; + Require(image.Width == 32 && image.Height == 32, "Window not selected."); + Near(image.GeoTransform!.C, 392000); + Near(image.GeoTransform.F, 5820000); + var decoded = RasterImageCodec.DecodeRgba( + File.ReadAllBytes(image.LocalFilePath), + image.Width, + image.Height + ); + Require( + decoded[0] == 1 && decoded[1] == 1 && decoded[2] == 170, + "TIFF tile row/color orientation changed." + ); + Require( + decoded[(10 * 32 + 10) * 4 + 3] == 0 + && decoded[(20 * 32 + 20) * 4 + 3] == 255, + "Internal mask lost." + ); + var mesh = RasterImageMeshBuilder.Build(image, context); + Require( + mesh.Faces.Count > 0 + && mesh.Vertices.All(v => + v.X >= -1e-6 && v.X <= 32.000001 && v.Y >= -1e-6 && v.Y <= 32.000001 + ), + "Shared local placement lost." + ); + } + ); + await Check( + "COG non-aligned crop uses correct tile sample offsets", + async () => + { + using var http = new HttpClient(new RangeHandler(imagery)); + var ctx = Lod2AtomChecks.Context( + "EPSG:25833", + new(392005, 5819960, 392025, 5819980) + ); + var result = await new CogReader(http).ReadImageAsync( + "https://fixture.test/image.tif", + ctx + ); + Require( + result.Image.Width == 20 && result.Image.Height == 20, + "Incorrect cropped dimensions." + ); + var decoded = RasterImageCodec.DecodeRgba( + File.ReadAllBytes(result.Image.LocalFilePath), + result.Image.Width, + result.Image.Height + ); + Require(decoded[0] == 6 && decoded[1] == 21, "Incorrect crop pixels."); + Near(result.Image.GeoTransform!.C, 392005); + Near(result.Image.GeoTransform.F, 5819980); + } + ); + await Check( + "COG overview chosen for bounded large-area texture", + async () => + { + var data = LowCompatibilityFixtures.Cog(2048, false, false); + using var http = new HttpClient(new RangeHandler(data)); + var ctx = Lod2AtomChecks.Context( + "EPSG:25833", + new(392000, 5817952, 394048, 5820000) + ); + var result = await new CogReader(http).ReadImageAsync( + "https://fixture.test/large.tif", + ctx + ); + Require( + result.OverviewWidth == 1024 + && result.Image.Width == 1024 + && result.TransferredBytes < data.Length, + "Overview was not used for range subset access." + ); + Near(result.Image.GeoTransform!.A, 2); + } + ); + await Check( + "remote DEM preserves elevation, NoData, mask and cell centers", + async () => + { + using var http = new HttpClient( + new RangeHandler(LowCompatibilityFixtures.Cog(128, true)) + ); + var result = await new CogReader(http).ReadTerrainAsync( + "https://fixture.test/dem.tif", + context + ); + var raster = result.Raster; + Require( + raster.Width == 32 && raster.Height == 32 && raster.NoDataValue == -9999, + $"Wrong DEM metadata: {raster.Width}x{raster.Height}, NoData {raster.NoDataValue}." + ); + Near(raster.Origin.X, 392000.5); + Near(raster.Origin.Y, 5819999.5); + Near(raster.Elevations[1], 100.25); + Require( + raster.Elevations[0] == -9999 + && float.IsNaN(raster.Elevations[10 * 32 + 10]), + "Missing DEM values fabricated." + ); + } + ); + await Check( + "RGB imagery rejected as a DEM", + async () => + { + using var http = new HttpClient(new RangeHandler(imagery)); + await Fails( + () => + new CogReader(http).ReadTerrainAsync( + "https://fixture.test/image.tif", + context + ), + "one contiguous" + ); + } + ); + await Check( + "COG non-overlap rejected", + async () => + { + using var http = new HttpClient(new RangeHandler(imagery)); + await Fails( + () => + new CogReader(http).ReadImageAsync( + "https://fixture.test/image.tif", + Lod2AtomChecks.Context("EPSG:25833", new(1, 1, 2, 2)) + ), + "overlap" + ); + } + ); + await Check( + "BigTIFF has explicit decoder limitation", + async () => + { + var big = (byte[])imagery.Clone(); + big[2] = 43; + using var http = new HttpClient(new RangeHandler(big)); + await Fails( + () => new CogReader(http).ReadMetadataAsync("https://fixture.test/big.tif"), + "BigTIFF" + ); + } + ); + await Check( + "TIFF field allocation bomb rejected before decoder allocation", + async () => + { + var bomb = (byte[])imagery.Clone(); + var count = BinaryPrimitives.ReadUInt16LittleEndian(bomb.AsSpan(8)); + for (var i = 0; i < count; i++) + { + var offset = 10 + i * 12; + if (BinaryPrimitives.ReadUInt16LittleEndian(bomb.AsSpan(offset)) == 324) + BinaryPrimitives.WriteUInt32LittleEndian( + bomb.AsSpan(offset + 4), + uint.MaxValue + ); + } + using var http = new HttpClient(new RangeHandler(bomb)); + await Fails( + () => + new CogReader(http).ReadMetadataAsync("https://fixture.test/bomb.tif"), + "allocation budget" + ); + } + ); + await Check( + "WGS84 UTM north/south codes transform without CRS substitution", + () => + Sync(() => + { + foreach ( + var test in new[] + { + (Srs: "EPSG:32632", Lon: 13.4, Lat: 52.5), + (Srs: "EPSG:32734", Lon: 18.4, Lat: -33.9), + } + ) + { + Require( + SpatialReferenceTransform.TryTransformXY( + "EPSG:4326", + test.Srs, + test.Lon, + test.Lat, + out var x, + out var y + ), + "UTM forward transform unavailable." + ); + Require( + SpatialReferenceTransform.TryTransformXY( + test.Srs, + "EPSG:4326", + x, + y, + out var lon, + out var lat + ), + "UTM inverse transform unavailable." + ); + Near(lon, test.Lon); + Near(lat, test.Lat); + } + }) + ); + await Check( + "COG alpha is converted from associated TIFF to straight PNG", + async () => + { + using var http = new HttpClient( + new RangeHandler(LowCompatibilityFixtures.Cog(128, false, false, true)) + ); + var result = await new CogReader(http).ReadImageAsync( + "https://fixture.test/alpha.tif", + context + ); + var pixels = RasterImageCodec.DecodeRgba( + File.ReadAllBytes(result.Image.LocalFilePath), + 32, + 32 + ); + Require( + pixels[3] == 128 && Math.Abs(pixels[2] - 170) <= 1, + "Semi-transparent pixels were darkened." + ); + } + ); + await Check( + "local GeoTIFF uses the GDAL ASCII NoData value rather than its count", + () => + Sync(() => + { + var path = Path.Combine(directory, "local-dem.tif"); + File.WriteAllBytes(path, LowCompatibilityFixtures.Cog(128, true)); + var raster = TerrainRasterReader.ReadRaster(path, "test", "EPSG:25833"); + Require(raster.NoDataValue == -9999, "Local NoData tag was misread."); + }) + ); + var gpkg = LowCompatibilityFixtures.GeoPackage(Path.Combine(directory, "indexed.gpkg")); + var before = SHA256.HashData(File.ReadAllBytes(gpkg)); + await Check( + "GeoPackage discovers feature layers, CRS and quoted identifiers", + () => + Sync(() => + { + var layers = GeoPackageReader.ReadLayers(gpkg); + Require( + layers.Count == 2 + && layers.Any(l => l.Name == "survey \"points\"") + && layers.All(l => l.SourceSrs == "EPSG:25833"), + "Catalog discovery lost layers or CRS." + ); + Require( + VectorSourceResolver.Resolve(gpkg).Kind + == VectorSourceKind.LocalGeoPackage, + "GeoPackage routed to service loader." + ); + }) + ); + await Check( + "GeoPackage RTree query preserves attributes and polygon holes", + () => + Sync(() => + { + var read = GeoPackageReader.ReadFeatures(gpkg, "buildings", context); + Require( + read.UsedSpatialIndex + && read.ScannedCount == 1 + && read.Features.Count == 1, + "Spatial index was not used." + ); + var feature = read.Features[0]; + Require( + feature.Geometry.OuterRings.Count == 2 + && feature.Attributes["name"] == "courtyard" + && feature.Attributes["nullable"] is null + && feature.Attributes["payload"] == "[BLOB 3 bytes]", + "Holes/attributes were lost." + ); + Near(feature.Geometry.OuterRings[1].Points[0].X, 392008); + }) + ); + await Check( + "GeoPackage quoted layer uses bounded fallback scan", + () => + Sync(() => + { + var read = GeoPackageReader.ReadFeatures( + gpkg, + "survey \"points\"", + context + ); + Require( + !read.UsedSpatialIndex + && read.Features.Single().Geometry.Points.Single() + == new Coordinate2D(392005, 5819990), + "Quoted layer was not read." + ); + }) + ); + await Check( + "GeoPackage without RTree filters context and null geometry", + () => + Sync(() => + { + var path = LowCompatibilityFixtures.GeoPackage( + Path.Combine(directory, "scan.gpkg"), + false + ); + var read = GeoPackageReader.ReadFeatures(path, "buildings", context); + Require( + !read.UsedSpatialIndex + && read.ScannedCount == 3 + && read.Features.Count == 1, + "Fallback scan did not filter." + ); + }) + ); + await Check( + "GeoPackage multiple layers require explicit selection", + () => + Fails( + () => Sync(() => GeoPackageReader.ReadFeatures(gpkg, "", context)), + "explicit selection" + ) + ); + await Check( + "GeoPackage sole feature layer can be selected automatically", + () => + Sync(() => + { + var path = LowCompatibilityFixtures.GeoPackage( + Path.Combine(directory, "sole.gpkg") + ); + LowCompatibilityFixtures.Sql( + path, + "DELETE FROM gpkg_contents WHERE table_name<>'buildings'; DELETE FROM gpkg_geometry_columns WHERE table_name<>'buildings'" + ); + Require( + GeoPackageReader.ReadFeatures(path, "", context).Features.Count == 1, + "Sole layer not selected." + ); + }) + ); + await Check( + "GeoPackage reference combines declared catalog metadata only", + () => + Sync(() => + { + var path = LowCompatibilityFixtures.GeoPackage( + Path.Combine(directory, "metadata.gpkg") + ); + LowCompatibilityFixtures.Sql(path, "UPDATE buildings SET geom=x'00'"); + var metadata = ReferenceSourceMetadataReader.Read(path); + Require( + metadata.SourceKind == "GeoPackage" + && metadata.SourceItemCount == 2 + && metadata.Wgs84BoundingBox is not null, + "Catalog reference failed or scanned geometry." + ); + }) + ); + await Check( + "GeoPackage missing extents do not trigger geometry scan", + () => + Fails( + () => + Sync(() => + { + var path = LowCompatibilityFixtures.GeoPackage( + Path.Combine(directory, "noextent.gpkg") + ); + LowCompatibilityFixtures.Sql( + path, + "UPDATE gpkg_contents SET min_x=NULL" + ); + ReferenceSourceMetadataReader.Read(path); + }), + "catalog extents" + ) + ); + await Check( + "GeoPackage custom table SRS ID honors EPSG organization code", + () => + Sync(() => + { + var path = LowCompatibilityFixtures.GeoPackage( + Path.Combine(directory, "custom.gpkg"), + customSrs: true + ); + Require( + GeoPackageReader.ReadFeatures(path, "buildings", context).Features.Count + == 1, + "Local SRS id was mistaken for EPSG code." + ); + }) + ); + await Check( + "GeoPackage unknown CRS is rejected without guessing", + () => + Fails( + () => + Sync(() => + { + var path = LowCompatibilityFixtures.GeoPackage( + Path.Combine(directory, "unknown.gpkg") + ); + LowCompatibilityFixtures.Sql( + path, + "UPDATE gpkg_spatial_ref_sys SET organization='LOCAL' WHERE srs_id=25833" + ); + GeoPackageReader.ReadFeatures(path, "buildings", context); + }), + "no CRS was guessed" + ) + ); + await Check( + "GeoPackage malformed binary geometry is rejected", + () => + Fails( + () => + Sync(() => + { + var path = LowCompatibilityFixtures.GeoPackage( + Path.Combine(directory, "badgeometry.gpkg") + ); + LowCompatibilityFixtures.Sql( + path, + "UPDATE buildings SET geom=x'4750000100000000' WHERE fid=1" + ); + GeoPackageReader.ReadFeatures(path, "buildings", context); + }), + "CRS/envelope" + ) + ); + await Check( + "GeoPackage malicious WKB count rejected before NTS allocation", + () => + Fails( + () => + Sync(() => + { + var path = LowCompatibilityFixtures.GeoPackage( + Path.Combine(directory, "badwkb.gpkg") + ); + var blob = Convert.FromHexString( + "47500001E96400000102000000FFFFFF7F" + ); + LowCompatibilityFixtures.Sql( + path, + "UPDATE buildings SET geom=x'" + + Convert.ToHexString(blob) + + "' WHERE fid=1" + ); + GeoPackageReader.ReadFeatures(path, "buildings", context); + }), + "Truncated WKB" + ) + ); + await Check( + "GeoPackage application ID validation", + () => + Fails( + () => + Sync(() => + { + var path = LowCompatibilityFixtures.GeoPackage( + Path.Combine(directory, "notgpkg.gpkg") + ); + LowCompatibilityFixtures.Sql(path, "PRAGMA application_id=0"); + GeoPackageReader.ReadLayers(path); + }), + "application ID" + ) + ); + await Check( + "GeoPackage explicit feature limit reported", + () => + Sync(() => + { + var result = GeoPackageReader.ReadFeatures(gpkg, "buildings", context, 1); + Require( + result.Features.Count == 1 && result.ReachedFeatureLimit, + "Feature cap not reported." + ); + }) + ); + await Check( + "GeoPackage cancellation honored", + () => + Fails(() => + Sync(() => + GeoPackageReader.ReadFeatures( + gpkg, + "buildings", + context, + token: new CancellationToken(true) + ) + ) + ) + ); + await Check( + "GeoPackage standard multi-geometries retain every curve and point", + () => + Sync(() => + { + var path = LowCompatibilityFixtures.GeoPackage( + Path.Combine(directory, "collection.gpkg") + ); + var factory = new NetTopologySuite.Geometries.GeometryFactory(); + var a = new NetTopologySuite.Geometries.Coordinate(392005, 5819990); + var b = new NetTopologySuite.Geometries.Coordinate(392006, 5819991); + var line = factory.CreateLineString(new[] { a, b }); + var points = factory.CreateMultiPoint( + new[] { factory.CreatePoint(a), factory.CreatePoint(b) } + ); + var collection = factory.CreateGeometryCollection( + new NetTopologySuite.Geometries.Geometry[] + { + factory.CreateMultiLineString(new[] { line, line }), + points, + } + ); + var blob = LowCompatibilityFixtures.BinaryGeometry(collection, 25833); + LowCompatibilityFixtures.Sql( + path, + "UPDATE gpkg_geometry_columns SET geometry_type_name='GEOMETRY' WHERE table_name='buildings'; UPDATE buildings SET geom=x'" + + Convert.ToHexString(blob) + + "' WHERE fid=1" + ); + var geometry = GeoPackageReader + .ReadFeatures(path, "buildings", context) + .Features.Single() + .Geometry; + Require( + geometry.LineStrings.Count == 2 && geometry.Points.Count == 2, + "Multi-geometries lost parts." + ); + }) + ); + await Check( + "GeoPackage big-endian header, XY envelope and WKB", + () => + Sync(() => + { + var path = LowCompatibilityFixtures.GeoPackage( + Path.Combine(directory, "big-endian.gpkg") + ); + var blob = new byte[61]; + blob[0] = 71; + blob[1] = 80; + blob[3] = 2; + BinaryPrimitives.WriteInt32BigEndian(blob.AsSpan(4, 4), 25833); + var values = new[] { 392005d, 392005d, 5819990d, 5819990d }; + for (var i = 0; i < 4; i++) + BinaryPrimitives.WriteInt64BigEndian( + blob.AsSpan(8 + i * 8, 8), + BitConverter.DoubleToInt64Bits(values[i]) + ); + blob[40] = 0; + BinaryPrimitives.WriteInt32BigEndian(blob.AsSpan(41, 4), 1); + BinaryPrimitives.WriteInt64BigEndian( + blob.AsSpan(45, 8), + BitConverter.DoubleToInt64Bits(392005) + ); + BinaryPrimitives.WriteInt64BigEndian( + blob.AsSpan(53, 8), + BitConverter.DoubleToInt64Bits(5819990) + ); + LowCompatibilityFixtures.Sql( + path, + "UPDATE gpkg_geometry_columns SET geometry_type_name='GEOMETRY' WHERE table_name='buildings'; UPDATE buildings SET geom=x'" + + Convert.ToHexString(blob) + + "' WHERE fid=1" + ); + Require( + GeoPackageReader + .ReadFeatures(path, "buildings", context) + .Features.Single() + .Geometry.Points.Single() == new Coordinate2D(392005, 5819990), + "Byte order/envelope offsets changed coordinates." + ); + }) + ); + await Check( + "GeoPackage unclosed polygon rings are rejected without repair", + () => + Fails( + () => + Sync(() => + { + var path = LowCompatibilityFixtures.GeoPackage( + Path.Combine(directory, "unclosed.gpkg") + ); + var ring = + new NetTopologySuite.Geometries.GeometryFactory().CreatePolygon( + new[] + { + new NetTopologySuite.Geometries.Coordinate( + 392001, + 5819990 + ), + new NetTopologySuite.Geometries.Coordinate( + 392002, + 5819990 + ), + new NetTopologySuite.Geometries.Coordinate( + 392002, + 5819991 + ), + new NetTopologySuite.Geometries.Coordinate( + 392001, + 5819990 + ), + } + ); + var blob = LowCompatibilityFixtures.BinaryGeometry(ring, 25833); + BitConverter.GetBytes(392003d).CopyTo(blob, 8 + 13 + 3 * 16); + LowCompatibilityFixtures.Sql( + path, + "UPDATE buildings SET geom=x'" + + Convert.ToHexString(blob) + + "' WHERE fid=1" + ); + GeoPackageReader.ReadFeatures(path, "buildings", context); + }), + "Unclosed" + ) + ); + await Check( + "GeoPackage extended binary geometries rejected explicitly", + () => + Fails( + () => + Sync(() => + { + var path = LowCompatibilityFixtures.GeoPackage( + Path.Combine(directory, "extended.gpkg") + ); + var geometry = + new NetTopologySuite.Geometries.GeometryFactory().CreatePoint( + new NetTopologySuite.Geometries.Coordinate(392005, 5819990) + ); + var blob = LowCompatibilityFixtures.BinaryGeometry(geometry, 25833); + blob[3] |= 32; + LowCompatibilityFixtures.Sql( + path, + "UPDATE buildings SET geom=x'" + + Convert.ToHexString(blob) + + "' WHERE fid=1" + ); + GeoPackageReader.ReadFeatures(path, "buildings", context); + }), + "Unsupported" + ) + ); + await Check( + "GeoPackage declared feature type must match its WKB", + () => + Fails( + () => + Sync(() => + { + var path = LowCompatibilityFixtures.GeoPackage( + Path.Combine(directory, "type-mismatch.gpkg") + ); + LowCompatibilityFixtures.Sql( + path, + "UPDATE gpkg_geometry_columns SET geometry_type_name='POINT' WHERE table_name='buildings'" + ); + GeoPackageReader.ReadFeatures(path, "buildings", context); + }), + "disagrees" + ) + ); + await Check( + "GeoPackage unsupported declared geometry extensions rejected", + () => + Fails( + () => + Sync(() => + { + var path = LowCompatibilityFixtures.GeoPackage( + Path.Combine(directory, "type-extension.gpkg") + ); + LowCompatibilityFixtures.Sql( + path, + "UPDATE gpkg_geometry_columns SET geometry_type_name='CIRCULARSTRING' WHERE table_name='buildings'" + ); + GeoPackageReader.ReadFeatures(path, "buildings", context); + }), + "Unsupported" + ) + ); + await Check( + "GeoPackage source remains byte-identical after reads", + () => + Sync(() => + Require( + before.SequenceEqual(SHA256.HashData(File.ReadAllBytes(gpkg))), + "Read operation mutated the database." + ) + ) + ); + Console.WriteLine($"Low compatibility checks passed: {_count}."); + } + finally + { + Directory.Delete(directory, true); + } + } +} diff --git a/sandbox/LowCompatibilityFixtures.cs b/sandbox/LowCompatibilityFixtures.cs new file mode 100644 index 0000000..379a688 --- /dev/null +++ b/sandbox/LowCompatibilityFixtures.cs @@ -0,0 +1,379 @@ +using System.IO.Compression; +using System.Text; +using Microsoft.Data.Sqlite; +using NetTopologySuite.Geometries; +using NetTopologySuite.IO; +using RhinoSpatial.Core; + +namespace RhinoSpatial.Sandbox; + +internal static class LowCompatibilityFixtures +{ + private sealed record Tag(ushort Id, ushort Type, uint Count, byte[] Data); + + private sealed class Page + { + public int Width, + Height, + Channels, + Bits, + Flags; + public byte[][] Tiles = Array.Empty(); + public uint[] Offsets = Array.Empty(); + public List Tags = new(); + public uint Ifd; + } + + internal static byte[] Cog( + int size = 128, + bool terrain = false, + bool masks = true, + bool alpha = false + ) + { + var pages = new List(); + foreach (var factor in new[] { 1, 2 }) + { + pages.Add( + new Page + { + Width = size / factor, + Height = size / factor, + Channels = + terrain ? 1 + : alpha ? 4 + : 3, + Bits = terrain ? 32 : 8, + Flags = factor == 1 ? 0 : 1, + } + ); + if (masks) + pages.Add( + new Page + { + Width = size / factor, + Height = size / factor, + Channels = 1, + Bits = 1, + Flags = factor == 1 ? 4 : 5, + } + ); + } + const int tile = 64; + foreach (var p in pages) + { + var isMask = (p.Flags & 4) != 0; + var tiles = new List(); + for (var y = 0; y < p.Height; y += tile) + for (var x = 0; x < p.Width; x += tile) + { + var raw = new byte[tile * tile * p.Channels * p.Bits / 8]; + for (var ly = 0; ly < tile; ly++) + for (var lx = 0; lx < tile; lx++) + { + var xx = x + lx; + var yy = y + ly; + var index = ly * tile + lx; + var factor = size / p.Width; + if (isMask) + { + if (!(xx * factor is >= 8 and < 16 && yy * factor is >= 8 and < 16)) + raw[ly * tile / 8 + lx / 8] |= (byte)(128 >> (lx % 8)); + } + else if (terrain) + BitConverter + .GetBytes( + xx == 0 && yy == 0 + ? -9999f + : 100f + xx * factor * .25f + yy * factor * .5f + ) + .CopyTo(raw, index * 4); + else + { + raw[index * p.Channels] = (byte)(xx % 200 + 1); + raw[index * p.Channels + 1] = (byte)(yy % 200 + 1); + raw[index * p.Channels + 2] = 170; + if (alpha) + raw[index * p.Channels + 3] = 128; + } + } + using var compressed = new MemoryStream(); + using (var z = new ZLibStream(compressed, CompressionLevel.Fastest, true)) + z.Write(raw); + tiles.Add(compressed.ToArray()); + } + p.Tiles = tiles.ToArray(); + p.Offsets = new uint[p.Tiles.Length]; + } + static byte[] Shorts(params ushort[] values) + { + using var m = new MemoryStream(); + using var w = new BinaryWriter(m); + foreach (var v in values) + w.Write(v); + return m.ToArray(); + } + static byte[] Longs(IEnumerable values) + { + using var m = new MemoryStream(); + using var w = new BinaryWriter(m); + foreach (var v in values) + w.Write(v); + return m.ToArray(); + } + static byte[] Doubles(params double[] values) + { + using var m = new MemoryStream(); + using var w = new BinaryWriter(m); + foreach (var v in values) + w.Write(v); + return m.ToArray(); + } + foreach (var p in pages) + { + var isMask = (p.Flags & 4) != 0; + var factor = (double)size / p.Width; + void S(ushort tag, ushort value) => p.Tags.Add(new(tag, 3, 1, Shorts(value))); + void L(ushort tag, uint value) => p.Tags.Add(new(tag, 4, 1, Longs(new[] { value }))); + L(254, (uint)p.Flags); + L(256, (uint)p.Width); + L(257, (uint)p.Height); + p.Tags.Add( + new( + 258, + 3, + (uint)p.Channels, + Shorts(Enumerable.Repeat((ushort)p.Bits, p.Channels).ToArray()) + ) + ); + S(259, 8); + S( + 262, + (ushort)( + isMask ? 4 + : terrain ? 1 + : 2 + ) + ); + S(274, 1); + S(277, (ushort)p.Channels); + S(284, 1); + L(322, tile); + L(323, tile); + p.Tags.Add(new(324, 4, (uint)p.Tiles.Length, Longs(p.Offsets))); + p.Tags.Add( + new(325, 4, (uint)p.Tiles.Length, Longs(p.Tiles.Select(t => (uint)t.Length))) + ); + S(339, (ushort)(terrain && !isMask ? 3 : 1)); + if (alpha && !isMask) + S(338, 2); + if (!isMask) + { + p.Tags.Add(new(33550, 12, 3, Doubles(factor, factor, 0))); + p.Tags.Add(new(33922, 12, 6, Doubles(0, 0, 0, 392000, 5820000, 0))); + var keys = Shorts(1, 1, 0, 3, 1024, 0, 1, 1, 1025, 0, 1, 1, 3072, 0, 1, 25833); + p.Tags.Add(new(34735, 3, (uint)keys.Length / 2, keys)); + if (terrain) + p.Tags.Add(new(42113, 2, 6, Encoding.ASCII.GetBytes("-9999\0"))); + } + p.Tags = p.Tags.OrderBy(t => t.Id).ToList(); + } + uint location = 8; + foreach (var p in pages) + { + p.Ifd = location; + location += checked((uint)(2 + p.Tags.Count * 12 + 4)); + } + var external = new Dictionary<(Page, ushort), uint>(); + foreach (var p in pages) + foreach (var tag in p.Tags) + if (tag.Data.Length > 4) + { + external[(p, tag.Id)] = location; + location += checked((uint)tag.Data.Length); + if (location % 2 != 0) + location++; + } + location = Math.Max(65536, location); + foreach (var p in pages.OrderBy(p => p.Width)) + for (var i = 0; i < p.Tiles.Length; i++) + { + p.Offsets[i] = location; + location += checked((uint)p.Tiles[i].Length); + } + using var output = new MemoryStream(new byte[location], true); + using var writer = new BinaryWriter(output); + writer.Write((ushort)0x4949); + writer.Write((ushort)42); + writer.Write(pages[0].Ifd); + for (var pageIndex = 0; pageIndex < pages.Count; pageIndex++) + { + var p = pages[pageIndex]; + output.Position = p.Ifd; + writer.Write((ushort)p.Tags.Count); + foreach (var tag in p.Tags) + { + writer.Write(tag.Id); + writer.Write(tag.Type); + writer.Write(tag.Count); + var data = tag.Id == 324 ? Longs(p.Offsets) : tag.Data; + if (data.Length <= 4) + { + writer.Write(data); + writer.Write(new byte[4 - data.Length]); + } + else + writer.Write(external[(p, tag.Id)]); + } + writer.Write(pageIndex + 1 < pages.Count ? pages[pageIndex + 1].Ifd : 0u); + foreach (var tag in p.Tags) + if (tag.Data.Length > 4) + { + output.Position = external[(p, tag.Id)]; + writer.Write(tag.Id == 324 ? Longs(p.Offsets) : tag.Data); + } + for (var i = 0; i < p.Tiles.Length; i++) + { + output.Position = p.Offsets[i]; + writer.Write(p.Tiles[i]); + } + } + return output.ToArray(); + } + + internal static string GeoPackage(string path, bool index = true, bool customSrs = false) + { + try + { + _ = SQLitePCL.raw.sqlite3_libversion_number(); + } + catch + { + if (OperatingSystem.IsWindows()) + SQLitePCL.raw.SetProvider(new SQLitePCL.SQLite3Provider_winsqlite3()); + else + SQLitePCL.raw.SetProvider(new SQLitePCL.SQLite3Provider_sqlite3()); + } + if (File.Exists(path)) + File.Delete(path); + using var connection = new SqliteConnection( + new SqliteConnectionStringBuilder { DataSource = path, Pooling = false }.ToString() + ); + connection.Open(); + var srs = customSrs ? 99001 : 25833; + using var command = connection.CreateCommand(); + command.CommandText = $"""" + PRAGMA application_id=1196444487; PRAGMA user_version=10300; + CREATE TABLE gpkg_spatial_ref_sys(srs_name TEXT NOT NULL,srs_id INTEGER PRIMARY KEY,organization TEXT NOT NULL,organization_coordsys_id INTEGER NOT NULL,definition TEXT NOT NULL,description TEXT); + INSERT INTO gpkg_spatial_ref_sys VALUES('Undefined Cartesian',-1,'NONE',-1,'undefined',NULL),('Undefined geographic',0,'NONE',0,'undefined',NULL),('WGS84',4326,'EPSG',4326,'GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433]]',NULL); + CREATE TABLE gpkg_contents(table_name TEXT PRIMARY KEY,data_type TEXT NOT NULL,identifier TEXT UNIQUE,description TEXT DEFAULT '',last_change DATETIME NOT NULL DEFAULT '2026-11-02T00:00:00.000Z',min_x DOUBLE,min_y DOUBLE,max_x DOUBLE,max_y DOUBLE,srs_id INTEGER); + CREATE TABLE gpkg_geometry_columns(table_name TEXT NOT NULL,column_name TEXT NOT NULL,geometry_type_name TEXT NOT NULL,srs_id INTEGER NOT NULL,z TINYINT NOT NULL,m TINYINT NOT NULL,PRIMARY KEY(table_name,column_name),UNIQUE(table_name)); + CREATE TABLE buildings(fid INTEGER PRIMARY KEY,geom POLYGON,name TEXT,nullable TEXT,payload BLOB); + CREATE TABLE "survey ""points"""(fid INTEGER PRIMARY KEY,geom POINT,name TEXT); + INSERT INTO gpkg_contents(table_name,data_type,identifier,min_x,min_y,max_x,max_y,srs_id) VALUES('buildings','features','Example buildings',392000,5819872,392300,5820200,{srs}),('survey "points"','features','Survey points',392005,5819990,392005,5819990,{srs}); + INSERT INTO gpkg_geometry_columns VALUES('buildings','geom','POLYGON',{srs},0,0),('survey "points"','geom','POINT',{srs},0,0); + """"; + command.ExecuteNonQuery(); + command.CommandText = + "INSERT INTO gpkg_spatial_ref_sys VALUES('ETRS89 / UTM zone 33N',@srs,'EPSG',25833,@wkt,NULL)"; + command.Parameters.AddWithValue("@srs", srs); + command.Parameters.AddWithValue("@wkt", MediumCompatibilityChecks.Wkt); + command.ExecuteNonQuery(); + command.Parameters.Clear(); + var factory = new GeometryFactory(); + NetTopologySuite.Geometries.LinearRing Ring(double x, double y, double size) => + factory.CreateLinearRing( + new[] + { + new Coordinate(x, y), + new Coordinate(x + size, y), + new Coordinate(x + size, y + size), + new Coordinate(x, y + size), + new Coordinate(x, y), + } + ); + var polygon = factory.CreatePolygon( + Ring(392000, 5819970, 30), + new[] { Ring(392008, 5819978, 5) } + ); + void Insert(string table, int id, Geometry? geometry, string name) + { + command.CommandText = $"INSERT INTO {table}(fid,geom,name) VALUES(@id,@geometry,@name)"; + command.Parameters.Clear(); + command.Parameters.AddWithValue("@id", id); + command.Parameters.AddWithValue( + "@geometry", + geometry is null ? DBNull.Value : BinaryGeometry(geometry, srs) + ); + command.Parameters.AddWithValue("@name", name); + command.ExecuteNonQuery(); + } + Insert("buildings", 1, polygon, "courtyard"); + Insert("buildings", 2, factory.CreatePolygon(Ring(392200, 5820100, 20)), "outside"); + Insert("buildings", 3, null, "null geometry"); + Insert( + "\"survey \"\"points\"\"\"", + 1, + factory.CreatePoint(new Coordinate(392005, 5819990)), + "marker" + ); + command.CommandText = "UPDATE buildings SET payload=x'010203' WHERE fid=1"; + command.Parameters.Clear(); + command.ExecuteNonQuery(); + if (index) + { + command.CommandText = """ + CREATE TABLE gpkg_extensions(table_name TEXT,column_name TEXT,extension_name TEXT NOT NULL,definition TEXT NOT NULL,scope TEXT NOT NULL,UNIQUE(table_name,column_name,extension_name)); + INSERT INTO gpkg_extensions VALUES('buildings','geom','gpkg_rtree_index','http://www.geopackage.org/spec/#extension_rtree','write-only'); + CREATE VIRTUAL TABLE rtree_buildings_geom USING rtree(id,minx,maxx,miny,maxy); + INSERT INTO rtree_buildings_geom VALUES(1,392000,392030,5819970,5820000),(2,392200,392220,5820100,5820120); + CREATE TRIGGER rtree_buildings_geom_insert AFTER INSERT ON buildings WHEN NEW.geom IS NOT NULL AND NOT ST_IsEmpty(NEW.geom) BEGIN INSERT OR REPLACE INTO rtree_buildings_geom VALUES(NEW.fid,ST_MinX(NEW.geom),ST_MaxX(NEW.geom),ST_MinY(NEW.geom),ST_MaxY(NEW.geom)); END; + CREATE TRIGGER rtree_buildings_geom_update1 AFTER UPDATE OF geom ON buildings WHEN OLD.fid=NEW.fid AND NEW.geom IS NOT NULL AND NOT ST_IsEmpty(NEW.geom) BEGIN INSERT OR REPLACE INTO rtree_buildings_geom VALUES(NEW.fid,ST_MinX(NEW.geom),ST_MaxX(NEW.geom),ST_MinY(NEW.geom),ST_MaxY(NEW.geom)); END; + CREATE TRIGGER rtree_buildings_geom_update2 AFTER UPDATE OF geom ON buildings WHEN OLD.fid=NEW.fid AND (NEW.geom IS NULL OR ST_IsEmpty(NEW.geom)) BEGIN DELETE FROM rtree_buildings_geom WHERE id=OLD.fid; END; + CREATE TRIGGER rtree_buildings_geom_update3 AFTER UPDATE ON buildings WHEN OLD.fid<>NEW.fid AND NEW.geom IS NOT NULL AND NOT ST_IsEmpty(NEW.geom) BEGIN DELETE FROM rtree_buildings_geom WHERE id=OLD.fid; INSERT OR REPLACE INTO rtree_buildings_geom VALUES(NEW.fid,ST_MinX(NEW.geom),ST_MaxX(NEW.geom),ST_MinY(NEW.geom),ST_MaxY(NEW.geom)); END; + CREATE TRIGGER rtree_buildings_geom_update4 AFTER UPDATE ON buildings WHEN OLD.fid<>NEW.fid AND (NEW.geom IS NULL OR ST_IsEmpty(NEW.geom)) BEGIN DELETE FROM rtree_buildings_geom WHERE id IN(OLD.fid,NEW.fid); END; + CREATE TRIGGER rtree_buildings_geom_delete AFTER DELETE ON buildings WHEN OLD.geom IS NOT NULL BEGIN DELETE FROM rtree_buildings_geom WHERE id=OLD.fid; END; + """; + command.ExecuteNonQuery(); + } + return path; + } + + internal static byte[] BinaryGeometry(Geometry geometry, int srs) + { + var wkb = new WKBWriter().Write(geometry); + var blob = new byte[8 + wkb.Length]; + blob[0] = 71; + blob[1] = 80; + blob[3] = (byte)(geometry.IsEmpty ? 17 : 1); + BitConverter.GetBytes(srs).CopyTo(blob, 4); + wkb.CopyTo(blob, 8); + return blob; + } + + internal static void Sql(string path, string sql) + { + using var connection = new SqliteConnection( + new SqliteConnectionStringBuilder { DataSource = path, Pooling = false }.ToString() + ); + connection.Open(); + using var command = connection.CreateCommand(); + command.CommandText = + "DROP TRIGGER IF EXISTS rtree_buildings_geom_insert; DROP TRIGGER IF EXISTS rtree_buildings_geom_update1; DROP TRIGGER IF EXISTS rtree_buildings_geom_update2; DROP TRIGGER IF EXISTS rtree_buildings_geom_update3; DROP TRIGGER IF EXISTS rtree_buildings_geom_update4;"; + command.ExecuteNonQuery(); + command.CommandText = sql; + command.ExecuteNonQuery(); + } + + internal static void WriteExamples(string directory) + { + Directory.CreateDirectory(directory); + File.WriteAllBytes(Path.Combine(directory, "imagery-mask.cog.tif"), Cog()); + File.WriteAllBytes(Path.Combine(directory, "elevation-mask.cog.tif"), Cog(128, true)); + GeoPackage(Path.Combine(directory, "local-layers.gpkg")); + Console.WriteLine( + "Wrote synthetic EPSG:25833 COG and GeoPackage examples to " + + Path.GetFullPath(directory) + ); + } +} diff --git a/sandbox/LowCompatibilityProbe.cs b/sandbox/LowCompatibilityProbe.cs new file mode 100644 index 0000000..a63f514 --- /dev/null +++ b/sandbox/LowCompatibilityProbe.cs @@ -0,0 +1,67 @@ +using System.Globalization; +using RhinoSpatial.Core; + +namespace RhinoSpatial.Sandbox; + +internal static class LowCompatibilityProbe +{ + internal static async Task RunAsync(string[] args) + { + if (args.Length < 3) + throw new ArgumentException( + "low-probe cog|dem [srs minX,minY,maxX,maxY] | gpkg [layer srs minX,minY,maxX,maxY] | examples " + ); + SpatialContext2D Context(int srsIndex, int boundsIndex) + { + var b = args[boundsIndex] + .Split(',') + .Select(v => double.Parse(v, CultureInfo.InvariantCulture)) + .ToArray(); + if (b.Length != 4) + throw new ArgumentException("Expected four bbox ordinates."); + return Lod2AtomChecks.Context(args[srsIndex], new(b[0], b[1], b[2], b[3])); + } + if (args[1] is "cog" or "dem") + { + var reader = new CogReader(); + var info = await reader.ReadMetadataAsync(args[2]); + Console.WriteLine( + $"{info.SrsName}; {info.Width}x{info.Height}; {info.BoundingBox}; affine {info.GeoTransform}" + ); + if (args.Length < 5) + return; + var context = Context(3, 4); + if (args[1] == "cog") + { + var result = await reader.ReadImageAsync(args[2], context); + var mesh = RasterImageMeshBuilder.Build(result.Image, context); + Console.WriteLine( + $"Window {result.Image.Width}x{result.Image.Height}; overview width {result.OverviewWidth}; {result.RangeRequests} requests, {result.TransferredBytes} bytes; {mesh.Faces.Count} mesh faces; PNG {result.Image.LocalFilePath}" + ); + } + else + { + var result = await reader.ReadTerrainAsync(args[2], context); + var raster = result.Raster; + Console.WriteLine( + $"DEM window {raster.Width}x{raster.Height}; overview width {result.OverviewWidth}; {result.RangeRequests} requests, {result.TransferredBytes} bytes; NoData {raster.NoDataValue}; finite samples {raster.Elevations.Count(float.IsFinite)}" + ); + } + } + else if (args[1] == "gpkg") + { + foreach (var layer in GeoPackageReader.ReadLayers(args[2])) + Console.WriteLine( + $"{layer.Name} | {layer.Title} | {layer.SourceSrs} | {layer.Bounds}" + ); + if (args.Length < 6) + return; + var result = GeoPackageReader.ReadFeatures(args[2], args[3], Context(4, 5)); + Console.WriteLine( + $"Features {result.Features.Count}; scanned {result.ScannedCount}; RTree {result.UsedSpatialIndex}; rings {result.Features.Sum(f => f.Geometry.OuterRings.Count)}" + ); + } + else + throw new ArgumentException("Unknown low-probe mode."); + } +} diff --git a/sandbox/MediumCompatibilityChecks.cs b/sandbox/MediumCompatibilityChecks.cs new file mode 100644 index 0000000..eba2e12 --- /dev/null +++ b/sandbox/MediumCompatibilityChecks.cs @@ -0,0 +1,145 @@ +using System.Net; +using System.Text; +using System.Text.Json; +using System.Xml.Linq; +using BitMiracle.LibTiff.Classic; +using RhinoSpatial.Core; + +namespace RhinoSpatial.Sandbox; + +internal static class MediumCompatibilityChecks +{ + private static int _count; + private static void Require(bool value, string message) { if (!value) throw new Exception(message); } + private static void Near(double actual, double expected) => Require(Math.Abs(actual - expected) < 1e-7, $"Expected {expected}, got {actual}."); + private static void Fails(Action action) { try { action(); } catch (Exception e) when (e is InvalidDataException or JsonException or KeyNotFoundException or InvalidOperationException) { return; } throw new Exception("Invalid source accepted."); } + private static async Task Check(string label, Func action) { await action(); _count++; Console.WriteLine("PASS " + label); } + private static Task Sync(Action action) { action(); return Task.CompletedTask; } + internal const string Wkt = "PROJCS[\"ETRS89 / UTM zone 33N\",GEOGCS[\"ETRS89\",DATUM[\"European_Terrestrial_Reference_System_1989\",SPHEROID[\"GRS 1980\",6378137,298.257222101]],PRIMEM[\"Greenwich\",0],UNIT[\"degree\",0.0174532925199433]],PROJECTION[\"Transverse_Mercator\"],PARAMETER[\"latitude_of_origin\",0],PARAMETER[\"central_meridian\",15],PARAMETER[\"scale_factor\",0.9996],PARAMETER[\"false_easting\",500000],PARAMETER[\"false_northing\",0],UNIT[\"metre\",1],AUTHORITY[\"EPSG\",\"25833\"]]"; + internal static string Geometry(string lod, string indices = "[[[0,1,2]]]") => "{\"type\":\"MultiSurface\",\"lod\":" + JsonSerializer.Serialize(lod) + ",\"boundaries\":" + indices + "}"; + internal static string Model(string geometry) => "{\"type\":\"CityJSON\",\"version\":\"2.0\",\"metadata\":{\"referenceSystem\":\"https://www.opengis.net/def/crs/EPSG/0/25833\"},\"CityObjects\":{\"building\":{\"type\":\"Building\",\"geometry\":[" + geometry + "]}},\"vertices\":[[0,0,0],[2,0,0],[0,2,1]]}"; + internal static string Sequence => "{\"type\":\"CityJSON\",\"version\":\"2.0\",\"metadata\":{\"referenceSystem\":\"https://www.opengis.net/def/crs/EPSG/0/25833\"},\"transform\":{\"scale\":[2,3,4],\"translate\":[10,20,30]},\"CityObjects\":{},\"vertices\":[]}\n" + + Model(Geometry("2.2")).Replace("\"CityJSON\"", "\"CityJSONFeature\"").Replace("\"metadata\":{\"referenceSystem\":\"https://www.opengis.net/def/crs/EPSG/0/25833\"},", "") + "\n"; + internal static string Capabilities(string srs = "EPSG:25833", string top = "0 4", string scale = "3571.4285714285716", string extra = "") => $""" + + + orthoOrtho-180 -90180 90image/png{extra}native + native{srs}level{scale}{top}2222 + """; + private sealed class Handler(Func fetch) : HttpMessageHandler + { public List Requests { get; } = new(); protected override Task SendAsync(HttpRequestMessage request, CancellationToken token) { token.ThrowIfCancellationRequested(); Requests.Add(request.RequestUri!); return Task.FromResult(fetch(request)); } } + private static MemoryStream Stream(string text) => new(Encoding.UTF8.GetBytes(text)); + public static async Task RunAsync() + { + var directory = Path.Combine(Path.GetTempPath(), "RhinoSpatial-Medium-" + Guid.NewGuid().ToString("N")); Directory.CreateDirectory(directory); + try + { + await Check("one suitable LoD is selected without overlaying alternate representations", () => Sync(() => + { var b = CityJsonReader.ReadBuildings(Model(Geometry("1") + "," + Geometry("2.2") + "," + Geometry("3")), "test").Single(); Require(b.Surfaces.Count == 1 && b.Attributes["source_lod"] == "2.2", "Mixed LoDs imported."); })); + await Check("same-LoD geometry parts are retained and higher-only LoDs have a deterministic fallback", () => Sync(() => + { var b = CityJsonReader.ReadBuildings(Model(Geometry("3") + "," + Geometry("3") + "," + Geometry("4")), "test").Single(); Require(b.Surfaces.Count == 2 && b.Attributes["source_lod"] == "3", "Geometry part or fallback lost."); })); + await Check("invalid indices fail instead of creating partial rings", () => Sync(() => Fails(() => CityJsonReader.ReadBuildings(Model(Geometry("2", "[[[0,1,99]]]")), "test")))); + await Check("sequence records inherit header transform and source CRS", () => Sync(() => + { using var s = Stream(Sequence); var b = CityJsonReader.ReadBuildings(s, "test").Single(); Near(b.Surfaces[0].OuterPoints[1].X, 14); Near(b.Surfaces[0].OuterPoints[2].Y, 26); Near(b.Surfaces[0].OuterPoints[2].Z, 34); var m = CityJsonReader.ReadSourceMetadata(Sequence); Require(m.SrsName == "EPSG:25833", "Header CRS lost."); Near(m.BoundingBox!.MaxY, 26); })); + await Check("sequence input rejects truncated later records and repeated building ids", () => Sync(() => + { Fails(() => CityJsonReader.ReadBuildings(Sequence + "{", "test")); Fails(() => CityJsonReader.ReadBuildings(Sequence + Sequence.Split('\n')[1] + "\n", "test")); })); + await Check("template vertices use row-major matrix then transformed instance anchor", () => Sync(() => + { + var model = Model("{\"type\":\"GeometryInstance\",\"template\":0,\"boundaries\":[0],\"transformationMatrix\":[0,-2,0,5,3,0,0,7,0,0,4,9,0,0,0,1]}"); + model = model.Replace("\"vertices\":", "\"transform\":{\"scale\":[10,10,10],\"translate\":[100,200,300]},\"geometry-templates\":{\"templates\":[" + Geometry("2.2") + "],\"vertices-templates\":[[0,0,0],[2,0,0],[0,2,1]]},\"vertices\":"); + var b = CityJsonReader.ReadBuildings(model, "test").Single(); Near(b.Surfaces[0].OuterPoints[1].X, 105); Near(b.Surfaces[0].OuterPoints[1].Y, 213); Near(b.Surfaces[0].OuterPoints[2].Z, 313); + var m = CityJsonReader.ReadSourceMetadata(model); Require(m.BoundingBox!.MaxY >= 213, "Template extent omitted."); + })); + await Check("sequence cancellation propagates", () => Sync(() => + { using var token = new CancellationTokenSource(); token.Cancel(); using var s = Stream(Sequence); try { CityJsonReader.ReadBuildings(s, "test", cancellationToken: token.Token); throw new Exception("Cancellation ignored."); } catch (OperationCanceledException) { } })); + await Check("direct sequence downloads validate and cache through shared building path", async () => + { using var http = new HttpClient(new Handler(_ => new(HttpStatusCode.OK) { Content = new StringContent(Sequence) })); var cache = new SourceDownloadCache(AtomDownloadKind.Buildings, http, Path.Combine(directory, "downloads")); var r = await cache.DownloadAsync(new(new Uri("https://fixture.test/model.city.jsonl"), "seq", "", null)); using var s = File.OpenRead(r.FilePath); Require(CityJsonReader.ReadBuildings(s, "test").Count == 1, "Sequence did not survive caching."); }); + var png = Path.Combine(directory, "rotated.png"); var rgba = Enumerable.Range(0, 64).Select(i => (byte)(i * 3)).ToArray(); RasterImageCodec.WritePng(png, 4, 4, rgba); File.WriteAllText(Path.ChangeExtension(png, ".pgw"), "2\n1\n0.5\n-3\n100\n200\n"); File.WriteAllText(Path.ChangeExtension(png, ".prj"), Wkt); + await Check("world-file centers, rotation and shear preserve the true footprint", () => Sync(() => + { var info = GeoTiffReader.ReadImageInfo(png); var g = info.GeoTransform!; Near(g.At(.5, .5).X, 100); Near(g.At(.5, .5).Y, 200); Near(g.At(4, 4).X, 108.75); Near(g.At(4, 4).Y, 193); Near(g.Pixel(100, 200).X, .5); Require(info.SrsName == "EPSG:25833", "Sidecar CRS lost."); })); + await Check("affine image clipping preserves UVs and localization", () => Sync(() => + { var info = GeoTiffReader.ReadImageInfo(png); var context = Lod2AtomChecks.Context("EPSG:25833", new(100, 191, 106, 202)); var mesh = RasterImageMeshBuilder.Build(info, context); Require(mesh.Faces.Count > 0, "Clipped image disappeared."); foreach (var v in mesh.Vertices) { var native = info.GeoTransform!.At(v.U * 4, (1 - v.V) * 4); Near(native.X - context.PlacementOrigin.X, v.X); Near(native.Y - context.PlacementOrigin.Y, v.Y); } Require(mesh.CoverageRatio < 1, "Clip ratio not retained."); })); + await Check("affine imagery reprojects through the shared context without bbox scaling", () => Sync(() => + { + var source = new RasterGeoTransform(2, 0.5, 391300, 0.5, -2, 5820100); var native = source.Bounds(4, 4); + var bounds = Lod2AtomFeedReader.TransformBounds(new("EPSG:25833", native), "EPSG:3857"); var context = Lod2AtomChecks.Context("EPSG:3857", bounds); + var info = new GeoReferencedRasterInfo("test", "EPSG:25833", 4, 4, native, 0) { GeoTransform = source }; var mesh = RasterImageMeshBuilder.Build(info, context); + Require(mesh.Faces.Count > 0, "Reprojected image disappeared."); foreach (var v in mesh.Vertices) { var p = source.At(v.U * 4, (1 - v.V) * 4); Require(SpatialReferenceTransform.TryTransformXY(info.SrsName, context.ResolvedSrs, p.X, p.Y, out var x, out var y), "Projection failed."); Near(v.X, x - context.PlacementOrigin.X); Near(v.Y, y - context.PlacementOrigin.Y); } + })); + await Check("global geographic imagery projects only the selected window with correct UVs",()=>Sync(()=> + { + var g=new RasterGeoTransform(1,0,-180,0,-1,90);var bounds=Lod2AtomFeedReader.TransformBounds(new("EPSG:4326",new(13.3,52.4,13.4,52.5)),"EPSG:3857");var context=Lod2AtomChecks.Context("EPSG:3857",bounds); + var info=new GeoReferencedRasterInfo("global","EPSG:4326",360,180,new(-180,-90,180,90),0){GeoTransform=g};var mesh=RasterImageMeshBuilder.Build(info,context);Require(mesh.Faces.Count>0&&mesh.CoverageRatio<0.001,"Global image was not windowed."); + foreach(var v in mesh.Vertices){var p=g.At(v.U*360,(1-v.V)*180);SpatialReferenceTransform.TryTransformXY("EPSG:4326","EPSG:3857",p.X,p.Y,out var x,out var y);Near(v.X,x-context.PlacementOrigin.X);Near(v.Y,y-context.PlacementOrigin.Y);} + })); + await Check("sidecar edits invalidate cached metadata even with unchanged raster", () => Sync(() => + { var old = GeoTiffInfoCache.GetOrRead(png); File.WriteAllText(Path.ChangeExtension(png, ".pgw"), "2\n1\n0.5\n-3\n110\n200\n"); var updated = GeoTiffInfoCache.GetOrRead(png); Near(updated.GeoTransform!.C - old.GeoTransform!.C, 10); })); + await Check("world file without a CRS and singular affine transforms fail", () => Sync(() => + { var prj = Path.ChangeExtension(png, ".prj"); File.Move(prj, prj + ".off"); Fails(() => GeoTiffReader.ReadImageInfo(png)); File.Move(prj + ".off", prj); Fails(() => new RasterGeoTransform(1, 2, 0, 2, 4, 0).Validate()); })); + await Check("terrain uses affine pixel centers including rotated rows", () => Sync(() => + { var g = new RasterGeoTransform(2, 1, 100, 1, -2, 200); var raster = new TerrainRasterData("test", "EPSG:25833", 2, 2, new(0, 0), new(1, 0), new(0, -1), null, new float[] { 1, 2, 3, 4 }) { GeoTransform = g }; var mesh = TerrainMeshBuilder.Build(raster, g.Bounds(2, 2), Lod2AtomChecks.Context("EPSG:25833", g.Bounds(2, 2)), 2, 0); Near(mesh.Vertices[0].X, g.At(.5, .5).X - 100); Near(mesh.Vertices[2].Y, g.At(.5, 1.5).Y - 196); })); + await Check("embedded affine GeoTIFF and PixelIsPoint retain centers and projected CRS", () => Sync(() => + { + foreach (var point in new[] { false, true }) + { + var path = Path.Combine(directory, "affine-" + point + ".tif"); WriteAffineTiff(path, point); + var info = GeoTiffReader.ReadImageInfo(path); Require(info.SrsName == "EPSG:25833", "Geographic key overrode projected CRS."); + Near(info.GeoTransform!.At(.5, .5).X, point ? 100 : 101.5); Near(info.GeoTransform.At(.5, .5).Y, point ? 200 : 199.5); + var raster = TerrainRasterReader.ReadRaster(path, "test", info.SrsName); Near(raster.Origin.X, point ? 100 : 101.5); Require(raster.Elevations[3] == 4, "TIFF samples changed."); + } + })); + await Check("small geographic affine resolutions remain invertible", () => Sync(() => new RasterGeoTransform(1e-8, 0, 13, 0, -1e-8, 52).Validate())); + var caps = WmtsClient.ReadCapabilities(XDocument.Parse(Capabilities()), new("https://fixture.test/wmts.xml")); var plan = WmtsClient.Plan(caps, "ortho", Lod2AtomChecks.Context("EPSG:25833", new(0, 0, 4, 4))); + await Check("projected WMTS selects exact tile ranges and native mosaic transform", () => Sync(() => + { Require(plan.Tiles.Count == 4 && plan.Width == 4 && plan.Height == 4, "Incorrect tile range."); Near(plan.Transform.At(0, 0).Y, 4); Near(plan.Transform.At(4, 4).X, 4); Require(plan.Tiles[3].Url.Contains("TileRow=1") && plan.Tiles[3].Url.Contains("TileCol=1"), "Wrong KVP tile indices."); })); + await Check("geographic WMTS top-left respects authority axis order", () => Sync(() => + { var c = WmtsClient.ReadCapabilities(XDocument.Parse(Capabilities("urn:ogc:def:crs:EPSG::4326", "90 -180", "279541132.0143589")), new("https://fixture.test/wmts.xml")); Near(c.Sets["native"].Matrices[0].TopLeft.X, -180); Near(c.Sets["native"].Matrices[0].TopLeft.Y, 90); })); + await Check("REST tile templates use default dimensions and URL-escaped identifiers", () => Sync(() => + { var c = WmtsClient.ReadCapabilities(XDocument.Parse(Capabilities(extra: "Time2024")), new("https://fixture.test/wmts.xml")); var p = WmtsClient.Plan(c, "ortho", Lod2AtomChecks.Context("EPSG:25833", new(0, 0, 4, 4))); Require(p.Tiles[0].Url == "https://fixture.test/tiles/2024/level/0/0.png", "REST template not resolved."); })); + await Check("matrix limits exclude unadvertised tiles and report clipped coverage", () => Sync(() => + { var text = Capabilities().Replace("", "level0001"); var p = WmtsClient.Plan(WmtsClient.ReadCapabilities(XDocument.Parse(text), new("https://fixture.test/wmts.xml")), "ortho", Lod2AtomChecks.Context("EPSG:25833", new(0, 0, 4, 4))); Require(p.Tiles.Count == 2 && p.CoverageClipped, "Limits ignored."); })); + await Check("unknown CRS, oversized selections and ambiguous dimensions fail explicitly", () => Sync(() => + { + var unsupported = WmtsClient.ReadCapabilities(XDocument.Parse(Capabilities("EPSG:999999")), new("https://fixture.test/wmts.xml")); Fails(() => WmtsClient.Plan(unsupported, "ortho", Lod2AtomChecks.Context("EPSG:25833", new(0, 0, 4, 4)))); + var wide = WmtsClient.ReadCapabilities(XDocument.Parse(Capabilities().Replace("2", "100").Replace("2", "100")), new("https://fixture.test/wmts.xml")); Fails(() => WmtsClient.Plan(wide, "ortho", Lod2AtomChecks.Context("EPSG:25833", new(0, -196, 200, 4)))); + var dim = WmtsClient.ReadCapabilities(XDocument.Parse(Capabilities(extra: "Time20232024")), new("https://fixture.test/wmts.xml")); Fails(() => WmtsClient.Plan(dim, "ortho", Lod2AtomChecks.Context("EPSG:25833", new(0, 0, 4, 4)))); + })); + var tile = Path.Combine(directory, "tile.png"); RasterImageCodec.WritePng(tile, 2, 2, Enumerable.Repeat(new byte[] { 12, 34, 56, 128 }, 4).SelectMany(x => x).ToArray()); var tileBytes = File.ReadAllBytes(tile); + await Check("WMTS PNG mosaic preserves decoded pixels/alpha and reuses completed cache", async () => + { var handler = new Handler(_ => new(HttpStatusCode.OK) { Content = new ByteArrayContent(tileBytes) }); using var http = new HttpClient(handler); var client = new WmtsClient(http, Path.Combine(directory, "wmts")); var first = await client.DownloadAsync(plan); var pixels = RasterImageCodec.DecodeRgba(File.ReadAllBytes(first.LocalFilePath), 4, 4); Require(pixels[0] == 12 && pixels[^1] == 128, "Mosaic pixel or alpha corrupted."); var second = await client.DownloadAsync(plan); Require(second.UsedCache && handler.Requests.Count == 4, "Completed mosaic was not reused."); }); + await Check("corrupt WMTS cache is reloaded and cancellation publishes no mosaic", async () => + { + var folder = Path.Combine(directory, "corruption"); var handler = new Handler(_ => new(HttpStatusCode.OK) { Content = new ByteArrayContent(tileBytes) }); using var http = new HttpClient(handler); var client = new WmtsClient(http, folder); var first = await client.DownloadAsync(plan); + File.AppendAllText(first.LocalFilePath, "corrupt"); var second = await client.DownloadAsync(plan); Require(!second.UsedCache && handler.Requests.Count == 8, "Corrupt mosaic cache accepted."); + using var token = new CancellationTokenSource(); token.Cancel(); try { await new WmtsClient(http, Path.Combine(directory, "cancelled")).DownloadAsync(plan, token.Token); throw new Exception("Cancellation ignored."); } catch (OperationCanceledException) { } + }); + await Check("tile row/column order is preserved in the mosaic", async () => + { + var images = new Dictionary(); + for (var row = 0; row < 2; row++) for (var col = 0; col < 2; col++) + { var file = Path.Combine(directory, $"tile-{row}-{col}.png"); RasterImageCodec.WritePng(file, 2, 2, Enumerable.Repeat(new byte[] { (byte)(row * 100 + col * 20), 0, 0, 255 }, 4).SelectMany(x => x).ToArray()); images[$"{row},{col}"] = File.ReadAllBytes(file); } + using var http = new HttpClient(new Handler(r => + { var q = r.RequestUri!.Query; var row = q.Contains("TileRow=1") ? 1 : 0; var col = q.Contains("TileCol=1") ? 1 : 0; return new(HttpStatusCode.OK) { Content = new ByteArrayContent(images[$"{row},{col}"]) }; })); + var image = await new WmtsClient(http, Path.Combine(directory, "ordering")).DownloadAsync(plan); var pixels = RasterImageCodec.DecodeRgba(File.ReadAllBytes(image.LocalFilePath), 4, 4); + Require(pixels[0] == 0 && pixels[3 * 4] == 20 && pixels[(3 * 4) * 4] == 100 && pixels[(3 * 4 + 3) * 4] == 120, "Mosaic tiles were flipped or transposed."); + }); + await Check("failed required WMTS tile does not publish a partial mosaic", async () => + { var folder = Path.Combine(directory, "failure"); using var http = new HttpClient(new Handler(r => r.RequestUri!.Query.Contains("TileRow=1") ? new(HttpStatusCode.NotFound) : new(HttpStatusCode.OK) { Content = new ByteArrayContent(tileBytes) })); try { await new WmtsClient(http, folder).DownloadAsync(plan); throw new Exception("Partial mosaic accepted."); } catch (HttpRequestException) { } Require(!Directory.EnumerateFiles(folder).Any(), "Partial output published."); }); + await Check("WMTS reference reads capabilities without requesting tile geometry", async () => + { var handler = new Handler(_ => new(HttpStatusCode.OK) { Content = new StringContent(Capabilities()) }); using var http = new HttpClient(handler); var metadata = await new RemoteReferenceMetadataReader(http).TryReadAsync("https://fixture.test/wmts.xml", "ortho"); Require(metadata?.SourceKind == "WMTS" && handler.Requests.Count == 1, "Reference fetched tiles."); }); + Console.WriteLine($"Medium compatibility checks passed: {_count}"); + } + finally { Directory.Delete(directory, true); } + } + private static void WriteAffineTiff(string path, bool point) + { + using var writer = new BinaryWriter(File.Create(path)); + const uint image = 146, matrix = 150, keys = 278; + writer.Write((ushort)0x4949); writer.Write((ushort)42); writer.Write((uint)8); writer.Write((ushort)11); + void Entry(ushort tag, ushort type, uint count, uint value) { writer.Write(tag); writer.Write(type); writer.Write(count); writer.Write(value); } + Entry(256, 3, 1, 2); Entry(257, 3, 1, 2); Entry(258, 3, 1, 8); Entry(259, 3, 1, 1); Entry(262, 3, 1, 1); Entry(273, 4, 1, image); Entry(277, 3, 1, 1); Entry(278, 4, 1, 2); Entry(279, 4, 1, 4); Entry(34264, 12, 16, matrix); Entry(34735, 3, 16, keys); writer.Write((uint)0); + writer.Write(new byte[] { 1, 2, 3, 4 }); foreach (var v in new double[] { 2, 1, 0, 100, 1, -2, 0, 200, 0, 0, 1, 0, 0, 0, 0, 1 }) writer.Write(v); + foreach (var v in new ushort[] { 1, 1, 0, 3, 1025, 0, 1, (ushort)(point ? 2 : 1), 2048, 0, 1, 4326, 3072, 0, 1, 25833 }) writer.Write(v); + } + +} diff --git a/sandbox/MediumCompatibilityProbe.cs b/sandbox/MediumCompatibilityProbe.cs new file mode 100644 index 0000000..d478ad0 --- /dev/null +++ b/sandbox/MediumCompatibilityProbe.cs @@ -0,0 +1,71 @@ +using System.Globalization; +using RhinoSpatial.Core; + +namespace RhinoSpatial.Sandbox; + +internal static class MediumCompatibilityProbe +{ + public static async Task RunAsync(string[] args) + { + if (args.Length < 3) throw new ArgumentException("medium-probe wmts [layer srs minX,minY,maxX,maxY] | cityjson | raster "); + if (args[1] == "examples") { WriteExamples(args[2]); return; } + if (args[1] == "wmts") + { + var client = new WmtsClient(); var caps = await client.TryLoadCapabilitiesAsync(args[2]) ?? throw new Exception("Not WMTS."); + Console.WriteLine($"Layers: {caps.Layers.Count}; supported matrix sets: {caps.Sets.Count}"); + foreach (var layer in caps.Layers.Take(8)) Console.WriteLine($"{layer.Id} | {layer.Title} | {string.Join(",", layer.Links.Select(l => l.Set))}"); + if (args.Length < 6) return; + var values = args[5].Split(',').Select(s => double.Parse(s, CultureInfo.InvariantCulture)).ToArray(); + var context = Lod2AtomChecks.Context(args[4], new(values[0], values[1], values[2], values[3])); + var plan = WmtsClient.Plan(caps, args[3], context); var result = await client.DownloadAsync(plan); + var mesh = RasterImageMeshBuilder.Build(result.Raster, context); + Console.WriteLine(result.Status); Console.WriteLine($"Image: {result.LocalFilePath}; clipped mesh: {mesh.Vertices.Count} vertices, {mesh.Faces.Count} triangles"); + } + else if (args[1] == "cityjson") + { + using var metadata = File.OpenRead(args[2]); var m = CityJsonReader.ReadSourceMetadata(metadata); + using var input = File.OpenRead(args[2]); var buildings = CityJsonReader.ReadBuildings(input, "probe"); + Console.WriteLine($"{m.SrsName}; buildings: {buildings.Count}; surfaces: {buildings.Sum(b => b.Surfaces.Count)}"); + Console.WriteLine("Selected LoDs: " + string.Join(", ", buildings.Select(b => b.Attributes["source_lod"]).Distinct())); + } + else if (args[1] == "raster") + { + var info = GeoTiffReader.ReadImageInfo(args[2]); var context = Lod2AtomChecks.Context(info.SrsName, info.BoundingBox); + var mesh = RasterImageMeshBuilder.Build(info, context); + Console.WriteLine($"{info.SrsName}; {info.Width}x{info.Height}; affine: {info.GeoTransform}; mesh: {mesh.Faces.Count} triangles"); + } + else throw new ArgumentException("Unknown medium probe mode."); + } + private static void WriteExamples(string directory) + { + Directory.CreateDirectory(directory); + var faces = new[] { new[] { 0, 3, 2, 1 }, new[] { 0, 1, 5, 4 }, new[] { 1, 2, 6, 5 }, new[] { 2, 3, 7, 6 }, new[] { 3, 0, 4, 7 }, new[] { 4, 5, 6, 7 } }; + object Boundaries(int offset) => new[] { faces.Select(face => new[] { face.Select(i => i + offset).ToArray() }).ToArray() }; + object Geometry(string lod, int offset) => new { type = "Solid", lod, boundaries = Boundaries(offset) }; + var vertices = new[] { new[] { 0, 0, 0 }, new[] { 8, 0, 0 }, new[] { 8, 6, 0 }, new[] { 0, 6, 0 }, new[] { 0, 0, 10 }, new[] { 8, 0, 10 }, new[] { 8, 6, 10 }, new[] { 0, 6, 10 }, new[] { 0, 0, 0 }, new[] { 8, 0, 0 }, new[] { 8, 6, 0 }, new[] { 0, 6, 0 }, new[] { 0, 0, 12 }, new[] { 8, 0, 12 }, new[] { 8, 6, 14 }, new[] { 0, 6, 14 } }; + var objects = new Dictionary { { "example-building", new { type = "Building", geometry = new[] { Geometry("1", 0), Geometry("2.2", 8) } } } }; + var metadata = new { referenceSystem = "https://www.opengis.net/def/crs/EPSG/0/25833", geographicalExtent = new[] { 391300d, 5820100d, 32d, 391308d, 5820106d, 46d } }; + var transform = new { scale = new[] { 1, 1, 1 }, translate = new[] { 391300, 5820100, 32 } }; + var model = new { type = "CityJSON", version = "2.0", metadata, transform, CityObjects = objects, vertices }; + var options = new System.Text.Json.JsonSerializerOptions { WriteIndented = true }; + void Json(string name, object value) => File.WriteAllText(Path.Combine(directory, name), System.Text.Json.JsonSerializer.Serialize(value, options) + "\n"); + Json("building-lods.city.json", model); + var header = new { type = "CityJSON", version = "2.0", metadata, transform, CityObjects = new Dictionary(), vertices = Array.Empty() }; + var feature = new { type = "CityJSONFeature", id = "example-building", CityObjects = objects, vertices }; + File.WriteAllText(Path.Combine(directory, "building-sequence.city.jsonl"), System.Text.Json.JsonSerializer.Serialize(header) + "\n" + System.Text.Json.JsonSerializer.Serialize(feature) + "\n"); + var instance = new { type = "GeometryInstance", template = 0, boundaries = new[] { 0 }, transformationMatrix = new[] { 8, 0, 0, 0, 0, 6, 0, 0, 0, 0, 12, 0, 0, 0, 0, 1 } }; + var template = new Dictionary(); + template["type"] = "CityJSON"; template["version"] = "2.0"; template["metadata"] = metadata; + template["CityObjects"] = new Dictionary { { "template-building", new { type = "Building", geometry = new[] { instance } } } }; + template["vertices"] = new[] { new[] { 391300, 5820100, 32 } }; + template["geometry-templates"] = new Dictionary { { "templates", new[] { Geometry("2.2", 0) } }, { "vertices-templates", new[] { new[] { 0, 0, 0 }, new[] { 1, 0, 0 }, new[] { 1, 1, 0 }, new[] { 0, 1, 0 }, new[] { 0, 0, 1 }, new[] { 1, 0, 1 }, new[] { 1, 1, 1 }, new[] { 0, 1, 1 } } } }; + Json("building-template.city.json", template); + var pixels = new byte[4 * 4 * 4]; + for (var row = 0; row < 4; row++) for (var col = 0; col < 4; col++) { var i = (row * 4 + col) * 4; pixels[i] = (byte)(col < 2 ? 240 : 40); pixels[i + 1] = (byte)(row < 2 ? 180 : 40); pixels[i + 2] = (byte)(col < 2 ? 40 : 220); pixels[i + 3] = 255; } + RasterImageCodec.WritePng(Path.Combine(directory, "rotated-raster.png"), 4, 4, pixels); + File.WriteAllText(Path.Combine(directory, "rotated-raster.pgw"), "2\n0.5\n1\n-2\n391300\n5820106\n"); + File.WriteAllText(Path.Combine(directory, "rotated-raster.prj"), MediumCompatibilityChecks.Wkt + "\n"); + Console.WriteLine("Wrote synthetic EPSG:25833 examples to " + Path.GetFullPath(directory)); + } + +} diff --git a/sandbox/Program.cs b/sandbox/Program.cs index d8e9772..08aecc4 100644 --- a/sandbox/Program.cs +++ b/sandbox/Program.cs @@ -9,6 +9,24 @@ internal class Program { static async Task Main(string[] args) { + if(args.Length>0&&args[0]=="low-compatibility-check"){await LowCompatibilityChecks.RunAsync();return;} + if(args.Length>2&&args[0]=="low-probe"&&args[1]=="examples"){LowCompatibilityFixtures.WriteExamples(args[2]);return;} + if(args.Length>0&&args[0]=="low-probe"){await LowCompatibilityProbe.RunAsync(args);return;} + if (args.Length > 0 && args[0].Equals("compatibility-probe", StringComparison.OrdinalIgnoreCase)) + { + try { await SourceCompatibilityProbe.RunAsync(args); } + catch (Exception ex) { Console.Error.WriteLine(ex.Message); Environment.ExitCode = 1; } + return; + } + if (args.Length > 0 && args[0].Equals("medium-probe", StringComparison.OrdinalIgnoreCase)) + { await MediumCompatibilityProbe.RunAsync(args); return; } + if (args.Length > 0 && args[0].Equals("medium-compatibility-check", StringComparison.OrdinalIgnoreCase)) + { await MediumCompatibilityChecks.RunAsync(); return; } + if (args.Length > 0 && args[0].Equals("source-compatibility-check", StringComparison.OrdinalIgnoreCase)) + { + await SourceCompatibilityChecks.RunAsync(); + return; + } if (args.Length > 0 && string.Equals(args[0], "lod2-atom-check", StringComparison.OrdinalIgnoreCase)) { await Lod2AtomChecks.RunAsync(); @@ -1206,7 +1224,9 @@ private static async Task RunOgcApiFeaturesAsync(string[] args) : 0; var startedAt = DateTime.UtcNow; var client = new OgcApiFeaturesClient(); - var features = await client.LoadFeaturesAsync(args[1], "ogcapi", bbox, maxFeatures); + var result = await client.LoadFeaturesWithStatusAsync(args[1], "ogcapi", bbox, maxFeatures); + var features = result.Features; + Console.WriteLine(result.StatusNote); var elapsed = DateTime.UtcNow - startedAt; Console.WriteLine("OGC API Features probe"); diff --git a/sandbox/SourceCompatibilityChecks.cs b/sandbox/SourceCompatibilityChecks.cs new file mode 100644 index 0000000..b3d210b --- /dev/null +++ b/sandbox/SourceCompatibilityChecks.cs @@ -0,0 +1,260 @@ +using System.Net; +using System.IO.Compression; +using System.Text; +using System.Text.Json; +using RhinoSpatial.Core; + +namespace RhinoSpatial.Sandbox; + +internal static class SourceCompatibilityChecks +{ + private static int _checks; + internal static void Require(bool condition, string message) + { if (!condition) throw new InvalidOperationException(message); } + private static async Task Check(string label, Func action) + { await action(); _checks++; Console.WriteLine("PASS " + label); } + internal sealed class Handler(Func response) : HttpMessageHandler + { + public List Requests { get; } = new(); + public Dictionary Payloads { get; } = new(); + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + Requests.Add(request.RequestUri!); + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { Content = Payloads.TryGetValue(request.RequestUri!.AbsolutePath, out var bytes) ? new ByteArrayContent(bytes) : new StringContent(response(request.RequestUri!), Encoding.UTF8, "application/json"), RequestMessage = request }); + } + } + private static string Feature(int id) => "{\"type\":\"Feature\",\"id\":" + id + ",\"properties\":{},\"geometry\":{\"type\":\"Point\",\"coordinates\":[13,52]}}"; + private static string Page(int[] ids, string? next = null, int? total = null) => + "{\"type\":\"FeatureCollection\",\"features\":[" + string.Join(",", ids.Select(Feature)) + "]" + + (total.HasValue ? ",\"numberMatched\":" + total : "") + + (next is null ? "" : ",\"links\":[{\"rel\":\"next\",\"href\":" + JsonSerializer.Serialize(next) + "}]") + "}"; + + public static async Task RunAsync() + { + await Check("JSON pages resolve relative next links and deduplicate IDs", async () => + { + var pages = new Dictionary { ["https://fixture.test/items"] = Page(new[]{1,2},"?page=2"), ["https://fixture.test/items?page=2"] = Page(new[]{2,3}) }; + var result = await FeatureCollectionPager.LoadAsync("https://fixture.test/items", (url,_) => Task.FromResult(pages[url]),0); + Require(result.PageCount == 2 && GeoJsonReader.ReadFeatures(result.Text,"test").Count == 3,"Wrong merged page result."); + }); + await Check("user feature cap stops before the next request", async () => + { + var calls=0; + var result = await FeatureCollectionPager.LoadAsync("https://fixture.test/items",(_,_)=>{calls++;return Task.FromResult(Page(new[]{1,2},"?page=2"));},1); + Require(calls==1 && GeoJsonReader.ReadFeatures(result.Text,"test").Count==1 && result.StatusNote.Contains("Max Features"),"Cap was not applied."); + }); + await Check("repeated pages fail instead of returning a partial selection", async () => + { + try { await FeatureCollectionPager.LoadAsync("https://fixture.test/items",(_,_)=>Task.FromResult(Page(new[]{1},"?page=2")),0); } + catch(InvalidDataException){return;} throw new Exception("Repeated page accepted."); + }); + await Check("missing next link for a known incomplete collection fails", async () => + { + try { await FeatureCollectionPager.LoadAsync("https://fixture.test/items",(_,_)=>Task.FromResult(Page(new[]{1},total:2)),0); } + catch(InvalidDataException){return;} throw new Exception("Incomplete result accepted."); + }); + await Check("cancellation propagates", async () => + { + using var cancel=new CancellationTokenSource();cancel.Cancel(); + try {await FeatureCollectionPager.LoadAsync("https://fixture.test/items",(_,_)=>Task.FromResult(Page(new[]{1})),0,cancellationToken:cancel.Token);} + catch(OperationCanceledException){return;}throw new Exception("Cancellation ignored."); + }); + await Check("WFS 2 pages advance by the server's actual returned count", async () => + { + var handler=new Handler(url=>{ + var query=url.Query; + return query.Contains("STARTINDEX=2")?Page(new[]{3},total:3):query.Contains("STARTINDEX=1")?Page(new[]{2},total:3):Page(new[]{1},total:3); + }); + using var http=new HttpClient(handler); + var client=new WfsClient(http); + var result=await client.LoadFeaturesWithStatusAsync(new WfsRequestOptions{BaseUrl="https://fixture.test/wfs",GetFeatureBaseUrl="https://fixture.test/wfs",TypeName="buildings",Version="2.0.0"}); + Require(result.Features.Count==3 && handler.Requests.Count==3,"WFS paging failed."); + }); + await Check("GML pages preserve feature IDs, namespaces and geometry", async () => + { + string Gml(int id,string next="")=>"13 52"; + var result=await FeatureCollectionPager.LoadAsync("https://fixture.test/wfs",(url,_)=>Task.FromResult(url.Contains("page=2")?Gml(2):Gml(1,"next='?page=2'")),0); + var features=GmlReader.ReadFeatures(result.Text,"test"); + Require(features.Count==2 && features[1].Id=="f2" && features[1].Geometry.Points.Count==1,"GML merge lost geometry."); + }); + await Check("landing page follows data and items links without fetching features", async () => + { + var handler=new Handler(url=>url.AbsolutePath=="/api/"?"{\"links\":[{\"rel\":\"data\",\"href\":\"datasets\"}]}":"{\"collections\":[{\"id\":\"buildings\",\"title\":\"Buildings\",\"links\":[{\"rel\":\"items\",\"href\":\"features/buildings\",\"type\":\"application/geo+json\"}],\"extent\":{\"spatial\":{\"bbox\":[[13,52,14,53]]}}}]}"); + using var http=new HttpClient(handler);var client=new OgcApiFeaturesClient(http); + var result=await client.TryDiscoverAsync("https://fixture.test/api/"); + Require(result?.Count==1 && result[0].ItemsUrl=="https://fixture.test/api/features/buildings" && result[0].Wgs84Bounds?.MinX==13 && handler.Requests.Count==2,"Discovery followed wrong links."); + }); + await Check("OGC items URL is not requested until a bounding box is attached", async () => + { + var handler=new Handler(url=> { Require(url.Query.Contains("bbox="),"Unbounded items probe.");return Page(new[]{1}); }); + using var http=new HttpClient(handler);var client=new OgcApiFeaturesClient(http); + var result=await client.LoadFeaturesWithStatusAsync("https://fixture.test/collections/a/items","a",new BoundingBox2D(12,51,14,53),5); + Require(result.Features.Count==1 && handler.Requests.Count==1,"Unexpected items discovery request."); + }); + await Check("feature total is retained when later pages omit it", async () => + { + var calls = 0; + try { await FeatureCollectionPager.LoadAsync("https://fixture.test/items", (_,_) => Task.FromResult(++calls == 1 ? Page(new[]{1}, "?page=2", 3) : Page(new[]{2})), 0); } + catch (InvalidDataException) { return; } + throw new Exception("An incomplete later page hid the original matching total."); + }); + await Check("collection fallback paths retain query parameters correctly", async () => + { + using var http = new HttpClient(new Handler(_ => "{\"collections\":[{\"id\":\"a\"}]}")); + var collections = await new OgcApiFeaturesClient(http).TryDiscoverAsync("https://fixture.test/collections?f=json"); + Require(collections![0].ItemsUrl == "https://fixture.test/collections/a/items?f=json", "Collection path was appended to a query string."); + }); + await Check("single GeoJSON Feature downloads use the ordinary reader", () => + { + Require(GeoJsonReader.ReadFeatures(Feature(1), "download").Count == 1, "Single Feature unsupported."); + return Task.CompletedTask; + }); + await Check("relative paging links resolve against the final redirected response URL", async () => + { + var result = await FeatureCollectionPager.LoadHttpAsync("https://fixture.test/old", + (url,_) => Task.FromResult(url.Contains("page2") + ? new FeaturePageContent(Page(new[]{2}), new Uri(url)) + : new FeaturePageContent(Page(new[]{1},"page2"),new Uri("https://fixture.test/new/page1"))),0); + Require(result.PageCount == 2, "Redirected paging did not resolve."); + }); + await Check("point and wrapped collection extents do not break API discovery", async () => + { + using var http = new HttpClient(new Handler(_=>"{\"collections\":[{\"id\":\"point\",\"extent\":{\"spatial\":{\"bbox\":[[13,52,13,52]]}}},{\"id\":\"wrapped\",\"extent\":{\"spatial\":{\"bbox\":[[177,-20,-178,-15]]}}}]}")); + var collections = await new OgcApiFeaturesClient(http).TryDiscoverAsync("https://fixture.test/collections"); + Require(collections?.Count == 2 && collections[1].Wgs84Bounds?.MinX == -180,"Valid collection extent rejected."); + }); + await Check("identical anonymous records remain separate features", async () => + { + var body = Page(new[]{1,1},total:2).Replace("\"id\":1,", ""); + var result = await FeatureCollectionPager.LoadAsync("https://fixture.test/items", (_,_)=>Task.FromResult(body),0); + Require(GeoJsonReader.ReadFeatures(result.Text,"test").Count == 2 && result.StatusNote.Contains("without stable IDs"),"Anonymous records were silently collapsed."); + }); + await Check("GeoJSON pages cannot silently mix coordinate systems", async () => + { + var first = Page(new[]{1},"?page=2"); + var second = Page(new[]{2}).Replace("\"features\":", "\"crs\":{\"type\":\"name\",\"properties\":{\"name\":\"EPSG:3857\"}},\"features\":"); + try { await FeatureCollectionPager.LoadAsync("https://fixture.test/items", (url,_)=>Task.FromResult(url.Contains("page=2")?second:first),0); throw new Exception("Mixed CRS accepted."); } + catch (InvalidDataException) { } + }); + await RunDownloadAndTerrainChecksAsync(); + Console.WriteLine($"Source compatibility checks passed: {_checks}"); + } + private static byte[] Zip(params (string Name, string Text)[] files) + { + using var output = new MemoryStream(); + using (var archive = new ZipArchive(output, ZipArchiveMode.Create, true)) + foreach (var file in files) { using var writer = new StreamWriter(archive.CreateEntry(file.Name).Open()); writer.Write(file.Text); } + return output.ToArray(); + } + + private static async Task RunDownloadAndTerrainChecksAsync() + { + var directory = Path.Combine(Path.GetTempPath(), "rhinospatial-compatibility-" + Guid.NewGuid().ToString("N")); + try + { + var handler = new Handler(_ => throw new Exception("Unexpected network request.")); + using var http = new HttpClient(handler); + var vector = new SourceDownloadCache(AtomDownloadKind.Vector, http, directory); + var link = new Lod2AtomLink(new Uri("https://fixture.test/download"), "vector", "", null); + handler.Payloads["/download"] = Encoding.UTF8.GetBytes(Page(new[]{1,2})); + await Check("extensionless downloads detect GeoJSON content and reuse validated cache", async () => + { + var first = await vector.DownloadAsync(link); + var second = await new SourceDownloadCache(AtomDownloadKind.Vector, http, directory).DownloadAsync(link); + Require(first.FilePath.EndsWith(".geojson") && second.UsedCache && handler.Requests.Count == 1, "Direct-file cache failed."); + }); + await Check("projected Shapefile CRS uses outer WKT authority, not ellipsoid EPSG", () => + { + var shape = Path.Combine(directory,"projection.shp"); File.WriteAllBytes(shape,Array.Empty()); + const string wkt = "PROJCS[\"ETRS89 / UTM zone 33N\",GEOGCS[\"ETRS89\",DATUM[\"European_Terrestrial_Reference_System_1989\",SPHEROID[\"GRS 1980\",6378137,298.257222101,AUTHORITY[\"EPSG\",\"7019\"]],AUTHORITY[\"EPSG\",\"6258\"]],PRIMEM[\"Greenwich\",0],UNIT[\"degree\",0.0174532925199433],AUTHORITY[\"EPSG\",\"4258\"]],PROJECTION[\"Transverse_Mercator\"],PARAMETER[\"latitude_of_origin\",0],PARAMETER[\"central_meridian\",15],PARAMETER[\"scale_factor\",0.9996],PARAMETER[\"false_easting\",500000],PARAMETER[\"false_northing\",0],UNIT[\"metre\",1],AUTHORITY[\"EPSG\",\"25833\"]]"; + File.WriteAllText(Path.ChangeExtension(shape,".prj"),wkt); + Require(ShapefileFeatureReader.ReadSourceMetadata(shape).SourceSrs == "EPSG:25833","Nested EPSG authority used as the source CRS."); + return Task.CompletedTask; + }); + await Check("uppercase Shapefile companions normalize within temporary extraction", () => + { + var archive = Path.Combine(directory,"uppercase.zip"); + File.WriteAllBytes(archive,Zip(("A.SHP","shape"),("A.DBF","dbf"),("A.SHX","shx"),("A.PRJ","prj"))); + using var lease = SourceFilesLease.Open(archive); + var shape = lease.Files.Single(f=>f.EndsWith(".SHP")); + Require(lease.Files.Contains(Path.ChangeExtension(shape,".prj")),"PRJ case was not normalized."); + return Task.CompletedTask; + }); + await Check("archive traversal is rejected before extraction", async () => + { + handler.Payloads["/unsafe.zip"] = Zip(("../outside.geojson",Page(new[]{1}))); + try { await vector.DownloadAsync(link with { Url = new Uri("https://fixture.test/unsafe.zip") }); } + catch (InvalidDataException) { return; } throw new Exception("Unsafe ZIP accepted."); + }); + await Check("Shapefile download requires companion files", async () => + { + handler.Payloads["/shape.zip"] = Zip(("a.shp","not a shapefile"),("a.dbf","dbf")); + try { await vector.DownloadAsync(link with { Url = new Uri("https://fixture.test/shape.zip") }); } + catch (InvalidDataException) { return; } throw new Exception("Incomplete Shapefile accepted."); + }); + await Check("downloaded CityJSON declares CRS and preserves its source coordinates", async () => + { + const string city = "{\"type\":\"CityJSON\",\"version\":\"2.0\",\"metadata\":{\"referenceSystem\":\"https://www.opengis.net/def/crs/EPSG/0/25833\"},\"CityObjects\":{},\"vertices\":[[391000,5820000,42]]}"; + handler.Payloads["/city.json"] = Encoding.UTF8.GetBytes(city); + var building = new SourceDownloadCache(AtomDownloadKind.Buildings,http,Path.Combine(directory,"buildings")); + var result = await building.DownloadAsync(link with { Url = new Uri("https://fixture.test/city.json") }); + var metadata = CityJsonReader.ReadSourceMetadata(File.ReadAllText(result.FilePath)); + Require(metadata.SrsName == "EPSG:25833" && metadata.BoundingBox?.MinX == 391000, "CityJSON source metadata changed."); + handler.Payloads["/missing.json"] = Encoding.UTF8.GetBytes(city.Replace("\"metadata\":", "\"ignored\":")); + try { await building.DownloadAsync(link with { Url = new Uri("https://fixture.test/missing.json") }); } + catch (InvalidDataException) { return; } throw new Exception("Missing downloaded CRS accepted."); + }); + const string start = ""; + string Entry(string name,int x) => ""+name+""+x+" 5820000"+(x+3)+" 5820003"; + string Grid(int x) + { + var rows = new List(); + for(var y=0;y<3;y++) for(var column=0;column<3;column++) + rows.Add($"{x+column};{5820000+y};{(column == 1 && y == 1 ? "NaN" : "10")}"); + return string.Join("\n",rows); + } + var terrainHandler = new Handler(url => url.AbsolutePath == "/terrain" ? start + Entry("left",391000) + Entry("right",391003) + "" : throw new Exception("Unexpected terrain request.")); + terrainHandler.Payloads["/left.zip"] = Zip(("left.csv",Grid(391000))); + terrainHandler.Payloads["/right.zip"] = Zip(("right.csv",Grid(391003))); + using var terrainHttp = new HttpClient(terrainHandler); + var context = Lod2AtomChecks.Context("EPSG:25833",new BoundingBox2D(391000,5820000,391005,5820002)); + await Check("terrain ATOM joins adjacent CSV grids and preserves NoData holes", async () => + { + var terrain = await new TerrainAtomLoader(terrainHttp,Path.Combine(directory,"terrain")).TryLoadAsync("https://fixture.test/terrain", context); + Require(terrain?.Raster.Width == 6 && terrain.Raster.Height == 3 && float.IsNaN(terrain.Raster.Elevations[7]),"Terrain mosaic or NoData changed."); + var mesh = TerrainMeshBuilder.Build(terrain!.Raster,terrain.Bounds,context,512,0); + Require(mesh.Faces.Count > 0, "Terrain produced no connected surface."); + }); + await Check("terrain mosaics reject conflicting elevations and mismatched grids", () => + { + var raster = new TerrainRasterData("test","EPSG:25833",2,2,new Coordinate2D(0,1),new Coordinate2D(1,0),new Coordinate2D(0,-1),double.NaN,new[]{1f,1f,1f,1f}); + try { TerrainRasterMosaic.Merge(new[]{raster,raster with { Elevations = new[]{2f,2f,2f,2f} }}); } + catch (InvalidDataException) { + try { TerrainRasterMosaic.Merge(new[]{raster,raster with { OffsetVectorX = new Coordinate2D(2,0) }}); } + catch (InvalidDataException) {return Task.CompletedTask;} + } + throw new Exception("Incompatible terrain grids accepted."); + }); + await Check("ATOM reference metadata reads no archives and infers native CRS", async () => + { + var reference = await new RemoteReferenceMetadataReader(terrainHttp).TryReadAsync("https://fixture.test/terrain"); + Require(reference?.SourceKind == "ATOM" && reference.SrsName == "EPSG:25833" && reference.Wgs84BoundingBox is not null,"ATOM reference metadata missing."); + Require(terrainHandler.Requests.Count(r=>r.AbsolutePath.EndsWith(".zip")) == 2,"Reference metadata downloaded geometry."); + }); + await Check("OGC API items reference reads parent collection metadata only", async () => + { + var metadataHandler = new Handler(url => { + Require(url.AbsolutePath == "/collections/a", "Reference requested feature data."); + return "{\"id\":\"a\",\"extent\":{\"spatial\":{\"bbox\":[[13,52,14,53]]}}}"; + }); + using var metadataHttp = new HttpClient(metadataHandler); + var reference = await new RemoteReferenceMetadataReader(metadataHttp).TryReadAsync("https://fixture.test/collections/a/items"); + Require(reference?.Wgs84BoundingBox?.MinX == 13 && metadataHandler.Requests.Count == 1,"Reference did not use the collection extent."); + }); + } + finally { if(Directory.Exists(directory)) Directory.Delete(directory,true); } + } + +} diff --git a/sandbox/SourceCompatibilityProbe.cs b/sandbox/SourceCompatibilityProbe.cs new file mode 100644 index 0000000..ba86ec0 --- /dev/null +++ b/sandbox/SourceCompatibilityProbe.cs @@ -0,0 +1,64 @@ +using System.Globalization; +using RhinoSpatial.Core; + +namespace RhinoSpatial.Sandbox; + +internal static class SourceCompatibilityProbe +{ + public static async Task RunAsync(string[] args) + { + if(args.Length < 3) throw new ArgumentException("Usage: compatibility-probe discover|reference|vector|buildings|terrain [srs bbox]"); + var mode=args[1]; var url=args[2]; + if(mode=="discover") + { + var collections=await new OgcApiFeaturesClient().TryDiscoverAsync(url) ?? throw new Exception("Not an OGC API."); + Console.WriteLine($"Collections: {collections.Count}"); + foreach(var c in collections.Take(12)) Console.WriteLine($"{c.Id} | {c.Title} | {c.ItemsUrl} | {c.Wgs84Bounds}"); + } + else if(mode=="reference") + { + var metadata=await new RemoteReferenceMetadataReader().TryReadAsync(url) ?? throw new Exception("No remote metadata."); + Console.WriteLine(metadata); + } + else if(mode=="terrain") + { + if(args.Length<5) throw new Exception("Terrain probe requires source SRS and minX,minY,maxX,maxY."); + var values=args[4].Split(',').Select(x=>double.Parse(x,CultureInfo.InvariantCulture)).ToArray(); + var context=Lod2AtomChecks.Context(args[3],new BoundingBox2D(values[0],values[1],values[2],values[3])); + var result=await new TerrainAtomLoader().TryLoadAsync(url,context,Console.WriteLine) ?? throw new Exception("Not an ATOM feed."); + var mesh=TerrainMeshBuilder.Build(result.Raster,result.Bounds,context,512,0); + var finite=result.Raster.Elevations.Where(x=>!TerrainMeshBuilder.IsNoData(result.Raster,x)).ToArray(); + Console.WriteLine(result.Status); + Console.WriteLine($"Bounds: {result.Bounds}; mesh: {mesh.Vertices.Count} vertices, {mesh.Faces.Count} faces; heights {finite.Min()}..{finite.Max()}"); + } + else + { + var kind=mode=="buildings"?AtomDownloadKind.Buildings:AtomDownloadKind.Vector; + var result=await new SourceDownloadCache(kind).DownloadAsync(new Lod2AtomLink(new Uri(url),mode,"",null)); + Console.WriteLine($"Validated {(result.UsedCache?"cached":"downloaded")} payload: {new FileInfo(result.FilePath).Length} bytes"); + using var files=SourceFilesLease.Open(result.FilePath); + foreach(var file in files.Files) + { + var extension=Path.GetExtension(file).ToLowerInvariant(); + if(kind==AtomDownloadKind.Buildings && extension is ".cityjson" or ".json") + { + var text=File.ReadAllText(file); var metadata=CityJsonReader.ReadSourceMetadata(text); + Console.WriteLine($"CityJSON: {metadata.SrsName}, bounds {metadata.BoundingBox}, buildings {CityJsonReader.ReadBuildings(text,"probe").Count}"); + } + else if(kind==AtomDownloadKind.Vector && extension is ".geojson" or ".json") + { + var text=File.ReadAllText(file); + Console.WriteLine($"GeoJSON: {GeoJsonReader.ReadFeatures(text,"probe").Count} features, {GeoJsonReader.TryReadSourceSrs(text)??"EPSG:4326"}"); + } + else if(kind==AtomDownloadKind.Vector && extension==".shp") + { + var metadata=ShapefileFeatureReader.ReadSourceMetadata(file); + if(string.IsNullOrWhiteSpace(metadata.SourceSrs) || metadata.BoundingBox is null) throw new Exception("Shapefile metadata unresolved."); + var context=Lod2AtomChecks.Context(metadata.SourceSrs,metadata.BoundingBox); + var read=ShapefileFeatureReader.ReadFeatures(file,"probe",context,3); + Console.WriteLine($"Shapefile: {metadata.SourceSrs}, {read.Features.Count} bounded sample features, bounds {metadata.BoundingBox}"); + } + } + } + } +}